From 5ff4c17907d5b19510a62e08fd8d3b11e62b431d Mon Sep 17 00:00:00 2001 From: Ondřej Surý Date: Tue, 13 Sep 2011 13:13:40 +0200 Subject: Imported Upstream version 60 --- src/pkg/Makefile | 312 + src/pkg/archive/tar/Makefile | 13 + src/pkg/archive/tar/common.go | 75 + src/pkg/archive/tar/reader.go | 221 + src/pkg/archive/tar/reader_test.go | 273 + src/pkg/archive/tar/testdata/gnu.tar | Bin 0 -> 3072 bytes src/pkg/archive/tar/testdata/small.txt | 1 + src/pkg/archive/tar/testdata/small2.txt | 1 + src/pkg/archive/tar/testdata/star.tar | Bin 0 -> 3072 bytes src/pkg/archive/tar/testdata/v7.tar | Bin 0 -> 3584 bytes src/pkg/archive/tar/testdata/writer-big.tar | Bin 0 -> 4096 bytes src/pkg/archive/tar/testdata/writer.tar | Bin 0 -> 3072 bytes src/pkg/archive/tar/writer.go | 205 + src/pkg/archive/tar/writer_test.go | 157 + src/pkg/archive/zip/Makefile | 13 + src/pkg/archive/zip/reader.go | 311 + src/pkg/archive/zip/reader_test.go | 233 + src/pkg/archive/zip/struct.go | 91 + src/pkg/archive/zip/testdata/dd.zip | Bin 0 -> 154 bytes src/pkg/archive/zip/testdata/gophercolor16x16.png | Bin 0 -> 785 bytes src/pkg/archive/zip/testdata/r.zip | Bin 0 -> 440 bytes src/pkg/archive/zip/testdata/readme.notzip | Bin 0 -> 1905 bytes src/pkg/archive/zip/testdata/readme.zip | Bin 0 -> 1885 bytes src/pkg/archive/zip/testdata/test.zip | Bin 0 -> 1170 bytes src/pkg/archive/zip/writer.go | 244 + src/pkg/archive/zip/writer_test.go | 73 + src/pkg/archive/zip/zip_test.go | 57 + src/pkg/asn1/Makefile | 13 + src/pkg/asn1/asn1.go | 840 +++ src/pkg/asn1/asn1_test.go | 689 ++ src/pkg/asn1/common.go | 158 + src/pkg/asn1/marshal.go | 544 ++ src/pkg/asn1/marshal_test.go | 129 + src/pkg/big/Makefile | 18 + src/pkg/big/arith.go | 231 + src/pkg/big/arith_386.s | 265 + src/pkg/big/arith_amd64.s | 263 + src/pkg/big/arith_arm.s | 312 + src/pkg/big/arith_decl.go | 18 + src/pkg/big/arith_test.go | 335 + src/pkg/big/calibrate_test.go | 88 + src/pkg/big/hilbert_test.go | 160 + src/pkg/big/int.go | 868 +++ src/pkg/big/int_test.go | 1391 +++++ src/pkg/big/nat.go | 1272 ++++ src/pkg/big/nat_test.go | 669 ++ src/pkg/big/rat.go | 371 ++ src/pkg/big/rat_test.go | 332 + src/pkg/bufio/Makefile | 11 + src/pkg/bufio/bufio.go | 549 ++ src/pkg/bufio/bufio_test.go | 699 +++ src/pkg/builtin/builtin.go | 135 + src/pkg/bytes/Makefile | 16 + src/pkg/bytes/asm_386.s | 17 + src/pkg/bytes/asm_amd64.s | 92 + src/pkg/bytes/asm_arm.s | 8 + src/pkg/bytes/buffer.go | 348 ++ src/pkg/bytes/buffer_test.go | 376 ++ src/pkg/bytes/bytes.go | 605 ++ src/pkg/bytes/bytes_decl.go | 8 + src/pkg/bytes/bytes_test.go | 858 +++ src/pkg/bytes/export_test.go | 8 + src/pkg/cmath/Makefile | 25 + src/pkg/cmath/abs.go | 12 + src/pkg/cmath/asin.go | 170 + src/pkg/cmath/cmath_test.go | 853 +++ src/pkg/cmath/conj.go | 8 + src/pkg/cmath/exp.go | 55 + src/pkg/cmath/isinf.go | 21 + src/pkg/cmath/isnan.go | 25 + src/pkg/cmath/log.go | 64 + src/pkg/cmath/phase.go | 11 + src/pkg/cmath/polar.go | 12 + src/pkg/cmath/pow.go | 60 + src/pkg/cmath/rect.go | 13 + src/pkg/cmath/sin.go | 132 + src/pkg/cmath/sqrt.go | 103 + src/pkg/cmath/tan.go | 184 + src/pkg/compress/bzip2/Makefile | 14 + src/pkg/compress/bzip2/bit_reader.go | 88 + src/pkg/compress/bzip2/bzip2.go | 390 ++ src/pkg/compress/bzip2/bzip2_test.go | 158 + src/pkg/compress/bzip2/huffman.go | 223 + src/pkg/compress/bzip2/move_to_front.go | 105 + src/pkg/compress/flate/Makefile | 17 + src/pkg/compress/flate/deflate.go | 493 ++ src/pkg/compress/flate/deflate_test.go | 321 + src/pkg/compress/flate/flate_test.go | 139 + src/pkg/compress/flate/huffman_bit_writer.go | 494 ++ src/pkg/compress/flate/huffman_code.go | 378 ++ src/pkg/compress/flate/inflate.go | 708 +++ src/pkg/compress/flate/reverse_bits.go | 48 + src/pkg/compress/flate/token.go | 103 + src/pkg/compress/flate/util.go | 72 + src/pkg/compress/gzip/Makefile | 12 + src/pkg/compress/gzip/gunzip.go | 230 + src/pkg/compress/gzip/gunzip_test.go | 305 + src/pkg/compress/gzip/gzip.go | 187 + src/pkg/compress/gzip/gzip_test.go | 84 + src/pkg/compress/lzw/Makefile | 12 + src/pkg/compress/lzw/reader.go | 253 + src/pkg/compress/lzw/reader_test.go | 145 + src/pkg/compress/lzw/writer.go | 259 + src/pkg/compress/lzw/writer_test.go | 132 + src/pkg/compress/testdata/e.txt | 1 + src/pkg/compress/testdata/pi.txt | 1 + src/pkg/compress/zlib/Makefile | 12 + src/pkg/compress/zlib/reader.go | 126 + src/pkg/compress/zlib/reader_test.go | 128 + src/pkg/compress/zlib/writer.go | 139 + src/pkg/compress/zlib/writer_test.go | 144 + src/pkg/container/heap/Makefile | 11 + src/pkg/container/heap/heap.go | 96 + src/pkg/container/heap/heap_test.go | 158 + src/pkg/container/list/Makefile | 11 + src/pkg/container/list/list.go | 211 + src/pkg/container/list/list_test.go | 209 + src/pkg/container/ring/Makefile | 11 + src/pkg/container/ring/ring.go | 141 + src/pkg/container/ring/ring_test.go | 220 + src/pkg/container/vector/Makefile | 69 + src/pkg/container/vector/defs.go | 43 + src/pkg/container/vector/intvector.go | 188 + src/pkg/container/vector/intvector_test.go | 331 + src/pkg/container/vector/nogen_test.go | 67 + src/pkg/container/vector/numbers_test.go | 123 + src/pkg/container/vector/stringvector.go | 188 + src/pkg/container/vector/stringvector_test.go | 331 + src/pkg/container/vector/vector.go | 188 + src/pkg/container/vector/vector_test.go | 331 + src/pkg/crypto/Makefile | 11 + src/pkg/crypto/aes/Makefile | 13 + src/pkg/crypto/aes/aes_test.go | 350 ++ src/pkg/crypto/aes/block.go | 176 + src/pkg/crypto/aes/cipher.go | 71 + src/pkg/crypto/aes/const.go | 362 ++ src/pkg/crypto/blowfish/Makefile | 13 + src/pkg/crypto/blowfish/block.go | 101 + src/pkg/crypto/blowfish/blowfish_test.go | 192 + src/pkg/crypto/blowfish/cipher.go | 79 + src/pkg/crypto/blowfish/const.go | 199 + src/pkg/crypto/cast5/Makefile | 11 + src/pkg/crypto/cast5/cast5.go | 536 ++ src/pkg/crypto/cast5/cast5_test.go | 104 + src/pkg/crypto/cipher/Makefile | 17 + src/pkg/crypto/cipher/cbc.go | 78 + src/pkg/crypto/cipher/cbc_aes_test.go | 89 + src/pkg/crypto/cipher/cfb.go | 64 + src/pkg/crypto/cipher/cfb_test.go | 35 + src/pkg/crypto/cipher/cipher.go | 63 + src/pkg/crypto/cipher/common_test.go | 28 + src/pkg/crypto/cipher/ctr.go | 55 + src/pkg/crypto/cipher/ctr_aes_test.go | 101 + src/pkg/crypto/cipher/io.go | 57 + src/pkg/crypto/cipher/ocfb.go | 138 + src/pkg/crypto/cipher/ocfb_test.go | 44 + src/pkg/crypto/cipher/ofb.go | 44 + src/pkg/crypto/cipher/ofb_test.go | 101 + src/pkg/crypto/crypto.go | 73 + src/pkg/crypto/des/Makefile | 13 + src/pkg/crypto/des/block.go | 98 + src/pkg/crypto/des/cipher.go | 103 + src/pkg/crypto/des/const.go | 139 + src/pkg/crypto/des/des_test.go | 1497 +++++ src/pkg/crypto/dsa/Makefile | 11 + src/pkg/crypto/dsa/dsa.go | 276 + src/pkg/crypto/dsa/dsa_test.go | 84 + src/pkg/crypto/ecdsa/Makefile | 11 + src/pkg/crypto/ecdsa/ecdsa.go | 149 + src/pkg/crypto/ecdsa/ecdsa_test.go | 227 + src/pkg/crypto/elliptic/Makefile | 11 + src/pkg/crypto/elliptic/elliptic.go | 381 ++ src/pkg/crypto/elliptic/elliptic_test.go | 334 + src/pkg/crypto/hmac/Makefile | 11 + src/pkg/crypto/hmac/hmac.go | 100 + src/pkg/crypto/hmac/hmac_test.go | 205 + src/pkg/crypto/md4/Makefile | 12 + src/pkg/crypto/md4/md4.go | 117 + src/pkg/crypto/md4/md4_test.go | 71 + src/pkg/crypto/md4/md4block.go | 89 + src/pkg/crypto/md5/Makefile | 12 + src/pkg/crypto/md5/md5.go | 117 + src/pkg/crypto/md5/md5_test.go | 71 + src/pkg/crypto/md5/md5block.go | 172 + src/pkg/crypto/ocsp/Makefile | 11 + src/pkg/crypto/ocsp/ocsp.go | 192 + src/pkg/crypto/ocsp/ocsp_test.go | 97 + src/pkg/crypto/openpgp/Makefile | 14 + src/pkg/crypto/openpgp/armor/Makefile | 12 + src/pkg/crypto/openpgp/armor/armor.go | 220 + src/pkg/crypto/openpgp/armor/armor_test.go | 95 + src/pkg/crypto/openpgp/armor/encode.go | 161 + src/pkg/crypto/openpgp/canonical_text.go | 58 + src/pkg/crypto/openpgp/canonical_text_test.go | 49 + src/pkg/crypto/openpgp/elgamal/Makefile | 11 + src/pkg/crypto/openpgp/elgamal/elgamal.go | 122 + src/pkg/crypto/openpgp/elgamal/elgamal_test.go | 49 + src/pkg/crypto/openpgp/error/Makefile | 11 + src/pkg/crypto/openpgp/error/error.go | 64 + src/pkg/crypto/openpgp/keys.go | 545 ++ src/pkg/crypto/openpgp/packet/Makefile | 22 + src/pkg/crypto/openpgp/packet/compressed.go | 39 + src/pkg/crypto/openpgp/packet/compressed_test.go | 41 + src/pkg/crypto/openpgp/packet/encrypted_key.go | 168 + .../crypto/openpgp/packet/encrypted_key_test.go | 126 + src/pkg/crypto/openpgp/packet/literal.go | 90 + .../crypto/openpgp/packet/one_pass_signature.go | 74 + src/pkg/crypto/openpgp/packet/packet.go | 483 ++ src/pkg/crypto/openpgp/packet/packet_test.go | 256 + src/pkg/crypto/openpgp/packet/private_key.go | 301 + src/pkg/crypto/openpgp/packet/private_key_test.go | 57 + src/pkg/crypto/openpgp/packet/public_key.go | 393 ++ src/pkg/crypto/openpgp/packet/public_key_test.go | 98 + src/pkg/crypto/openpgp/packet/reader.go | 63 + src/pkg/crypto/openpgp/packet/signature.go | 558 ++ src/pkg/crypto/openpgp/packet/signature_test.go | 42 + .../openpgp/packet/symmetric_key_encrypted.go | 162 + .../openpgp/packet/symmetric_key_encrypted_test.go | 101 + .../openpgp/packet/symmetrically_encrypted.go | 291 + .../openpgp/packet/symmetrically_encrypted_test.go | 124 + src/pkg/crypto/openpgp/packet/userid.go | 161 + src/pkg/crypto/openpgp/packet/userid_test.go | 87 + src/pkg/crypto/openpgp/read.go | 415 ++ src/pkg/crypto/openpgp/read_test.go | 361 ++ src/pkg/crypto/openpgp/s2k/Makefile | 11 + src/pkg/crypto/openpgp/s2k/s2k.go | 180 + src/pkg/crypto/openpgp/s2k/s2k_test.go | 118 + src/pkg/crypto/openpgp/write.go | 308 + src/pkg/crypto/openpgp/write_test.go | 233 + src/pkg/crypto/rand/Makefile | 30 + src/pkg/crypto/rand/rand.go | 21 + src/pkg/crypto/rand/rand_test.go | 31 + src/pkg/crypto/rand/rand_unix.go | 125 + src/pkg/crypto/rand/rand_windows.go | 43 + src/pkg/crypto/rand/util.go | 80 + src/pkg/crypto/rc4/Makefile | 11 + src/pkg/crypto/rc4/rc4.go | 66 + src/pkg/crypto/rc4/rc4_test.go | 59 + src/pkg/crypto/ripemd160/Makefile | 12 + src/pkg/crypto/ripemd160/ripemd160.go | 118 + src/pkg/crypto/ripemd160/ripemd160_test.go | 64 + src/pkg/crypto/ripemd160/ripemd160block.go | 161 + src/pkg/crypto/rsa/Makefile | 12 + src/pkg/crypto/rsa/pkcs1v15.go | 242 + src/pkg/crypto/rsa/pkcs1v15_test.go | 221 + src/pkg/crypto/rsa/rsa.go | 502 ++ src/pkg/crypto/rsa/rsa_test.go | 348 ++ src/pkg/crypto/sha1/Makefile | 12 + src/pkg/crypto/sha1/sha1.go | 119 + src/pkg/crypto/sha1/sha1_test.go | 73 + src/pkg/crypto/sha1/sha1block.go | 81 + src/pkg/crypto/sha256/Makefile | 12 + src/pkg/crypto/sha256/sha256.go | 166 + src/pkg/crypto/sha256/sha256_test.go | 125 + src/pkg/crypto/sha256/sha256block.go | 129 + src/pkg/crypto/sha512/Makefile | 12 + src/pkg/crypto/sha512/sha512.go | 170 + src/pkg/crypto/sha512/sha512_test.go | 125 + src/pkg/crypto/sha512/sha512block.go | 144 + src/pkg/crypto/subtle/Makefile | 11 + src/pkg/crypto/subtle/constant_time.go | 57 + src/pkg/crypto/subtle/constant_time_test.go | 105 + src/pkg/crypto/tls/Makefile | 20 + src/pkg/crypto/tls/alert.go | 73 + src/pkg/crypto/tls/cipher_suites.go | 102 + src/pkg/crypto/tls/common.go | 267 + src/pkg/crypto/tls/conn.go | 799 +++ src/pkg/crypto/tls/conn_test.go | 52 + src/pkg/crypto/tls/generate_cert.go | 72 + src/pkg/crypto/tls/handshake_client.go | 315 + src/pkg/crypto/tls/handshake_client_test.go | 211 + src/pkg/crypto/tls/handshake_messages.go | 904 +++ src/pkg/crypto/tls/handshake_messages_test.go | 206 + src/pkg/crypto/tls/handshake_server.go | 298 + src/pkg/crypto/tls/handshake_server_test.go | 517 ++ src/pkg/crypto/tls/key_agreement.go | 245 + src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py | 55 + src/pkg/crypto/tls/prf.go | 153 + src/pkg/crypto/tls/prf_test.go | 104 + src/pkg/crypto/tls/tls.go | 181 + src/pkg/crypto/twofish/Makefile | 11 + src/pkg/crypto/twofish/twofish.go | 358 ++ src/pkg/crypto/twofish/twofish_test.go | 129 + src/pkg/crypto/x509/Makefile | 13 + src/pkg/crypto/x509/cert_pool.go | 106 + src/pkg/crypto/x509/pkix/Makefile | 11 + src/pkg/crypto/x509/pkix/pkix.go | 167 + src/pkg/crypto/x509/verify.go | 244 + src/pkg/crypto/x509/verify_test.go | 391 ++ src/pkg/crypto/x509/x509.go | 1073 ++++ src/pkg/crypto/x509/x509_test.go | 431 ++ src/pkg/crypto/xtea/Makefile | 12 + src/pkg/crypto/xtea/block.go | 66 + src/pkg/crypto/xtea/cipher.go | 92 + src/pkg/crypto/xtea/xtea_test.go | 246 + src/pkg/csv/Makefile | 12 + src/pkg/csv/reader.go | 372 ++ src/pkg/csv/reader_test.go | 265 + src/pkg/csv/writer.go | 122 + src/pkg/csv/writer_test.go | 44 + src/pkg/debug/dwarf/Makefile | 16 + src/pkg/debug/dwarf/buf.go | 154 + src/pkg/debug/dwarf/const.go | 433 ++ src/pkg/debug/dwarf/entry.go | 343 + src/pkg/debug/dwarf/open.go | 80 + src/pkg/debug/dwarf/testdata/typedef.c | 79 + src/pkg/debug/dwarf/testdata/typedef.elf | Bin 0 -> 10837 bytes src/pkg/debug/dwarf/testdata/typedef.macho | Bin 0 -> 5256 bytes src/pkg/debug/dwarf/type.go | 584 ++ src/pkg/debug/dwarf/type_test.go | 111 + src/pkg/debug/dwarf/unit.go | 62 + src/pkg/debug/elf/Makefile | 12 + src/pkg/debug/elf/elf.go | 1521 +++++ src/pkg/debug/elf/elf_test.go | 49 + src/pkg/debug/elf/file.go | 761 +++ src/pkg/debug/elf/file_test.go | 241 + src/pkg/debug/elf/testdata/gcc-386-freebsd-exec | Bin 0 -> 5742 bytes src/pkg/debug/elf/testdata/gcc-amd64-linux-exec | Bin 0 -> 8844 bytes .../testdata/go-relocation-test-gcc424-x86-64.obj | Bin 0 -> 3088 bytes .../testdata/go-relocation-test-gcc441-x86-64.obj | Bin 0 -> 2936 bytes .../elf/testdata/go-relocation-test-gcc441-x86.obj | Bin 0 -> 1884 bytes src/pkg/debug/gosym/Makefile | 19 + src/pkg/debug/gosym/pclinetest.h | 7 + src/pkg/debug/gosym/pclinetest.s | 58 + src/pkg/debug/gosym/pclntab.go | 82 + src/pkg/debug/gosym/pclntab_test.go | 204 + src/pkg/debug/gosym/symtab.go | 548 ++ src/pkg/debug/macho/Makefile | 12 + src/pkg/debug/macho/file.go | 517 ++ src/pkg/debug/macho/file_test.go | 167 + src/pkg/debug/macho/macho.go | 305 + src/pkg/debug/macho/testdata/gcc-386-darwin-exec | Bin 0 -> 12588 bytes src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec | Bin 0 -> 8512 bytes .../macho/testdata/gcc-amd64-darwin-exec-debug | Bin 0 -> 4540 bytes src/pkg/debug/macho/testdata/hello.c | 8 + src/pkg/debug/pe/Makefile | 12 + src/pkg/debug/pe/file.go | 315 + src/pkg/debug/pe/file_test.go | 99 + src/pkg/debug/pe/pe.go | 51 + src/pkg/debug/pe/testdata/gcc-386-mingw-exec | Bin 0 -> 29941 bytes src/pkg/debug/pe/testdata/gcc-386-mingw-obj | Bin 0 -> 2372 bytes src/pkg/debug/pe/testdata/hello.c | 8 + src/pkg/deps.bash | 49 + src/pkg/ebnf/Makefile | 12 + src/pkg/ebnf/ebnf.go | 245 + src/pkg/ebnf/ebnf_test.go | 87 + src/pkg/ebnf/parser.go | 197 + src/pkg/encoding/ascii85/Makefile | 11 + src/pkg/encoding/ascii85/ascii85.go | 300 + src/pkg/encoding/ascii85/ascii85_test.go | 188 + src/pkg/encoding/base32/Makefile | 11 + src/pkg/encoding/base32/base32.go | 368 ++ src/pkg/encoding/base32/base32_test.go | 193 + src/pkg/encoding/base64/Makefile | 11 + src/pkg/encoding/base64/base64.go | 343 + src/pkg/encoding/base64/base64_test.go | 199 + src/pkg/encoding/binary/Makefile | 11 + src/pkg/encoding/binary/binary.go | 498 ++ src/pkg/encoding/binary/binary_test.go | 235 + src/pkg/encoding/git85/Makefile | 11 + src/pkg/encoding/git85/git.go | 277 + src/pkg/encoding/git85/git_test.go | 194 + src/pkg/encoding/hex/Makefile | 11 + src/pkg/encoding/hex/hex.go | 216 + src/pkg/encoding/hex/hex_test.go | 192 + src/pkg/encoding/pem/Makefile | 11 + src/pkg/encoding/pem/pem.go | 258 + src/pkg/encoding/pem/pem_test.go | 390 ++ src/pkg/exec/Makefile | 31 + src/pkg/exec/exec.go | 377 ++ src/pkg/exec/exec_test.go | 214 + src/pkg/exec/lp_plan9.go | 51 + src/pkg/exec/lp_test.go | 33 + src/pkg/exec/lp_unix.go | 52 + src/pkg/exec/lp_windows.go | 77 + src/pkg/exp/README | 3 + src/pkg/exp/datafmt/Makefile | 12 + src/pkg/exp/datafmt/datafmt.go | 710 +++ src/pkg/exp/datafmt/datafmt_test.go | 330 + src/pkg/exp/datafmt/parser.go | 368 ++ src/pkg/exp/gui/Makefile | 11 + src/pkg/exp/gui/gui.go | 58 + src/pkg/exp/gui/x11/Makefile | 12 + src/pkg/exp/gui/x11/auth.go | 93 + src/pkg/exp/gui/x11/conn.go | 627 ++ src/pkg/exp/norm/Makefile | 44 + src/pkg/exp/norm/composition.go | 344 + src/pkg/exp/norm/composition_test.go | 138 + src/pkg/exp/norm/forminfo.go | 188 + src/pkg/exp/norm/maketables.go | 855 +++ src/pkg/exp/norm/maketesttables.go | 42 + src/pkg/exp/norm/norm_test.go | 14 + src/pkg/exp/norm/normalize.go | 99 + src/pkg/exp/norm/tables.go | 6580 ++++++++++++++++++++ src/pkg/exp/norm/trie.go | 234 + src/pkg/exp/norm/trie_test.go | 107 + src/pkg/exp/norm/triedata_test.go | 63 + src/pkg/exp/norm/triegen.go | 211 + src/pkg/exp/regexp/Makefile | 12 + src/pkg/exp/regexp/all_test.go | 429 ++ src/pkg/exp/regexp/exec.go | 295 + src/pkg/exp/regexp/find_test.go | 472 ++ src/pkg/exp/regexp/regexp.go | 795 +++ src/pkg/exp/regexp/syntax/Makefile | 16 + src/pkg/exp/regexp/syntax/compile.go | 269 + src/pkg/exp/regexp/syntax/make_perl_groups.pl | 103 + src/pkg/exp/regexp/syntax/parse.go | 1797 ++++++ src/pkg/exp/regexp/syntax/parse_test.go | 350 ++ src/pkg/exp/regexp/syntax/perl_groups.go | 130 + src/pkg/exp/regexp/syntax/prog.go | 237 + src/pkg/exp/regexp/syntax/prog_test.go | 91 + src/pkg/exp/regexp/syntax/regexp.go | 284 + src/pkg/exp/regexp/syntax/simplify.go | 151 + src/pkg/exp/regexp/syntax/simplify_test.go | 151 + src/pkg/exp/template/html/Makefile | 11 + src/pkg/exp/template/html/context.go | 98 + src/pkg/exp/template/html/escape.go | 105 + src/pkg/exp/template/html/escape_test.go | 75 + src/pkg/exp/wingui/Makefile | 29 + src/pkg/exp/wingui/gui.go | 153 + src/pkg/exp/wingui/winapi.go | 131 + src/pkg/exp/wingui/zwinapi.go | 211 + src/pkg/expvar/Makefile | 11 + src/pkg/expvar/expvar.go | 287 + src/pkg/expvar/expvar_test.go | 128 + src/pkg/flag/Makefile | 11 + src/pkg/flag/export_test.go | 22 + src/pkg/flag/flag.go | 705 +++ src/pkg/flag/flag_test.go | 255 + src/pkg/fmt/Makefile | 14 + src/pkg/fmt/doc.go | 181 + src/pkg/fmt/fmt_test.go | 746 +++ src/pkg/fmt/format.go | 452 ++ src/pkg/fmt/print.go | 996 +++ src/pkg/fmt/scan.go | 1112 ++++ src/pkg/fmt/scan_test.go | 923 +++ src/pkg/fmt/stringer_test.go | 61 + src/pkg/go/ast/Makefile | 16 + src/pkg/go/ast/ast.go | 914 +++ src/pkg/go/ast/filter.go | 475 ++ src/pkg/go/ast/print.go | 224 + src/pkg/go/ast/print_test.go | 77 + src/pkg/go/ast/resolve.go | 174 + src/pkg/go/ast/scope.go | 156 + src/pkg/go/ast/walk.go | 382 ++ src/pkg/go/build/Makefile | 22 + src/pkg/go/build/build.go | 444 ++ src/pkg/go/build/build_test.go | 61 + src/pkg/go/build/cgotest/cgotest.c | 9 + src/pkg/go/build/cgotest/cgotest.go | 19 + src/pkg/go/build/cgotest/cgotest.h | 5 + src/pkg/go/build/cmdtest/main.go | 12 + src/pkg/go/build/dir.go | 172 + src/pkg/go/build/path.go | 182 + src/pkg/go/build/pkgtest/pkgtest.go | 9 + src/pkg/go/build/pkgtest/sqrt_386.s | 10 + src/pkg/go/build/pkgtest/sqrt_amd64.s | 9 + src/pkg/go/build/pkgtest/sqrt_arm.s | 10 + src/pkg/go/build/syslist_test.go | 62 + src/pkg/go/doc/Makefile | 12 + src/pkg/go/doc/comment.go | 345 + src/pkg/go/doc/doc.go | 641 ++ src/pkg/go/parser/Makefile | 12 + src/pkg/go/parser/interface.go | 214 + src/pkg/go/parser/parser.go | 2161 +++++++ src/pkg/go/parser/parser_test.go | 130 + src/pkg/go/printer/Makefile | 12 + src/pkg/go/printer/nodes.go | 1510 +++++ src/pkg/go/printer/performance_test.go | 58 + src/pkg/go/printer/printer.go | 1012 +++ src/pkg/go/printer/printer_test.go | 194 + src/pkg/go/printer/testdata/comments.golden | 473 ++ src/pkg/go/printer/testdata/comments.input | 483 ++ src/pkg/go/printer/testdata/comments.x | 56 + src/pkg/go/printer/testdata/declarations.golden | 747 +++ src/pkg/go/printer/testdata/declarations.input | 757 +++ src/pkg/go/printer/testdata/empty.golden | 5 + src/pkg/go/printer/testdata/empty.input | 5 + src/pkg/go/printer/testdata/expressions.golden | 627 ++ src/pkg/go/printer/testdata/expressions.input | 656 ++ src/pkg/go/printer/testdata/expressions.raw | 627 ++ src/pkg/go/printer/testdata/linebreaks.golden | 223 + src/pkg/go/printer/testdata/linebreaks.input | 223 + src/pkg/go/printer/testdata/parser.go | 2153 +++++++ src/pkg/go/printer/testdata/slow.golden | 85 + src/pkg/go/printer/testdata/slow.input | 85 + src/pkg/go/printer/testdata/statements.golden | 412 ++ src/pkg/go/printer/testdata/statements.input | 351 ++ src/pkg/go/scanner/Makefile | 12 + src/pkg/go/scanner/errors.go | 168 + src/pkg/go/scanner/scanner.go | 670 ++ src/pkg/go/scanner/scanner_test.go | 672 ++ src/pkg/go/token/Makefile | 12 + src/pkg/go/token/position.go | 424 ++ src/pkg/go/token/position_test.go | 180 + src/pkg/go/token/token.go | 310 + src/pkg/go/typechecker/Makefile | 14 + src/pkg/go/typechecker/scope.go | 69 + src/pkg/go/typechecker/testdata/test0.src | 94 + src/pkg/go/typechecker/testdata/test1.src | 13 + src/pkg/go/typechecker/testdata/test3.src | 41 + src/pkg/go/typechecker/testdata/test4.src | 11 + src/pkg/go/typechecker/type.go | 118 + src/pkg/go/typechecker/typechecker.go | 468 ++ src/pkg/go/typechecker/typechecker_test.go | 163 + src/pkg/go/typechecker/universe.go | 36 + src/pkg/go/types/Makefile | 16 + src/pkg/go/types/check.go | 226 + src/pkg/go/types/check_test.go | 215 + src/pkg/go/types/const.go | 332 + src/pkg/go/types/exportdata.go | 132 + src/pkg/go/types/gcimporter.go | 799 +++ src/pkg/go/types/gcimporter_test.go | 101 + src/pkg/go/types/testdata/exports.go | 84 + src/pkg/go/types/testdata/test0.src | 154 + src/pkg/go/types/types.go | 255 + src/pkg/go/types/universe.go | 108 + src/pkg/gob/Makefile | 25 + src/pkg/gob/codec_test.go | 1406 +++++ src/pkg/gob/debug.go | 686 ++ src/pkg/gob/decode.go | 1275 ++++ src/pkg/gob/decoder.go | 202 + src/pkg/gob/doc.go | 360 ++ src/pkg/gob/dump.go | 22 + src/pkg/gob/encode.go | 717 +++ src/pkg/gob/encoder.go | 243 + src/pkg/gob/encoder_test.go | 580 ++ src/pkg/gob/error.go | 42 + src/pkg/gob/gobencdec_test.go | 490 ++ src/pkg/gob/timing_test.go | 94 + src/pkg/gob/type.go | 786 +++ src/pkg/gob/type_test.go | 153 + src/pkg/hash/Makefile | 11 + src/pkg/hash/adler32/Makefile | 11 + src/pkg/hash/adler32/adler32.go | 88 + src/pkg/hash/adler32/adler32_test.go | 77 + src/pkg/hash/crc32/Makefile | 20 + src/pkg/hash/crc32/crc32.go | 139 + src/pkg/hash/crc32/crc32_amd64.go | 25 + src/pkg/hash/crc32/crc32_amd64.s | 62 + src/pkg/hash/crc32/crc32_generic.go | 12 + src/pkg/hash/crc32/crc32_test.go | 97 + src/pkg/hash/crc64/Makefile | 11 + src/pkg/hash/crc64/crc64.go | 97 + src/pkg/hash/crc64/crc64_test.go | 78 + src/pkg/hash/fnv/Makefile | 11 + src/pkg/hash/fnv/fnv.go | 131 + src/pkg/hash/fnv/fnv_test.go | 167 + src/pkg/hash/hash.go | 37 + src/pkg/hash/test_cases.txt | 31 + src/pkg/hash/test_gen.awk | 14 + src/pkg/html/Makefile | 17 + src/pkg/html/const.go | 90 + src/pkg/html/doc.go | 110 + src/pkg/html/entity.go | 2253 +++++++ src/pkg/html/entity_test.go | 29 + src/pkg/html/escape.go | 238 + src/pkg/html/node.go | 147 + src/pkg/html/parse.go | 800 +++ src/pkg/html/parse_test.go | 156 + src/pkg/html/testdata/webkit/README | 28 + src/pkg/html/testdata/webkit/adoption01.dat | 194 + src/pkg/html/testdata/webkit/adoption02.dat | 31 + src/pkg/html/testdata/webkit/comments01.dat | 135 + src/pkg/html/testdata/webkit/doctype01.dat | 370 ++ src/pkg/html/testdata/webkit/entities01.dat | 603 ++ src/pkg/html/testdata/webkit/entities02.dat | 249 + src/pkg/html/testdata/webkit/html5test-com.dat | 246 + src/pkg/html/testdata/webkit/inbody01.dat | 43 + src/pkg/html/testdata/webkit/isindex.dat | 40 + .../pending-spec-changes-plain-text-unsafe.dat | Bin 0 -> 115 bytes .../html/testdata/webkit/pending-spec-changes.dat | 28 + src/pkg/html/testdata/webkit/plain-text-unsafe.dat | 8 + src/pkg/html/testdata/webkit/scriptdata01.dat | 308 + .../html/testdata/webkit/scripted/adoption01.dat | 15 + src/pkg/html/testdata/webkit/scripted/webkit01.dat | 28 + src/pkg/html/testdata/webkit/tables01.dat | 197 + src/pkg/html/testdata/webkit/tests1.dat | 1952 ++++++ src/pkg/html/testdata/webkit/tests10.dat | 799 +++ src/pkg/html/testdata/webkit/tests11.dat | 482 ++ src/pkg/html/testdata/webkit/tests12.dat | 62 + src/pkg/html/testdata/webkit/tests14.dat | 74 + src/pkg/html/testdata/webkit/tests15.dat | 208 + src/pkg/html/testdata/webkit/tests16.dat | 2277 +++++++ src/pkg/html/testdata/webkit/tests17.dat | 153 + src/pkg/html/testdata/webkit/tests18.dat | 269 + src/pkg/html/testdata/webkit/tests19.dat | 1220 ++++ src/pkg/html/testdata/webkit/tests2.dat | 763 +++ src/pkg/html/testdata/webkit/tests20.dat | 455 ++ src/pkg/html/testdata/webkit/tests21.dat | 221 + src/pkg/html/testdata/webkit/tests22.dat | 157 + src/pkg/html/testdata/webkit/tests23.dat | 155 + src/pkg/html/testdata/webkit/tests24.dat | 79 + src/pkg/html/testdata/webkit/tests25.dat | 219 + src/pkg/html/testdata/webkit/tests26.dat | 195 + src/pkg/html/testdata/webkit/tests3.dat | 305 + src/pkg/html/testdata/webkit/tests4.dat | 59 + src/pkg/html/testdata/webkit/tests5.dat | 191 + src/pkg/html/testdata/webkit/tests6.dat | 663 ++ src/pkg/html/testdata/webkit/tests7.dat | 390 ++ src/pkg/html/testdata/webkit/tests8.dat | 148 + src/pkg/html/testdata/webkit/tests9.dat | 457 ++ src/pkg/html/testdata/webkit/tests_innerHTML_1.dat | 733 +++ src/pkg/html/testdata/webkit/tricky01.dat | 261 + src/pkg/html/testdata/webkit/webkit01.dat | 609 ++ src/pkg/html/testdata/webkit/webkit02.dat | 104 + src/pkg/html/token.go | 575 ++ src/pkg/html/token_test.go | 340 + src/pkg/http/Makefile | 26 + src/pkg/http/cgi/Makefile | 12 + src/pkg/http/cgi/child.go | 191 + src/pkg/http/cgi/child_test.go | 93 + src/pkg/http/cgi/host.go | 323 + src/pkg/http/cgi/host_test.go | 449 ++ src/pkg/http/cgi/matryoshka_test.go | 74 + src/pkg/http/cgi/testdata/test.cgi | 96 + src/pkg/http/chunked.go | 77 + src/pkg/http/client.go | 290 + src/pkg/http/client_test.go | 292 + src/pkg/http/cookie.go | 268 + src/pkg/http/cookie_test.go | 201 + src/pkg/http/dump.go | 78 + src/pkg/http/export_test.go | 41 + src/pkg/http/fcgi/Makefile | 12 + src/pkg/http/fcgi/child.go | 258 + src/pkg/http/fcgi/fcgi.go | 271 + src/pkg/http/fcgi/fcgi_test.go | 114 + src/pkg/http/fs.go | 323 + src/pkg/http/fs_test.go | 308 + src/pkg/http/header.go | 78 + src/pkg/http/header_test.go | 81 + src/pkg/http/httptest/Makefile | 12 + src/pkg/http/httptest/recorder.go | 59 + src/pkg/http/httptest/server.go | 151 + src/pkg/http/lex.go | 144 + src/pkg/http/lex_test.go | 70 + src/pkg/http/persist.go | 420 ++ src/pkg/http/pprof/Makefile | 11 + src/pkg/http/pprof/pprof.go | 132 + src/pkg/http/proxy_test.go | 48 + src/pkg/http/range_test.go | 57 + src/pkg/http/readrequest_test.go | 183 + src/pkg/http/request.go | 745 +++ src/pkg/http/request_test.go | 336 + src/pkg/http/requestwrite_test.go | 370 ++ src/pkg/http/response.go | 214 + src/pkg/http/response_test.go | 397 ++ src/pkg/http/responsewrite_test.go | 109 + src/pkg/http/reverseproxy.go | 159 + src/pkg/http/reverseproxy_test.go | 66 + src/pkg/http/serve_test.go | 959 +++ src/pkg/http/server.go | 1183 ++++ src/pkg/http/sniff.go | 214 + src/pkg/http/sniff_test.go | 80 + src/pkg/http/spdy/Makefile | 13 + src/pkg/http/spdy/read.go | 313 + src/pkg/http/spdy/spdy_test.go | 497 ++ src/pkg/http/spdy/types.go | 370 ++ src/pkg/http/spdy/write.go | 286 + src/pkg/http/status.go | 106 + src/pkg/http/testdata/file | 1 + src/pkg/http/testdata/index.html | 1 + src/pkg/http/testdata/style.css | 1 + src/pkg/http/transfer.go | 520 ++ src/pkg/http/transport.go | 717 +++ src/pkg/http/transport_test.go | 662 ++ src/pkg/http/triv.go | 149 + src/pkg/image/Makefile | 15 + src/pkg/image/bmp/Makefile | 11 + src/pkg/image/bmp/reader.go | 151 + src/pkg/image/color.go | 251 + src/pkg/image/decode_test.go | 123 + src/pkg/image/draw/Makefile | 11 + src/pkg/image/draw/bench_test.go | 206 + src/pkg/image/draw/clip_test.go | 193 + src/pkg/image/draw/draw.go | 493 ++ src/pkg/image/draw/draw_test.go | 354 ++ src/pkg/image/format.go | 100 + src/pkg/image/geom.go | 234 + src/pkg/image/gif/Makefile | 11 + src/pkg/image/gif/reader.go | 426 ++ src/pkg/image/image.go | 850 +++ src/pkg/image/image_test.go | 112 + src/pkg/image/jpeg/Makefile | 15 + src/pkg/image/jpeg/fdct.go | 190 + src/pkg/image/jpeg/huffman.go | 190 + src/pkg/image/jpeg/idct.go | 204 + src/pkg/image/jpeg/reader.go | 476 ++ src/pkg/image/jpeg/writer.go | 549 ++ src/pkg/image/jpeg/writer_test.go | 115 + src/pkg/image/names.go | 67 + src/pkg/image/png/Makefile | 12 + src/pkg/image/png/reader.go | 708 +++ src/pkg/image/png/reader_test.go | 241 + src/pkg/image/png/testdata/pngsuite/README | 21 + .../image/png/testdata/pngsuite/README.original | 85 + .../image/png/testdata/pngsuite/basn0g01-30.png | Bin 0 -> 162 bytes .../image/png/testdata/pngsuite/basn0g01-30.sng | 39 + src/pkg/image/png/testdata/pngsuite/basn0g01.png | Bin 0 -> 164 bytes src/pkg/image/png/testdata/pngsuite/basn0g01.sng | 41 + .../image/png/testdata/pngsuite/basn0g02-29.png | Bin 0 -> 110 bytes .../image/png/testdata/pngsuite/basn0g02-29.sng | 38 + src/pkg/image/png/testdata/pngsuite/basn0g02.png | Bin 0 -> 104 bytes src/pkg/image/png/testdata/pngsuite/basn0g02.sng | 41 + .../image/png/testdata/pngsuite/basn0g04-31.png | Bin 0 -> 153 bytes .../image/png/testdata/pngsuite/basn0g04-31.sng | 40 + src/pkg/image/png/testdata/pngsuite/basn0g04.png | Bin 0 -> 145 bytes src/pkg/image/png/testdata/pngsuite/basn0g04.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn0g08.png | Bin 0 -> 138 bytes src/pkg/image/png/testdata/pngsuite/basn0g08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn0g16.png | Bin 0 -> 167 bytes src/pkg/image/png/testdata/pngsuite/basn0g16.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn2c08.png | Bin 0 -> 145 bytes src/pkg/image/png/testdata/pngsuite/basn2c08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn2c16.png | Bin 0 -> 302 bytes src/pkg/image/png/testdata/pngsuite/basn2c16.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn3p01.png | Bin 0 -> 112 bytes src/pkg/image/png/testdata/pngsuite/basn3p01.sng | 45 + src/pkg/image/png/testdata/pngsuite/basn3p02.png | Bin 0 -> 146 bytes src/pkg/image/png/testdata/pngsuite/basn3p02.sng | 47 + src/pkg/image/png/testdata/pngsuite/basn3p04.png | Bin 0 -> 216 bytes src/pkg/image/png/testdata/pngsuite/basn3p04.sng | 58 + .../image/png/testdata/pngsuite/basn3p08-trns.png | Bin 0 -> 1538 bytes .../image/png/testdata/pngsuite/basn3p08-trns.sng | 301 + src/pkg/image/png/testdata/pngsuite/basn3p08.png | Bin 0 -> 1286 bytes src/pkg/image/png/testdata/pngsuite/basn3p08.sng | 299 + src/pkg/image/png/testdata/pngsuite/basn4a08.png | Bin 0 -> 126 bytes src/pkg/image/png/testdata/pngsuite/basn4a08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn4a16.png | Bin 0 -> 2206 bytes src/pkg/image/png/testdata/pngsuite/basn4a16.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn6a08.png | Bin 0 -> 184 bytes src/pkg/image/png/testdata/pngsuite/basn6a08.sng | 41 + src/pkg/image/png/testdata/pngsuite/basn6a16.png | Bin 0 -> 3435 bytes src/pkg/image/png/testdata/pngsuite/basn6a16.sng | 41 + src/pkg/image/png/writer.go | 468 ++ src/pkg/image/png/writer_test.go | 149 + src/pkg/image/testdata/video-001.5bpp.gif | Bin 0 -> 6214 bytes src/pkg/image/testdata/video-001.bmp | Bin 0 -> 46610 bytes src/pkg/image/testdata/video-001.gif | Bin 0 -> 13106 bytes src/pkg/image/testdata/video-001.interlaced.gif | Bin 0 -> 14142 bytes src/pkg/image/testdata/video-001.jpeg | Bin 0 -> 21459 bytes src/pkg/image/testdata/video-001.png | Bin 0 -> 29228 bytes src/pkg/image/testdata/video-001.tiff | Bin 0 -> 30810 bytes src/pkg/image/testdata/video-005.gray.jpeg | Bin 0 -> 5618 bytes src/pkg/image/testdata/video-005.gray.png | Bin 0 -> 14974 bytes src/pkg/image/tiff/Makefile | 13 + src/pkg/image/tiff/buffer.go | 57 + src/pkg/image/tiff/buffer_test.go | 36 + src/pkg/image/tiff/consts.go | 103 + src/pkg/image/tiff/reader.go | 423 ++ src/pkg/image/tiff/reader_test.go | 25 + src/pkg/image/tiff/testdata/no_rps.tiff | Bin 0 -> 1294 bytes src/pkg/image/ycbcr/Makefile | 11 + src/pkg/image/ycbcr/ycbcr.go | 183 + src/pkg/image/ycbcr/ycbcr_test.go | 33 + src/pkg/index/suffixarray/Makefile | 12 + src/pkg/index/suffixarray/qsufsort.go | 166 + src/pkg/index/suffixarray/suffixarray.go | 181 + src/pkg/index/suffixarray/suffixarray_test.go | 230 + src/pkg/io/Makefile | 13 + src/pkg/io/io.go | 439 ++ src/pkg/io/io_test.go | 179 + src/pkg/io/ioutil/Makefile | 12 + src/pkg/io/ioutil/ioutil.go | 130 + src/pkg/io/ioutil/ioutil_test.go | 91 + src/pkg/io/ioutil/tempfile.go | 91 + src/pkg/io/ioutil/tempfile_test.go | 54 + src/pkg/io/multi.go | 58 + src/pkg/io/multi_test.go | 89 + src/pkg/io/pipe.go | 182 + src/pkg/io/pipe_test.go | 271 + src/pkg/json/Makefile | 15 + src/pkg/json/decode.go | 920 +++ src/pkg/json/decode_test.go | 552 ++ src/pkg/json/encode.go | 428 ++ src/pkg/json/encode_test.go | 44 + src/pkg/json/indent.go | 116 + src/pkg/json/scanner.go | 624 ++ src/pkg/json/scanner_test.go | 278 + src/pkg/json/stream.go | 185 + src/pkg/json/stream_test.go | 122 + src/pkg/json/tagkey_test.go | 95 + src/pkg/log/Makefile | 11 + src/pkg/log/log.go | 327 + src/pkg/log/log_test.go | 119 + src/pkg/mail/Makefile | 11 + src/pkg/mail/message.go | 524 ++ src/pkg/mail/message_test.go | 278 + src/pkg/math/Makefile | 102 + src/pkg/math/acosh.go | 61 + src/pkg/math/all_test.go | 2737 ++++++++ src/pkg/math/asin.go | 49 + src/pkg/math/asin_386.s | 28 + src/pkg/math/asin_decl.go | 8 + src/pkg/math/asinh.go | 71 + src/pkg/math/atan.go | 62 + src/pkg/math/atan2.go | 71 + src/pkg/math/atan2_386.s | 11 + src/pkg/math/atan2_decl.go | 7 + src/pkg/math/atan_386.s | 11 + src/pkg/math/atan_decl.go | 7 + src/pkg/math/atanh.go | 78 + src/pkg/math/bits.go | 59 + src/pkg/math/cbrt.go | 79 + src/pkg/math/const.go | 53 + src/pkg/math/copysign.go | 12 + src/pkg/math/erf.go | 339 + src/pkg/math/exp.go | 14 + src/pkg/math/exp2.go | 10 + src/pkg/math/exp2_386.s | 38 + src/pkg/math/exp2_decl.go | 7 + src/pkg/math/exp_386.s | 39 + src/pkg/math/exp_amd64.s | 112 + src/pkg/math/exp_decl.go | 7 + src/pkg/math/exp_port.go | 191 + src/pkg/math/exp_test.go | 10 + src/pkg/math/expm1.go | 237 + src/pkg/math/expm1_386.s | 55 + src/pkg/math/expm1_decl.go | 7 + src/pkg/math/fabs.go | 21 + src/pkg/math/fabs_386.s | 10 + src/pkg/math/fabs_amd64.s | 12 + src/pkg/math/fabs_decl.go | 7 + src/pkg/math/fdim.go | 29 + src/pkg/math/fdim_amd64.s | 26 + src/pkg/math/fdim_decl.go | 9 + src/pkg/math/floor.go | 52 + src/pkg/math/floor_386.s | 44 + src/pkg/math/floor_decl.go | 9 + src/pkg/math/fltasm_amd64.s | 67 + src/pkg/math/fmod.go | 47 + src/pkg/math/fmod_386.s | 15 + src/pkg/math/fmod_decl.go | 7 + src/pkg/math/frexp.go | 33 + src/pkg/math/frexp_386.s | 23 + src/pkg/math/frexp_decl.go | 7 + src/pkg/math/gamma.go | 188 + src/pkg/math/hypot.go | 41 + src/pkg/math/hypot_386.s | 57 + src/pkg/math/hypot_amd64.s | 50 + src/pkg/math/hypot_decl.go | 7 + src/pkg/math/hypot_port.go | 63 + src/pkg/math/hypot_test.go | 9 + src/pkg/math/j0.go | 433 ++ src/pkg/math/j1.go | 426 ++ src/pkg/math/jn.go | 310 + src/pkg/math/ldexp.go | 45 + src/pkg/math/ldexp_386.s | 12 + src/pkg/math/ldexp_decl.go | 7 + src/pkg/math/lgamma.go | 350 ++ src/pkg/math/log.go | 123 + src/pkg/math/log10.go | 13 + src/pkg/math/log10_386.s | 19 + src/pkg/math/log10_decl.go | 8 + src/pkg/math/log1p.go | 199 + src/pkg/math/log1p_386.s | 25 + src/pkg/math/log1p_decl.go | 7 + src/pkg/math/log_386.s | 11 + src/pkg/math/log_amd64.s | 109 + src/pkg/math/log_decl.go | 7 + src/pkg/math/logb.go | 54 + src/pkg/math/modf.go | 33 + src/pkg/math/modf_386.s | 19 + src/pkg/math/modf_decl.go | 7 + src/pkg/math/nextafter.go | 29 + src/pkg/math/pow.go | 139 + src/pkg/math/pow10.go | 30 + src/pkg/math/remainder.go | 85 + src/pkg/math/remainder_386.s | 15 + src/pkg/math/remainder_decl.go | 7 + src/pkg/math/signbit.go | 10 + src/pkg/math/sin.go | 65 + src/pkg/math/sin_386.s | 45 + src/pkg/math/sin_decl.go | 8 + src/pkg/math/sincos.go | 13 + src/pkg/math/sincos_386.s | 26 + src/pkg/math/sincos_amd64.s | 143 + src/pkg/math/sincos_decl.go | 7 + src/pkg/math/sinh.go | 67 + src/pkg/math/sqrt.go | 14 + src/pkg/math/sqrt_386.s | 10 + src/pkg/math/sqrt_amd64.s | 9 + src/pkg/math/sqrt_arm.s | 10 + src/pkg/math/sqrt_decl.go | 7 + src/pkg/math/sqrt_port.go | 147 + src/pkg/math/sqrt_test.go | 9 + src/pkg/math/tan.go | 64 + src/pkg/math/tan_386.s | 26 + src/pkg/math/tan_decl.go | 7 + src/pkg/math/tanh.go | 27 + src/pkg/math/unsafe.go | 21 + src/pkg/mime/Makefile | 13 + src/pkg/mime/grammar.go | 36 + src/pkg/mime/mediatype.go | 295 + src/pkg/mime/mediatype_test.go | 248 + src/pkg/mime/mime_test.go | 27 + src/pkg/mime/multipart/Makefile | 13 + src/pkg/mime/multipart/formdata.go | 166 + src/pkg/mime/multipart/formdata_test.go | 89 + src/pkg/mime/multipart/multipart.go | 268 + src/pkg/mime/multipart/multipart_test.go | 380 ++ src/pkg/mime/multipart/writer.go | 157 + src/pkg/mime/multipart/writer_test.go | 78 + src/pkg/mime/test.types | 8 + src/pkg/mime/type.go | 104 + src/pkg/net/Makefile | 157 + src/pkg/net/cgo_bsd.go | 14 + src/pkg/net/cgo_linux.go | 14 + src/pkg/net/cgo_stub.go | 25 + src/pkg/net/cgo_unix.go | 148 + src/pkg/net/dial.go | 153 + src/pkg/net/dialgoogle_test.go | 169 + src/pkg/net/dict/Makefile | 7 + src/pkg/net/dict/dict.go | 211 + src/pkg/net/dnsclient.go | 247 + src/pkg/net/dnsclient_unix.go | 261 + src/pkg/net/dnsconfig.go | 119 + src/pkg/net/dnsmsg.go | 779 +++ src/pkg/net/dnsmsg_test.go | 100 + src/pkg/net/dnsname_test.go | 65 + src/pkg/net/fd.go | 645 ++ src/pkg/net/fd_darwin.go | 120 + src/pkg/net/fd_freebsd.go | 116 + src/pkg/net/fd_linux.go | 182 + src/pkg/net/fd_openbsd.go | 116 + src/pkg/net/fd_windows.go | 532 ++ src/pkg/net/file.go | 119 + src/pkg/net/file_plan9.go | 33 + src/pkg/net/file_test.go | 133 + src/pkg/net/file_windows.go | 25 + src/pkg/net/hosts.go | 86 + src/pkg/net/hosts_test.go | 68 + src/pkg/net/hosts_testdata | 12 + src/pkg/net/interface.go | 132 + src/pkg/net/interface_bsd.go | 171 + src/pkg/net/interface_darwin.go | 80 + src/pkg/net/interface_freebsd.go | 80 + src/pkg/net/interface_linux.go | 262 + src/pkg/net/interface_openbsd.go | 16 + src/pkg/net/interface_stub.go | 30 + src/pkg/net/interface_test.go | 73 + src/pkg/net/interface_windows.go | 158 + src/pkg/net/ip.go | 614 ++ src/pkg/net/ip_test.go | 215 + src/pkg/net/ipraw_test.go | 120 + src/pkg/net/iprawsock.go | 69 + src/pkg/net/iprawsock_plan9.go | 99 + src/pkg/net/iprawsock_posix.go | 305 + src/pkg/net/ipsock.go | 158 + src/pkg/net/ipsock_plan9.go | 305 + src/pkg/net/ipsock_posix.go | 174 + src/pkg/net/lookup_plan9.go | 226 + src/pkg/net/lookup_test.go | 57 + src/pkg/net/lookup_unix.go | 111 + src/pkg/net/lookup_windows.go | 130 + src/pkg/net/multicast_test.go | 73 + src/pkg/net/net.go | 188 + src/pkg/net/net_test.go | 121 + src/pkg/net/newpollserver.go | 43 + src/pkg/net/parse.go | 204 + src/pkg/net/parse_test.go | 50 + src/pkg/net/pipe.go | 62 + src/pkg/net/pipe_test.go | 57 + src/pkg/net/port.go | 70 + src/pkg/net/port_test.go | 53 + src/pkg/net/sendfile_linux.go | 84 + src/pkg/net/sendfile_stub.go | 14 + src/pkg/net/sendfile_windows.go | 68 + src/pkg/net/server_test.go | 243 + src/pkg/net/sock.go | 166 + src/pkg/net/sock_bsd.go | 31 + src/pkg/net/sock_linux.go | 25 + src/pkg/net/sock_windows.go | 25 + src/pkg/net/tcpsock.go | 40 + src/pkg/net/tcpsock_plan9.go | 63 + src/pkg/net/tcpsock_posix.go | 283 + src/pkg/net/textproto/Makefile | 15 + src/pkg/net/textproto/header.go | 43 + src/pkg/net/textproto/pipeline.go | 117 + src/pkg/net/textproto/reader.go | 514 ++ src/pkg/net/textproto/reader_test.go | 140 + src/pkg/net/textproto/textproto.go | 121 + src/pkg/net/textproto/writer.go | 119 + src/pkg/net/textproto/writer_test.go | 35 + src/pkg/net/timeout_test.go | 57 + src/pkg/net/udpsock.go | 40 + src/pkg/net/udpsock_plan9.go | 187 + src/pkg/net/udpsock_posix.go | 294 + src/pkg/net/unixsock.go | 50 + src/pkg/net/unixsock_plan9.go | 105 + src/pkg/net/unixsock_posix.go | 418 ++ src/pkg/netchan/Makefile | 13 + src/pkg/netchan/common.go | 336 + src/pkg/netchan/export.go | 400 ++ src/pkg/netchan/import.go | 287 + src/pkg/netchan/netchan_test.go | 435 ++ src/pkg/old/template/Makefile | 14 + src/pkg/old/template/doc.go | 91 + src/pkg/old/template/execute.go | 346 + src/pkg/old/template/format.go | 77 + src/pkg/old/template/parse.go | 743 +++ src/pkg/old/template/template_test.go | 804 +++ src/pkg/os/Makefile | 100 + src/pkg/os/dir_plan9.go | 300 + src/pkg/os/dir_unix.go | 67 + src/pkg/os/dir_windows.go | 14 + src/pkg/os/env.go | 75 + src/pkg/os/env_plan9.go | 96 + src/pkg/os/env_test.go | 59 + src/pkg/os/env_unix.go | 109 + src/pkg/os/env_windows.go | 127 + src/pkg/os/error.go | 31 + src/pkg/os/error_plan9.go | 61 + src/pkg/os/error_posix.go | 90 + src/pkg/os/exec.go | 58 + src/pkg/os/exec_plan9.go | 153 + src/pkg/os/exec_posix.go | 147 + src/pkg/os/exec_unix.go | 77 + src/pkg/os/exec_windows.go | 66 + src/pkg/os/file.go | 211 + src/pkg/os/file_plan9.go | 331 + src/pkg/os/file_posix.go | 226 + src/pkg/os/file_unix.go | 208 + src/pkg/os/file_windows.go | 317 + src/pkg/os/getwd.go | 92 + src/pkg/os/inotify/Makefile | 14 + src/pkg/os/inotify/inotify_linux.go | 288 + src/pkg/os/inotify/inotify_linux_test.go | 96 + src/pkg/os/mkunixsignals.sh | 24 + src/pkg/os/os_test.go | 1058 ++++ src/pkg/os/path.go | 118 + src/pkg/os/path_plan9.go | 15 + src/pkg/os/path_test.go | 208 + src/pkg/os/path_unix.go | 15 + src/pkg/os/path_windows.go | 16 + src/pkg/os/proc.go | 34 + src/pkg/os/signal/Makefile | 11 + src/pkg/os/signal/signal.go | 33 + src/pkg/os/signal/signal_test.go | 20 + src/pkg/os/stat_darwin.go | 32 + src/pkg/os/stat_freebsd.go | 32 + src/pkg/os/stat_linux.go | 32 + src/pkg/os/stat_openbsd.go | 32 + src/pkg/os/stat_plan9.go | 90 + src/pkg/os/stat_windows.go | 46 + src/pkg/os/str.go | 20 + src/pkg/os/sys_bsd.go | 19 + src/pkg/os/sys_linux.go | 27 + src/pkg/os/sys_plan9.go | 26 + src/pkg/os/sys_windows.go | 15 + src/pkg/os/time.go | 19 + src/pkg/os/types.go | 56 + src/pkg/os/user/Makefile | 26 + src/pkg/os/user/lookup_stubs.go | 19 + src/pkg/os/user/lookup_unix.go | 108 + src/pkg/os/user/user.go | 37 + src/pkg/os/user/user_test.go | 61 + src/pkg/patch/Makefile | 14 + src/pkg/patch/apply.go | 54 + src/pkg/patch/git.go | 121 + src/pkg/patch/patch.go | 322 + src/pkg/patch/patch_test.go | 390 ++ src/pkg/patch/textdiff.go | 175 + src/pkg/path/Makefile | 14 + src/pkg/path/filepath/Makefile | 32 + src/pkg/path/filepath/match.go | 294 + src/pkg/path/filepath/match_test.go | 134 + src/pkg/path/filepath/path.go | 358 ++ src/pkg/path/filepath/path_plan9.go | 23 + src/pkg/path/filepath/path_test.go | 576 ++ src/pkg/path/filepath/path_unix.go | 23 + src/pkg/path/filepath/path_windows.go | 46 + src/pkg/path/match.go | 207 + src/pkg/path/match_test.go | 77 + src/pkg/path/path.go | 162 + src/pkg/path/path_test.go | 196 + src/pkg/rand/Makefile | 15 + src/pkg/rand/exp.go | 223 + src/pkg/rand/normal.go | 158 + src/pkg/rand/rand.go | 179 + src/pkg/rand/rand_test.go | 350 ++ src/pkg/rand/rng.go | 246 + src/pkg/rand/zipf.go | 73 + src/pkg/reflect/Makefile | 13 + src/pkg/reflect/all_test.go | 1564 +++++ src/pkg/reflect/deepequal.go | 126 + src/pkg/reflect/set_test.go | 211 + src/pkg/reflect/tostring_test.go | 96 + src/pkg/reflect/type.go | 1165 ++++ src/pkg/reflect/value.go | 1736 ++++++ src/pkg/regexp/Makefile | 11 + src/pkg/regexp/all_test.go | 426 ++ src/pkg/regexp/find_test.go | 472 ++ src/pkg/regexp/regexp.go | 1488 +++++ src/pkg/rpc/Makefile | 13 + src/pkg/rpc/client.go | 286 + src/pkg/rpc/debug.go | 90 + src/pkg/rpc/jsonrpc/Makefile | 12 + src/pkg/rpc/jsonrpc/all_test.go | 156 + src/pkg/rpc/jsonrpc/client.go | 124 + src/pkg/rpc/jsonrpc/server.go | 136 + src/pkg/rpc/server.go | 637 ++ src/pkg/rpc/server_test.go | 501 ++ src/pkg/runtime/386/arch.h | 3 + src/pkg/runtime/386/asm.s | 549 ++ src/pkg/runtime/386/atomic.c | 19 + src/pkg/runtime/386/closure.c | 105 + src/pkg/runtime/386/memmove.s | 86 + src/pkg/runtime/386/vlop.s | 48 + src/pkg/runtime/386/vlrt.c | 815 +++ src/pkg/runtime/Makefile | 168 + src/pkg/runtime/amd64/arch.h | 3 + src/pkg/runtime/amd64/asm.s | 577 ++ src/pkg/runtime/amd64/atomic.c | 19 + src/pkg/runtime/amd64/closure.c | 123 + src/pkg/runtime/amd64/memmove.s | 88 + src/pkg/runtime/amd64/traceback.c | 295 + src/pkg/runtime/append_test.go | 52 + src/pkg/runtime/arm/arch.h | 3 + src/pkg/runtime/arm/asm.s | 316 + src/pkg/runtime/arm/atomic.c | 83 + src/pkg/runtime/arm/closure.c | 129 + src/pkg/runtime/arm/memmove.s | 255 + src/pkg/runtime/arm/memset.s | 94 + src/pkg/runtime/arm/softfloat.c | 525 ++ src/pkg/runtime/arm/traceback.c | 213 + src/pkg/runtime/arm/vlop.s | 190 + src/pkg/runtime/arm/vlrt.c | 816 +++ src/pkg/runtime/cgo/386.S | 67 + src/pkg/runtime/cgo/Makefile | 60 + src/pkg/runtime/cgo/amd64.S | 73 + src/pkg/runtime/cgo/arm.S | 1 + src/pkg/runtime/cgo/callbacks.c | 73 + src/pkg/runtime/cgo/cgo.go | 17 + src/pkg/runtime/cgo/darwin_386.c | 149 + src/pkg/runtime/cgo/darwin_amd64.c | 119 + src/pkg/runtime/cgo/freebsd.c | 13 + src/pkg/runtime/cgo/freebsd_386.c | 64 + src/pkg/runtime/cgo/freebsd_amd64.c | 63 + src/pkg/runtime/cgo/iscgo.c | 14 + src/pkg/runtime/cgo/libcgo.h | 60 + src/pkg/runtime/cgo/linux_386.c | 73 + src/pkg/runtime/cgo/linux_amd64.c | 63 + src/pkg/runtime/cgo/linux_arm.c | 19 + src/pkg/runtime/cgo/setenv.c | 16 + src/pkg/runtime/cgo/util.c | 51 + src/pkg/runtime/cgo/windows_386.c | 62 + src/pkg/runtime/cgo/windows_amd64.c | 60 + src/pkg/runtime/cgocall.c | 249 + src/pkg/runtime/cgocall.h | 12 + src/pkg/runtime/chan.c | 1161 ++++ src/pkg/runtime/chan_test.go | 322 + src/pkg/runtime/closure_test.go | 53 + src/pkg/runtime/complex.c | 60 + src/pkg/runtime/cpuprof.c | 425 ++ src/pkg/runtime/darwin/386/defs.h | 289 + src/pkg/runtime/darwin/386/rt0.s | 8 + src/pkg/runtime/darwin/386/signal.c | 194 + src/pkg/runtime/darwin/386/sys.s | 311 + src/pkg/runtime/darwin/amd64/defs.h | 305 + src/pkg/runtime/darwin/amd64/rt0.s | 10 + src/pkg/runtime/darwin/amd64/signal.c | 204 + src/pkg/runtime/darwin/amd64/sys.s | 295 + src/pkg/runtime/darwin/defs.c | 159 + src/pkg/runtime/darwin/mem.c | 55 + src/pkg/runtime/darwin/os.h | 31 + src/pkg/runtime/darwin/signals.h | 51 + src/pkg/runtime/darwin/thread.c | 484 ++ src/pkg/runtime/debug.go | 115 + src/pkg/runtime/debug/Makefile | 11 + src/pkg/runtime/debug/stack.go | 90 + src/pkg/runtime/debug/stack_test.go | 55 + src/pkg/runtime/error.go | 138 + src/pkg/runtime/export_test.go | 23 + src/pkg/runtime/extern.go | 192 + src/pkg/runtime/float.c | 173 + src/pkg/runtime/freebsd/386/defs.h | 187 + src/pkg/runtime/freebsd/386/rt0.s | 9 + src/pkg/runtime/freebsd/386/signal.c | 193 + src/pkg/runtime/freebsd/386/sys.s | 239 + src/pkg/runtime/freebsd/amd64/defs.h | 198 + src/pkg/runtime/freebsd/amd64/rt0.s | 9 + src/pkg/runtime/freebsd/amd64/signal.c | 201 + src/pkg/runtime/freebsd/amd64/sys.s | 182 + src/pkg/runtime/freebsd/defs.c | 108 + src/pkg/runtime/freebsd/mem.c | 74 + src/pkg/runtime/freebsd/os.h | 12 + src/pkg/runtime/freebsd/signals.h | 52 + src/pkg/runtime/freebsd/thread.c | 201 + src/pkg/runtime/goc2c.c | 727 +++ src/pkg/runtime/hashmap.c | 1180 ++++ src/pkg/runtime/hashmap.h | 159 + src/pkg/runtime/iface.c | 788 +++ src/pkg/runtime/linux/386/defs.h | 191 + src/pkg/runtime/linux/386/rt0.s | 9 + src/pkg/runtime/linux/386/signal.c | 184 + src/pkg/runtime/linux/386/sys.s | 344 + src/pkg/runtime/linux/amd64/defs.h | 236 + src/pkg/runtime/linux/amd64/rt0.s | 10 + src/pkg/runtime/linux/amd64/signal.c | 194 + src/pkg/runtime/linux/amd64/sys.s | 252 + src/pkg/runtime/linux/arm/defs.h | 149 + src/pkg/runtime/linux/arm/rt0.s | 6 + src/pkg/runtime/linux/arm/signal.c | 189 + src/pkg/runtime/linux/arm/sys.s | 319 + src/pkg/runtime/linux/defs.c | 95 + src/pkg/runtime/linux/defs1.c | 24 + src/pkg/runtime/linux/defs2.c | 120 + src/pkg/runtime/linux/defs_arm.c | 122 + src/pkg/runtime/linux/mem.c | 113 + src/pkg/runtime/linux/os.h | 19 + src/pkg/runtime/linux/signals.h | 51 + src/pkg/runtime/linux/thread.c | 320 + src/pkg/runtime/malloc.goc | 482 ++ src/pkg/runtime/malloc.h | 422 ++ src/pkg/runtime/mcache.c | 133 + src/pkg/runtime/mcentral.c | 200 + src/pkg/runtime/mem.go | 74 + src/pkg/runtime/mfinal.c | 181 + src/pkg/runtime/mfixalloc.c | 62 + src/pkg/runtime/mgc0.c | 910 +++ src/pkg/runtime/mheap.c | 374 ++ src/pkg/runtime/mkasmh.sh | 112 + src/pkg/runtime/mkgodefs.sh | 39 + src/pkg/runtime/mkversion.c | 15 + src/pkg/runtime/mprof.goc | 274 + src/pkg/runtime/msize.c | 168 + src/pkg/runtime/openbsd/amd64/defs.h | 149 + src/pkg/runtime/openbsd/amd64/rt0.s | 8 + src/pkg/runtime/openbsd/amd64/signal.c | 199 + src/pkg/runtime/openbsd/amd64/sys.s | 221 + src/pkg/runtime/openbsd/defs.c | 103 + src/pkg/runtime/openbsd/mem.c | 74 + src/pkg/runtime/openbsd/os.h | 12 + src/pkg/runtime/openbsd/signals.h | 52 + src/pkg/runtime/openbsd/thread.c | 156 + src/pkg/runtime/plan9/386/defs.h | 2 + src/pkg/runtime/plan9/386/rt0.s | 32 + src/pkg/runtime/plan9/386/signal.c | 24 + src/pkg/runtime/plan9/386/sys.s | 82 + src/pkg/runtime/plan9/mem.c | 67 + src/pkg/runtime/plan9/os.h | 57 + src/pkg/runtime/plan9/signals.h | 1 + src/pkg/runtime/plan9/thread.c | 174 + src/pkg/runtime/pprof/Makefile | 11 + src/pkg/runtime/pprof/pprof.go | 176 + src/pkg/runtime/pprof/pprof_test.go | 77 + src/pkg/runtime/print.c | 351 ++ src/pkg/runtime/proc.c | 1568 +++++ src/pkg/runtime/proc.p | 526 ++ src/pkg/runtime/proc_test.go | 125 + src/pkg/runtime/rune.c | 224 + src/pkg/runtime/runtime-gdb.py | 400 ++ src/pkg/runtime/runtime.c | 728 +++ src/pkg/runtime/runtime.h | 635 ++ src/pkg/runtime/runtime1.goc | 10 + src/pkg/runtime/sema.goc | 180 + src/pkg/runtime/sema_test.go | 100 + src/pkg/runtime/sig.go | 16 + src/pkg/runtime/sigqueue.goc | 99 + src/pkg/runtime/slice.c | 330 + src/pkg/runtime/softfloat64.go | 498 ++ src/pkg/runtime/softfloat64_test.go | 198 + src/pkg/runtime/stack.h | 97 + src/pkg/runtime/string.goc | 360 ++ src/pkg/runtime/symtab.c | 466 ++ src/pkg/runtime/symtab_test.go | 47 + src/pkg/runtime/type.go | 208 + src/pkg/runtime/type.h | 131 + src/pkg/runtime/windows/386/defs.h | 81 + src/pkg/runtime/windows/386/rt0.s | 14 + src/pkg/runtime/windows/386/signal.c | 98 + src/pkg/runtime/windows/386/sys.s | 256 + src/pkg/runtime/windows/amd64/defs.h | 40 + src/pkg/runtime/windows/amd64/rt0.s | 13 + src/pkg/runtime/windows/amd64/signal.c | 20 + src/pkg/runtime/windows/amd64/sys.s | 130 + src/pkg/runtime/windows/defs.c | 37 + src/pkg/runtime/windows/mem.c | 70 + src/pkg/runtime/windows/os.h | 30 + src/pkg/runtime/windows/signals.h | 3 + src/pkg/runtime/windows/syscall.goc | 67 + src/pkg/runtime/windows/thread.c | 432 ++ src/pkg/scanner/Makefile | 11 + src/pkg/scanner/scanner.go | 660 ++ src/pkg/scanner/scanner_test.go | 566 ++ src/pkg/smtp/Makefile | 12 + src/pkg/smtp/auth.go | 69 + src/pkg/smtp/smtp.go | 294 + src/pkg/smtp/smtp_test.go | 182 + src/pkg/sort/Makefile | 12 + src/pkg/sort/search.go | 104 + src/pkg/sort/search_test.go | 132 + src/pkg/sort/sort.go | 202 + src/pkg/sort/sort_test.go | 282 + src/pkg/strconv/Makefile | 17 + src/pkg/strconv/atob.go | 28 + src/pkg/strconv/atob_test.go | 58 + src/pkg/strconv/atof.go | 415 ++ src/pkg/strconv/atof_test.go | 191 + src/pkg/strconv/atoi.go | 202 + src/pkg/strconv/atoi_test.go | 303 + src/pkg/strconv/decimal.go | 371 ++ src/pkg/strconv/decimal_test.go | 117 + src/pkg/strconv/fp_test.go | 149 + src/pkg/strconv/ftoa.go | 405 ++ src/pkg/strconv/ftoa_test.go | 150 + src/pkg/strconv/internal_test.go | 15 + src/pkg/strconv/itoa.go | 57 + src/pkg/strconv/itoa_test.go | 174 + src/pkg/strconv/quote.go | 310 + src/pkg/strconv/quote_test.go | 213 + src/pkg/strconv/testfp.txt | 181 + src/pkg/strings/Makefile | 12 + src/pkg/strings/reader.go | 91 + src/pkg/strings/strings.go | 585 ++ src/pkg/strings/strings_test.go | 927 +++ src/pkg/sync/Makefile | 15 + src/pkg/sync/atomic/Makefile | 18 + src/pkg/sync/atomic/asm_386.s | 96 + src/pkg/sync/atomic/asm_amd64.s | 69 + src/pkg/sync/atomic/asm_arm.s | 122 + src/pkg/sync/atomic/asm_linux_arm.s | 98 + src/pkg/sync/atomic/atomic_test.go | 641 ++ src/pkg/sync/atomic/doc.go | 68 + src/pkg/sync/cond.go | 113 + src/pkg/sync/cond_test.go | 126 + src/pkg/sync/mutex.go | 95 + src/pkg/sync/mutex_test.go | 166 + src/pkg/sync/once.go | 43 + src/pkg/sync/once_test.go | 62 + src/pkg/sync/rwmutex.go | 93 + src/pkg/sync/rwmutex_test.go | 237 + src/pkg/sync/waitgroup.go | 97 + src/pkg/sync/waitgroup_test.go | 165 + src/pkg/syscall/Makefile | 58 + src/pkg/syscall/asm_darwin_386.s | 137 + src/pkg/syscall/asm_darwin_amd64.s | 101 + src/pkg/syscall/asm_freebsd_386.s | 137 + src/pkg/syscall/asm_freebsd_amd64.s | 97 + src/pkg/syscall/asm_linux_386.s | 181 + src/pkg/syscall/asm_linux_amd64.s | 129 + src/pkg/syscall/asm_linux_arm.s | 153 + src/pkg/syscall/asm_plan9_386.s | 151 + src/pkg/syscall/asm_windows_386.s | 7 + src/pkg/syscall/asm_windows_amd64.s | 7 + src/pkg/syscall/bpf_bsd.go | 167 + src/pkg/syscall/exec_plan9.go | 525 ++ src/pkg/syscall/exec_unix.go | 425 ++ src/pkg/syscall/exec_windows.go | 327 + src/pkg/syscall/lsf_linux.go | 78 + src/pkg/syscall/mkall.sh | 186 + src/pkg/syscall/mkerrors.sh | 235 + src/pkg/syscall/mkerrors_windows.sh | 202 + src/pkg/syscall/mksyscall.pl | 234 + src/pkg/syscall/mksyscall_windows.pl | 294 + src/pkg/syscall/mksysnum_darwin.pl | 32 + src/pkg/syscall/mksysnum_freebsd.pl | 43 + src/pkg/syscall/mksysnum_linux.pl | 38 + src/pkg/syscall/mksysnum_plan9.sh | 25 + src/pkg/syscall/netlink_linux.go | 227 + src/pkg/syscall/route_bsd.go | 152 + src/pkg/syscall/route_darwin.go | 77 + src/pkg/syscall/route_freebsd.go | 77 + src/pkg/syscall/sockcmsg_linux.go | 38 + src/pkg/syscall/sockcmsg_unix.go | 113 + src/pkg/syscall/str.go | 20 + src/pkg/syscall/syscall.go | 30 + src/pkg/syscall/syscall_386.go | 7 + src/pkg/syscall/syscall_amd64.go | 7 + src/pkg/syscall/syscall_arm.go | 7 + src/pkg/syscall/syscall_bsd.go | 627 ++ src/pkg/syscall/syscall_darwin.go | 377 ++ src/pkg/syscall/syscall_darwin_386.go | 55 + src/pkg/syscall/syscall_darwin_amd64.go | 53 + src/pkg/syscall/syscall_freebsd.go | 366 ++ src/pkg/syscall/syscall_freebsd_386.go | 44 + src/pkg/syscall/syscall_freebsd_amd64.go | 42 + src/pkg/syscall/syscall_linux.go | 999 +++ src/pkg/syscall/syscall_linux_386.go | 218 + src/pkg/syscall/syscall_linux_amd64.go | 92 + src/pkg/syscall/syscall_linux_arm.go | 138 + src/pkg/syscall/syscall_plan9.go | 357 ++ src/pkg/syscall/syscall_plan9_386.go | 5 + src/pkg/syscall/syscall_unix.go | 90 + src/pkg/syscall/syscall_windows.go | 770 +++ src/pkg/syscall/syscall_windows_386.go | 5 + src/pkg/syscall/syscall_windows_amd64.go | 5 + src/pkg/syscall/types_darwin.c | 180 + src/pkg/syscall/types_freebsd.c | 190 + src/pkg/syscall/types_linux.c | 266 + src/pkg/syscall/types_plan9.c | 113 + src/pkg/syscall/zerrors_darwin_386.go | 1195 ++++ src/pkg/syscall/zerrors_darwin_amd64.go | 1195 ++++ src/pkg/syscall/zerrors_freebsd_386.go | 1405 +++++ src/pkg/syscall/zerrors_freebsd_amd64.go | 1405 +++++ src/pkg/syscall/zerrors_linux_386.go | 1329 ++++ src/pkg/syscall/zerrors_linux_amd64.go | 1330 ++++ src/pkg/syscall/zerrors_linux_arm.go | 1319 ++++ src/pkg/syscall/zerrors_plan9_386.go | 23 + src/pkg/syscall/zerrors_windows.go | 283 + src/pkg/syscall/zerrors_windows_386.go | 5 + src/pkg/syscall/zerrors_windows_amd64.go | 5 + src/pkg/syscall/zsyscall_darwin_386.go | 977 +++ src/pkg/syscall/zsyscall_darwin_amd64.go | 977 +++ src/pkg/syscall/zsyscall_freebsd_386.go | 959 +++ src/pkg/syscall/zsyscall_freebsd_amd64.go | 959 +++ src/pkg/syscall/zsyscall_linux_386.go | 1128 ++++ src/pkg/syscall/zsyscall_linux_amd64.go | 1272 ++++ src/pkg/syscall/zsyscall_linux_arm.go | 1210 ++++ src/pkg/syscall/zsyscall_plan9_386.go | 267 + src/pkg/syscall/zsyscall_windows_386.go | 1323 ++++ src/pkg/syscall/zsyscall_windows_amd64.go | 1323 ++++ src/pkg/syscall/zsysnum_darwin_386.go | 355 ++ src/pkg/syscall/zsysnum_darwin_amd64.go | 355 ++ src/pkg/syscall/zsysnum_freebsd_386.go | 321 + src/pkg/syscall/zsysnum_freebsd_amd64.go | 321 + src/pkg/syscall/zsysnum_linux_386.go | 341 + src/pkg/syscall/zsysnum_linux_amd64.go | 306 + src/pkg/syscall/zsysnum_linux_arm.go | 336 + src/pkg/syscall/zsysnum_plan9_386.go | 47 + src/pkg/syscall/zsysnum_windows_386.go | 3 + src/pkg/syscall/zsysnum_windows_amd64.go | 3 + src/pkg/syscall/ztypes_darwin_386.go | 406 ++ src/pkg/syscall/ztypes_darwin_amd64.go | 417 ++ src/pkg/syscall/ztypes_freebsd_386.go | 401 ++ src/pkg/syscall/ztypes_freebsd_amd64.go | 405 ++ src/pkg/syscall/ztypes_linux_386.go | 486 ++ src/pkg/syscall/ztypes_linux_amd64.go | 502 ++ src/pkg/syscall/ztypes_linux_arm.go | 476 ++ src/pkg/syscall/ztypes_plan9_386.go | 75 + src/pkg/syscall/ztypes_windows.go | 603 ++ src/pkg/syscall/ztypes_windows_386.go | 22 + src/pkg/syscall/ztypes_windows_amd64.go | 22 + src/pkg/syslog/Makefile | 12 + src/pkg/syslog/syslog.go | 149 + src/pkg/syslog/syslog_test.go | 113 + src/pkg/syslog/syslog_unix.go | 31 + src/pkg/tabwriter/Makefile | 11 + src/pkg/tabwriter/tabwriter.go | 561 ++ src/pkg/tabwriter/tabwriter_test.go | 615 ++ src/pkg/template/Makefile | 15 + src/pkg/template/doc.go | 313 + src/pkg/template/exec.go | 674 ++ src/pkg/template/exec_test.go | 613 ++ src/pkg/template/funcs.go | 368 ++ src/pkg/template/helper.go | 238 + src/pkg/template/parse.go | 85 + src/pkg/template/parse/Makefile | 14 + src/pkg/template/parse/lex.go | 472 ++ src/pkg/template/parse/lex_test.go | 223 + src/pkg/template/parse/node.go | 470 ++ src/pkg/template/parse/parse.go | 436 ++ src/pkg/template/parse/parse_test.go | 259 + src/pkg/template/parse/set.go | 50 + src/pkg/template/set.go | 108 + src/pkg/template/set_test.go | 239 + src/pkg/template/testdata/file1.tmpl | 2 + src/pkg/template/testdata/file2.tmpl | 2 + src/pkg/template/testdata/tmpl1.tmpl | 1 + src/pkg/template/testdata/tmpl2.tmpl | 1 + src/pkg/testing/Makefile | 12 + src/pkg/testing/benchmark.go | 236 + src/pkg/testing/iotest/Makefile | 13 + src/pkg/testing/iotest/logger.go | 55 + src/pkg/testing/iotest/reader.go | 86 + src/pkg/testing/iotest/writer.go | 38 + src/pkg/testing/quick/Makefile | 11 + src/pkg/testing/quick/quick.go | 362 ++ src/pkg/testing/quick/quick_test.go | 147 + src/pkg/testing/script/Makefile | 11 + src/pkg/testing/script/script.go | 359 ++ src/pkg/testing/script/script_test.go | 75 + src/pkg/testing/testing.go | 298 + src/pkg/time/Makefile | 46 + src/pkg/time/format.go | 734 +++ src/pkg/time/sleep.go | 177 + src/pkg/time/sleep_test.go | 189 + src/pkg/time/sys.go | 51 + src/pkg/time/sys_plan9.go | 18 + src/pkg/time/sys_posix.go | 18 + src/pkg/time/tick.go | 177 + src/pkg/time/tick_test.go | 58 + src/pkg/time/time.go | 230 + src/pkg/time/time_test.go | 506 ++ src/pkg/time/zoneinfo_plan9.go | 59 + src/pkg/time/zoneinfo_posix.go | 62 + src/pkg/time/zoneinfo_unix.go | 213 + src/pkg/time/zoneinfo_windows.go | 192 + src/pkg/try/Makefile | 11 + src/pkg/try/try.go | 174 + src/pkg/try/try_test.go | 60 + src/pkg/unicode/Makefile | 36 + src/pkg/unicode/casetables.go | 20 + src/pkg/unicode/digit.go | 13 + src/pkg/unicode/digit_test.go | 126 + src/pkg/unicode/graphic.go | 132 + src/pkg/unicode/graphic_test.go | 122 + src/pkg/unicode/letter.go | 326 + src/pkg/unicode/letter_test.go | 428 ++ src/pkg/unicode/maketables.go | 1276 ++++ src/pkg/unicode/script_test.go | 257 + src/pkg/unicode/tables.go | 6026 ++++++++++++++++++ src/pkg/unsafe/unsafe.go | 61 + src/pkg/url/Makefile | 11 + src/pkg/url/url.go | 677 ++ src/pkg/url/url_test.go | 698 +++ src/pkg/utf16/Makefile | 11 + src/pkg/utf16/utf16.go | 101 + src/pkg/utf16/utf16_test.go | 118 + src/pkg/utf8/Makefile | 12 + src/pkg/utf8/string.go | 211 + src/pkg/utf8/string_test.go | 114 + src/pkg/utf8/utf8.go | 356 ++ src/pkg/utf8/utf8_test.go | 315 + src/pkg/websocket/Makefile | 9 + src/pkg/websocket/client.go | 323 + src/pkg/websocket/server.go | 219 + src/pkg/websocket/websocket.go | 192 + src/pkg/websocket/websocket_test.go | 271 + src/pkg/xml/Makefile | 14 + src/pkg/xml/atom_test.go | 50 + src/pkg/xml/embed_test.go | 124 + src/pkg/xml/marshal.go | 228 + src/pkg/xml/marshal_test.go | 305 + src/pkg/xml/read.go | 631 ++ src/pkg/xml/read_test.go | 371 ++ src/pkg/xml/xml.go | 1694 +++++ src/pkg/xml/xml_test.go | 609 ++ 1529 files changed, 322728 insertions(+) create mode 100644 src/pkg/Makefile create mode 100644 src/pkg/archive/tar/Makefile create mode 100644 src/pkg/archive/tar/common.go create mode 100644 src/pkg/archive/tar/reader.go create mode 100644 src/pkg/archive/tar/reader_test.go create mode 100644 src/pkg/archive/tar/testdata/gnu.tar create mode 100644 src/pkg/archive/tar/testdata/small.txt create mode 100644 src/pkg/archive/tar/testdata/small2.txt create mode 100644 src/pkg/archive/tar/testdata/star.tar create mode 100644 src/pkg/archive/tar/testdata/v7.tar create mode 100644 src/pkg/archive/tar/testdata/writer-big.tar create mode 100644 src/pkg/archive/tar/testdata/writer.tar create mode 100644 src/pkg/archive/tar/writer.go create mode 100644 src/pkg/archive/tar/writer_test.go create mode 100644 src/pkg/archive/zip/Makefile create mode 100644 src/pkg/archive/zip/reader.go create mode 100644 src/pkg/archive/zip/reader_test.go create mode 100644 src/pkg/archive/zip/struct.go create mode 100644 src/pkg/archive/zip/testdata/dd.zip create mode 100644 src/pkg/archive/zip/testdata/gophercolor16x16.png create mode 100644 src/pkg/archive/zip/testdata/r.zip create mode 100644 src/pkg/archive/zip/testdata/readme.notzip create mode 100644 src/pkg/archive/zip/testdata/readme.zip create mode 100644 src/pkg/archive/zip/testdata/test.zip create mode 100644 src/pkg/archive/zip/writer.go create mode 100644 src/pkg/archive/zip/writer_test.go create mode 100644 src/pkg/archive/zip/zip_test.go create mode 100644 src/pkg/asn1/Makefile create mode 100644 src/pkg/asn1/asn1.go create mode 100644 src/pkg/asn1/asn1_test.go create mode 100644 src/pkg/asn1/common.go create mode 100644 src/pkg/asn1/marshal.go create mode 100644 src/pkg/asn1/marshal_test.go create mode 100644 src/pkg/big/Makefile create mode 100644 src/pkg/big/arith.go create mode 100644 src/pkg/big/arith_386.s create mode 100644 src/pkg/big/arith_amd64.s create mode 100644 src/pkg/big/arith_arm.s create mode 100644 src/pkg/big/arith_decl.go create mode 100644 src/pkg/big/arith_test.go create mode 100644 src/pkg/big/calibrate_test.go create mode 100644 src/pkg/big/hilbert_test.go create mode 100755 src/pkg/big/int.go create mode 100755 src/pkg/big/int_test.go create mode 100755 src/pkg/big/nat.go create mode 100755 src/pkg/big/nat_test.go create mode 100644 src/pkg/big/rat.go create mode 100644 src/pkg/big/rat_test.go create mode 100644 src/pkg/bufio/Makefile create mode 100644 src/pkg/bufio/bufio.go create mode 100644 src/pkg/bufio/bufio_test.go create mode 100644 src/pkg/builtin/builtin.go create mode 100644 src/pkg/bytes/Makefile create mode 100644 src/pkg/bytes/asm_386.s create mode 100644 src/pkg/bytes/asm_amd64.s create mode 100644 src/pkg/bytes/asm_arm.s create mode 100644 src/pkg/bytes/buffer.go create mode 100644 src/pkg/bytes/buffer_test.go create mode 100644 src/pkg/bytes/bytes.go create mode 100644 src/pkg/bytes/bytes_decl.go create mode 100644 src/pkg/bytes/bytes_test.go create mode 100644 src/pkg/bytes/export_test.go create mode 100644 src/pkg/cmath/Makefile create mode 100644 src/pkg/cmath/abs.go create mode 100644 src/pkg/cmath/asin.go create mode 100644 src/pkg/cmath/cmath_test.go create mode 100644 src/pkg/cmath/conj.go create mode 100644 src/pkg/cmath/exp.go create mode 100644 src/pkg/cmath/isinf.go create mode 100644 src/pkg/cmath/isnan.go create mode 100644 src/pkg/cmath/log.go create mode 100644 src/pkg/cmath/phase.go create mode 100644 src/pkg/cmath/polar.go create mode 100644 src/pkg/cmath/pow.go create mode 100644 src/pkg/cmath/rect.go create mode 100644 src/pkg/cmath/sin.go create mode 100644 src/pkg/cmath/sqrt.go create mode 100644 src/pkg/cmath/tan.go create mode 100644 src/pkg/compress/bzip2/Makefile create mode 100644 src/pkg/compress/bzip2/bit_reader.go create mode 100644 src/pkg/compress/bzip2/bzip2.go create mode 100644 src/pkg/compress/bzip2/bzip2_test.go create mode 100644 src/pkg/compress/bzip2/huffman.go create mode 100644 src/pkg/compress/bzip2/move_to_front.go create mode 100644 src/pkg/compress/flate/Makefile create mode 100644 src/pkg/compress/flate/deflate.go create mode 100644 src/pkg/compress/flate/deflate_test.go create mode 100644 src/pkg/compress/flate/flate_test.go create mode 100644 src/pkg/compress/flate/huffman_bit_writer.go create mode 100644 src/pkg/compress/flate/huffman_code.go create mode 100644 src/pkg/compress/flate/inflate.go create mode 100644 src/pkg/compress/flate/reverse_bits.go create mode 100644 src/pkg/compress/flate/token.go create mode 100644 src/pkg/compress/flate/util.go create mode 100644 src/pkg/compress/gzip/Makefile create mode 100644 src/pkg/compress/gzip/gunzip.go create mode 100644 src/pkg/compress/gzip/gunzip_test.go create mode 100644 src/pkg/compress/gzip/gzip.go create mode 100644 src/pkg/compress/gzip/gzip_test.go create mode 100644 src/pkg/compress/lzw/Makefile create mode 100644 src/pkg/compress/lzw/reader.go create mode 100644 src/pkg/compress/lzw/reader_test.go create mode 100644 src/pkg/compress/lzw/writer.go create mode 100644 src/pkg/compress/lzw/writer_test.go create mode 100644 src/pkg/compress/testdata/e.txt create mode 100644 src/pkg/compress/testdata/pi.txt create mode 100644 src/pkg/compress/zlib/Makefile create mode 100644 src/pkg/compress/zlib/reader.go create mode 100644 src/pkg/compress/zlib/reader_test.go create mode 100644 src/pkg/compress/zlib/writer.go create mode 100644 src/pkg/compress/zlib/writer_test.go create mode 100644 src/pkg/container/heap/Makefile create mode 100644 src/pkg/container/heap/heap.go create mode 100644 src/pkg/container/heap/heap_test.go create mode 100644 src/pkg/container/list/Makefile create mode 100644 src/pkg/container/list/list.go create mode 100644 src/pkg/container/list/list_test.go create mode 100644 src/pkg/container/ring/Makefile create mode 100644 src/pkg/container/ring/ring.go create mode 100644 src/pkg/container/ring/ring_test.go create mode 100644 src/pkg/container/vector/Makefile create mode 100644 src/pkg/container/vector/defs.go create mode 100644 src/pkg/container/vector/intvector.go create mode 100644 src/pkg/container/vector/intvector_test.go create mode 100644 src/pkg/container/vector/nogen_test.go create mode 100644 src/pkg/container/vector/numbers_test.go create mode 100644 src/pkg/container/vector/stringvector.go create mode 100644 src/pkg/container/vector/stringvector_test.go create mode 100644 src/pkg/container/vector/vector.go create mode 100644 src/pkg/container/vector/vector_test.go create mode 100644 src/pkg/crypto/Makefile create mode 100644 src/pkg/crypto/aes/Makefile create mode 100644 src/pkg/crypto/aes/aes_test.go create mode 100644 src/pkg/crypto/aes/block.go create mode 100644 src/pkg/crypto/aes/cipher.go create mode 100644 src/pkg/crypto/aes/const.go create mode 100644 src/pkg/crypto/blowfish/Makefile create mode 100644 src/pkg/crypto/blowfish/block.go create mode 100644 src/pkg/crypto/blowfish/blowfish_test.go create mode 100644 src/pkg/crypto/blowfish/cipher.go create mode 100644 src/pkg/crypto/blowfish/const.go create mode 100644 src/pkg/crypto/cast5/Makefile create mode 100644 src/pkg/crypto/cast5/cast5.go create mode 100644 src/pkg/crypto/cast5/cast5_test.go create mode 100644 src/pkg/crypto/cipher/Makefile create mode 100644 src/pkg/crypto/cipher/cbc.go create mode 100644 src/pkg/crypto/cipher/cbc_aes_test.go create mode 100644 src/pkg/crypto/cipher/cfb.go create mode 100644 src/pkg/crypto/cipher/cfb_test.go create mode 100644 src/pkg/crypto/cipher/cipher.go create mode 100644 src/pkg/crypto/cipher/common_test.go create mode 100644 src/pkg/crypto/cipher/ctr.go create mode 100644 src/pkg/crypto/cipher/ctr_aes_test.go create mode 100644 src/pkg/crypto/cipher/io.go create mode 100644 src/pkg/crypto/cipher/ocfb.go create mode 100644 src/pkg/crypto/cipher/ocfb_test.go create mode 100644 src/pkg/crypto/cipher/ofb.go create mode 100644 src/pkg/crypto/cipher/ofb_test.go create mode 100644 src/pkg/crypto/crypto.go create mode 100644 src/pkg/crypto/des/Makefile create mode 100644 src/pkg/crypto/des/block.go create mode 100644 src/pkg/crypto/des/cipher.go create mode 100644 src/pkg/crypto/des/const.go create mode 100644 src/pkg/crypto/des/des_test.go create mode 100644 src/pkg/crypto/dsa/Makefile create mode 100644 src/pkg/crypto/dsa/dsa.go create mode 100644 src/pkg/crypto/dsa/dsa_test.go create mode 100644 src/pkg/crypto/ecdsa/Makefile create mode 100644 src/pkg/crypto/ecdsa/ecdsa.go create mode 100644 src/pkg/crypto/ecdsa/ecdsa_test.go create mode 100644 src/pkg/crypto/elliptic/Makefile create mode 100644 src/pkg/crypto/elliptic/elliptic.go create mode 100644 src/pkg/crypto/elliptic/elliptic_test.go create mode 100644 src/pkg/crypto/hmac/Makefile create mode 100644 src/pkg/crypto/hmac/hmac.go create mode 100644 src/pkg/crypto/hmac/hmac_test.go create mode 100644 src/pkg/crypto/md4/Makefile create mode 100644 src/pkg/crypto/md4/md4.go create mode 100644 src/pkg/crypto/md4/md4_test.go create mode 100644 src/pkg/crypto/md4/md4block.go create mode 100644 src/pkg/crypto/md5/Makefile create mode 100644 src/pkg/crypto/md5/md5.go create mode 100644 src/pkg/crypto/md5/md5_test.go create mode 100644 src/pkg/crypto/md5/md5block.go create mode 100644 src/pkg/crypto/ocsp/Makefile create mode 100644 src/pkg/crypto/ocsp/ocsp.go create mode 100644 src/pkg/crypto/ocsp/ocsp_test.go create mode 100644 src/pkg/crypto/openpgp/Makefile create mode 100644 src/pkg/crypto/openpgp/armor/Makefile create mode 100644 src/pkg/crypto/openpgp/armor/armor.go create mode 100644 src/pkg/crypto/openpgp/armor/armor_test.go create mode 100644 src/pkg/crypto/openpgp/armor/encode.go create mode 100644 src/pkg/crypto/openpgp/canonical_text.go create mode 100644 src/pkg/crypto/openpgp/canonical_text_test.go create mode 100644 src/pkg/crypto/openpgp/elgamal/Makefile create mode 100644 src/pkg/crypto/openpgp/elgamal/elgamal.go create mode 100644 src/pkg/crypto/openpgp/elgamal/elgamal_test.go create mode 100644 src/pkg/crypto/openpgp/error/Makefile create mode 100644 src/pkg/crypto/openpgp/error/error.go create mode 100644 src/pkg/crypto/openpgp/keys.go create mode 100644 src/pkg/crypto/openpgp/packet/Makefile create mode 100644 src/pkg/crypto/openpgp/packet/compressed.go create mode 100644 src/pkg/crypto/openpgp/packet/compressed_test.go create mode 100644 src/pkg/crypto/openpgp/packet/encrypted_key.go create mode 100644 src/pkg/crypto/openpgp/packet/encrypted_key_test.go create mode 100644 src/pkg/crypto/openpgp/packet/literal.go create mode 100644 src/pkg/crypto/openpgp/packet/one_pass_signature.go create mode 100644 src/pkg/crypto/openpgp/packet/packet.go create mode 100644 src/pkg/crypto/openpgp/packet/packet_test.go create mode 100644 src/pkg/crypto/openpgp/packet/private_key.go create mode 100644 src/pkg/crypto/openpgp/packet/private_key_test.go create mode 100644 src/pkg/crypto/openpgp/packet/public_key.go create mode 100644 src/pkg/crypto/openpgp/packet/public_key_test.go create mode 100644 src/pkg/crypto/openpgp/packet/reader.go create mode 100644 src/pkg/crypto/openpgp/packet/signature.go create mode 100644 src/pkg/crypto/openpgp/packet/signature_test.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go create mode 100644 src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go create mode 100644 src/pkg/crypto/openpgp/packet/userid.go create mode 100644 src/pkg/crypto/openpgp/packet/userid_test.go create mode 100644 src/pkg/crypto/openpgp/read.go create mode 100644 src/pkg/crypto/openpgp/read_test.go create mode 100644 src/pkg/crypto/openpgp/s2k/Makefile create mode 100644 src/pkg/crypto/openpgp/s2k/s2k.go create mode 100644 src/pkg/crypto/openpgp/s2k/s2k_test.go create mode 100644 src/pkg/crypto/openpgp/write.go create mode 100644 src/pkg/crypto/openpgp/write_test.go create mode 100644 src/pkg/crypto/rand/Makefile create mode 100644 src/pkg/crypto/rand/rand.go create mode 100644 src/pkg/crypto/rand/rand_test.go create mode 100644 src/pkg/crypto/rand/rand_unix.go create mode 100755 src/pkg/crypto/rand/rand_windows.go create mode 100644 src/pkg/crypto/rand/util.go create mode 100644 src/pkg/crypto/rc4/Makefile create mode 100644 src/pkg/crypto/rc4/rc4.go create mode 100644 src/pkg/crypto/rc4/rc4_test.go create mode 100644 src/pkg/crypto/ripemd160/Makefile create mode 100644 src/pkg/crypto/ripemd160/ripemd160.go create mode 100644 src/pkg/crypto/ripemd160/ripemd160_test.go create mode 100644 src/pkg/crypto/ripemd160/ripemd160block.go create mode 100644 src/pkg/crypto/rsa/Makefile create mode 100644 src/pkg/crypto/rsa/pkcs1v15.go create mode 100644 src/pkg/crypto/rsa/pkcs1v15_test.go create mode 100644 src/pkg/crypto/rsa/rsa.go create mode 100644 src/pkg/crypto/rsa/rsa_test.go create mode 100644 src/pkg/crypto/sha1/Makefile create mode 100644 src/pkg/crypto/sha1/sha1.go create mode 100644 src/pkg/crypto/sha1/sha1_test.go create mode 100644 src/pkg/crypto/sha1/sha1block.go create mode 100644 src/pkg/crypto/sha256/Makefile create mode 100644 src/pkg/crypto/sha256/sha256.go create mode 100644 src/pkg/crypto/sha256/sha256_test.go create mode 100644 src/pkg/crypto/sha256/sha256block.go create mode 100644 src/pkg/crypto/sha512/Makefile create mode 100644 src/pkg/crypto/sha512/sha512.go create mode 100644 src/pkg/crypto/sha512/sha512_test.go create mode 100644 src/pkg/crypto/sha512/sha512block.go create mode 100644 src/pkg/crypto/subtle/Makefile create mode 100644 src/pkg/crypto/subtle/constant_time.go create mode 100644 src/pkg/crypto/subtle/constant_time_test.go create mode 100644 src/pkg/crypto/tls/Makefile create mode 100644 src/pkg/crypto/tls/alert.go create mode 100644 src/pkg/crypto/tls/cipher_suites.go create mode 100644 src/pkg/crypto/tls/common.go create mode 100644 src/pkg/crypto/tls/conn.go create mode 100644 src/pkg/crypto/tls/conn_test.go create mode 100644 src/pkg/crypto/tls/generate_cert.go create mode 100644 src/pkg/crypto/tls/handshake_client.go create mode 100644 src/pkg/crypto/tls/handshake_client_test.go create mode 100644 src/pkg/crypto/tls/handshake_messages.go create mode 100644 src/pkg/crypto/tls/handshake_messages_test.go create mode 100644 src/pkg/crypto/tls/handshake_server.go create mode 100644 src/pkg/crypto/tls/handshake_server_test.go create mode 100644 src/pkg/crypto/tls/key_agreement.go create mode 100644 src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py create mode 100644 src/pkg/crypto/tls/prf.go create mode 100644 src/pkg/crypto/tls/prf_test.go create mode 100644 src/pkg/crypto/tls/tls.go create mode 100644 src/pkg/crypto/twofish/Makefile create mode 100644 src/pkg/crypto/twofish/twofish.go create mode 100644 src/pkg/crypto/twofish/twofish_test.go create mode 100644 src/pkg/crypto/x509/Makefile create mode 100644 src/pkg/crypto/x509/cert_pool.go create mode 100644 src/pkg/crypto/x509/pkix/Makefile create mode 100644 src/pkg/crypto/x509/pkix/pkix.go create mode 100644 src/pkg/crypto/x509/verify.go create mode 100644 src/pkg/crypto/x509/verify_test.go create mode 100644 src/pkg/crypto/x509/x509.go create mode 100644 src/pkg/crypto/x509/x509_test.go create mode 100644 src/pkg/crypto/xtea/Makefile create mode 100644 src/pkg/crypto/xtea/block.go create mode 100644 src/pkg/crypto/xtea/cipher.go create mode 100644 src/pkg/crypto/xtea/xtea_test.go create mode 100644 src/pkg/csv/Makefile create mode 100644 src/pkg/csv/reader.go create mode 100644 src/pkg/csv/reader_test.go create mode 100644 src/pkg/csv/writer.go create mode 100644 src/pkg/csv/writer_test.go create mode 100644 src/pkg/debug/dwarf/Makefile create mode 100644 src/pkg/debug/dwarf/buf.go create mode 100644 src/pkg/debug/dwarf/const.go create mode 100644 src/pkg/debug/dwarf/entry.go create mode 100644 src/pkg/debug/dwarf/open.go create mode 100644 src/pkg/debug/dwarf/testdata/typedef.c create mode 100755 src/pkg/debug/dwarf/testdata/typedef.elf create mode 100644 src/pkg/debug/dwarf/testdata/typedef.macho create mode 100644 src/pkg/debug/dwarf/type.go create mode 100644 src/pkg/debug/dwarf/type_test.go create mode 100644 src/pkg/debug/dwarf/unit.go create mode 100644 src/pkg/debug/elf/Makefile create mode 100644 src/pkg/debug/elf/elf.go create mode 100644 src/pkg/debug/elf/elf_test.go create mode 100644 src/pkg/debug/elf/file.go create mode 100644 src/pkg/debug/elf/file_test.go create mode 100755 src/pkg/debug/elf/testdata/gcc-386-freebsd-exec create mode 100755 src/pkg/debug/elf/testdata/gcc-amd64-linux-exec create mode 100644 src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj create mode 100644 src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj create mode 100644 src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj create mode 100644 src/pkg/debug/gosym/Makefile create mode 100644 src/pkg/debug/gosym/pclinetest.h create mode 100644 src/pkg/debug/gosym/pclinetest.s create mode 100644 src/pkg/debug/gosym/pclntab.go create mode 100644 src/pkg/debug/gosym/pclntab_test.go create mode 100644 src/pkg/debug/gosym/symtab.go create mode 100644 src/pkg/debug/macho/Makefile create mode 100644 src/pkg/debug/macho/file.go create mode 100644 src/pkg/debug/macho/file_test.go create mode 100644 src/pkg/debug/macho/macho.go create mode 100755 src/pkg/debug/macho/testdata/gcc-386-darwin-exec create mode 100755 src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec create mode 100644 src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug create mode 100644 src/pkg/debug/macho/testdata/hello.c create mode 100644 src/pkg/debug/pe/Makefile create mode 100644 src/pkg/debug/pe/file.go create mode 100644 src/pkg/debug/pe/file_test.go create mode 100644 src/pkg/debug/pe/pe.go create mode 100644 src/pkg/debug/pe/testdata/gcc-386-mingw-exec create mode 100644 src/pkg/debug/pe/testdata/gcc-386-mingw-obj create mode 100644 src/pkg/debug/pe/testdata/hello.c create mode 100755 src/pkg/deps.bash create mode 100644 src/pkg/ebnf/Makefile create mode 100644 src/pkg/ebnf/ebnf.go create mode 100644 src/pkg/ebnf/ebnf_test.go create mode 100644 src/pkg/ebnf/parser.go create mode 100644 src/pkg/encoding/ascii85/Makefile create mode 100644 src/pkg/encoding/ascii85/ascii85.go create mode 100644 src/pkg/encoding/ascii85/ascii85_test.go create mode 100644 src/pkg/encoding/base32/Makefile create mode 100644 src/pkg/encoding/base32/base32.go create mode 100644 src/pkg/encoding/base32/base32_test.go create mode 100644 src/pkg/encoding/base64/Makefile create mode 100644 src/pkg/encoding/base64/base64.go create mode 100644 src/pkg/encoding/base64/base64_test.go create mode 100644 src/pkg/encoding/binary/Makefile create mode 100644 src/pkg/encoding/binary/binary.go create mode 100644 src/pkg/encoding/binary/binary_test.go create mode 100644 src/pkg/encoding/git85/Makefile create mode 100644 src/pkg/encoding/git85/git.go create mode 100644 src/pkg/encoding/git85/git_test.go create mode 100644 src/pkg/encoding/hex/Makefile create mode 100644 src/pkg/encoding/hex/hex.go create mode 100644 src/pkg/encoding/hex/hex_test.go create mode 100644 src/pkg/encoding/pem/Makefile create mode 100644 src/pkg/encoding/pem/pem.go create mode 100644 src/pkg/encoding/pem/pem_test.go create mode 100644 src/pkg/exec/Makefile create mode 100644 src/pkg/exec/exec.go create mode 100644 src/pkg/exec/exec_test.go create mode 100644 src/pkg/exec/lp_plan9.go create mode 100644 src/pkg/exec/lp_test.go create mode 100644 src/pkg/exec/lp_unix.go create mode 100644 src/pkg/exec/lp_windows.go create mode 100644 src/pkg/exp/README create mode 100644 src/pkg/exp/datafmt/Makefile create mode 100644 src/pkg/exp/datafmt/datafmt.go create mode 100644 src/pkg/exp/datafmt/datafmt_test.go create mode 100644 src/pkg/exp/datafmt/parser.go create mode 100644 src/pkg/exp/gui/Makefile create mode 100644 src/pkg/exp/gui/gui.go create mode 100644 src/pkg/exp/gui/x11/Makefile create mode 100644 src/pkg/exp/gui/x11/auth.go create mode 100644 src/pkg/exp/gui/x11/conn.go create mode 100644 src/pkg/exp/norm/Makefile create mode 100644 src/pkg/exp/norm/composition.go create mode 100644 src/pkg/exp/norm/composition_test.go create mode 100644 src/pkg/exp/norm/forminfo.go create mode 100644 src/pkg/exp/norm/maketables.go create mode 100644 src/pkg/exp/norm/maketesttables.go create mode 100644 src/pkg/exp/norm/norm_test.go create mode 100644 src/pkg/exp/norm/normalize.go create mode 100644 src/pkg/exp/norm/tables.go create mode 100644 src/pkg/exp/norm/trie.go create mode 100644 src/pkg/exp/norm/trie_test.go create mode 100644 src/pkg/exp/norm/triedata_test.go create mode 100644 src/pkg/exp/norm/triegen.go create mode 100644 src/pkg/exp/regexp/Makefile create mode 100644 src/pkg/exp/regexp/all_test.go create mode 100644 src/pkg/exp/regexp/exec.go create mode 100644 src/pkg/exp/regexp/find_test.go create mode 100644 src/pkg/exp/regexp/regexp.go create mode 100644 src/pkg/exp/regexp/syntax/Makefile create mode 100644 src/pkg/exp/regexp/syntax/compile.go create mode 100755 src/pkg/exp/regexp/syntax/make_perl_groups.pl create mode 100644 src/pkg/exp/regexp/syntax/parse.go create mode 100644 src/pkg/exp/regexp/syntax/parse_test.go create mode 100644 src/pkg/exp/regexp/syntax/perl_groups.go create mode 100644 src/pkg/exp/regexp/syntax/prog.go create mode 100644 src/pkg/exp/regexp/syntax/prog_test.go create mode 100644 src/pkg/exp/regexp/syntax/regexp.go create mode 100644 src/pkg/exp/regexp/syntax/simplify.go create mode 100644 src/pkg/exp/regexp/syntax/simplify_test.go create mode 100644 src/pkg/exp/template/html/Makefile create mode 100644 src/pkg/exp/template/html/context.go create mode 100644 src/pkg/exp/template/html/escape.go create mode 100644 src/pkg/exp/template/html/escape_test.go create mode 100644 src/pkg/exp/wingui/Makefile create mode 100644 src/pkg/exp/wingui/gui.go create mode 100644 src/pkg/exp/wingui/winapi.go create mode 100644 src/pkg/exp/wingui/zwinapi.go create mode 100644 src/pkg/expvar/Makefile create mode 100644 src/pkg/expvar/expvar.go create mode 100644 src/pkg/expvar/expvar_test.go create mode 100644 src/pkg/flag/Makefile create mode 100644 src/pkg/flag/export_test.go create mode 100644 src/pkg/flag/flag.go create mode 100644 src/pkg/flag/flag_test.go create mode 100644 src/pkg/fmt/Makefile create mode 100644 src/pkg/fmt/doc.go create mode 100644 src/pkg/fmt/fmt_test.go create mode 100644 src/pkg/fmt/format.go create mode 100644 src/pkg/fmt/print.go create mode 100644 src/pkg/fmt/scan.go create mode 100644 src/pkg/fmt/scan_test.go create mode 100644 src/pkg/fmt/stringer_test.go create mode 100644 src/pkg/go/ast/Makefile create mode 100644 src/pkg/go/ast/ast.go create mode 100644 src/pkg/go/ast/filter.go create mode 100644 src/pkg/go/ast/print.go create mode 100644 src/pkg/go/ast/print_test.go create mode 100644 src/pkg/go/ast/resolve.go create mode 100644 src/pkg/go/ast/scope.go create mode 100644 src/pkg/go/ast/walk.go create mode 100644 src/pkg/go/build/Makefile create mode 100644 src/pkg/go/build/build.go create mode 100644 src/pkg/go/build/build_test.go create mode 100644 src/pkg/go/build/cgotest/cgotest.c create mode 100644 src/pkg/go/build/cgotest/cgotest.go create mode 100644 src/pkg/go/build/cgotest/cgotest.h create mode 100644 src/pkg/go/build/cmdtest/main.go create mode 100644 src/pkg/go/build/dir.go create mode 100644 src/pkg/go/build/path.go create mode 100644 src/pkg/go/build/pkgtest/pkgtest.go create mode 100644 src/pkg/go/build/pkgtest/sqrt_386.s create mode 100644 src/pkg/go/build/pkgtest/sqrt_amd64.s create mode 100644 src/pkg/go/build/pkgtest/sqrt_arm.s create mode 100644 src/pkg/go/build/syslist_test.go create mode 100644 src/pkg/go/doc/Makefile create mode 100644 src/pkg/go/doc/comment.go create mode 100644 src/pkg/go/doc/doc.go create mode 100644 src/pkg/go/parser/Makefile create mode 100644 src/pkg/go/parser/interface.go create mode 100644 src/pkg/go/parser/parser.go create mode 100644 src/pkg/go/parser/parser_test.go create mode 100644 src/pkg/go/printer/Makefile create mode 100644 src/pkg/go/printer/nodes.go create mode 100644 src/pkg/go/printer/performance_test.go create mode 100644 src/pkg/go/printer/printer.go create mode 100644 src/pkg/go/printer/printer_test.go create mode 100644 src/pkg/go/printer/testdata/comments.golden create mode 100644 src/pkg/go/printer/testdata/comments.input create mode 100644 src/pkg/go/printer/testdata/comments.x create mode 100644 src/pkg/go/printer/testdata/declarations.golden create mode 100644 src/pkg/go/printer/testdata/declarations.input create mode 100644 src/pkg/go/printer/testdata/empty.golden create mode 100644 src/pkg/go/printer/testdata/empty.input create mode 100644 src/pkg/go/printer/testdata/expressions.golden create mode 100644 src/pkg/go/printer/testdata/expressions.input create mode 100644 src/pkg/go/printer/testdata/expressions.raw create mode 100644 src/pkg/go/printer/testdata/linebreaks.golden create mode 100644 src/pkg/go/printer/testdata/linebreaks.input create mode 100644 src/pkg/go/printer/testdata/parser.go create mode 100644 src/pkg/go/printer/testdata/slow.golden create mode 100644 src/pkg/go/printer/testdata/slow.input create mode 100644 src/pkg/go/printer/testdata/statements.golden create mode 100644 src/pkg/go/printer/testdata/statements.input create mode 100644 src/pkg/go/scanner/Makefile create mode 100644 src/pkg/go/scanner/errors.go create mode 100644 src/pkg/go/scanner/scanner.go create mode 100644 src/pkg/go/scanner/scanner_test.go create mode 100644 src/pkg/go/token/Makefile create mode 100644 src/pkg/go/token/position.go create mode 100644 src/pkg/go/token/position_test.go create mode 100644 src/pkg/go/token/token.go create mode 100644 src/pkg/go/typechecker/Makefile create mode 100644 src/pkg/go/typechecker/scope.go create mode 100644 src/pkg/go/typechecker/testdata/test0.src create mode 100644 src/pkg/go/typechecker/testdata/test1.src create mode 100644 src/pkg/go/typechecker/testdata/test3.src create mode 100644 src/pkg/go/typechecker/testdata/test4.src create mode 100644 src/pkg/go/typechecker/type.go create mode 100644 src/pkg/go/typechecker/typechecker.go create mode 100644 src/pkg/go/typechecker/typechecker_test.go create mode 100644 src/pkg/go/typechecker/universe.go create mode 100644 src/pkg/go/types/Makefile create mode 100644 src/pkg/go/types/check.go create mode 100644 src/pkg/go/types/check_test.go create mode 100644 src/pkg/go/types/const.go create mode 100644 src/pkg/go/types/exportdata.go create mode 100644 src/pkg/go/types/gcimporter.go create mode 100644 src/pkg/go/types/gcimporter_test.go create mode 100644 src/pkg/go/types/testdata/exports.go create mode 100644 src/pkg/go/types/testdata/test0.src create mode 100644 src/pkg/go/types/types.go create mode 100644 src/pkg/go/types/universe.go create mode 100644 src/pkg/gob/Makefile create mode 100644 src/pkg/gob/codec_test.go create mode 100644 src/pkg/gob/debug.go create mode 100644 src/pkg/gob/decode.go create mode 100644 src/pkg/gob/decoder.go create mode 100644 src/pkg/gob/doc.go create mode 100644 src/pkg/gob/dump.go create mode 100644 src/pkg/gob/encode.go create mode 100644 src/pkg/gob/encoder.go create mode 100644 src/pkg/gob/encoder_test.go create mode 100644 src/pkg/gob/error.go create mode 100644 src/pkg/gob/gobencdec_test.go create mode 100644 src/pkg/gob/timing_test.go create mode 100644 src/pkg/gob/type.go create mode 100644 src/pkg/gob/type_test.go create mode 100644 src/pkg/hash/Makefile create mode 100644 src/pkg/hash/adler32/Makefile create mode 100644 src/pkg/hash/adler32/adler32.go create mode 100644 src/pkg/hash/adler32/adler32_test.go create mode 100644 src/pkg/hash/crc32/Makefile create mode 100644 src/pkg/hash/crc32/crc32.go create mode 100644 src/pkg/hash/crc32/crc32_amd64.go create mode 100644 src/pkg/hash/crc32/crc32_amd64.s create mode 100644 src/pkg/hash/crc32/crc32_generic.go create mode 100644 src/pkg/hash/crc32/crc32_test.go create mode 100644 src/pkg/hash/crc64/Makefile create mode 100644 src/pkg/hash/crc64/crc64.go create mode 100644 src/pkg/hash/crc64/crc64_test.go create mode 100644 src/pkg/hash/fnv/Makefile create mode 100644 src/pkg/hash/fnv/fnv.go create mode 100644 src/pkg/hash/fnv/fnv_test.go create mode 100644 src/pkg/hash/hash.go create mode 100644 src/pkg/hash/test_cases.txt create mode 100644 src/pkg/hash/test_gen.awk create mode 100644 src/pkg/html/Makefile create mode 100644 src/pkg/html/const.go create mode 100644 src/pkg/html/doc.go create mode 100644 src/pkg/html/entity.go create mode 100644 src/pkg/html/entity_test.go create mode 100644 src/pkg/html/escape.go create mode 100644 src/pkg/html/node.go create mode 100644 src/pkg/html/parse.go create mode 100644 src/pkg/html/parse_test.go create mode 100644 src/pkg/html/testdata/webkit/README create mode 100644 src/pkg/html/testdata/webkit/adoption01.dat create mode 100644 src/pkg/html/testdata/webkit/adoption02.dat create mode 100644 src/pkg/html/testdata/webkit/comments01.dat create mode 100644 src/pkg/html/testdata/webkit/doctype01.dat create mode 100644 src/pkg/html/testdata/webkit/entities01.dat create mode 100644 src/pkg/html/testdata/webkit/entities02.dat create mode 100644 src/pkg/html/testdata/webkit/html5test-com.dat create mode 100644 src/pkg/html/testdata/webkit/inbody01.dat create mode 100644 src/pkg/html/testdata/webkit/isindex.dat create mode 100644 src/pkg/html/testdata/webkit/pending-spec-changes-plain-text-unsafe.dat create mode 100644 src/pkg/html/testdata/webkit/pending-spec-changes.dat create mode 100644 src/pkg/html/testdata/webkit/plain-text-unsafe.dat create mode 100644 src/pkg/html/testdata/webkit/scriptdata01.dat create mode 100644 src/pkg/html/testdata/webkit/scripted/adoption01.dat create mode 100644 src/pkg/html/testdata/webkit/scripted/webkit01.dat create mode 100644 src/pkg/html/testdata/webkit/tables01.dat create mode 100644 src/pkg/html/testdata/webkit/tests1.dat create mode 100644 src/pkg/html/testdata/webkit/tests10.dat create mode 100644 src/pkg/html/testdata/webkit/tests11.dat create mode 100644 src/pkg/html/testdata/webkit/tests12.dat create mode 100644 src/pkg/html/testdata/webkit/tests14.dat create mode 100644 src/pkg/html/testdata/webkit/tests15.dat create mode 100644 src/pkg/html/testdata/webkit/tests16.dat create mode 100644 src/pkg/html/testdata/webkit/tests17.dat create mode 100644 src/pkg/html/testdata/webkit/tests18.dat create mode 100644 src/pkg/html/testdata/webkit/tests19.dat create mode 100644 src/pkg/html/testdata/webkit/tests2.dat create mode 100644 src/pkg/html/testdata/webkit/tests20.dat create mode 100644 src/pkg/html/testdata/webkit/tests21.dat create mode 100644 src/pkg/html/testdata/webkit/tests22.dat create mode 100644 src/pkg/html/testdata/webkit/tests23.dat create mode 100644 src/pkg/html/testdata/webkit/tests24.dat create mode 100644 src/pkg/html/testdata/webkit/tests25.dat create mode 100644 src/pkg/html/testdata/webkit/tests26.dat create mode 100644 src/pkg/html/testdata/webkit/tests3.dat create mode 100644 src/pkg/html/testdata/webkit/tests4.dat create mode 100644 src/pkg/html/testdata/webkit/tests5.dat create mode 100644 src/pkg/html/testdata/webkit/tests6.dat create mode 100644 src/pkg/html/testdata/webkit/tests7.dat create mode 100644 src/pkg/html/testdata/webkit/tests8.dat create mode 100644 src/pkg/html/testdata/webkit/tests9.dat create mode 100644 src/pkg/html/testdata/webkit/tests_innerHTML_1.dat create mode 100644 src/pkg/html/testdata/webkit/tricky01.dat create mode 100644 src/pkg/html/testdata/webkit/webkit01.dat create mode 100644 src/pkg/html/testdata/webkit/webkit02.dat create mode 100644 src/pkg/html/token.go create mode 100644 src/pkg/html/token_test.go create mode 100644 src/pkg/http/Makefile create mode 100644 src/pkg/http/cgi/Makefile create mode 100644 src/pkg/http/cgi/child.go create mode 100644 src/pkg/http/cgi/child_test.go create mode 100644 src/pkg/http/cgi/host.go create mode 100644 src/pkg/http/cgi/host_test.go create mode 100644 src/pkg/http/cgi/matryoshka_test.go create mode 100755 src/pkg/http/cgi/testdata/test.cgi create mode 100644 src/pkg/http/chunked.go create mode 100644 src/pkg/http/client.go create mode 100644 src/pkg/http/client_test.go create mode 100644 src/pkg/http/cookie.go create mode 100644 src/pkg/http/cookie_test.go create mode 100644 src/pkg/http/dump.go create mode 100644 src/pkg/http/export_test.go create mode 100644 src/pkg/http/fcgi/Makefile create mode 100644 src/pkg/http/fcgi/child.go create mode 100644 src/pkg/http/fcgi/fcgi.go create mode 100644 src/pkg/http/fcgi/fcgi_test.go create mode 100644 src/pkg/http/fs.go create mode 100644 src/pkg/http/fs_test.go create mode 100644 src/pkg/http/header.go create mode 100644 src/pkg/http/header_test.go create mode 100644 src/pkg/http/httptest/Makefile create mode 100644 src/pkg/http/httptest/recorder.go create mode 100644 src/pkg/http/httptest/server.go create mode 100644 src/pkg/http/lex.go create mode 100644 src/pkg/http/lex_test.go create mode 100644 src/pkg/http/persist.go create mode 100644 src/pkg/http/pprof/Makefile create mode 100644 src/pkg/http/pprof/pprof.go create mode 100644 src/pkg/http/proxy_test.go create mode 100644 src/pkg/http/range_test.go create mode 100644 src/pkg/http/readrequest_test.go create mode 100644 src/pkg/http/request.go create mode 100644 src/pkg/http/request_test.go create mode 100644 src/pkg/http/requestwrite_test.go create mode 100644 src/pkg/http/response.go create mode 100644 src/pkg/http/response_test.go create mode 100644 src/pkg/http/responsewrite_test.go create mode 100644 src/pkg/http/reverseproxy.go create mode 100644 src/pkg/http/reverseproxy_test.go create mode 100644 src/pkg/http/serve_test.go create mode 100644 src/pkg/http/server.go create mode 100644 src/pkg/http/sniff.go create mode 100644 src/pkg/http/sniff_test.go create mode 100644 src/pkg/http/spdy/Makefile create mode 100644 src/pkg/http/spdy/read.go create mode 100644 src/pkg/http/spdy/spdy_test.go create mode 100644 src/pkg/http/spdy/types.go create mode 100644 src/pkg/http/spdy/write.go create mode 100644 src/pkg/http/status.go create mode 100644 src/pkg/http/testdata/file create mode 100644 src/pkg/http/testdata/index.html create mode 100644 src/pkg/http/testdata/style.css create mode 100644 src/pkg/http/transfer.go create mode 100644 src/pkg/http/transport.go create mode 100644 src/pkg/http/transport_test.go create mode 100644 src/pkg/http/triv.go create mode 100644 src/pkg/image/Makefile create mode 100644 src/pkg/image/bmp/Makefile create mode 100644 src/pkg/image/bmp/reader.go create mode 100644 src/pkg/image/color.go create mode 100644 src/pkg/image/decode_test.go create mode 100644 src/pkg/image/draw/Makefile create mode 100644 src/pkg/image/draw/bench_test.go create mode 100644 src/pkg/image/draw/clip_test.go create mode 100644 src/pkg/image/draw/draw.go create mode 100644 src/pkg/image/draw/draw_test.go create mode 100644 src/pkg/image/format.go create mode 100644 src/pkg/image/geom.go create mode 100644 src/pkg/image/gif/Makefile create mode 100644 src/pkg/image/gif/reader.go create mode 100644 src/pkg/image/image.go create mode 100644 src/pkg/image/image_test.go create mode 100644 src/pkg/image/jpeg/Makefile create mode 100644 src/pkg/image/jpeg/fdct.go create mode 100644 src/pkg/image/jpeg/huffman.go create mode 100644 src/pkg/image/jpeg/idct.go create mode 100644 src/pkg/image/jpeg/reader.go create mode 100644 src/pkg/image/jpeg/writer.go create mode 100644 src/pkg/image/jpeg/writer_test.go create mode 100644 src/pkg/image/names.go create mode 100644 src/pkg/image/png/Makefile create mode 100644 src/pkg/image/png/reader.go create mode 100644 src/pkg/image/png/reader_test.go create mode 100644 src/pkg/image/png/testdata/pngsuite/README create mode 100644 src/pkg/image/png/testdata/pngsuite/README.original create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01-30.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01-30.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g01.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02-29.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02-29.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g02.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04-31.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04-31.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g04.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn0g16.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn2c16.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p01.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p01.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p02.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p02.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p04.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p04.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08-trns.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08-trns.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn3p08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn4a16.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a08.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a08.sng create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a16.png create mode 100644 src/pkg/image/png/testdata/pngsuite/basn6a16.sng create mode 100644 src/pkg/image/png/writer.go create mode 100644 src/pkg/image/png/writer_test.go create mode 100644 src/pkg/image/testdata/video-001.5bpp.gif create mode 100644 src/pkg/image/testdata/video-001.bmp create mode 100644 src/pkg/image/testdata/video-001.gif create mode 100644 src/pkg/image/testdata/video-001.interlaced.gif create mode 100644 src/pkg/image/testdata/video-001.jpeg create mode 100644 src/pkg/image/testdata/video-001.png create mode 100644 src/pkg/image/testdata/video-001.tiff create mode 100644 src/pkg/image/testdata/video-005.gray.jpeg create mode 100644 src/pkg/image/testdata/video-005.gray.png create mode 100644 src/pkg/image/tiff/Makefile create mode 100644 src/pkg/image/tiff/buffer.go create mode 100644 src/pkg/image/tiff/buffer_test.go create mode 100644 src/pkg/image/tiff/consts.go create mode 100644 src/pkg/image/tiff/reader.go create mode 100644 src/pkg/image/tiff/reader_test.go create mode 100644 src/pkg/image/tiff/testdata/no_rps.tiff create mode 100644 src/pkg/image/ycbcr/Makefile create mode 100644 src/pkg/image/ycbcr/ycbcr.go create mode 100644 src/pkg/image/ycbcr/ycbcr_test.go create mode 100644 src/pkg/index/suffixarray/Makefile create mode 100644 src/pkg/index/suffixarray/qsufsort.go create mode 100644 src/pkg/index/suffixarray/suffixarray.go create mode 100644 src/pkg/index/suffixarray/suffixarray_test.go create mode 100644 src/pkg/io/Makefile create mode 100644 src/pkg/io/io.go create mode 100644 src/pkg/io/io_test.go create mode 100644 src/pkg/io/ioutil/Makefile create mode 100644 src/pkg/io/ioutil/ioutil.go create mode 100644 src/pkg/io/ioutil/ioutil_test.go create mode 100644 src/pkg/io/ioutil/tempfile.go create mode 100644 src/pkg/io/ioutil/tempfile_test.go create mode 100644 src/pkg/io/multi.go create mode 100644 src/pkg/io/multi_test.go create mode 100644 src/pkg/io/pipe.go create mode 100644 src/pkg/io/pipe_test.go create mode 100644 src/pkg/json/Makefile create mode 100644 src/pkg/json/decode.go create mode 100644 src/pkg/json/decode_test.go create mode 100644 src/pkg/json/encode.go create mode 100644 src/pkg/json/encode_test.go create mode 100644 src/pkg/json/indent.go create mode 100644 src/pkg/json/scanner.go create mode 100644 src/pkg/json/scanner_test.go create mode 100644 src/pkg/json/stream.go create mode 100644 src/pkg/json/stream_test.go create mode 100644 src/pkg/json/tagkey_test.go create mode 100644 src/pkg/log/Makefile create mode 100644 src/pkg/log/log.go create mode 100644 src/pkg/log/log_test.go create mode 100644 src/pkg/mail/Makefile create mode 100644 src/pkg/mail/message.go create mode 100644 src/pkg/mail/message_test.go create mode 100644 src/pkg/math/Makefile create mode 100644 src/pkg/math/acosh.go create mode 100644 src/pkg/math/all_test.go create mode 100644 src/pkg/math/asin.go create mode 100644 src/pkg/math/asin_386.s create mode 100644 src/pkg/math/asin_decl.go create mode 100644 src/pkg/math/asinh.go create mode 100644 src/pkg/math/atan.go create mode 100644 src/pkg/math/atan2.go create mode 100755 src/pkg/math/atan2_386.s create mode 100755 src/pkg/math/atan2_decl.go create mode 100644 src/pkg/math/atan_386.s create mode 100644 src/pkg/math/atan_decl.go create mode 100644 src/pkg/math/atanh.go create mode 100644 src/pkg/math/bits.go create mode 100644 src/pkg/math/cbrt.go create mode 100644 src/pkg/math/const.go create mode 100644 src/pkg/math/copysign.go create mode 100644 src/pkg/math/erf.go create mode 100644 src/pkg/math/exp.go create mode 100644 src/pkg/math/exp2.go create mode 100644 src/pkg/math/exp2_386.s create mode 100644 src/pkg/math/exp2_decl.go create mode 100644 src/pkg/math/exp_386.s create mode 100644 src/pkg/math/exp_amd64.s create mode 100644 src/pkg/math/exp_decl.go create mode 100644 src/pkg/math/exp_port.go create mode 100644 src/pkg/math/exp_test.go create mode 100644 src/pkg/math/expm1.go create mode 100644 src/pkg/math/expm1_386.s create mode 100644 src/pkg/math/expm1_decl.go create mode 100644 src/pkg/math/fabs.go create mode 100644 src/pkg/math/fabs_386.s create mode 100644 src/pkg/math/fabs_amd64.s create mode 100644 src/pkg/math/fabs_decl.go create mode 100644 src/pkg/math/fdim.go create mode 100644 src/pkg/math/fdim_amd64.s create mode 100644 src/pkg/math/fdim_decl.go create mode 100644 src/pkg/math/floor.go create mode 100644 src/pkg/math/floor_386.s create mode 100644 src/pkg/math/floor_decl.go create mode 100644 src/pkg/math/fltasm_amd64.s create mode 100644 src/pkg/math/fmod.go create mode 100644 src/pkg/math/fmod_386.s create mode 100644 src/pkg/math/fmod_decl.go create mode 100644 src/pkg/math/frexp.go create mode 100644 src/pkg/math/frexp_386.s create mode 100644 src/pkg/math/frexp_decl.go create mode 100644 src/pkg/math/gamma.go create mode 100644 src/pkg/math/hypot.go create mode 100644 src/pkg/math/hypot_386.s create mode 100644 src/pkg/math/hypot_amd64.s create mode 100644 src/pkg/math/hypot_decl.go create mode 100644 src/pkg/math/hypot_port.go create mode 100644 src/pkg/math/hypot_test.go create mode 100644 src/pkg/math/j0.go create mode 100644 src/pkg/math/j1.go create mode 100644 src/pkg/math/jn.go create mode 100644 src/pkg/math/ldexp.go create mode 100644 src/pkg/math/ldexp_386.s create mode 100644 src/pkg/math/ldexp_decl.go create mode 100644 src/pkg/math/lgamma.go create mode 100644 src/pkg/math/log.go create mode 100644 src/pkg/math/log10.go create mode 100644 src/pkg/math/log10_386.s create mode 100644 src/pkg/math/log10_decl.go create mode 100644 src/pkg/math/log1p.go create mode 100644 src/pkg/math/log1p_386.s create mode 100644 src/pkg/math/log1p_decl.go create mode 100644 src/pkg/math/log_386.s create mode 100644 src/pkg/math/log_amd64.s create mode 100644 src/pkg/math/log_decl.go create mode 100644 src/pkg/math/logb.go create mode 100644 src/pkg/math/modf.go create mode 100644 src/pkg/math/modf_386.s create mode 100644 src/pkg/math/modf_decl.go create mode 100644 src/pkg/math/nextafter.go create mode 100644 src/pkg/math/pow.go create mode 100644 src/pkg/math/pow10.go create mode 100644 src/pkg/math/remainder.go create mode 100644 src/pkg/math/remainder_386.s create mode 100644 src/pkg/math/remainder_decl.go create mode 100644 src/pkg/math/signbit.go create mode 100644 src/pkg/math/sin.go create mode 100644 src/pkg/math/sin_386.s create mode 100644 src/pkg/math/sin_decl.go create mode 100644 src/pkg/math/sincos.go create mode 100644 src/pkg/math/sincos_386.s create mode 100644 src/pkg/math/sincos_amd64.s create mode 100644 src/pkg/math/sincos_decl.go create mode 100644 src/pkg/math/sinh.go create mode 100644 src/pkg/math/sqrt.go create mode 100644 src/pkg/math/sqrt_386.s create mode 100644 src/pkg/math/sqrt_amd64.s create mode 100644 src/pkg/math/sqrt_arm.s create mode 100644 src/pkg/math/sqrt_decl.go create mode 100644 src/pkg/math/sqrt_port.go create mode 100644 src/pkg/math/sqrt_test.go create mode 100644 src/pkg/math/tan.go create mode 100644 src/pkg/math/tan_386.s create mode 100644 src/pkg/math/tan_decl.go create mode 100644 src/pkg/math/tanh.go create mode 100644 src/pkg/math/unsafe.go create mode 100644 src/pkg/mime/Makefile create mode 100644 src/pkg/mime/grammar.go create mode 100644 src/pkg/mime/mediatype.go create mode 100644 src/pkg/mime/mediatype_test.go create mode 100644 src/pkg/mime/mime_test.go create mode 100644 src/pkg/mime/multipart/Makefile create mode 100644 src/pkg/mime/multipart/formdata.go create mode 100644 src/pkg/mime/multipart/formdata_test.go create mode 100644 src/pkg/mime/multipart/multipart.go create mode 100644 src/pkg/mime/multipart/multipart_test.go create mode 100644 src/pkg/mime/multipart/writer.go create mode 100644 src/pkg/mime/multipart/writer_test.go create mode 100644 src/pkg/mime/test.types create mode 100644 src/pkg/mime/type.go create mode 100644 src/pkg/net/Makefile create mode 100644 src/pkg/net/cgo_bsd.go create mode 100644 src/pkg/net/cgo_linux.go create mode 100644 src/pkg/net/cgo_stub.go create mode 100644 src/pkg/net/cgo_unix.go create mode 100644 src/pkg/net/dial.go create mode 100644 src/pkg/net/dialgoogle_test.go create mode 100644 src/pkg/net/dict/Makefile create mode 100644 src/pkg/net/dict/dict.go create mode 100644 src/pkg/net/dnsclient.go create mode 100644 src/pkg/net/dnsclient_unix.go create mode 100644 src/pkg/net/dnsconfig.go create mode 100644 src/pkg/net/dnsmsg.go create mode 100644 src/pkg/net/dnsmsg_test.go create mode 100644 src/pkg/net/dnsname_test.go create mode 100644 src/pkg/net/fd.go create mode 100644 src/pkg/net/fd_darwin.go create mode 100644 src/pkg/net/fd_freebsd.go create mode 100644 src/pkg/net/fd_linux.go create mode 100644 src/pkg/net/fd_openbsd.go create mode 100644 src/pkg/net/fd_windows.go create mode 100644 src/pkg/net/file.go create mode 100644 src/pkg/net/file_plan9.go create mode 100644 src/pkg/net/file_test.go create mode 100644 src/pkg/net/file_windows.go create mode 100644 src/pkg/net/hosts.go create mode 100644 src/pkg/net/hosts_test.go create mode 100644 src/pkg/net/hosts_testdata create mode 100644 src/pkg/net/interface.go create mode 100644 src/pkg/net/interface_bsd.go create mode 100644 src/pkg/net/interface_darwin.go create mode 100644 src/pkg/net/interface_freebsd.go create mode 100644 src/pkg/net/interface_linux.go create mode 100644 src/pkg/net/interface_openbsd.go create mode 100644 src/pkg/net/interface_stub.go create mode 100644 src/pkg/net/interface_test.go create mode 100644 src/pkg/net/interface_windows.go create mode 100644 src/pkg/net/ip.go create mode 100644 src/pkg/net/ip_test.go create mode 100644 src/pkg/net/ipraw_test.go create mode 100644 src/pkg/net/iprawsock.go create mode 100644 src/pkg/net/iprawsock_plan9.go create mode 100644 src/pkg/net/iprawsock_posix.go create mode 100644 src/pkg/net/ipsock.go create mode 100644 src/pkg/net/ipsock_plan9.go create mode 100644 src/pkg/net/ipsock_posix.go create mode 100644 src/pkg/net/lookup_plan9.go create mode 100644 src/pkg/net/lookup_test.go create mode 100644 src/pkg/net/lookup_unix.go create mode 100644 src/pkg/net/lookup_windows.go create mode 100644 src/pkg/net/multicast_test.go create mode 100644 src/pkg/net/net.go create mode 100644 src/pkg/net/net_test.go create mode 100644 src/pkg/net/newpollserver.go create mode 100644 src/pkg/net/parse.go create mode 100644 src/pkg/net/parse_test.go create mode 100644 src/pkg/net/pipe.go create mode 100644 src/pkg/net/pipe_test.go create mode 100644 src/pkg/net/port.go create mode 100644 src/pkg/net/port_test.go create mode 100644 src/pkg/net/sendfile_linux.go create mode 100644 src/pkg/net/sendfile_stub.go create mode 100644 src/pkg/net/sendfile_windows.go create mode 100644 src/pkg/net/server_test.go create mode 100644 src/pkg/net/sock.go create mode 100644 src/pkg/net/sock_bsd.go create mode 100644 src/pkg/net/sock_linux.go create mode 100644 src/pkg/net/sock_windows.go create mode 100644 src/pkg/net/tcpsock.go create mode 100644 src/pkg/net/tcpsock_plan9.go create mode 100644 src/pkg/net/tcpsock_posix.go create mode 100644 src/pkg/net/textproto/Makefile create mode 100644 src/pkg/net/textproto/header.go create mode 100644 src/pkg/net/textproto/pipeline.go create mode 100644 src/pkg/net/textproto/reader.go create mode 100644 src/pkg/net/textproto/reader_test.go create mode 100644 src/pkg/net/textproto/textproto.go create mode 100644 src/pkg/net/textproto/writer.go create mode 100644 src/pkg/net/textproto/writer_test.go create mode 100644 src/pkg/net/timeout_test.go create mode 100644 src/pkg/net/udpsock.go create mode 100644 src/pkg/net/udpsock_plan9.go create mode 100644 src/pkg/net/udpsock_posix.go create mode 100644 src/pkg/net/unixsock.go create mode 100644 src/pkg/net/unixsock_plan9.go create mode 100644 src/pkg/net/unixsock_posix.go create mode 100644 src/pkg/netchan/Makefile create mode 100644 src/pkg/netchan/common.go create mode 100644 src/pkg/netchan/export.go create mode 100644 src/pkg/netchan/import.go create mode 100644 src/pkg/netchan/netchan_test.go create mode 100644 src/pkg/old/template/Makefile create mode 100644 src/pkg/old/template/doc.go create mode 100644 src/pkg/old/template/execute.go create mode 100644 src/pkg/old/template/format.go create mode 100644 src/pkg/old/template/parse.go create mode 100644 src/pkg/old/template/template_test.go create mode 100644 src/pkg/os/Makefile create mode 100644 src/pkg/os/dir_plan9.go create mode 100644 src/pkg/os/dir_unix.go create mode 100644 src/pkg/os/dir_windows.go create mode 100644 src/pkg/os/env.go create mode 100644 src/pkg/os/env_plan9.go create mode 100644 src/pkg/os/env_test.go create mode 100644 src/pkg/os/env_unix.go create mode 100644 src/pkg/os/env_windows.go create mode 100644 src/pkg/os/error.go create mode 100644 src/pkg/os/error_plan9.go create mode 100644 src/pkg/os/error_posix.go create mode 100644 src/pkg/os/exec.go create mode 100644 src/pkg/os/exec_plan9.go create mode 100644 src/pkg/os/exec_posix.go create mode 100644 src/pkg/os/exec_unix.go create mode 100644 src/pkg/os/exec_windows.go create mode 100644 src/pkg/os/file.go create mode 100644 src/pkg/os/file_plan9.go create mode 100644 src/pkg/os/file_posix.go create mode 100644 src/pkg/os/file_unix.go create mode 100644 src/pkg/os/file_windows.go create mode 100644 src/pkg/os/getwd.go create mode 100644 src/pkg/os/inotify/Makefile create mode 100644 src/pkg/os/inotify/inotify_linux.go create mode 100644 src/pkg/os/inotify/inotify_linux_test.go create mode 100755 src/pkg/os/mkunixsignals.sh create mode 100644 src/pkg/os/os_test.go create mode 100644 src/pkg/os/path.go create mode 100644 src/pkg/os/path_plan9.go create mode 100644 src/pkg/os/path_test.go create mode 100644 src/pkg/os/path_unix.go create mode 100644 src/pkg/os/path_windows.go create mode 100644 src/pkg/os/proc.go create mode 100644 src/pkg/os/signal/Makefile create mode 100644 src/pkg/os/signal/signal.go create mode 100644 src/pkg/os/signal/signal_test.go create mode 100644 src/pkg/os/stat_darwin.go create mode 100644 src/pkg/os/stat_freebsd.go create mode 100644 src/pkg/os/stat_linux.go create mode 100644 src/pkg/os/stat_openbsd.go create mode 100644 src/pkg/os/stat_plan9.go create mode 100644 src/pkg/os/stat_windows.go create mode 100644 src/pkg/os/str.go create mode 100644 src/pkg/os/sys_bsd.go create mode 100644 src/pkg/os/sys_linux.go create mode 100644 src/pkg/os/sys_plan9.go create mode 100644 src/pkg/os/sys_windows.go create mode 100644 src/pkg/os/time.go create mode 100644 src/pkg/os/types.go create mode 100644 src/pkg/os/user/Makefile create mode 100644 src/pkg/os/user/lookup_stubs.go create mode 100644 src/pkg/os/user/lookup_unix.go create mode 100644 src/pkg/os/user/user.go create mode 100644 src/pkg/os/user/user_test.go create mode 100644 src/pkg/patch/Makefile create mode 100644 src/pkg/patch/apply.go create mode 100644 src/pkg/patch/git.go create mode 100644 src/pkg/patch/patch.go create mode 100644 src/pkg/patch/patch_test.go create mode 100644 src/pkg/patch/textdiff.go create mode 100644 src/pkg/path/Makefile create mode 100644 src/pkg/path/filepath/Makefile create mode 100644 src/pkg/path/filepath/match.go create mode 100644 src/pkg/path/filepath/match_test.go create mode 100644 src/pkg/path/filepath/path.go create mode 100644 src/pkg/path/filepath/path_plan9.go create mode 100644 src/pkg/path/filepath/path_test.go create mode 100644 src/pkg/path/filepath/path_unix.go create mode 100644 src/pkg/path/filepath/path_windows.go create mode 100644 src/pkg/path/match.go create mode 100644 src/pkg/path/match_test.go create mode 100644 src/pkg/path/path.go create mode 100644 src/pkg/path/path_test.go create mode 100644 src/pkg/rand/Makefile create mode 100644 src/pkg/rand/exp.go create mode 100644 src/pkg/rand/normal.go create mode 100644 src/pkg/rand/rand.go create mode 100644 src/pkg/rand/rand_test.go create mode 100644 src/pkg/rand/rng.go create mode 100644 src/pkg/rand/zipf.go create mode 100644 src/pkg/reflect/Makefile create mode 100644 src/pkg/reflect/all_test.go create mode 100644 src/pkg/reflect/deepequal.go create mode 100644 src/pkg/reflect/set_test.go create mode 100644 src/pkg/reflect/tostring_test.go create mode 100644 src/pkg/reflect/type.go create mode 100644 src/pkg/reflect/value.go create mode 100644 src/pkg/regexp/Makefile create mode 100644 src/pkg/regexp/all_test.go create mode 100644 src/pkg/regexp/find_test.go create mode 100644 src/pkg/regexp/regexp.go create mode 100644 src/pkg/rpc/Makefile create mode 100644 src/pkg/rpc/client.go create mode 100644 src/pkg/rpc/debug.go create mode 100644 src/pkg/rpc/jsonrpc/Makefile create mode 100644 src/pkg/rpc/jsonrpc/all_test.go create mode 100644 src/pkg/rpc/jsonrpc/client.go create mode 100644 src/pkg/rpc/jsonrpc/server.go create mode 100644 src/pkg/rpc/server.go create mode 100644 src/pkg/rpc/server_test.go create mode 100644 src/pkg/runtime/386/arch.h create mode 100644 src/pkg/runtime/386/asm.s create mode 100644 src/pkg/runtime/386/atomic.c create mode 100644 src/pkg/runtime/386/closure.c create mode 100644 src/pkg/runtime/386/memmove.s create mode 100644 src/pkg/runtime/386/vlop.s create mode 100644 src/pkg/runtime/386/vlrt.c create mode 100644 src/pkg/runtime/Makefile create mode 100644 src/pkg/runtime/amd64/arch.h create mode 100644 src/pkg/runtime/amd64/asm.s create mode 100644 src/pkg/runtime/amd64/atomic.c create mode 100644 src/pkg/runtime/amd64/closure.c create mode 100644 src/pkg/runtime/amd64/memmove.s create mode 100644 src/pkg/runtime/amd64/traceback.c create mode 100644 src/pkg/runtime/append_test.go create mode 100644 src/pkg/runtime/arm/arch.h create mode 100644 src/pkg/runtime/arm/asm.s create mode 100644 src/pkg/runtime/arm/atomic.c create mode 100644 src/pkg/runtime/arm/closure.c create mode 100644 src/pkg/runtime/arm/memmove.s create mode 100644 src/pkg/runtime/arm/memset.s create mode 100644 src/pkg/runtime/arm/softfloat.c create mode 100644 src/pkg/runtime/arm/traceback.c create mode 100644 src/pkg/runtime/arm/vlop.s create mode 100644 src/pkg/runtime/arm/vlrt.c create mode 100755 src/pkg/runtime/cgo/386.S create mode 100644 src/pkg/runtime/cgo/Makefile create mode 100644 src/pkg/runtime/cgo/amd64.S create mode 100644 src/pkg/runtime/cgo/arm.S create mode 100644 src/pkg/runtime/cgo/callbacks.c create mode 100644 src/pkg/runtime/cgo/cgo.go create mode 100644 src/pkg/runtime/cgo/darwin_386.c create mode 100644 src/pkg/runtime/cgo/darwin_amd64.c create mode 100644 src/pkg/runtime/cgo/freebsd.c create mode 100644 src/pkg/runtime/cgo/freebsd_386.c create mode 100644 src/pkg/runtime/cgo/freebsd_amd64.c create mode 100644 src/pkg/runtime/cgo/iscgo.c create mode 100644 src/pkg/runtime/cgo/libcgo.h create mode 100644 src/pkg/runtime/cgo/linux_386.c create mode 100644 src/pkg/runtime/cgo/linux_amd64.c create mode 100644 src/pkg/runtime/cgo/linux_arm.c create mode 100644 src/pkg/runtime/cgo/setenv.c create mode 100644 src/pkg/runtime/cgo/util.c create mode 100755 src/pkg/runtime/cgo/windows_386.c create mode 100755 src/pkg/runtime/cgo/windows_amd64.c create mode 100644 src/pkg/runtime/cgocall.c create mode 100644 src/pkg/runtime/cgocall.h create mode 100644 src/pkg/runtime/chan.c create mode 100644 src/pkg/runtime/chan_test.go create mode 100644 src/pkg/runtime/closure_test.go create mode 100644 src/pkg/runtime/complex.c create mode 100644 src/pkg/runtime/cpuprof.c create mode 100644 src/pkg/runtime/darwin/386/defs.h create mode 100644 src/pkg/runtime/darwin/386/rt0.s create mode 100644 src/pkg/runtime/darwin/386/signal.c create mode 100644 src/pkg/runtime/darwin/386/sys.s create mode 100644 src/pkg/runtime/darwin/amd64/defs.h create mode 100644 src/pkg/runtime/darwin/amd64/rt0.s create mode 100644 src/pkg/runtime/darwin/amd64/signal.c create mode 100644 src/pkg/runtime/darwin/amd64/sys.s create mode 100644 src/pkg/runtime/darwin/defs.c create mode 100644 src/pkg/runtime/darwin/mem.c create mode 100644 src/pkg/runtime/darwin/os.h create mode 100644 src/pkg/runtime/darwin/signals.h create mode 100644 src/pkg/runtime/darwin/thread.c create mode 100644 src/pkg/runtime/debug.go create mode 100644 src/pkg/runtime/debug/Makefile create mode 100644 src/pkg/runtime/debug/stack.go create mode 100644 src/pkg/runtime/debug/stack_test.go create mode 100644 src/pkg/runtime/error.go create mode 100644 src/pkg/runtime/export_test.go create mode 100644 src/pkg/runtime/extern.go create mode 100644 src/pkg/runtime/float.c create mode 100644 src/pkg/runtime/freebsd/386/defs.h create mode 100644 src/pkg/runtime/freebsd/386/rt0.s create mode 100644 src/pkg/runtime/freebsd/386/signal.c create mode 100644 src/pkg/runtime/freebsd/386/sys.s create mode 100644 src/pkg/runtime/freebsd/amd64/defs.h create mode 100644 src/pkg/runtime/freebsd/amd64/rt0.s create mode 100644 src/pkg/runtime/freebsd/amd64/signal.c create mode 100644 src/pkg/runtime/freebsd/amd64/sys.s create mode 100644 src/pkg/runtime/freebsd/defs.c create mode 100644 src/pkg/runtime/freebsd/mem.c create mode 100644 src/pkg/runtime/freebsd/os.h create mode 100644 src/pkg/runtime/freebsd/signals.h create mode 100644 src/pkg/runtime/freebsd/thread.c create mode 100644 src/pkg/runtime/goc2c.c create mode 100644 src/pkg/runtime/hashmap.c create mode 100644 src/pkg/runtime/hashmap.h create mode 100644 src/pkg/runtime/iface.c create mode 100644 src/pkg/runtime/linux/386/defs.h create mode 100644 src/pkg/runtime/linux/386/rt0.s create mode 100644 src/pkg/runtime/linux/386/signal.c create mode 100644 src/pkg/runtime/linux/386/sys.s create mode 100644 src/pkg/runtime/linux/amd64/defs.h create mode 100644 src/pkg/runtime/linux/amd64/rt0.s create mode 100644 src/pkg/runtime/linux/amd64/signal.c create mode 100644 src/pkg/runtime/linux/amd64/sys.s create mode 100644 src/pkg/runtime/linux/arm/defs.h create mode 100644 src/pkg/runtime/linux/arm/rt0.s create mode 100644 src/pkg/runtime/linux/arm/signal.c create mode 100644 src/pkg/runtime/linux/arm/sys.s create mode 100644 src/pkg/runtime/linux/defs.c create mode 100644 src/pkg/runtime/linux/defs1.c create mode 100644 src/pkg/runtime/linux/defs2.c create mode 100644 src/pkg/runtime/linux/defs_arm.c create mode 100644 src/pkg/runtime/linux/mem.c create mode 100644 src/pkg/runtime/linux/os.h create mode 100644 src/pkg/runtime/linux/signals.h create mode 100644 src/pkg/runtime/linux/thread.c create mode 100644 src/pkg/runtime/malloc.goc create mode 100644 src/pkg/runtime/malloc.h create mode 100644 src/pkg/runtime/mcache.c create mode 100644 src/pkg/runtime/mcentral.c create mode 100644 src/pkg/runtime/mem.go create mode 100644 src/pkg/runtime/mfinal.c create mode 100644 src/pkg/runtime/mfixalloc.c create mode 100644 src/pkg/runtime/mgc0.c create mode 100644 src/pkg/runtime/mheap.c create mode 100755 src/pkg/runtime/mkasmh.sh create mode 100755 src/pkg/runtime/mkgodefs.sh create mode 100644 src/pkg/runtime/mkversion.c create mode 100644 src/pkg/runtime/mprof.goc create mode 100644 src/pkg/runtime/msize.c create mode 100644 src/pkg/runtime/openbsd/amd64/defs.h create mode 100644 src/pkg/runtime/openbsd/amd64/rt0.s create mode 100644 src/pkg/runtime/openbsd/amd64/signal.c create mode 100644 src/pkg/runtime/openbsd/amd64/sys.s create mode 100644 src/pkg/runtime/openbsd/defs.c create mode 100644 src/pkg/runtime/openbsd/mem.c create mode 100644 src/pkg/runtime/openbsd/os.h create mode 100644 src/pkg/runtime/openbsd/signals.h create mode 100644 src/pkg/runtime/openbsd/thread.c create mode 100644 src/pkg/runtime/plan9/386/defs.h create mode 100644 src/pkg/runtime/plan9/386/rt0.s create mode 100644 src/pkg/runtime/plan9/386/signal.c create mode 100644 src/pkg/runtime/plan9/386/sys.s create mode 100644 src/pkg/runtime/plan9/mem.c create mode 100644 src/pkg/runtime/plan9/os.h create mode 100644 src/pkg/runtime/plan9/signals.h create mode 100644 src/pkg/runtime/plan9/thread.c create mode 100644 src/pkg/runtime/pprof/Makefile create mode 100644 src/pkg/runtime/pprof/pprof.go create mode 100644 src/pkg/runtime/pprof/pprof_test.go create mode 100644 src/pkg/runtime/print.c create mode 100644 src/pkg/runtime/proc.c create mode 100644 src/pkg/runtime/proc.p create mode 100644 src/pkg/runtime/proc_test.go create mode 100644 src/pkg/runtime/rune.c create mode 100644 src/pkg/runtime/runtime-gdb.py create mode 100644 src/pkg/runtime/runtime.c create mode 100644 src/pkg/runtime/runtime.h create mode 100644 src/pkg/runtime/runtime1.goc create mode 100644 src/pkg/runtime/sema.goc create mode 100644 src/pkg/runtime/sema_test.go create mode 100644 src/pkg/runtime/sig.go create mode 100644 src/pkg/runtime/sigqueue.goc create mode 100644 src/pkg/runtime/slice.c create mode 100644 src/pkg/runtime/softfloat64.go create mode 100644 src/pkg/runtime/softfloat64_test.go create mode 100644 src/pkg/runtime/stack.h create mode 100644 src/pkg/runtime/string.goc create mode 100644 src/pkg/runtime/symtab.c create mode 100644 src/pkg/runtime/symtab_test.go create mode 100644 src/pkg/runtime/type.go create mode 100644 src/pkg/runtime/type.h create mode 100644 src/pkg/runtime/windows/386/defs.h create mode 100644 src/pkg/runtime/windows/386/rt0.s create mode 100644 src/pkg/runtime/windows/386/signal.c create mode 100644 src/pkg/runtime/windows/386/sys.s create mode 100644 src/pkg/runtime/windows/amd64/defs.h create mode 100644 src/pkg/runtime/windows/amd64/rt0.s create mode 100644 src/pkg/runtime/windows/amd64/signal.c create mode 100644 src/pkg/runtime/windows/amd64/sys.s create mode 100644 src/pkg/runtime/windows/defs.c create mode 100644 src/pkg/runtime/windows/mem.c create mode 100644 src/pkg/runtime/windows/os.h create mode 100644 src/pkg/runtime/windows/signals.h create mode 100644 src/pkg/runtime/windows/syscall.goc create mode 100644 src/pkg/runtime/windows/thread.c create mode 100644 src/pkg/scanner/Makefile create mode 100644 src/pkg/scanner/scanner.go create mode 100644 src/pkg/scanner/scanner_test.go create mode 100644 src/pkg/smtp/Makefile create mode 100644 src/pkg/smtp/auth.go create mode 100644 src/pkg/smtp/smtp.go create mode 100644 src/pkg/smtp/smtp_test.go create mode 100644 src/pkg/sort/Makefile create mode 100644 src/pkg/sort/search.go create mode 100644 src/pkg/sort/search_test.go create mode 100644 src/pkg/sort/sort.go create mode 100644 src/pkg/sort/sort_test.go create mode 100644 src/pkg/strconv/Makefile create mode 100644 src/pkg/strconv/atob.go create mode 100644 src/pkg/strconv/atob_test.go create mode 100644 src/pkg/strconv/atof.go create mode 100644 src/pkg/strconv/atof_test.go create mode 100644 src/pkg/strconv/atoi.go create mode 100644 src/pkg/strconv/atoi_test.go create mode 100644 src/pkg/strconv/decimal.go create mode 100644 src/pkg/strconv/decimal_test.go create mode 100644 src/pkg/strconv/fp_test.go create mode 100644 src/pkg/strconv/ftoa.go create mode 100644 src/pkg/strconv/ftoa_test.go create mode 100644 src/pkg/strconv/internal_test.go create mode 100644 src/pkg/strconv/itoa.go create mode 100644 src/pkg/strconv/itoa_test.go create mode 100644 src/pkg/strconv/quote.go create mode 100644 src/pkg/strconv/quote_test.go create mode 100644 src/pkg/strconv/testfp.txt create mode 100644 src/pkg/strings/Makefile create mode 100644 src/pkg/strings/reader.go create mode 100644 src/pkg/strings/strings.go create mode 100644 src/pkg/strings/strings_test.go create mode 100644 src/pkg/sync/Makefile create mode 100644 src/pkg/sync/atomic/Makefile create mode 100644 src/pkg/sync/atomic/asm_386.s create mode 100644 src/pkg/sync/atomic/asm_amd64.s create mode 100644 src/pkg/sync/atomic/asm_arm.s create mode 100644 src/pkg/sync/atomic/asm_linux_arm.s create mode 100644 src/pkg/sync/atomic/atomic_test.go create mode 100644 src/pkg/sync/atomic/doc.go create mode 100644 src/pkg/sync/cond.go create mode 100644 src/pkg/sync/cond_test.go create mode 100644 src/pkg/sync/mutex.go create mode 100644 src/pkg/sync/mutex_test.go create mode 100644 src/pkg/sync/once.go create mode 100644 src/pkg/sync/once_test.go create mode 100644 src/pkg/sync/rwmutex.go create mode 100644 src/pkg/sync/rwmutex_test.go create mode 100644 src/pkg/sync/waitgroup.go create mode 100644 src/pkg/sync/waitgroup_test.go create mode 100644 src/pkg/syscall/Makefile create mode 100644 src/pkg/syscall/asm_darwin_386.s create mode 100644 src/pkg/syscall/asm_darwin_amd64.s create mode 100644 src/pkg/syscall/asm_freebsd_386.s create mode 100644 src/pkg/syscall/asm_freebsd_amd64.s create mode 100644 src/pkg/syscall/asm_linux_386.s create mode 100644 src/pkg/syscall/asm_linux_amd64.s create mode 100644 src/pkg/syscall/asm_linux_arm.s create mode 100644 src/pkg/syscall/asm_plan9_386.s create mode 100644 src/pkg/syscall/asm_windows_386.s create mode 100644 src/pkg/syscall/asm_windows_amd64.s create mode 100644 src/pkg/syscall/bpf_bsd.go create mode 100644 src/pkg/syscall/exec_plan9.go create mode 100644 src/pkg/syscall/exec_unix.go create mode 100644 src/pkg/syscall/exec_windows.go create mode 100644 src/pkg/syscall/lsf_linux.go create mode 100755 src/pkg/syscall/mkall.sh create mode 100755 src/pkg/syscall/mkerrors.sh create mode 100755 src/pkg/syscall/mkerrors_windows.sh create mode 100755 src/pkg/syscall/mksyscall.pl create mode 100755 src/pkg/syscall/mksyscall_windows.pl create mode 100755 src/pkg/syscall/mksysnum_darwin.pl create mode 100755 src/pkg/syscall/mksysnum_freebsd.pl create mode 100755 src/pkg/syscall/mksysnum_linux.pl create mode 100755 src/pkg/syscall/mksysnum_plan9.sh create mode 100644 src/pkg/syscall/netlink_linux.go create mode 100644 src/pkg/syscall/route_bsd.go create mode 100644 src/pkg/syscall/route_darwin.go create mode 100644 src/pkg/syscall/route_freebsd.go create mode 100644 src/pkg/syscall/sockcmsg_linux.go create mode 100644 src/pkg/syscall/sockcmsg_unix.go create mode 100644 src/pkg/syscall/str.go create mode 100644 src/pkg/syscall/syscall.go create mode 100644 src/pkg/syscall/syscall_386.go create mode 100644 src/pkg/syscall/syscall_amd64.go create mode 100644 src/pkg/syscall/syscall_arm.go create mode 100644 src/pkg/syscall/syscall_bsd.go create mode 100644 src/pkg/syscall/syscall_darwin.go create mode 100644 src/pkg/syscall/syscall_darwin_386.go create mode 100644 src/pkg/syscall/syscall_darwin_amd64.go create mode 100644 src/pkg/syscall/syscall_freebsd.go create mode 100644 src/pkg/syscall/syscall_freebsd_386.go create mode 100644 src/pkg/syscall/syscall_freebsd_amd64.go create mode 100644 src/pkg/syscall/syscall_linux.go create mode 100644 src/pkg/syscall/syscall_linux_386.go create mode 100644 src/pkg/syscall/syscall_linux_amd64.go create mode 100644 src/pkg/syscall/syscall_linux_arm.go create mode 100644 src/pkg/syscall/syscall_plan9.go create mode 100644 src/pkg/syscall/syscall_plan9_386.go create mode 100644 src/pkg/syscall/syscall_unix.go create mode 100644 src/pkg/syscall/syscall_windows.go create mode 100644 src/pkg/syscall/syscall_windows_386.go create mode 100644 src/pkg/syscall/syscall_windows_amd64.go create mode 100644 src/pkg/syscall/types_darwin.c create mode 100644 src/pkg/syscall/types_freebsd.c create mode 100644 src/pkg/syscall/types_linux.c create mode 100644 src/pkg/syscall/types_plan9.c create mode 100644 src/pkg/syscall/zerrors_darwin_386.go create mode 100644 src/pkg/syscall/zerrors_darwin_amd64.go create mode 100644 src/pkg/syscall/zerrors_freebsd_386.go create mode 100644 src/pkg/syscall/zerrors_freebsd_amd64.go create mode 100644 src/pkg/syscall/zerrors_linux_386.go create mode 100644 src/pkg/syscall/zerrors_linux_amd64.go create mode 100644 src/pkg/syscall/zerrors_linux_arm.go create mode 100644 src/pkg/syscall/zerrors_plan9_386.go create mode 100644 src/pkg/syscall/zerrors_windows.go create mode 100644 src/pkg/syscall/zerrors_windows_386.go create mode 100644 src/pkg/syscall/zerrors_windows_amd64.go create mode 100644 src/pkg/syscall/zsyscall_darwin_386.go create mode 100644 src/pkg/syscall/zsyscall_darwin_amd64.go create mode 100644 src/pkg/syscall/zsyscall_freebsd_386.go create mode 100644 src/pkg/syscall/zsyscall_freebsd_amd64.go create mode 100644 src/pkg/syscall/zsyscall_linux_386.go create mode 100644 src/pkg/syscall/zsyscall_linux_amd64.go create mode 100644 src/pkg/syscall/zsyscall_linux_arm.go create mode 100644 src/pkg/syscall/zsyscall_plan9_386.go create mode 100644 src/pkg/syscall/zsyscall_windows_386.go create mode 100644 src/pkg/syscall/zsyscall_windows_amd64.go create mode 100644 src/pkg/syscall/zsysnum_darwin_386.go create mode 100644 src/pkg/syscall/zsysnum_darwin_amd64.go create mode 100644 src/pkg/syscall/zsysnum_freebsd_386.go create mode 100644 src/pkg/syscall/zsysnum_freebsd_amd64.go create mode 100644 src/pkg/syscall/zsysnum_linux_386.go create mode 100644 src/pkg/syscall/zsysnum_linux_amd64.go create mode 100644 src/pkg/syscall/zsysnum_linux_arm.go create mode 100644 src/pkg/syscall/zsysnum_plan9_386.go create mode 100644 src/pkg/syscall/zsysnum_windows_386.go create mode 100644 src/pkg/syscall/zsysnum_windows_amd64.go create mode 100644 src/pkg/syscall/ztypes_darwin_386.go create mode 100644 src/pkg/syscall/ztypes_darwin_amd64.go create mode 100644 src/pkg/syscall/ztypes_freebsd_386.go create mode 100644 src/pkg/syscall/ztypes_freebsd_amd64.go create mode 100644 src/pkg/syscall/ztypes_linux_386.go create mode 100644 src/pkg/syscall/ztypes_linux_amd64.go create mode 100644 src/pkg/syscall/ztypes_linux_arm.go create mode 100644 src/pkg/syscall/ztypes_plan9_386.go create mode 100644 src/pkg/syscall/ztypes_windows.go create mode 100644 src/pkg/syscall/ztypes_windows_386.go create mode 100644 src/pkg/syscall/ztypes_windows_amd64.go create mode 100644 src/pkg/syslog/Makefile create mode 100644 src/pkg/syslog/syslog.go create mode 100644 src/pkg/syslog/syslog_test.go create mode 100644 src/pkg/syslog/syslog_unix.go create mode 100644 src/pkg/tabwriter/Makefile create mode 100644 src/pkg/tabwriter/tabwriter.go create mode 100644 src/pkg/tabwriter/tabwriter_test.go create mode 100644 src/pkg/template/Makefile create mode 100644 src/pkg/template/doc.go create mode 100644 src/pkg/template/exec.go create mode 100644 src/pkg/template/exec_test.go create mode 100644 src/pkg/template/funcs.go create mode 100644 src/pkg/template/helper.go create mode 100644 src/pkg/template/parse.go create mode 100644 src/pkg/template/parse/Makefile create mode 100644 src/pkg/template/parse/lex.go create mode 100644 src/pkg/template/parse/lex_test.go create mode 100644 src/pkg/template/parse/node.go create mode 100644 src/pkg/template/parse/parse.go create mode 100644 src/pkg/template/parse/parse_test.go create mode 100644 src/pkg/template/parse/set.go create mode 100644 src/pkg/template/set.go create mode 100644 src/pkg/template/set_test.go create mode 100644 src/pkg/template/testdata/file1.tmpl create mode 100644 src/pkg/template/testdata/file2.tmpl create mode 100644 src/pkg/template/testdata/tmpl1.tmpl create mode 100644 src/pkg/template/testdata/tmpl2.tmpl create mode 100644 src/pkg/testing/Makefile create mode 100644 src/pkg/testing/benchmark.go create mode 100644 src/pkg/testing/iotest/Makefile create mode 100644 src/pkg/testing/iotest/logger.go create mode 100644 src/pkg/testing/iotest/reader.go create mode 100644 src/pkg/testing/iotest/writer.go create mode 100644 src/pkg/testing/quick/Makefile create mode 100644 src/pkg/testing/quick/quick.go create mode 100644 src/pkg/testing/quick/quick_test.go create mode 100644 src/pkg/testing/script/Makefile create mode 100644 src/pkg/testing/script/script.go create mode 100644 src/pkg/testing/script/script_test.go create mode 100644 src/pkg/testing/testing.go create mode 100644 src/pkg/time/Makefile create mode 100644 src/pkg/time/format.go create mode 100644 src/pkg/time/sleep.go create mode 100644 src/pkg/time/sleep_test.go create mode 100644 src/pkg/time/sys.go create mode 100644 src/pkg/time/sys_plan9.go create mode 100644 src/pkg/time/sys_posix.go create mode 100644 src/pkg/time/tick.go create mode 100644 src/pkg/time/tick_test.go create mode 100644 src/pkg/time/time.go create mode 100644 src/pkg/time/time_test.go create mode 100644 src/pkg/time/zoneinfo_plan9.go create mode 100644 src/pkg/time/zoneinfo_posix.go create mode 100644 src/pkg/time/zoneinfo_unix.go create mode 100644 src/pkg/time/zoneinfo_windows.go create mode 100644 src/pkg/try/Makefile create mode 100644 src/pkg/try/try.go create mode 100644 src/pkg/try/try_test.go create mode 100644 src/pkg/unicode/Makefile create mode 100644 src/pkg/unicode/casetables.go create mode 100644 src/pkg/unicode/digit.go create mode 100644 src/pkg/unicode/digit_test.go create mode 100644 src/pkg/unicode/graphic.go create mode 100644 src/pkg/unicode/graphic_test.go create mode 100644 src/pkg/unicode/letter.go create mode 100644 src/pkg/unicode/letter_test.go create mode 100644 src/pkg/unicode/maketables.go create mode 100644 src/pkg/unicode/script_test.go create mode 100644 src/pkg/unicode/tables.go create mode 100644 src/pkg/unsafe/unsafe.go create mode 100644 src/pkg/url/Makefile create mode 100644 src/pkg/url/url.go create mode 100644 src/pkg/url/url_test.go create mode 100644 src/pkg/utf16/Makefile create mode 100644 src/pkg/utf16/utf16.go create mode 100644 src/pkg/utf16/utf16_test.go create mode 100644 src/pkg/utf8/Makefile create mode 100644 src/pkg/utf8/string.go create mode 100644 src/pkg/utf8/string_test.go create mode 100644 src/pkg/utf8/utf8.go create mode 100644 src/pkg/utf8/utf8_test.go create mode 100644 src/pkg/websocket/Makefile create mode 100644 src/pkg/websocket/client.go create mode 100644 src/pkg/websocket/server.go create mode 100644 src/pkg/websocket/websocket.go create mode 100644 src/pkg/websocket/websocket_test.go create mode 100644 src/pkg/xml/Makefile create mode 100644 src/pkg/xml/atom_test.go create mode 100644 src/pkg/xml/embed_test.go create mode 100644 src/pkg/xml/marshal.go create mode 100644 src/pkg/xml/marshal_test.go create mode 100644 src/pkg/xml/read.go create mode 100644 src/pkg/xml/read_test.go create mode 100644 src/pkg/xml/xml.go create mode 100644 src/pkg/xml/xml_test.go (limited to 'src/pkg') diff --git a/src/pkg/Makefile b/src/pkg/Makefile new file mode 100644 index 000000000..991e3cbde --- /dev/null +++ b/src/pkg/Makefile @@ -0,0 +1,312 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# After editing the DIRS= list or adding imports to any Go files +# in any of those directories, run: +# +# ./deps.bash +# +# to rebuild the dependency information in Make.deps. + +include ../Make.inc + +all: install + +DIRS=\ + archive/tar\ + archive/zip\ + asn1\ + big\ + bufio\ + bytes\ + cmath\ + compress/bzip2\ + compress/flate\ + compress/gzip\ + compress/lzw \ + compress/zlib\ + container/heap\ + container/list\ + container/ring\ + container/vector\ + crypto\ + crypto/aes\ + crypto/blowfish\ + crypto/cast5\ + crypto/cipher\ + crypto/des\ + crypto/dsa\ + crypto/ecdsa\ + crypto/elliptic\ + crypto/hmac\ + crypto/md4\ + crypto/md5\ + crypto/ocsp\ + crypto/openpgp\ + crypto/openpgp/armor\ + crypto/openpgp/elgamal\ + crypto/openpgp/error\ + crypto/openpgp/packet\ + crypto/openpgp/s2k\ + crypto/rand\ + crypto/rc4\ + crypto/ripemd160\ + crypto/rsa\ + crypto/sha1\ + crypto/sha256\ + crypto/sha512\ + crypto/subtle\ + crypto/tls\ + crypto/twofish\ + crypto/x509\ + crypto/x509/pkix\ + crypto/xtea\ + csv\ + debug/dwarf\ + debug/macho\ + debug/elf\ + debug/gosym\ + debug/pe\ + ebnf\ + encoding/ascii85\ + encoding/base32\ + encoding/base64\ + encoding/binary\ + encoding/git85\ + encoding/hex\ + encoding/pem\ + exec\ + exp/datafmt\ + exp/gui\ + exp/gui/x11\ + exp/norm\ + exp/regexp/syntax\ + exp/template/html\ + expvar\ + flag\ + fmt\ + go/ast\ + go/build\ + go/doc\ + go/parser\ + go/printer\ + go/scanner\ + go/token\ + go/typechecker\ + go/types\ + gob\ + hash\ + hash/adler32\ + hash/crc32\ + hash/crc64\ + hash/fnv\ + html\ + http\ + http/cgi\ + http/fcgi\ + http/pprof\ + http/httptest\ + http/spdy\ + image\ + image/bmp\ + image/draw\ + image/gif\ + image/jpeg\ + image/png\ + image/tiff\ + image/ycbcr\ + index/suffixarray\ + io\ + io/ioutil\ + json\ + log\ + mail\ + math\ + mime\ + mime/multipart\ + net\ + net/dict\ + net/textproto\ + netchan\ + old/template\ + os\ + os/signal\ + os/user\ + patch\ + path\ + path/filepath\ + rand\ + reflect\ + regexp\ + rpc\ + rpc/jsonrpc\ + runtime\ + runtime/cgo\ + runtime/debug\ + runtime/pprof\ + scanner\ + smtp\ + sort\ + strconv\ + strings\ + sync\ + sync/atomic\ + syscall\ + syslog\ + tabwriter\ + template\ + template/parse\ + testing\ + testing/iotest\ + testing/quick\ + testing/script\ + time\ + try\ + unicode\ + url\ + utf16\ + utf8\ + websocket\ + xml\ + ../cmd/cgo\ + ../cmd/ebnflint\ + ../cmd/godoc\ + ../cmd/gofix\ + ../cmd/gofmt\ + ../cmd/goinstall\ + ../cmd/gotest\ + ../cmd/gotype\ + ../cmd/govet\ + ../cmd/goyacc\ + ../cmd/hgpatch\ + +ifeq ($(GOOS),linux) +DIRS+=\ + os/inotify\ + +endif + +ifeq ($(GOOS),plan9) +NOPLAN9BUILD=\ + crypto/tls\ + exp/gui/x11\ + expvar\ + http\ + http/cgi\ + http/fcgi\ + http/httptest\ + http/pprof\ + http/spdy\ + mail\ + mime/multipart\ + net\ + net/dict\ + net/textproto\ + netchan\ + os/signal\ + rpc\ + rpc/jsonrpc\ + smtp\ + syslog\ + websocket\ + ../cmd/godoc\ + ../cmd/goinstall\ + +DIRS:=$(filter-out $(NOPLAN9BUILD),$(DIRS)) +endif + +NOTEST+=\ + crypto\ + crypto/openpgp/error\ + crypto/x509/pkix\ + exp/gui\ + exp/gui/x11\ + go/ast\ + go/doc\ + go/token\ + hash\ + http/pprof\ + http/httptest\ + image/bmp\ + image/gif\ + net/dict\ + rand\ + runtime/cgo\ + syscall\ + testing\ + testing/iotest\ + try\ + ../cmd/cgo\ + ../cmd/ebnflint\ + ../cmd/godoc\ + ../cmd/gotest\ + ../cmd/goyacc\ + ../cmd/hgpatch\ + +NOBENCH+=\ + container/vector\ + +# Disable tests that windows cannot run yet. +ifeq ($(GOOS),windows) +NOTEST+=os/signal # no signals +NOTEST+=syslog # no network +NOTEST+=time # no syscall.Kill, syscall.SIGCHLD for sleep tests +endif + +TEST=\ + $(filter-out $(NOTEST),$(DIRS)) + +BENCH=\ + $(filter-out $(NOBENCH),$(TEST)) + +clean.dirs: $(addsuffix .clean, $(DIRS)) +install.dirs: $(addsuffix .install, $(DIRS)) +nuke.dirs: $(addsuffix .nuke, $(DIRS)) +test.dirs: $(addsuffix .test, $(TEST)) +testshort.dirs: $(addsuffix .testshort, $(TEST)) +bench.dirs: $(addsuffix .bench, $(BENCH)) + +%.clean: + +$(MAKE) -C $* clean + +%.install: + +@echo install $* + +@$(MAKE) -C $* install.clean >$*/build.out 2>&1 || (echo INSTALL FAIL $*; cat $*/build.out; exit 1) + +%.nuke: + +$(MAKE) -C $* nuke + +%.test: + +@echo test $* + +@$(MAKE) -C $* test.clean >$*/test.out 2>&1 || (echo TEST FAIL $*; cat $*/test.out; exit 1) + +%.testshort: + +@echo test $* + +@$(MAKE) -C $* testshort.clean >$*/test.out 2>&1 || (echo TEST FAIL $*; cat $*/test.out; exit 1) + +%.bench: + +$(MAKE) -C $* bench + +clean: clean.dirs + +install: install.dirs + +test: test.dirs + +testshort: testshort.dirs + +bench: bench.dirs ../../test/garbage.bench + +nuke: nuke.dirs + rm -rf "$(GOROOT)"/pkg/* + +deps: + ./deps.bash + +echo-dirs: + @echo $(DIRS) + +-include Make.deps + +runtime/cgo.install: ../cmd/cgo.install diff --git a/src/pkg/archive/tar/Makefile b/src/pkg/archive/tar/Makefile new file mode 100644 index 000000000..8897e883e --- /dev/null +++ b/src/pkg/archive/tar/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=archive/tar +GOFILES=\ + common.go\ + reader.go\ + writer.go\ + +include ../../../Make.pkg diff --git a/src/pkg/archive/tar/common.go b/src/pkg/archive/tar/common.go new file mode 100644 index 000000000..528858765 --- /dev/null +++ b/src/pkg/archive/tar/common.go @@ -0,0 +1,75 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tar implements access to tar archives. +// It aims to cover most of the variations, including those produced +// by GNU and BSD tars. +// +// References: +// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 +// http://www.gnu.org/software/tar/manual/html_node/Standard.html +package tar + +const ( + blockSize = 512 + + // Types + TypeReg = '0' + TypeRegA = '\x00' + TypeLink = '1' + TypeSymlink = '2' + TypeChar = '3' + TypeBlock = '4' + TypeDir = '5' + TypeFifo = '6' + TypeCont = '7' + TypeXHeader = 'x' + TypeXGlobalHeader = 'g' +) + +// A Header represents a single header in a tar archive. +// Some fields may not be populated. +type Header struct { + Name string + Mode int64 + Uid int + Gid int + Size int64 + Mtime int64 + Typeflag byte + Linkname string + Uname string + Gname string + Devmajor int64 + Devminor int64 + Atime int64 + Ctime int64 +} + +var zeroBlock = make([]byte, blockSize) + +// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values. +// We compute and return both. +func checksum(header []byte) (unsigned int64, signed int64) { + for i := 0; i < len(header); i++ { + if i == 148 { + // The chksum field (header[148:156]) is special: it should be treated as space bytes. + unsigned += ' ' * 8 + signed += ' ' * 8 + i += 7 + continue + } + unsigned += int64(header[i]) + signed += int64(int8(header[i])) + } + return +} + +type slicer []byte + +func (sp *slicer) next(n int) (b []byte) { + s := *sp + b, *sp = s[0:n], s[n:] + return +} diff --git a/src/pkg/archive/tar/reader.go b/src/pkg/archive/tar/reader.go new file mode 100644 index 000000000..45d95c3df --- /dev/null +++ b/src/pkg/archive/tar/reader.go @@ -0,0 +1,221 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - pax extensions + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strconv" +) + +var ( + HeaderError = os.NewError("invalid tar header") +) + +// A Reader provides sequential access to the contents of a tar archive. +// A tar archive consists of a sequence of files. +// The Next method advances to the next file in the archive (including the first), +// and then it can be treated as an io.Reader to access the file's data. +// +// Example: +// tr := tar.NewReader(r) +// for { +// hdr, err := tr.Next() +// if err == os.EOF { +// // end of tar archive +// break +// } +// if err != nil { +// // handle error +// } +// io.Copy(data, tr) +// } +type Reader struct { + r io.Reader + err os.Error + nb int64 // number of unread bytes for current file entry + pad int64 // amount of padding (ignored) after current file entry +} + +// NewReader creates a new Reader reading from r. +func NewReader(r io.Reader) *Reader { return &Reader{r: r} } + +// Next advances to the next entry in the tar archive. +func (tr *Reader) Next() (*Header, os.Error) { + var hdr *Header + if tr.err == nil { + tr.skipUnread() + } + if tr.err == nil { + hdr = tr.readHeader() + } + return hdr, tr.err +} + +// Parse bytes as a NUL-terminated C-style string. +// If a NUL byte is not found then the whole slice is returned as a string. +func cString(b []byte) string { + n := 0 + for n < len(b) && b[n] != 0 { + n++ + } + return string(b[0:n]) +} + +func (tr *Reader) octal(b []byte) int64 { + // Removing leading spaces. + for len(b) > 0 && b[0] == ' ' { + b = b[1:] + } + // Removing trailing NULs and spaces. + for len(b) > 0 && (b[len(b)-1] == ' ' || b[len(b)-1] == '\x00') { + b = b[0 : len(b)-1] + } + x, err := strconv.Btoui64(cString(b), 8) + if err != nil { + tr.err = err + } + return int64(x) +} + +// Skip any unread bytes in the existing file entry, as well as any alignment padding. +func (tr *Reader) skipUnread() { + nr := tr.nb + tr.pad // number of bytes to skip + tr.nb, tr.pad = 0, 0 + if sr, ok := tr.r.(io.Seeker); ok { + if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil { + return + } + } + _, tr.err = io.Copyn(ioutil.Discard, tr.r, nr) +} + +func (tr *Reader) verifyChecksum(header []byte) bool { + if tr.err != nil { + return false + } + + given := tr.octal(header[148:156]) + unsigned, signed := checksum(header) + return given == unsigned || given == signed +} + +func (tr *Reader) readHeader() *Header { + header := make([]byte, blockSize) + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil + } + + // Two blocks of zero bytes marks the end of the archive. + if bytes.Equal(header, zeroBlock[0:blockSize]) { + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil + } + if bytes.Equal(header, zeroBlock[0:blockSize]) { + tr.err = os.EOF + } else { + tr.err = HeaderError // zero block and then non-zero block + } + return nil + } + + if !tr.verifyChecksum(header) { + tr.err = HeaderError + return nil + } + + // Unpack + hdr := new(Header) + s := slicer(header) + + hdr.Name = cString(s.next(100)) + hdr.Mode = tr.octal(s.next(8)) + hdr.Uid = int(tr.octal(s.next(8))) + hdr.Gid = int(tr.octal(s.next(8))) + hdr.Size = tr.octal(s.next(12)) + hdr.Mtime = tr.octal(s.next(12)) + s.next(8) // chksum + hdr.Typeflag = s.next(1)[0] + hdr.Linkname = cString(s.next(100)) + + // The remainder of the header depends on the value of magic. + // The original (v7) version of tar had no explicit magic field, + // so its magic bytes, like the rest of the block, are NULs. + magic := string(s.next(8)) // contains version field as well. + var format string + switch magic { + case "ustar\x0000": // POSIX tar (1003.1-1988) + if string(header[508:512]) == "tar\x00" { + format = "star" + } else { + format = "posix" + } + case "ustar \x00": // old GNU tar + format = "gnu" + } + + switch format { + case "posix", "gnu", "star": + hdr.Uname = cString(s.next(32)) + hdr.Gname = cString(s.next(32)) + devmajor := s.next(8) + devminor := s.next(8) + if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { + hdr.Devmajor = tr.octal(devmajor) + hdr.Devminor = tr.octal(devminor) + } + var prefix string + switch format { + case "posix", "gnu": + prefix = cString(s.next(155)) + case "star": + prefix = cString(s.next(131)) + hdr.Atime = tr.octal(s.next(12)) + hdr.Ctime = tr.octal(s.next(12)) + } + if len(prefix) > 0 { + hdr.Name = prefix + "/" + hdr.Name + } + } + + if tr.err != nil { + tr.err = HeaderError + return nil + } + + // Maximum value of hdr.Size is 64 GB (12 octal digits), + // so there's no risk of int64 overflowing. + tr.nb = int64(hdr.Size) + tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two + + return hdr +} + +// Read reads from the current entry in the tar archive. +// It returns 0, os.EOF when it reaches the end of that entry, +// until Next is called to advance to the next entry. +func (tr *Reader) Read(b []byte) (n int, err os.Error) { + if tr.nb == 0 { + // file consumed + return 0, os.EOF + } + + if int64(len(b)) > tr.nb { + b = b[0:tr.nb] + } + n, err = tr.r.Read(b) + tr.nb -= int64(n) + + if err == os.EOF && tr.nb > 0 { + err = io.ErrUnexpectedEOF + } + tr.err = err + return +} diff --git a/src/pkg/archive/tar/reader_test.go b/src/pkg/archive/tar/reader_test.go new file mode 100644 index 000000000..f473c900f --- /dev/null +++ b/src/pkg/archive/tar/reader_test.go @@ -0,0 +1,273 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "os" + "reflect" + "testing" +) + +type untarTest struct { + file string + headers []*Header + cksums []string +} + +var gnuTarTest = &untarTest{ + file: "testdata/gnu.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244428340, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244436044, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + }, + cksums: []string{ + "e38b27eaccb4391bdec553a7f3ae6b2f", + "c65bd2e50a56a2138bf1716f2fd56fe9", + }, +} + +var untarTests = []*untarTest{ + gnuTarTest, + &untarTest{ + file: "testdata/star.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + }, + }, + &untarTest{ + file: "testdata/v7.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244593104, + Typeflag: '\x00', + }, + &Header{ + Name: "small2.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244593104, + Typeflag: '\x00', + }, + }, + }, +} + +func TestReader(t *testing.T) { +testLoop: + for i, test := range untarTests { + f, err := os.Open(test.file) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + tr := NewReader(f) + for j, header := range test.headers { + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) + f.Close() + continue testLoop + } + if !reflect.DeepEqual(hdr, header) { + t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", + i, j, *hdr, *header) + } + } + hdr, err := tr.Next() + if err == os.EOF { + break + } + if hdr != nil || err != nil { + t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err) + } + f.Close() + } +} + +func TestPartialRead(t *testing.T) { + f, err := os.Open("testdata/gnu.tar") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + // Read the first four bytes; Next() should skip the last byte. + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Fatalf("Didn't get first file: %v", err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if expected := []byte("Kilt"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected) + } + + // Second file + hdr, err = tr.Next() + if err != nil || hdr == nil { + t.Fatalf("Didn't get second file: %v", err) + } + buf = make([]byte, 6) + if _, err := io.ReadFull(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if expected := []byte("Google"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected) + } +} + +func TestIncrementalRead(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + headers := test.headers + cksums := test.cksums + nread := 0 + + // loop over all files + for ; ; nread++ { + hdr, err := tr.Next() + if hdr == nil || err == os.EOF { + break + } + + // check the header + if !reflect.DeepEqual(hdr, headers[nread]) { + t.Errorf("Incorrect header:\nhave %+v\nwant %+v", + *hdr, headers[nread]) + } + + // read file contents in little chunks EOF, + // checksumming all the way + h := md5.New() + rdbuf := make([]uint8, 8) + for { + nr, err := tr.Read(rdbuf) + if err == os.EOF { + break + } + if err != nil { + t.Errorf("Read: unexpected error %v\n", err) + break + } + h.Write(rdbuf[0:nr]) + } + // verify checksum + have := fmt.Sprintf("%x", h.Sum()) + want := cksums[nread] + if want != have { + t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want) + } + } + if nread != len(headers) { + t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread) + } +} + +func TestNonSeekable(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + // pipe the data in + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("Unexpected error %s", err) + } + go func() { + rdbuf := make([]uint8, 1<<16) + for { + nr, err := f.Read(rdbuf) + w.Write(rdbuf[0:nr]) + if err == os.EOF { + break + } + } + w.Close() + }() + + tr := NewReader(r) + nread := 0 + + for ; ; nread++ { + hdr, err := tr.Next() + if hdr == nil || err == os.EOF { + break + } + } + + if nread != len(test.headers) { + t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread) + } +} diff --git a/src/pkg/archive/tar/testdata/gnu.tar b/src/pkg/archive/tar/testdata/gnu.tar new file mode 100644 index 000000000..fc899dc8d Binary files /dev/null and b/src/pkg/archive/tar/testdata/gnu.tar differ diff --git a/src/pkg/archive/tar/testdata/small.txt b/src/pkg/archive/tar/testdata/small.txt new file mode 100644 index 000000000..b249bfc51 --- /dev/null +++ b/src/pkg/archive/tar/testdata/small.txt @@ -0,0 +1 @@ +Kilts \ No newline at end of file diff --git a/src/pkg/archive/tar/testdata/small2.txt b/src/pkg/archive/tar/testdata/small2.txt new file mode 100644 index 000000000..394ee3ecd --- /dev/null +++ b/src/pkg/archive/tar/testdata/small2.txt @@ -0,0 +1 @@ +Google.com diff --git a/src/pkg/archive/tar/testdata/star.tar b/src/pkg/archive/tar/testdata/star.tar new file mode 100644 index 000000000..59e2d4e60 Binary files /dev/null and b/src/pkg/archive/tar/testdata/star.tar differ diff --git a/src/pkg/archive/tar/testdata/v7.tar b/src/pkg/archive/tar/testdata/v7.tar new file mode 100644 index 000000000..eb65fc941 Binary files /dev/null and b/src/pkg/archive/tar/testdata/v7.tar differ diff --git a/src/pkg/archive/tar/testdata/writer-big.tar b/src/pkg/archive/tar/testdata/writer-big.tar new file mode 100644 index 000000000..753e883ce Binary files /dev/null and b/src/pkg/archive/tar/testdata/writer-big.tar differ diff --git a/src/pkg/archive/tar/testdata/writer.tar b/src/pkg/archive/tar/testdata/writer.tar new file mode 100644 index 000000000..0358f91b9 Binary files /dev/null and b/src/pkg/archive/tar/testdata/writer.tar differ diff --git a/src/pkg/archive/tar/writer.go b/src/pkg/archive/tar/writer.go new file mode 100644 index 000000000..8673bad31 --- /dev/null +++ b/src/pkg/archive/tar/writer.go @@ -0,0 +1,205 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - catch more errors (no first header, write after close, etc.) + +import ( + "io" + "os" + "strconv" +) + +var ( + ErrWriteTooLong = os.NewError("write too long") + ErrFieldTooLong = os.NewError("header field too long") + ErrWriteAfterClose = os.NewError("write after close") +) + +// A Writer provides sequential writing of a tar archive in POSIX.1 format. +// A tar archive consists of a sequence of files. +// Call WriteHeader to begin a new file, and then call Write to supply that file's data, +// writing at most hdr.Size bytes in total. +// +// Example: +// tw := tar.NewWriter(w) +// hdr := new(Header) +// hdr.Size = length of data in bytes +// // populate other hdr fields as desired +// if err := tw.WriteHeader(hdr); err != nil { +// // handle error +// } +// io.Copy(tw, data) +// tw.Close() +type Writer struct { + w io.Writer + err os.Error + nb int64 // number of unwritten bytes for current file entry + pad int64 // amount of padding to write after current file entry + closed bool + usedBinary bool // whether the binary numeric field extension was used +} + +// NewWriter creates a new Writer writing to w. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } + +// Flush finishes writing the current file (optional). +func (tw *Writer) Flush() os.Error { + n := tw.nb + tw.pad + for n > 0 && tw.err == nil { + nr := n + if nr > blockSize { + nr = blockSize + } + var nw int + nw, tw.err = tw.w.Write(zeroBlock[0:nr]) + n -= int64(nw) + } + tw.nb = 0 + tw.pad = 0 + return tw.err +} + +// Write s into b, terminating it with a NUL if there is room. +func (tw *Writer) cString(b []byte, s string) { + if len(s) > len(b) { + if tw.err == nil { + tw.err = ErrFieldTooLong + } + return + } + copy(b, s) + if len(s) < len(b) { + b[len(s)] = 0 + } +} + +// Encode x as an octal ASCII string and write it into b with leading zeros. +func (tw *Writer) octal(b []byte, x int64) { + s := strconv.Itob64(x, 8) + // leading zeros, but leave room for a NUL. + for len(s)+1 < len(b) { + s = "0" + s + } + tw.cString(b, s) +} + +// Write x into b, either as octal or as binary (GNUtar/star extension). +func (tw *Writer) numeric(b []byte, x int64) { + // Try octal first. + s := strconv.Itob64(x, 8) + if len(s) < len(b) { + tw.octal(b, x) + return + } + // Too big: use binary (big-endian). + tw.usedBinary = true + for i := len(b) - 1; x > 0 && i >= 0; i-- { + b[i] = byte(x) + x >>= 8 + } + b[0] |= 0x80 // highest bit indicates binary format +} + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +func (tw *Writer) WriteHeader(hdr *Header) os.Error { + if tw.closed { + return ErrWriteAfterClose + } + if tw.err == nil { + tw.Flush() + } + if tw.err != nil { + return tw.err + } + + tw.nb = int64(hdr.Size) + tw.pad = -tw.nb & (blockSize - 1) // blockSize is a power of two + + header := make([]byte, blockSize) + s := slicer(header) + + // TODO(dsymonds): handle names longer than 100 chars + copy(s.next(100), []byte(hdr.Name)) + + tw.octal(s.next(8), hdr.Mode) // 100:108 + tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116 + tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124 + tw.numeric(s.next(12), hdr.Size) // 124:136 + tw.numeric(s.next(12), hdr.Mtime) // 136:148 + s.next(8) // chksum (148:156) + s.next(1)[0] = hdr.Typeflag // 156:157 + s.next(100) // linkname (157:257) + copy(s.next(8), []byte("ustar\x0000")) // 257:265 + tw.cString(s.next(32), hdr.Uname) // 265:297 + tw.cString(s.next(32), hdr.Gname) // 297:329 + tw.numeric(s.next(8), hdr.Devmajor) // 329:337 + tw.numeric(s.next(8), hdr.Devminor) // 337:345 + + // Use the GNU magic instead of POSIX magic if we used any GNU extensions. + if tw.usedBinary { + copy(header[257:265], []byte("ustar \x00")) + } + + // The chksum field is terminated by a NUL and a space. + // This is different from the other octal fields. + chksum, _ := checksum(header) + tw.octal(header[148:155], chksum) + header[155] = ' ' + + if tw.err != nil { + // problem with header; probably integer too big for a field. + return tw.err + } + + _, tw.err = tw.w.Write(header) + + return tw.err +} + +// Write writes to the current entry in the tar archive. +// Write returns the error ErrWriteTooLong if more than +// hdr.Size bytes are written after WriteHeader. +func (tw *Writer) Write(b []byte) (n int, err os.Error) { + if tw.closed { + err = ErrWriteTooLong + return + } + overwrite := false + if int64(len(b)) > tw.nb { + b = b[0:tw.nb] + overwrite = true + } + n, err = tw.w.Write(b) + tw.nb -= int64(n) + if err == nil && overwrite { + err = ErrWriteTooLong + return + } + tw.err = err + return +} + +// Close closes the tar archive, flushing any unwritten +// data to the underlying writer. +func (tw *Writer) Close() os.Error { + if tw.err != nil || tw.closed { + return tw.err + } + tw.Flush() + tw.closed = true + + // trailer: two zero blocks + for i := 0; i < 2; i++ { + _, tw.err = tw.w.Write(zeroBlock) + if tw.err != nil { + break + } + } + return tw.err +} diff --git a/src/pkg/archive/tar/writer_test.go b/src/pkg/archive/tar/writer_test.go new file mode 100644 index 000000000..838cb7e1f --- /dev/null +++ b/src/pkg/archive/tar/writer_test.go @@ -0,0 +1,157 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "testing" + "testing/iotest" +) + +type writerTestEntry struct { + header *Header + contents string +} + +type writerTest struct { + file string // filename of expected output + entries []*writerTestEntry +} + +var writerTests = []*writerTest{ + &writerTest{ + file: "testdata/writer.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1246508266, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Kilts", + }, + &writerTestEntry{ + header: &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1245217492, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Google.com\n", + }, + }, + }, + // The truncated test file was produced using these commands: + // dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt + // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar + &writerTest{ + file: "testdata/writer-big.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "tmp/16gig.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 16 << 30, + Mtime: 1254699560, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + // no contents + }, + }, + }, +} + +// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection. +func bytestr(offset int, b []byte) string { + const rowLen = 32 + s := fmt.Sprintf("%04x ", offset) + for _, ch := range b { + switch { + case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z': + s += fmt.Sprintf(" %c", ch) + default: + s += fmt.Sprintf(" %02x", ch) + } + } + return s +} + +// Render a pseudo-diff between two blocks of bytes. +func bytediff(a []byte, b []byte) string { + const rowLen = 32 + s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b)) + for offset := 0; len(a)+len(b) > 0; offset += rowLen { + na, nb := rowLen, rowLen + if na > len(a) { + na = len(a) + } + if nb > len(b) { + nb = len(b) + } + sa := bytestr(offset, a[0:na]) + sb := bytestr(offset, b[0:nb]) + if sa != sb { + s += fmt.Sprintf("-%v\n+%v\n", sa, sb) + } + a = a[na:] + b = b[nb:] + } + return s +} + +func TestWriter(t *testing.T) { +testLoop: + for i, test := range writerTests { + expected, err := ioutil.ReadFile(test.file) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + + buf := new(bytes.Buffer) + tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB + for j, entry := range test.entries { + if err := tw.WriteHeader(entry.header); err != nil { + t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err) + continue testLoop + } + if _, err := io.WriteString(tw, entry.contents); err != nil { + t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err) + continue testLoop + } + } + if err := tw.Close(); err != nil { + t.Errorf("test %d: Failed closing archive: %v", i, err) + continue testLoop + } + + actual := buf.Bytes() + if !bytes.Equal(expected, actual) { + t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v", + i, bytediff(expected, actual)) + } + if testing.Short() { // The second test is expensive. + break + } + } +} diff --git a/src/pkg/archive/zip/Makefile b/src/pkg/archive/zip/Makefile new file mode 100644 index 000000000..9071690f0 --- /dev/null +++ b/src/pkg/archive/zip/Makefile @@ -0,0 +1,13 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=archive/zip +GOFILES=\ + reader.go\ + struct.go\ + writer.go\ + +include ../../../Make.pkg diff --git a/src/pkg/archive/zip/reader.go b/src/pkg/archive/zip/reader.go new file mode 100644 index 000000000..f92f9297a --- /dev/null +++ b/src/pkg/archive/zip/reader.go @@ -0,0 +1,311 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bufio" + "compress/flate" + "hash" + "hash/crc32" + "encoding/binary" + "io" + "io/ioutil" + "os" +) + +var ( + FormatError = os.NewError("zip: not a valid zip file") + UnsupportedMethod = os.NewError("zip: unsupported compression algorithm") + ChecksumError = os.NewError("zip: checksum error") +) + +type Reader struct { + r io.ReaderAt + File []*File + Comment string +} + +type ReadCloser struct { + f *os.File + Reader +} + +type File struct { + FileHeader + zipr io.ReaderAt + zipsize int64 + headerOffset int64 +} + +func (f *File) hasDataDescriptor() bool { + return f.Flags&0x8 != 0 +} + +// OpenReader will open the Zip file specified by name and return a ReadCloser. +func OpenReader(name string) (*ReadCloser, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + r := new(ReadCloser) + if err := r.init(f, fi.Size); err != nil { + f.Close() + return nil, err + } + return r, nil +} + +// NewReader returns a new Reader reading from r, which is assumed to +// have the given size in bytes. +func NewReader(r io.ReaderAt, size int64) (*Reader, os.Error) { + zr := new(Reader) + if err := zr.init(r, size); err != nil { + return nil, err + } + return zr, nil +} + +func (z *Reader) init(r io.ReaderAt, size int64) os.Error { + end, err := readDirectoryEnd(r, size) + if err != nil { + return err + } + z.r = r + z.File = make([]*File, 0, end.directoryRecords) + z.Comment = end.comment + rs := io.NewSectionReader(r, 0, size) + if _, err = rs.Seek(int64(end.directoryOffset), os.SEEK_SET); err != nil { + return err + } + buf := bufio.NewReader(rs) + + // The count of files inside a zip is truncated to fit in a uint16. + // Gloss over this by reading headers until we encounter + // a bad one, and then only report a FormatError or UnexpectedEOF if + // the file count modulo 65536 is incorrect. + for { + f := &File{zipr: r, zipsize: size} + err = readDirectoryHeader(f, buf) + if err == FormatError || err == io.ErrUnexpectedEOF { + break + } + if err != nil { + return err + } + z.File = append(z.File, f) + } + if uint16(len(z.File)) != end.directoryRecords { + // Return the readDirectoryHeader error if we read + // the wrong number of directory entries. + return err + } + return nil +} + +// Close closes the Zip file, rendering it unusable for I/O. +func (rc *ReadCloser) Close() os.Error { + return rc.f.Close() +} + +// Open returns a ReadCloser that provides access to the File's contents. +// It is safe to Open and Read from files concurrently. +func (f *File) Open() (rc io.ReadCloser, err os.Error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return + } + size := int64(f.CompressedSize) + if size == 0 && f.hasDataDescriptor() { + // permit SectionReader to see the rest of the file + size = f.zipsize - (f.headerOffset + bodyOffset) + } + r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size) + switch f.Method { + case Store: // (no compression) + rc = ioutil.NopCloser(r) + case Deflate: + rc = flate.NewReader(r) + default: + err = UnsupportedMethod + } + if rc != nil { + rc = &checksumReader{rc, crc32.NewIEEE(), f, r} + } + return +} + +type checksumReader struct { + rc io.ReadCloser + hash hash.Hash32 + f *File + zipr io.Reader // for reading the data descriptor +} + +func (r *checksumReader) Read(b []byte) (n int, err os.Error) { + n, err = r.rc.Read(b) + r.hash.Write(b[:n]) + if err != os.EOF { + return + } + if r.f.hasDataDescriptor() { + if err = readDataDescriptor(r.zipr, r.f); err != nil { + return + } + } + if r.hash.Sum32() != r.f.CRC32 { + err = ChecksumError + } + return +} + +func (r *checksumReader) Close() os.Error { return r.rc.Close() } + +func readFileHeader(f *File, r io.Reader) os.Error { + var b [fileHeaderLen]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + c := binary.LittleEndian + if sig := c.Uint32(b[:4]); sig != fileHeaderSignature { + return FormatError + } + f.ReaderVersion = c.Uint16(b[4:6]) + f.Flags = c.Uint16(b[6:8]) + f.Method = c.Uint16(b[8:10]) + f.ModifiedTime = c.Uint16(b[10:12]) + f.ModifiedDate = c.Uint16(b[12:14]) + f.CRC32 = c.Uint32(b[14:18]) + f.CompressedSize = c.Uint32(b[18:22]) + f.UncompressedSize = c.Uint32(b[22:26]) + filenameLen := int(c.Uint16(b[26:28])) + extraLen := int(c.Uint16(b[28:30])) + d := make([]byte, filenameLen+extraLen) + if _, err := io.ReadFull(r, d); err != nil { + return err + } + f.Name = string(d[:filenameLen]) + f.Extra = d[filenameLen:] + return nil +} + +// findBodyOffset does the minimum work to verify the file has a header +// and returns the file body offset. +func (f *File) findBodyOffset() (int64, os.Error) { + r := io.NewSectionReader(f.zipr, f.headerOffset, f.zipsize-f.headerOffset) + var b [fileHeaderLen]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return 0, err + } + c := binary.LittleEndian + if sig := c.Uint32(b[:4]); sig != fileHeaderSignature { + return 0, FormatError + } + filenameLen := int(c.Uint16(b[26:28])) + extraLen := int(c.Uint16(b[28:30])) + return int64(fileHeaderLen + filenameLen + extraLen), nil +} + +// readDirectoryHeader attempts to read a directory header from r. +// It returns io.ErrUnexpectedEOF if it cannot read a complete header, +// and FormatError if it doesn't find a valid header signature. +func readDirectoryHeader(f *File, r io.Reader) os.Error { + var b [directoryHeaderLen]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + c := binary.LittleEndian + if sig := c.Uint32(b[:4]); sig != directoryHeaderSignature { + return FormatError + } + f.CreatorVersion = c.Uint16(b[4:6]) + f.ReaderVersion = c.Uint16(b[6:8]) + f.Flags = c.Uint16(b[8:10]) + f.Method = c.Uint16(b[10:12]) + f.ModifiedTime = c.Uint16(b[12:14]) + f.ModifiedDate = c.Uint16(b[14:16]) + f.CRC32 = c.Uint32(b[16:20]) + f.CompressedSize = c.Uint32(b[20:24]) + f.UncompressedSize = c.Uint32(b[24:28]) + filenameLen := int(c.Uint16(b[28:30])) + extraLen := int(c.Uint16(b[30:32])) + commentLen := int(c.Uint16(b[32:34])) + // startDiskNumber := c.Uint16(b[34:36]) // Unused + // internalAttributes := c.Uint16(b[36:38]) // Unused + // externalAttributes := c.Uint32(b[38:42]) // Unused + f.headerOffset = int64(c.Uint32(b[42:46])) + d := make([]byte, filenameLen+extraLen+commentLen) + if _, err := io.ReadFull(r, d); err != nil { + return err + } + f.Name = string(d[:filenameLen]) + f.Extra = d[filenameLen : filenameLen+extraLen] + f.Comment = string(d[filenameLen+extraLen:]) + return nil +} + +func readDataDescriptor(r io.Reader, f *File) os.Error { + var b [dataDescriptorLen]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + c := binary.LittleEndian + f.CRC32 = c.Uint32(b[:4]) + f.CompressedSize = c.Uint32(b[4:8]) + f.UncompressedSize = c.Uint32(b[8:12]) + return nil +} + +func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err os.Error) { + // look for directoryEndSignature in the last 1k, then in the last 65k + var b []byte + for i, bLen := range []int64{1024, 65 * 1024} { + if bLen > size { + bLen = size + } + b = make([]byte, int(bLen)) + if _, err := r.ReadAt(b, size-bLen); err != nil && err != os.EOF { + return nil, err + } + if p := findSignatureInBlock(b); p >= 0 { + b = b[p:] + break + } + if i == 1 || bLen == size { + return nil, FormatError + } + } + + // read header into struct + c := binary.LittleEndian + d := new(directoryEnd) + d.diskNbr = c.Uint16(b[4:6]) + d.dirDiskNbr = c.Uint16(b[6:8]) + d.dirRecordsThisDisk = c.Uint16(b[8:10]) + d.directoryRecords = c.Uint16(b[10:12]) + d.directorySize = c.Uint32(b[12:16]) + d.directoryOffset = c.Uint32(b[16:20]) + d.commentLen = c.Uint16(b[20:22]) + d.comment = string(b[22 : 22+int(d.commentLen)]) + return d, nil +} + +func findSignatureInBlock(b []byte) int { + for i := len(b) - directoryEndLen; i >= 0; i-- { + // defined from directoryEndSignature in struct.go + if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 { + // n is length of comment + n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8 + if n+directoryEndLen+i == len(b) { + return i + } + } + } + return -1 +} diff --git a/src/pkg/archive/zip/reader_test.go b/src/pkg/archive/zip/reader_test.go new file mode 100644 index 000000000..fd5fed2af --- /dev/null +++ b/src/pkg/archive/zip/reader_test.go @@ -0,0 +1,233 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bytes" + "encoding/binary" + "io" + "io/ioutil" + "os" + "testing" + "time" +) + +type ZipTest struct { + Name string + Comment string + File []ZipTestFile + Error os.Error // the error that Opening this file should return +} + +type ZipTestFile struct { + Name string + Content []byte // if blank, will attempt to compare against File + File string // name of file to compare to (relative to testdata/) + Mtime string // modified time in format "mm-dd-yy hh:mm:ss" +} + +// Caution: The Mtime values found for the test files should correspond to +// the values listed with unzip -l . However, the values +// listed by unzip appear to be off by some hours. When creating +// fresh test files and testing them, this issue is not present. +// The test files were created in Sydney, so there might be a time +// zone issue. The time zone information does have to be encoded +// somewhere, because otherwise unzip -l could not provide a different +// time from what the archive/zip package provides, but there appears +// to be no documentation about this. + +var tests = []ZipTest{ + { + Name: "test.zip", + Comment: "This is a zipfile comment.", + File: []ZipTestFile{ + { + Name: "test.txt", + Content: []byte("This is a test text file.\n"), + Mtime: "09-05-10 12:12:02", + }, + { + Name: "gophercolor16x16.png", + File: "gophercolor16x16.png", + Mtime: "09-05-10 15:52:58", + }, + }, + }, + { + Name: "r.zip", + File: []ZipTestFile{ + { + Name: "r/r.zip", + File: "r.zip", + Mtime: "03-04-10 00:24:16", + }, + }, + }, + {Name: "readme.zip"}, + {Name: "readme.notzip", Error: FormatError}, + { + Name: "dd.zip", + File: []ZipTestFile{ + { + Name: "filename", + Content: []byte("This is a test textfile.\n"), + Mtime: "02-02-11 13:06:20", + }, + }, + }, +} + +func TestReader(t *testing.T) { + for _, zt := range tests { + readTestZip(t, zt) + } +} + +func readTestZip(t *testing.T, zt ZipTest) { + z, err := OpenReader("testdata/" + zt.Name) + if err != zt.Error { + t.Errorf("error=%v, want %v", err, zt.Error) + return + } + + // bail if file is not zip + if err == FormatError { + return + } + defer z.Close() + + // bail here if no Files expected to be tested + // (there may actually be files in the zip, but we don't care) + if zt.File == nil { + return + } + + if z.Comment != zt.Comment { + t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment) + } + if len(z.File) != len(zt.File) { + t.Errorf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File)) + } + + // test read of each file + for i, ft := range zt.File { + readTestFile(t, ft, z.File[i]) + } + + // test simultaneous reads + n := 0 + done := make(chan bool) + for i := 0; i < 5; i++ { + for j, ft := range zt.File { + go func() { + readTestFile(t, ft, z.File[j]) + done <- true + }() + n++ + } + } + for ; n > 0; n-- { + <-done + } + + // test invalid checksum + if !z.File[0].hasDataDescriptor() { // skip test when crc32 in dd + z.File[0].CRC32++ // invalidate + r, err := z.File[0].Open() + if err != nil { + t.Error(err) + return + } + var b bytes.Buffer + _, err = io.Copy(&b, r) + if err != ChecksumError { + t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError) + } + } +} + +func readTestFile(t *testing.T, ft ZipTestFile, f *File) { + if f.Name != ft.Name { + t.Errorf("name=%q, want %q", f.Name, ft.Name) + } + + mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime) + if err != nil { + t.Error(err) + return + } + if got, want := f.Mtime_ns()/1e9, mtime.Seconds(); got != want { + t.Errorf("%s: mtime=%s (%d); want %s (%d)", f.Name, time.SecondsToUTC(got), got, mtime, want) + } + + size0 := f.UncompressedSize + + var b bytes.Buffer + r, err := f.Open() + if err != nil { + t.Error(err) + return + } + + if size1 := f.UncompressedSize; size0 != size1 { + t.Errorf("file %q changed f.UncompressedSize from %d to %d", f.Name, size0, size1) + } + + _, err = io.Copy(&b, r) + if err != nil { + t.Error(err) + return + } + r.Close() + + var c []byte + if len(ft.Content) != 0 { + c = ft.Content + } else if c, err = ioutil.ReadFile("testdata/" + ft.File); err != nil { + t.Error(err) + return + } + + if b.Len() != len(c) { + t.Errorf("%s: len=%d, want %d", f.Name, b.Len(), len(c)) + return + } + + for i, b := range b.Bytes() { + if b != c[i] { + t.Errorf("%s: content[%d]=%q want %q", f.Name, i, b, c[i]) + return + } + } +} + +func TestInvalidFiles(t *testing.T) { + const size = 1024 * 70 // 70kb + b := make([]byte, size) + + // zeroes + _, err := NewReader(sliceReaderAt(b), size) + if err != FormatError { + t.Errorf("zeroes: error=%v, want %v", err, FormatError) + } + + // repeated directoryEndSignatures + sig := make([]byte, 4) + binary.LittleEndian.PutUint32(sig, directoryEndSignature) + for i := 0; i < size-4; i += 4 { + copy(b[i:i+4], sig) + } + _, err = NewReader(sliceReaderAt(b), size) + if err != FormatError { + t.Errorf("sigs: error=%v, want %v", err, FormatError) + } +} + +type sliceReaderAt []byte + +func (r sliceReaderAt) ReadAt(b []byte, off int64) (int, os.Error) { + copy(b, r[int(off):int(off)+len(b)]) + return len(b), nil +} diff --git a/src/pkg/archive/zip/struct.go b/src/pkg/archive/zip/struct.go new file mode 100644 index 000000000..1d6e70f10 --- /dev/null +++ b/src/pkg/archive/zip/struct.go @@ -0,0 +1,91 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package zip provides support for reading and writing ZIP archives. + +See: http://www.pkware.com/documents/casestudies/APPNOTE.TXT + +This package does not support ZIP64 or disk spanning. +*/ +package zip + +import "os" +import "time" + +// Compression methods. +const ( + Store uint16 = 0 + Deflate uint16 = 8 +) + +const ( + fileHeaderSignature = 0x04034b50 + directoryHeaderSignature = 0x02014b50 + directoryEndSignature = 0x06054b50 + fileHeaderLen = 30 // + filename + extra + directoryHeaderLen = 46 // + filename + extra + comment + directoryEndLen = 22 // + comment + dataDescriptorLen = 12 +) + +type FileHeader struct { + Name string + CreatorVersion uint16 + ReaderVersion uint16 + Flags uint16 + Method uint16 + ModifiedTime uint16 // MS-DOS time + ModifiedDate uint16 // MS-DOS date + CRC32 uint32 + CompressedSize uint32 + UncompressedSize uint32 + Extra []byte + Comment string +} + +type directoryEnd struct { + diskNbr uint16 // unused + dirDiskNbr uint16 // unused + dirRecordsThisDisk uint16 // unused + directoryRecords uint16 + directorySize uint32 + directoryOffset uint32 // relative to file + commentLen uint16 + comment string +} + +func recoverError(err *os.Error) { + if e := recover(); e != nil { + if osErr, ok := e.(os.Error); ok { + *err = osErr + return + } + panic(e) + } +} + +// msDosTimeToTime converts an MS-DOS date and time into a time.Time. +// The resolution is 2s. +// See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx +func msDosTimeToTime(dosDate, dosTime uint16) time.Time { + return time.Time{ + // date bits 0-4: day of month; 5-8: month; 9-15: years since 1980 + Year: int64(dosDate>>9 + 1980), + Month: int(dosDate >> 5 & 0xf), + Day: int(dosDate & 0x1f), + + // time bits 0-4: second/2; 5-10: minute; 11-15: hour + Hour: int(dosTime >> 11), + Minute: int(dosTime >> 5 & 0x3f), + Second: int(dosTime & 0x1f * 2), + } +} + +// Mtime_ns returns the modified time in ns since epoch. +// The resolution is 2s. +func (h *FileHeader) Mtime_ns() int64 { + t := msDosTimeToTime(h.ModifiedDate, h.ModifiedTime) + return t.Seconds() * 1e9 +} diff --git a/src/pkg/archive/zip/testdata/dd.zip b/src/pkg/archive/zip/testdata/dd.zip new file mode 100644 index 000000000..e53378b0b Binary files /dev/null and b/src/pkg/archive/zip/testdata/dd.zip differ diff --git a/src/pkg/archive/zip/testdata/gophercolor16x16.png b/src/pkg/archive/zip/testdata/gophercolor16x16.png new file mode 100644 index 000000000..48854ff3b Binary files /dev/null and b/src/pkg/archive/zip/testdata/gophercolor16x16.png differ diff --git a/src/pkg/archive/zip/testdata/r.zip b/src/pkg/archive/zip/testdata/r.zip new file mode 100644 index 000000000..ea0fa2ffc Binary files /dev/null and b/src/pkg/archive/zip/testdata/r.zip differ diff --git a/src/pkg/archive/zip/testdata/readme.notzip b/src/pkg/archive/zip/testdata/readme.notzip new file mode 100644 index 000000000..06668c4c1 Binary files /dev/null and b/src/pkg/archive/zip/testdata/readme.notzip differ diff --git a/src/pkg/archive/zip/testdata/readme.zip b/src/pkg/archive/zip/testdata/readme.zip new file mode 100644 index 000000000..db3bb900e Binary files /dev/null and b/src/pkg/archive/zip/testdata/readme.zip differ diff --git a/src/pkg/archive/zip/testdata/test.zip b/src/pkg/archive/zip/testdata/test.zip new file mode 100644 index 000000000..03890c05d Binary files /dev/null and b/src/pkg/archive/zip/testdata/test.zip differ diff --git a/src/pkg/archive/zip/writer.go b/src/pkg/archive/zip/writer.go new file mode 100644 index 000000000..2065b06da --- /dev/null +++ b/src/pkg/archive/zip/writer.go @@ -0,0 +1,244 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bufio" + "compress/flate" + "encoding/binary" + "hash" + "hash/crc32" + "io" + "os" +) + +// TODO(adg): support zip file comments +// TODO(adg): support specifying deflate level + +// Writer implements a zip file writer. +type Writer struct { + *countWriter + dir []*header + last *fileWriter + closed bool +} + +type header struct { + *FileHeader + offset uint32 +} + +// NewWriter returns a new Writer writing a zip file to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{countWriter: &countWriter{w: bufio.NewWriter(w)}} +} + +// Close finishes writing the zip file by writing the central directory. +// It does not (and can not) close the underlying writer. +func (w *Writer) Close() (err os.Error) { + if w.last != nil && !w.last.closed { + if err = w.last.close(); err != nil { + return + } + w.last = nil + } + if w.closed { + return os.NewError("zip: writer closed twice") + } + w.closed = true + + defer recoverError(&err) + + // write central directory + start := w.count + for _, h := range w.dir { + write(w, uint32(directoryHeaderSignature)) + write(w, h.CreatorVersion) + write(w, h.ReaderVersion) + write(w, h.Flags) + write(w, h.Method) + write(w, h.ModifiedTime) + write(w, h.ModifiedDate) + write(w, h.CRC32) + write(w, h.CompressedSize) + write(w, h.UncompressedSize) + write(w, uint16(len(h.Name))) + write(w, uint16(len(h.Extra))) + write(w, uint16(len(h.Comment))) + write(w, uint16(0)) // disk number start + write(w, uint16(0)) // internal file attributes + write(w, uint32(0)) // external file attributes + write(w, h.offset) + writeBytes(w, []byte(h.Name)) + writeBytes(w, h.Extra) + writeBytes(w, []byte(h.Comment)) + } + end := w.count + + // write end record + write(w, uint32(directoryEndSignature)) + write(w, uint16(0)) // disk number + write(w, uint16(0)) // disk number where directory starts + write(w, uint16(len(w.dir))) // number of entries this disk + write(w, uint16(len(w.dir))) // number of entries total + write(w, uint32(end-start)) // size of directory + write(w, uint32(start)) // start of directory + write(w, uint16(0)) // size of comment + + return w.w.(*bufio.Writer).Flush() +} + +// Create adds a file to the zip file using the provided name. +// It returns a Writer to which the file contents should be written. +// The file's contents must be written to the io.Writer before the next +// call to Create, CreateHeader, or Close. +func (w *Writer) Create(name string) (io.Writer, os.Error) { + header := &FileHeader{ + Name: name, + Method: Deflate, + } + return w.CreateHeader(header) +} + +// CreateHeader adds a file to the zip file using the provided FileHeader +// for the file metadata. +// It returns a Writer to which the file contents should be written. +// The file's contents must be written to the io.Writer before the next +// call to Create, CreateHeader, or Close. +func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, os.Error) { + if w.last != nil && !w.last.closed { + if err := w.last.close(); err != nil { + return nil, err + } + } + + fh.Flags |= 0x8 // we will write a data descriptor + fh.CreatorVersion = 0x14 + fh.ReaderVersion = 0x14 + + fw := &fileWriter{ + zipw: w, + compCount: &countWriter{w: w}, + crc32: crc32.NewIEEE(), + } + switch fh.Method { + case Store: + fw.comp = nopCloser{fw.compCount} + case Deflate: + fw.comp = flate.NewWriter(fw.compCount, 5) + default: + return nil, UnsupportedMethod + } + fw.rawCount = &countWriter{w: fw.comp} + + h := &header{ + FileHeader: fh, + offset: uint32(w.count), + } + w.dir = append(w.dir, h) + fw.header = h + + if err := writeHeader(w, fh); err != nil { + return nil, err + } + + w.last = fw + return fw, nil +} + +func writeHeader(w io.Writer, h *FileHeader) (err os.Error) { + defer recoverError(&err) + write(w, uint32(fileHeaderSignature)) + write(w, h.ReaderVersion) + write(w, h.Flags) + write(w, h.Method) + write(w, h.ModifiedTime) + write(w, h.ModifiedDate) + write(w, h.CRC32) + write(w, h.CompressedSize) + write(w, h.UncompressedSize) + write(w, uint16(len(h.Name))) + write(w, uint16(len(h.Extra))) + writeBytes(w, []byte(h.Name)) + writeBytes(w, h.Extra) + return nil +} + +type fileWriter struct { + *header + zipw io.Writer + rawCount *countWriter + comp io.WriteCloser + compCount *countWriter + crc32 hash.Hash32 + closed bool +} + +func (w *fileWriter) Write(p []byte) (int, os.Error) { + if w.closed { + return 0, os.NewError("zip: write to closed file") + } + w.crc32.Write(p) + return w.rawCount.Write(p) +} + +func (w *fileWriter) close() (err os.Error) { + if w.closed { + return os.NewError("zip: file closed twice") + } + w.closed = true + if err = w.comp.Close(); err != nil { + return + } + + // update FileHeader + fh := w.header.FileHeader + fh.CRC32 = w.crc32.Sum32() + fh.CompressedSize = uint32(w.compCount.count) + fh.UncompressedSize = uint32(w.rawCount.count) + + // write data descriptor + defer recoverError(&err) + write(w.zipw, fh.CRC32) + write(w.zipw, fh.CompressedSize) + write(w.zipw, fh.UncompressedSize) + + return nil +} + +type countWriter struct { + w io.Writer + count int64 +} + +func (w *countWriter) Write(p []byte) (int, os.Error) { + n, err := w.w.Write(p) + w.count += int64(n) + return n, err +} + +type nopCloser struct { + io.Writer +} + +func (w nopCloser) Close() os.Error { + return nil +} + +func write(w io.Writer, data interface{}) { + if err := binary.Write(w, binary.LittleEndian, data); err != nil { + panic(err) + } +} + +func writeBytes(w io.Writer, b []byte) { + n, err := w.Write(b) + if err != nil { + panic(err) + } + if n != len(b) { + panic(io.ErrShortWrite) + } +} diff --git a/src/pkg/archive/zip/writer_test.go b/src/pkg/archive/zip/writer_test.go new file mode 100644 index 000000000..eb2a80c3f --- /dev/null +++ b/src/pkg/archive/zip/writer_test.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bytes" + "io/ioutil" + "rand" + "testing" +) + +// TODO(adg): a more sophisticated test suite + +const testString = "Rabbits, guinea pigs, gophers, marsupial rats, and quolls." + +func TestWriter(t *testing.T) { + largeData := make([]byte, 1<<17) + for i := range largeData { + largeData[i] = byte(rand.Int()) + } + + // write a zip file + buf := new(bytes.Buffer) + w := NewWriter(buf) + testCreate(t, w, "foo", []byte(testString), Store) + testCreate(t, w, "bar", largeData, Deflate) + if err := w.Close(); err != nil { + t.Fatal(err) + } + + // read it back + r, err := NewReader(sliceReaderAt(buf.Bytes()), int64(buf.Len())) + if err != nil { + t.Fatal(err) + } + testReadFile(t, r.File[0], []byte(testString)) + testReadFile(t, r.File[1], largeData) +} + +func testCreate(t *testing.T, w *Writer, name string, data []byte, method uint16) { + header := &FileHeader{ + Name: name, + Method: method, + } + f, err := w.CreateHeader(header) + if err != nil { + t.Fatal(err) + } + _, err = f.Write(data) + if err != nil { + t.Fatal(err) + } +} + +func testReadFile(t *testing.T, f *File, data []byte) { + rc, err := f.Open() + if err != nil { + t.Fatal("opening:", err) + } + b, err := ioutil.ReadAll(rc) + if err != nil { + t.Fatal("reading:", err) + } + err = rc.Close() + if err != nil { + t.Fatal("closing:", err) + } + if !bytes.Equal(b, data) { + t.Errorf("File contents %q, want %q", b, data) + } +} diff --git a/src/pkg/archive/zip/zip_test.go b/src/pkg/archive/zip/zip_test.go new file mode 100644 index 000000000..0f71fdfac --- /dev/null +++ b/src/pkg/archive/zip/zip_test.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests that involve both reading and writing. + +package zip + +import ( + "bytes" + "fmt" + "os" + "testing" +) + +type stringReaderAt string + +func (s stringReaderAt) ReadAt(p []byte, off int64) (n int, err os.Error) { + if off >= int64(len(s)) { + return 0, os.EOF + } + n = copy(p, s[off:]) + return +} + +func TestOver65kFiles(t *testing.T) { + if testing.Short() { + t.Logf("slow test; skipping") + return + } + buf := new(bytes.Buffer) + w := NewWriter(buf) + const nFiles = (1 << 16) + 42 + for i := 0; i < nFiles; i++ { + _, err := w.Create(fmt.Sprintf("%d.dat", i)) + if err != nil { + t.Fatalf("creating file %d: %v", i, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("Writer.Close: %v", err) + } + rat := stringReaderAt(buf.String()) + zr, err := NewReader(rat, int64(len(rat))) + if err != nil { + t.Fatalf("NewReader: %v", err) + } + if got := len(zr.File); got != nFiles { + t.Fatalf("File contains %d files, want %d", got, nFiles) + } + for i := 0; i < nFiles; i++ { + want := fmt.Sprintf("%d.dat", i) + if zr.File[i].Name != want { + t.Fatalf("File(%d) = %q, want %q", i, zr.File[i].Name, want) + } + } +} diff --git a/src/pkg/asn1/Makefile b/src/pkg/asn1/Makefile new file mode 100644 index 000000000..6b7770e82 --- /dev/null +++ b/src/pkg/asn1/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=asn1 +GOFILES=\ + asn1.go\ + common.go\ + marshal.go\ + +include ../../Make.pkg diff --git a/src/pkg/asn1/asn1.go b/src/pkg/asn1/asn1.go new file mode 100644 index 000000000..39b676b41 --- /dev/null +++ b/src/pkg/asn1/asn1.go @@ -0,0 +1,840 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package asn1 implements parsing of DER-encoded ASN.1 data structures, +// as defined in ITU-T Rec X.690. +// +// See also ``A Layman's Guide to a Subset of ASN.1, BER, and DER,'' +// http://luca.ntop.org/Teaching/Appunti/asn1.html. +package asn1 + +// ASN.1 is a syntax for specifying abstract objects and BER, DER, PER, XER etc +// are different encoding formats for those objects. Here, we'll be dealing +// with DER, the Distinguished Encoding Rules. DER is used in X.509 because +// it's fast to parse and, unlike BER, has a unique encoding for every object. +// When calculating hashes over objects, it's important that the resulting +// bytes be the same at both ends and DER removes this margin of error. +// +// ASN.1 is very complex and this package doesn't attempt to implement +// everything by any means. + +import ( + "big" + "fmt" + "os" + "reflect" + "time" +) + +// A StructuralError suggests that the ASN.1 data is valid, but the Go type +// which is receiving it doesn't match. +type StructuralError struct { + Msg string +} + +func (e StructuralError) String() string { return "ASN.1 structure error: " + e.Msg } + +// A SyntaxError suggests that the ASN.1 data is invalid. +type SyntaxError struct { + Msg string +} + +func (e SyntaxError) String() string { return "ASN.1 syntax error: " + e.Msg } + +// We start by dealing with each of the primitive types in turn. + +// BOOLEAN + +func parseBool(bytes []byte) (ret bool, err os.Error) { + if len(bytes) != 1 { + err = SyntaxError{"invalid boolean"} + return + } + + return bytes[0] != 0, nil +} + +// INTEGER + +// parseInt64 treats the given bytes as a big-endian, signed integer and +// returns the result. +func parseInt64(bytes []byte) (ret int64, err os.Error) { + if len(bytes) > 8 { + // We'll overflow an int64 in this case. + err = StructuralError{"integer too large"} + return + } + for bytesRead := 0; bytesRead < len(bytes); bytesRead++ { + ret <<= 8 + ret |= int64(bytes[bytesRead]) + } + + // Shift up and down in order to sign extend the result. + ret <<= 64 - uint8(len(bytes))*8 + ret >>= 64 - uint8(len(bytes))*8 + return +} + +// parseInt treats the given bytes as a big-endian, signed integer and returns +// the result. +func parseInt(bytes []byte) (int, os.Error) { + ret64, err := parseInt64(bytes) + if err != nil { + return 0, err + } + if ret64 != int64(int(ret64)) { + return 0, StructuralError{"integer too large"} + } + return int(ret64), nil +} + +var bigOne = big.NewInt(1) + +// parseBigInt treats the given bytes as a big-endian, signed integer and returns +// the result. +func parseBigInt(bytes []byte) *big.Int { + ret := new(big.Int) + if len(bytes) > 0 && bytes[0]&0x80 == 0x80 { + // This is a negative number. + notBytes := make([]byte, len(bytes)) + for i := range notBytes { + notBytes[i] = ^bytes[i] + } + ret.SetBytes(notBytes) + ret.Add(ret, bigOne) + ret.Neg(ret) + return ret + } + ret.SetBytes(bytes) + return ret +} + +// BIT STRING + +// BitString is the structure to use when you want an ASN.1 BIT STRING type. A +// bit string is padded up to the nearest byte in memory and the number of +// valid bits is recorded. Padding bits will be zero. +type BitString struct { + Bytes []byte // bits packed into bytes. + BitLength int // length in bits. +} + +// At returns the bit at the given index. If the index is out of range it +// returns false. +func (b BitString) At(i int) int { + if i < 0 || i >= b.BitLength { + return 0 + } + x := i / 8 + y := 7 - uint(i%8) + return int(b.Bytes[x]>>y) & 1 +} + +// RightAlign returns a slice where the padding bits are at the beginning. The +// slice may share memory with the BitString. +func (b BitString) RightAlign() []byte { + shift := uint(8 - (b.BitLength % 8)) + if shift == 8 || len(b.Bytes) == 0 { + return b.Bytes + } + + a := make([]byte, len(b.Bytes)) + a[0] = b.Bytes[0] >> shift + for i := 1; i < len(b.Bytes); i++ { + a[i] = b.Bytes[i-1] << (8 - shift) + a[i] |= b.Bytes[i] >> shift + } + + return a +} + +// parseBitString parses an ASN.1 bit string from the given byte slice and returns it. +func parseBitString(bytes []byte) (ret BitString, err os.Error) { + if len(bytes) == 0 { + err = SyntaxError{"zero length BIT STRING"} + return + } + paddingBits := int(bytes[0]) + if paddingBits > 7 || + len(bytes) == 1 && paddingBits > 0 || + bytes[len(bytes)-1]&((1< 4 { + err = StructuralError{"base 128 integer too large"} + return + } + ret <<= 7 + b := bytes[offset] + ret |= int(b & 0x7f) + offset++ + if b&0x80 == 0 { + return + } + } + err = SyntaxError{"truncated base 128 integer"} + return +} + +// UTCTime + +func parseUTCTime(bytes []byte) (ret *time.Time, err os.Error) { + s := string(bytes) + ret, err = time.Parse("0601021504Z0700", s) + if err == nil { + return + } + ret, err = time.Parse("060102150405Z0700", s) + return +} + +// parseGeneralizedTime parses the GeneralizedTime from the given byte slice +// and returns the resulting time. +func parseGeneralizedTime(bytes []byte) (ret *time.Time, err os.Error) { + return time.Parse("20060102150405Z0700", string(bytes)) +} + +// PrintableString + +// parsePrintableString parses a ASN.1 PrintableString from the given byte +// array and returns it. +func parsePrintableString(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if !isPrintable(b) { + err = SyntaxError{"PrintableString contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// isPrintable returns true iff the given b is in the ASN.1 PrintableString set. +func isPrintable(b byte) bool { + return 'a' <= b && b <= 'z' || + 'A' <= b && b <= 'Z' || + '0' <= b && b <= '9' || + '\'' <= b && b <= ')' || + '+' <= b && b <= '/' || + b == ' ' || + b == ':' || + b == '=' || + b == '?' || + // This is technically not allowed in a PrintableString. + // However, x509 certificates with wildcard strings don't + // always use the correct string type so we permit it. + b == '*' +} + +// IA5String + +// parseIA5String parses a ASN.1 IA5String (ASCII string) from the given +// byte slice and returns it. +func parseIA5String(bytes []byte) (ret string, err os.Error) { + for _, b := range bytes { + if b >= 0x80 { + err = SyntaxError{"IA5String contains invalid character"} + return + } + } + ret = string(bytes) + return +} + +// T61String + +// parseT61String parses a ASN.1 T61String (8-bit clean string) from the given +// byte slice and returns it. +func parseT61String(bytes []byte) (ret string, err os.Error) { + return string(bytes), nil +} + +// UTF8String + +// parseUTF8String parses a ASN.1 UTF8String (raw UTF-8) from the given byte +// array and returns it. +func parseUTF8String(bytes []byte) (ret string, err os.Error) { + return string(bytes), nil +} + +// A RawValue represents an undecoded ASN.1 object. +type RawValue struct { + Class, Tag int + IsCompound bool + Bytes []byte + FullBytes []byte // includes the tag and length +} + +// RawContent is used to signal that the undecoded, DER data needs to be +// preserved for a struct. To use it, the first field of the struct must have +// this type. It's an error for any of the other fields to have this type. +type RawContent []byte + +// Tagging + +// parseTagAndLength parses an ASN.1 tag and length pair from the given offset +// into a byte slice. It returns the parsed data and the new offset. SET and +// SET OF (tag 17) are mapped to SEQUENCE and SEQUENCE OF (tag 16) since we +// don't distinguish between ordered and unordered objects in this code. +func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset int, err os.Error) { + offset = initOffset + b := bytes[offset] + offset++ + ret.class = int(b >> 6) + ret.isCompound = b&0x20 == 0x20 + ret.tag = int(b & 0x1f) + + // If the bottom five bits are set, then the tag number is actually base 128 + // encoded afterwards + if ret.tag == 0x1f { + ret.tag, offset, err = parseBase128Int(bytes, offset) + if err != nil { + return + } + } + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + if b&0x80 == 0 { + // The length is encoded in the bottom 7 bits. + ret.length = int(b & 0x7f) + } else { + // Bottom 7 bits give the number of length bytes to follow. + numBytes := int(b & 0x7f) + // We risk overflowing a signed 32-bit number if we accept more than 3 bytes. + if numBytes > 3 { + err = StructuralError{"length too large"} + return + } + if numBytes == 0 { + err = SyntaxError{"indefinite length found (not DER)"} + return + } + ret.length = 0 + for i := 0; i < numBytes; i++ { + if offset >= len(bytes) { + err = SyntaxError{"truncated tag or length"} + return + } + b = bytes[offset] + offset++ + ret.length <<= 8 + ret.length |= int(b) + } + } + + return +} + +// parseSequenceOf is used for SEQUENCE OF and SET OF values. It tries to parse +// a number of ASN.1 values from the given byte slice and returns them as a +// slice of Go values of the given type. +func parseSequenceOf(bytes []byte, sliceType reflect.Type, elemType reflect.Type) (ret reflect.Value, err os.Error) { + expectedTag, compoundType, ok := getUniversalType(elemType) + if !ok { + err = StructuralError{"unknown Go type for slice"} + return + } + + // First we iterate over the input and count the number of elements, + // checking that the types are correct in each case. + numElements := 0 + for offset := 0; offset < len(bytes); { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + // We pretend that GENERAL STRINGs are PRINTABLE STRINGs so + // that a sequence of them can be parsed into a []string. + if t.tag == tagGeneralString { + t.tag = tagPrintableString + } + if t.class != classUniversal || t.isCompound != compoundType || t.tag != expectedTag { + err = StructuralError{"sequence tag mismatch"} + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"truncated sequence"} + return + } + offset += t.length + numElements++ + } + ret = reflect.MakeSlice(sliceType, numElements, numElements) + params := fieldParameters{} + offset := 0 + for i := 0; i < numElements; i++ { + offset, err = parseField(ret.Index(i), bytes, offset, params) + if err != nil { + return + } + } + return +} + +var ( + bitStringType = reflect.TypeOf(BitString{}) + objectIdentifierType = reflect.TypeOf(ObjectIdentifier{}) + enumeratedType = reflect.TypeOf(Enumerated(0)) + flagType = reflect.TypeOf(Flag(false)) + timeType = reflect.TypeOf(&time.Time{}) + rawValueType = reflect.TypeOf(RawValue{}) + rawContentsType = reflect.TypeOf(RawContent(nil)) + bigIntType = reflect.TypeOf(new(big.Int)) +) + +// invalidLength returns true iff offset + length > sliceLength, or if the +// addition would overflow. +func invalidLength(offset, length, sliceLength int) bool { + return offset+length < offset || offset+length > sliceLength +} + +// parseField is the main parsing function. Given a byte slice and an offset +// into the array, it will try to parse a suitable ASN.1 value out and store it +// in the given Value. +func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParameters) (offset int, err os.Error) { + offset = initOffset + fieldType := v.Type() + + // If we have run out of data, it may be that there are optional elements at the end. + if offset == len(bytes) { + if !setDefaultValue(v, params) { + err = SyntaxError{"sequence truncated"} + } + return + } + + // Deal with raw values. + if fieldType == rawValueType { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + result := RawValue{t.class, t.tag, t.isCompound, bytes[offset : offset+t.length], bytes[initOffset : offset+t.length]} + offset += t.length + v.Set(reflect.ValueOf(result)) + return + } + + // Deal with the ANY type. + if ifaceType := fieldType; ifaceType.Kind() == reflect.Interface && ifaceType.NumMethod() == 0 { + var t tagAndLength + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + var result interface{} + if !t.isCompound && t.class == classUniversal { + innerBytes := bytes[offset : offset+t.length] + switch t.tag { + case tagPrintableString: + result, err = parsePrintableString(innerBytes) + case tagIA5String: + result, err = parseIA5String(innerBytes) + case tagT61String: + result, err = parseT61String(innerBytes) + case tagInteger: + result, err = parseInt64(innerBytes) + case tagBitString: + result, err = parseBitString(innerBytes) + case tagOID: + result, err = parseObjectIdentifier(innerBytes) + case tagUTCTime: + result, err = parseUTCTime(innerBytes) + case tagOctetString: + result = innerBytes + default: + // If we don't know how to handle the type, we just leave Value as nil. + } + } + offset += t.length + if err != nil { + return + } + if result != nil { + v.Set(reflect.ValueOf(result)) + } + return + } + universalTag, compoundType, ok1 := getUniversalType(fieldType) + if !ok1 { + err = StructuralError{fmt.Sprintf("unknown Go type: %v", fieldType)} + return + } + + t, offset, err := parseTagAndLength(bytes, offset) + if err != nil { + return + } + if params.explicit { + expectedClass := classContextSpecific + if params.application { + expectedClass = classApplication + } + if t.class == expectedClass && t.tag == *params.tag && (t.length == 0 || t.isCompound) { + if t.length > 0 { + t, offset, err = parseTagAndLength(bytes, offset) + if err != nil { + return + } + } else { + if fieldType != flagType { + err = StructuralError{"Zero length explicit tag was not an asn1.Flag"} + return + } + v.SetBool(true) + return + } + } else { + // The tags didn't match, it might be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{"explicitly tagged member didn't match"} + } + return + } + } + + // Special case for strings: all the ASN.1 string types map to the Go + // type string. getUniversalType returns the tag for PrintableString + // when it sees a string, so if we see a different string type on the + // wire, we change the universal type to match. + if universalTag == tagPrintableString { + switch t.tag { + case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: + universalTag = t.tag + } + } + + // Special case for time: UTCTime and GeneralizedTime both map to the + // Go type time.Time. + if universalTag == tagUTCTime && t.tag == tagGeneralizedTime { + universalTag = tagGeneralizedTime + } + + expectedClass := classUniversal + expectedTag := universalTag + + if !params.explicit && params.tag != nil { + expectedClass = classContextSpecific + expectedTag = *params.tag + } + + if !params.explicit && params.application && params.tag != nil { + expectedClass = classApplication + expectedTag = *params.tag + } + + // We have unwrapped any explicit tagging at this point. + if t.class != expectedClass || t.tag != expectedTag || t.isCompound != compoundType { + // Tags don't match. Again, it could be an optional element. + ok := setDefaultValue(v, params) + if ok { + offset = initOffset + } else { + err = StructuralError{fmt.Sprintf("tags don't match (%d vs %+v) %+v %s @%d", expectedTag, t, params, fieldType.Name(), offset)} + } + return + } + if invalidLength(offset, t.length, len(bytes)) { + err = SyntaxError{"data truncated"} + return + } + innerBytes := bytes[offset : offset+t.length] + offset += t.length + + // We deal with the structures defined in this package first. + switch fieldType { + case objectIdentifierType: + newSlice, err1 := parseObjectIdentifier(innerBytes) + v.Set(reflect.MakeSlice(v.Type(), len(newSlice), len(newSlice))) + if err1 == nil { + reflect.Copy(v, reflect.ValueOf(newSlice)) + } + err = err1 + return + case bitStringType: + bs, err1 := parseBitString(innerBytes) + if err1 == nil { + v.Set(reflect.ValueOf(bs)) + } + err = err1 + return + case timeType: + var time *time.Time + var err1 os.Error + if universalTag == tagUTCTime { + time, err1 = parseUTCTime(innerBytes) + } else { + time, err1 = parseGeneralizedTime(innerBytes) + } + if err1 == nil { + v.Set(reflect.ValueOf(time)) + } + err = err1 + return + case enumeratedType: + parsedInt, err1 := parseInt(innerBytes) + if err1 == nil { + v.SetInt(int64(parsedInt)) + } + err = err1 + return + case flagType: + v.SetBool(true) + return + case bigIntType: + parsedInt := parseBigInt(innerBytes) + v.Set(reflect.ValueOf(parsedInt)) + return + } + switch val := v; val.Kind() { + case reflect.Bool: + parsedBool, err1 := parseBool(innerBytes) + if err1 == nil { + val.SetBool(parsedBool) + } + err = err1 + return + case reflect.Int, reflect.Int32: + parsedInt, err1 := parseInt(innerBytes) + if err1 == nil { + val.SetInt(int64(parsedInt)) + } + err = err1 + return + case reflect.Int64: + parsedInt, err1 := parseInt64(innerBytes) + if err1 == nil { + val.SetInt(parsedInt) + } + err = err1 + return + // TODO(dfc) Add support for the remaining integer types + case reflect.Struct: + structType := fieldType + + if structType.NumField() > 0 && + structType.Field(0).Type == rawContentsType { + bytes := bytes[initOffset:offset] + val.Field(0).Set(reflect.ValueOf(RawContent(bytes))) + } + + innerOffset := 0 + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if i == 0 && field.Type == rawContentsType { + continue + } + innerOffset, err = parseField(val.Field(i), innerBytes, innerOffset, parseFieldParameters(field.Tag.Get("asn1"))) + if err != nil { + return + } + } + // We allow extra bytes at the end of the SEQUENCE because + // adding elements to the end has been used in X.509 as the + // version numbers have increased. + return + case reflect.Slice: + sliceType := fieldType + if sliceType.Elem().Kind() == reflect.Uint8 { + val.Set(reflect.MakeSlice(sliceType, len(innerBytes), len(innerBytes))) + reflect.Copy(val, reflect.ValueOf(innerBytes)) + return + } + newSlice, err1 := parseSequenceOf(innerBytes, sliceType, sliceType.Elem()) + if err1 == nil { + val.Set(newSlice) + } + err = err1 + return + case reflect.String: + var v string + switch universalTag { + case tagPrintableString: + v, err = parsePrintableString(innerBytes) + case tagIA5String: + v, err = parseIA5String(innerBytes) + case tagT61String: + v, err = parseT61String(innerBytes) + case tagUTF8String: + v, err = parseUTF8String(innerBytes) + case tagGeneralString: + // GeneralString is specified in ISO-2022/ECMA-35, + // A brief review suggests that it includes structures + // that allow the encoding to change midstring and + // such. We give up and pass it as an 8-bit string. + v, err = parseT61String(innerBytes) + default: + err = SyntaxError{fmt.Sprintf("internal error: unknown string type %d", universalTag)} + } + if err == nil { + val.SetString(v) + } + return + } + err = StructuralError{"unsupported: " + v.Type().String()} + return +} + +// setDefaultValue is used to install a default value, from a tag string, into +// a Value. It is successful is the field was optional, even if a default value +// wasn't provided or it failed to install it into the Value. +func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { + if !params.optional { + return + } + ok = true + if params.defaultValue == nil { + return + } + switch val := v; val.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val.SetInt(*params.defaultValue) + } + return +} + +// Unmarshal parses the DER-encoded ASN.1 data structure b +// and uses the reflect package to fill in an arbitrary value pointed at by val. +// Because Unmarshal uses the reflect package, the structs +// being written to must use upper case field names. +// +// An ASN.1 INTEGER can be written to an int, int32 or int64. +// If the encoded value does not fit in the Go type, +// Unmarshal returns a parse error. +// +// An ASN.1 BIT STRING can be written to a BitString. +// +// An ASN.1 OCTET STRING can be written to a []byte. +// +// An ASN.1 OBJECT IDENTIFIER can be written to an +// ObjectIdentifier. +// +// An ASN.1 ENUMERATED can be written to an Enumerated. +// +// An ASN.1 UTCTIME or GENERALIZEDTIME can be written to a *time.Time. +// +// An ASN.1 PrintableString or IA5String can be written to a string. +// +// Any of the above ASN.1 values can be written to an interface{}. +// The value stored in the interface has the corresponding Go type. +// For integers, that type is int64. +// +// An ASN.1 SEQUENCE OF x or SET OF x can be written +// to a slice if an x can be written to the slice's element type. +// +// An ASN.1 SEQUENCE or SET can be written to a struct +// if each of the elements in the sequence can be +// written to the corresponding element in the struct. +// +// The following tags on struct fields have special meaning to Unmarshal: +// +// optional marks the field as ASN.1 OPTIONAL +// [explicit] tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC +// default:x sets the default value for optional integer fields +// +// If the type of the first field of a structure is RawContent then the raw +// ASN1 contents of the struct will be stored in it. +// +// Other ASN.1 types are not supported; if it encounters them, +// Unmarshal returns a parse error. +func Unmarshal(b []byte, val interface{}) (rest []byte, err os.Error) { + return UnmarshalWithParams(b, val, "") +} + +// UnmarshalWithParams allows field parameters to be specified for the +// top-level element. The form of the params is the same as the field tags. +func UnmarshalWithParams(b []byte, val interface{}, params string) (rest []byte, err os.Error) { + v := reflect.ValueOf(val).Elem() + offset, err := parseField(v, b, 0, parseFieldParameters(params)) + if err != nil { + return nil, err + } + return b[offset:], nil +} diff --git a/src/pkg/asn1/asn1_test.go b/src/pkg/asn1/asn1_test.go new file mode 100644 index 000000000..9f48f7bdd --- /dev/null +++ b/src/pkg/asn1/asn1_test.go @@ -0,0 +1,689 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "bytes" + "reflect" + "testing" + "time" +) + +type int64Test struct { + in []byte + ok bool + out int64 +} + +var int64TestData = []int64Test{ + {[]byte{0x00}, true, 0}, + {[]byte{0x7f}, true, 127}, + {[]byte{0x00, 0x80}, true, 128}, + {[]byte{0x01, 0x00}, true, 256}, + {[]byte{0x80}, true, -128}, + {[]byte{0xff, 0x7f}, true, -129}, + {[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, true, -1}, + {[]byte{0xff}, true, -1}, + {[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, true, -9223372036854775808}, + {[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, false, 0}, +} + +func TestParseInt64(t *testing.T) { + for i, test := range int64TestData { + ret, err := parseInt64(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if test.ok && ret != test.out { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } +} + +type int32Test struct { + in []byte + ok bool + out int32 +} + +var int32TestData = []int32Test{ + {[]byte{0x00}, true, 0}, + {[]byte{0x7f}, true, 127}, + {[]byte{0x00, 0x80}, true, 128}, + {[]byte{0x01, 0x00}, true, 256}, + {[]byte{0x80}, true, -128}, + {[]byte{0xff, 0x7f}, true, -129}, + {[]byte{0xff, 0xff, 0xff, 0xff}, true, -1}, + {[]byte{0xff}, true, -1}, + {[]byte{0x80, 0x00, 0x00, 0x00}, true, -2147483648}, + {[]byte{0x80, 0x00, 0x00, 0x00, 0x00}, false, 0}, +} + +func TestParseInt32(t *testing.T) { + for i, test := range int32TestData { + ret, err := parseInt(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if test.ok && int32(ret) != test.out { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } +} + +var bigIntTests = []struct { + in []byte + base10 string +}{ + {[]byte{0xff}, "-1"}, + {[]byte{0x00}, "0"}, + {[]byte{0x01}, "1"}, + {[]byte{0x00, 0xff}, "255"}, + {[]byte{0xff, 0x00}, "-256"}, + {[]byte{0x01, 0x00}, "256"}, +} + +func TestParseBigInt(t *testing.T) { + for i, test := range bigIntTests { + ret := parseBigInt(test.in) + if ret.String() != test.base10 { + t.Errorf("#%d: bad result from %x, got %s want %s", i, test.in, ret.String(), test.base10) + } + fw := newForkableWriter() + marshalBigInt(fw, ret) + result := fw.Bytes() + if !bytes.Equal(result, test.in) { + t.Errorf("#%d: got %x from marshaling %s, want %x", i, result, ret, test.in) + } + } +} + +type bitStringTest struct { + in []byte + ok bool + out []byte + bitLength int +} + +var bitStringTestData = []bitStringTest{ + {[]byte{}, false, []byte{}, 0}, + {[]byte{0x00}, true, []byte{}, 0}, + {[]byte{0x07, 0x00}, true, []byte{0x00}, 1}, + {[]byte{0x07, 0x01}, false, []byte{}, 0}, + {[]byte{0x07, 0x40}, false, []byte{}, 0}, + {[]byte{0x08, 0x00}, false, []byte{}, 0}, +} + +func TestBitString(t *testing.T) { + for i, test := range bitStringTestData { + ret, err := parseBitString(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if test.bitLength != ret.BitLength || bytes.Compare(ret.Bytes, test.out) != 0 { + t.Errorf("#%d: Bad result: %v (expected %v %v)", i, ret, test.out, test.bitLength) + } + } + } +} + +func TestBitStringAt(t *testing.T) { + bs := BitString{[]byte{0x82, 0x40}, 16} + if bs.At(0) != 1 { + t.Error("#1: Failed") + } + if bs.At(1) != 0 { + t.Error("#2: Failed") + } + if bs.At(6) != 1 { + t.Error("#3: Failed") + } + if bs.At(9) != 1 { + t.Error("#4: Failed") + } +} + +type bitStringRightAlignTest struct { + in []byte + inlen int + out []byte +} + +var bitStringRightAlignTests = []bitStringRightAlignTest{ + {[]byte{0x80}, 1, []byte{0x01}}, + {[]byte{0x80, 0x80}, 9, []byte{0x01, 0x01}}, + {[]byte{}, 0, []byte{}}, + {[]byte{0xce}, 8, []byte{0xce}}, + {[]byte{0xce, 0x47}, 16, []byte{0xce, 0x47}}, + {[]byte{0x34, 0x50}, 12, []byte{0x03, 0x45}}, +} + +func TestBitStringRightAlign(t *testing.T) { + for i, test := range bitStringRightAlignTests { + bs := BitString{test.in, test.inlen} + out := bs.RightAlign() + if bytes.Compare(out, test.out) != 0 { + t.Errorf("#%d got: %x want: %x", i, out, test.out) + } + } +} + +type objectIdentifierTest struct { + in []byte + ok bool + out []int +} + +var objectIdentifierTestData = []objectIdentifierTest{ + {[]byte{}, false, []int{}}, + {[]byte{85}, true, []int{2, 5}}, + {[]byte{85, 0x02}, true, []int{2, 5, 2}}, + {[]byte{85, 0x02, 0xc0, 0x00}, true, []int{2, 5, 2, 0x2000}}, + {[]byte{85, 0x02, 0xc0, 0x80, 0x80, 0x80, 0x80}, false, []int{}}, +} + +func TestObjectIdentifier(t *testing.T) { + for i, test := range objectIdentifierTestData { + ret, err := parseObjectIdentifier(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if !reflect.DeepEqual(test.out, ret) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } + } +} + +type timeTest struct { + in string + ok bool + out *time.Time +} + +var utcTestData = []timeTest{ + {"910506164540-0700", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, 0, -7 * 60 * 60, ""}}, + {"910506164540+0730", true, &time.Time{1991, 05, 06, 16, 45, 40, 0, 0, 7*60*60 + 30*60, ""}}, + {"910506234540Z", true, &time.Time{1991, 05, 06, 23, 45, 40, 0, 0, 0, "UTC"}}, + {"9105062345Z", true, &time.Time{1991, 05, 06, 23, 45, 0, 0, 0, 0, "UTC"}}, + {"a10506234540Z", false, nil}, + {"91a506234540Z", false, nil}, + {"9105a6234540Z", false, nil}, + {"910506a34540Z", false, nil}, + {"910506334a40Z", false, nil}, + {"91050633444aZ", false, nil}, + {"910506334461Z", false, nil}, + {"910506334400Za", false, nil}, +} + +func TestUTCTime(t *testing.T) { + for i, test := range utcTestData { + ret, err := parseUTCTime([]byte(test.in)) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if !reflect.DeepEqual(test.out, ret) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } + } +} + +var generalizedTimeTestData = []timeTest{ + {"20100102030405Z", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, 0, "UTC"}}, + {"20100102030405", false, nil}, + {"20100102030405+0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, 6*60*60 + 7*60, ""}}, + {"20100102030405-0607", true, &time.Time{2010, 01, 02, 03, 04, 05, 0, 0, -6*60*60 - 7*60, ""}}, +} + +func TestGeneralizedTime(t *testing.T) { + for i, test := range generalizedTimeTestData { + ret, err := parseGeneralizedTime([]byte(test.in)) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil { + if !reflect.DeepEqual(test.out, ret) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, ret, test.out) + } + } + } +} + +type tagAndLengthTest struct { + in []byte + ok bool + out tagAndLength +} + +var tagAndLengthData = []tagAndLengthTest{ + {[]byte{0x80, 0x01}, true, tagAndLength{2, 0, 1, false}}, + {[]byte{0xa0, 0x01}, true, tagAndLength{2, 0, 1, true}}, + {[]byte{0x02, 0x00}, true, tagAndLength{0, 2, 0, false}}, + {[]byte{0xfe, 0x00}, true, tagAndLength{3, 30, 0, true}}, + {[]byte{0x1f, 0x01, 0x00}, true, tagAndLength{0, 1, 0, false}}, + {[]byte{0x1f, 0x81, 0x00, 0x00}, true, tagAndLength{0, 128, 0, false}}, + {[]byte{0x1f, 0x81, 0x80, 0x01, 0x00}, true, tagAndLength{0, 0x4001, 0, false}}, + {[]byte{0x00, 0x81, 0x01}, true, tagAndLength{0, 0, 1, false}}, + {[]byte{0x00, 0x82, 0x01, 0x00}, true, tagAndLength{0, 0, 256, false}}, + {[]byte{0x00, 0x83, 0x01, 0x00}, false, tagAndLength{}}, + {[]byte{0x1f, 0x85}, false, tagAndLength{}}, + {[]byte{0x30, 0x80}, false, tagAndLength{}}, +} + +func TestParseTagAndLength(t *testing.T) { + for i, test := range tagAndLengthData { + tagAndLength, _, err := parseTagAndLength(test.in, 0) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did pass? %v, expected: %v)", i, err == nil, test.ok) + } + if err == nil && !reflect.DeepEqual(test.out, tagAndLength) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, tagAndLength, test.out) + } + } +} + +type parseFieldParametersTest struct { + in string + out fieldParameters +} + +func newInt(n int) *int { return &n } + +func newInt64(n int64) *int64 { return &n } + +func newString(s string) *string { return &s } + +func newBool(b bool) *bool { return &b } + +var parseFieldParametersTestData []parseFieldParametersTest = []parseFieldParametersTest{ + {"", fieldParameters{}}, + {"ia5", fieldParameters{stringType: tagIA5String}}, + {"printable", fieldParameters{stringType: tagPrintableString}}, + {"optional", fieldParameters{optional: true}}, + {"explicit", fieldParameters{explicit: true, tag: new(int)}}, + {"application", fieldParameters{application: true, tag: new(int)}}, + {"optional,explicit", fieldParameters{optional: true, explicit: true, tag: new(int)}}, + {"default:42", fieldParameters{defaultValue: newInt64(42)}}, + {"tag:17", fieldParameters{tag: newInt(17)}}, + {"optional,explicit,default:42,tag:17", fieldParameters{optional: true, explicit: true, defaultValue: newInt64(42), tag: newInt(17)}}, + {"optional,explicit,default:42,tag:17,rubbish1", fieldParameters{true, true, false, newInt64(42), newInt(17), 0, false}}, + {"set", fieldParameters{set: true}}, +} + +func TestParseFieldParameters(t *testing.T) { + for i, test := range parseFieldParametersTestData { + f := parseFieldParameters(test.in) + if !reflect.DeepEqual(f, test.out) { + t.Errorf("#%d: Bad result: %v (expected %v)", i, f, test.out) + } + } +} + +type TestObjectIdentifierStruct struct { + OID ObjectIdentifier +} + +type TestContextSpecificTags struct { + A int `asn1:"tag:1"` +} + +type TestContextSpecificTags2 struct { + A int `asn1:"explicit,tag:1"` + B int +} + +type TestElementsAfterString struct { + S string + A, B int +} + +var unmarshalTestData = []struct { + in []byte + out interface{} +}{ + {[]byte{0x02, 0x01, 0x42}, newInt(0x42)}, + {[]byte{0x30, 0x08, 0x06, 0x06, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d}, &TestObjectIdentifierStruct{[]int{1, 2, 840, 113549}}}, + {[]byte{0x03, 0x04, 0x06, 0x6e, 0x5d, 0xc0}, &BitString{[]byte{110, 93, 192}, 18}}, + {[]byte{0x30, 0x09, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x01, 0x03}, &[]int{1, 2, 3}}, + {[]byte{0x02, 0x01, 0x10}, newInt(16)}, + {[]byte{0x13, 0x04, 't', 'e', 's', 't'}, newString("test")}, + {[]byte{0x16, 0x04, 't', 'e', 's', 't'}, newString("test")}, + {[]byte{0x16, 0x04, 't', 'e', 's', 't'}, &RawValue{0, 22, false, []byte("test"), []byte("\x16\x04test")}}, + {[]byte{0x04, 0x04, 1, 2, 3, 4}, &RawValue{0, 4, false, []byte{1, 2, 3, 4}, []byte{4, 4, 1, 2, 3, 4}}}, + {[]byte{0x30, 0x03, 0x81, 0x01, 0x01}, &TestContextSpecificTags{1}}, + {[]byte{0x30, 0x08, 0xa1, 0x03, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02}, &TestContextSpecificTags2{1, 2}}, + {[]byte{0x01, 0x01, 0x00}, newBool(false)}, + {[]byte{0x01, 0x01, 0x01}, newBool(true)}, + {[]byte{0x30, 0x0b, 0x13, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x01, 0x22, 0x02, 0x01, 0x33}, &TestElementsAfterString{"foo", 0x22, 0x33}}, +} + +func TestUnmarshal(t *testing.T) { + for i, test := range unmarshalTestData { + pv := reflect.New(reflect.TypeOf(test.out).Elem()) + val := pv.Interface() + _, err := Unmarshal(test.in, val) + if err != nil { + t.Errorf("Unmarshal failed at index %d %v", i, err) + } + if !reflect.DeepEqual(val, test.out) { + t.Errorf("#%d:\nhave %#v\nwant %#v", i, val, test.out) + } + } +} + +type Certificate struct { + TBSCertificate TBSCertificate + SignatureAlgorithm AlgorithmIdentifier + SignatureValue BitString +} + +type TBSCertificate struct { + Version int `asn1:"optional,explicit,default:0,tag:0"` + SerialNumber RawValue + SignatureAlgorithm AlgorithmIdentifier + Issuer RDNSequence + Validity Validity + Subject RDNSequence + PublicKey PublicKeyInfo +} + +type AlgorithmIdentifier struct { + Algorithm ObjectIdentifier +} + +type RDNSequence []RelativeDistinguishedNameSET + +type RelativeDistinguishedNameSET []AttributeTypeAndValue + +type AttributeTypeAndValue struct { + Type ObjectIdentifier + Value interface{} +} + +type Validity struct { + NotBefore, NotAfter *time.Time +} + +type PublicKeyInfo struct { + Algorithm AlgorithmIdentifier + PublicKey BitString +} + +func TestCertificate(t *testing.T) { + // This is a minimal, self-signed certificate that should parse correctly. + var cert Certificate + if _, err := Unmarshal(derEncodedSelfSignedCertBytes, &cert); err != nil { + t.Errorf("Unmarshal failed: %v", err) + } + if !reflect.DeepEqual(cert, derEncodedSelfSignedCert) { + t.Errorf("Bad result:\ngot: %+v\nwant: %+v", cert, derEncodedSelfSignedCert) + } +} + +func TestCertificateWithNUL(t *testing.T) { + // This is the paypal NUL-hack certificate. It should fail to parse because + // NUL isn't a permitted character in a PrintableString. + + var cert Certificate + if _, err := Unmarshal(derEncodedPaypalNULCertBytes, &cert); err == nil { + t.Error("Unmarshal succeeded, should not have") + } +} + +type rawStructTest struct { + Raw RawContent + A int +} + +func TestRawStructs(t *testing.T) { + var s rawStructTest + input := []byte{0x30, 0x03, 0x02, 0x01, 0x50} + + rest, err := Unmarshal(input, &s) + if len(rest) != 0 { + t.Errorf("incomplete parse: %x", rest) + return + } + if err != nil { + t.Error(err) + return + } + if s.A != 0x50 { + t.Errorf("bad value for A: got %d want %d", s.A, 0x50) + } + if bytes.Compare([]byte(s.Raw), input) != 0 { + t.Errorf("bad value for Raw: got %x want %x", s.Raw, input) + } +} + +var derEncodedSelfSignedCert = Certificate{ + TBSCertificate: TBSCertificate{ + Version: 0, + SerialNumber: RawValue{Class: 0, Tag: 2, IsCompound: false, Bytes: []uint8{0x0, 0x8c, 0xc3, 0x37, 0x92, 0x10, 0xec, 0x2c, 0x98}, FullBytes: []byte{2, 9, 0x0, 0x8c, 0xc3, 0x37, 0x92, 0x10, 0xec, 0x2c, 0x98}}, + SignatureAlgorithm: AlgorithmIdentifier{Algorithm: ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5}}, + Issuer: RDNSequence{ + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 6}, Value: "XX"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 8}, Value: "Some-State"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 7}, Value: "City"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 10}, Value: "Internet Widgits Pty Ltd"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 3}, Value: "false.example.com"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{1, 2, 840, 113549, 1, 9, 1}, Value: "false@example.com"}}, + }, + Validity: Validity{NotBefore: &time.Time{Year: 2009, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NotAfter: &time.Time{Year: 2010, Month: 10, Day: 8, Hour: 0, Minute: 25, Second: 53, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}}, + Subject: RDNSequence{ + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 6}, Value: "XX"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 8}, Value: "Some-State"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 7}, Value: "City"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 10}, Value: "Internet Widgits Pty Ltd"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{2, 5, 4, 3}, Value: "false.example.com"}}, + RelativeDistinguishedNameSET{AttributeTypeAndValue{Type: ObjectIdentifier{1, 2, 840, 113549, 1, 9, 1}, Value: "false@example.com"}}, + }, + PublicKey: PublicKeyInfo{ + Algorithm: AlgorithmIdentifier{Algorithm: ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1}}, + PublicKey: BitString{ + Bytes: []uint8{ + 0x30, 0x48, 0x2, 0x41, 0x0, 0xcd, 0xb7, + 0x63, 0x9c, 0x32, 0x78, 0xf0, 0x6, 0xaa, 0x27, 0x7f, 0x6e, 0xaf, 0x42, + 0x90, 0x2b, 0x59, 0x2d, 0x8c, 0xbc, 0xbe, 0x38, 0xa1, 0xc9, 0x2b, 0xa4, + 0x69, 0x5a, 0x33, 0x1b, 0x1d, 0xea, 0xde, 0xad, 0xd8, 0xe9, 0xa5, 0xc2, + 0x7e, 0x8c, 0x4c, 0x2f, 0xd0, 0xa8, 0x88, 0x96, 0x57, 0x72, 0x2a, 0x4f, + 0x2a, 0xf7, 0x58, 0x9c, 0xf2, 0xc7, 0x70, 0x45, 0xdc, 0x8f, 0xde, 0xec, + 0x35, 0x7d, 0x2, 0x3, 0x1, 0x0, 0x1, + }, + BitLength: 592, + }, + }, + }, + SignatureAlgorithm: AlgorithmIdentifier{Algorithm: ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5}}, + SignatureValue: BitString{ + Bytes: []uint8{ + 0xa6, 0x7b, 0x6, 0xec, 0x5e, 0xce, + 0x92, 0x77, 0x2c, 0xa4, 0x13, 0xcb, 0xa3, 0xca, 0x12, 0x56, 0x8f, 0xdc, 0x6c, + 0x7b, 0x45, 0x11, 0xcd, 0x40, 0xa7, 0xf6, 0x59, 0x98, 0x4, 0x2, 0xdf, 0x2b, + 0x99, 0x8b, 0xb9, 0xa4, 0xa8, 0xcb, 0xeb, 0x34, 0xc0, 0xf0, 0xa7, 0x8c, 0xf8, + 0xd9, 0x1e, 0xde, 0x14, 0xa5, 0xed, 0x76, 0xbf, 0x11, 0x6f, 0xe3, 0x60, 0xaa, + 0xfa, 0x88, 0x21, 0x49, 0x4, 0x35, + }, + BitLength: 512, + }, +} + +var derEncodedSelfSignedCertBytes = []byte{ + 0x30, 0x82, 0x02, 0x18, 0x30, + 0x82, 0x01, 0xc2, 0x02, 0x09, 0x00, 0x8c, 0xc3, 0x37, 0x92, 0x10, 0xec, 0x2c, + 0x98, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, + 0x05, 0x05, 0x00, 0x30, 0x81, 0x92, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x58, 0x58, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, + 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x0d, 0x30, 0x0b, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x04, 0x43, + 0x69, 0x74, 0x79, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, + 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, + 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x11, 0x66, 0x61, 0x6c, + 0x73, 0x65, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, + 0x6d, 0x31, 0x20, 0x30, 0x1e, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x09, 0x01, 0x16, 0x11, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x40, 0x65, 0x78, + 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, 0x0d, + 0x30, 0x39, 0x31, 0x30, 0x30, 0x38, 0x30, 0x30, 0x32, 0x35, 0x35, 0x33, 0x5a, + 0x17, 0x0d, 0x31, 0x30, 0x31, 0x30, 0x30, 0x38, 0x30, 0x30, 0x32, 0x35, 0x35, + 0x33, 0x5a, 0x30, 0x81, 0x92, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, + 0x06, 0x13, 0x02, 0x58, 0x58, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, + 0x08, 0x13, 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x0d, 0x30, 0x0b, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x04, 0x43, 0x69, + 0x74, 0x79, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x1a, + 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x11, 0x66, 0x61, 0x6c, 0x73, + 0x65, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, + 0x31, 0x20, 0x30, 0x1e, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, + 0x09, 0x01, 0x16, 0x11, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x40, 0x65, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x5c, 0x30, 0x0d, 0x06, + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, + 0x4b, 0x00, 0x30, 0x48, 0x02, 0x41, 0x00, 0xcd, 0xb7, 0x63, 0x9c, 0x32, 0x78, + 0xf0, 0x06, 0xaa, 0x27, 0x7f, 0x6e, 0xaf, 0x42, 0x90, 0x2b, 0x59, 0x2d, 0x8c, + 0xbc, 0xbe, 0x38, 0xa1, 0xc9, 0x2b, 0xa4, 0x69, 0x5a, 0x33, 0x1b, 0x1d, 0xea, + 0xde, 0xad, 0xd8, 0xe9, 0xa5, 0xc2, 0x7e, 0x8c, 0x4c, 0x2f, 0xd0, 0xa8, 0x88, + 0x96, 0x57, 0x72, 0x2a, 0x4f, 0x2a, 0xf7, 0x58, 0x9c, 0xf2, 0xc7, 0x70, 0x45, + 0xdc, 0x8f, 0xde, 0xec, 0x35, 0x7d, 0x02, 0x03, 0x01, 0x00, 0x01, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, + 0x03, 0x41, 0x00, 0xa6, 0x7b, 0x06, 0xec, 0x5e, 0xce, 0x92, 0x77, 0x2c, 0xa4, + 0x13, 0xcb, 0xa3, 0xca, 0x12, 0x56, 0x8f, 0xdc, 0x6c, 0x7b, 0x45, 0x11, 0xcd, + 0x40, 0xa7, 0xf6, 0x59, 0x98, 0x04, 0x02, 0xdf, 0x2b, 0x99, 0x8b, 0xb9, 0xa4, + 0xa8, 0xcb, 0xeb, 0x34, 0xc0, 0xf0, 0xa7, 0x8c, 0xf8, 0xd9, 0x1e, 0xde, 0x14, + 0xa5, 0xed, 0x76, 0xbf, 0x11, 0x6f, 0xe3, 0x60, 0xaa, 0xfa, 0x88, 0x21, 0x49, + 0x04, 0x35, +} + +var derEncodedPaypalNULCertBytes = []byte{ + 0x30, 0x82, 0x06, 0x44, 0x30, + 0x82, 0x05, 0xad, 0xa0, 0x03, 0x02, 0x01, 0x02, 0x02, 0x03, 0x00, 0xf0, 0x9b, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, + 0x05, 0x00, 0x30, 0x82, 0x01, 0x12, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x45, 0x53, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, + 0x04, 0x08, 0x13, 0x09, 0x42, 0x61, 0x72, 0x63, 0x65, 0x6c, 0x6f, 0x6e, 0x61, + 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x09, 0x42, 0x61, + 0x72, 0x63, 0x65, 0x6c, 0x6f, 0x6e, 0x61, 0x31, 0x29, 0x30, 0x27, 0x06, 0x03, + 0x55, 0x04, 0x0a, 0x13, 0x20, 0x49, 0x50, 0x53, 0x20, 0x43, 0x65, 0x72, 0x74, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x20, 0x73, 0x2e, 0x6c, 0x2e, 0x31, 0x2e, + 0x30, 0x2c, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x14, 0x25, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x6c, 0x40, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, + 0x20, 0x43, 0x2e, 0x49, 0x2e, 0x46, 0x2e, 0x20, 0x20, 0x42, 0x2d, 0x42, 0x36, + 0x32, 0x32, 0x31, 0x30, 0x36, 0x39, 0x35, 0x31, 0x2e, 0x30, 0x2c, 0x06, 0x03, + 0x55, 0x04, 0x0b, 0x13, 0x25, 0x69, 0x70, 0x73, 0x43, 0x41, 0x20, 0x43, 0x4c, + 0x41, 0x53, 0x45, 0x41, 0x31, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x31, 0x2e, 0x30, 0x2c, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, + 0x25, 0x69, 0x70, 0x73, 0x43, 0x41, 0x20, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, + 0x31, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x31, + 0x20, 0x30, 0x1e, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x09, + 0x01, 0x16, 0x11, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x40, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, 0x0d, 0x30, 0x39, + 0x30, 0x32, 0x32, 0x34, 0x32, 0x33, 0x30, 0x34, 0x31, 0x37, 0x5a, 0x17, 0x0d, + 0x31, 0x31, 0x30, 0x32, 0x32, 0x34, 0x32, 0x33, 0x30, 0x34, 0x31, 0x37, 0x5a, + 0x30, 0x81, 0x94, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x55, 0x53, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x43, 0x61, 0x6c, 0x69, 0x66, 0x6f, 0x72, 0x6e, 0x69, 0x61, 0x31, 0x16, + 0x30, 0x14, 0x06, 0x03, 0x55, 0x04, 0x07, 0x13, 0x0d, 0x53, 0x61, 0x6e, 0x20, + 0x46, 0x72, 0x61, 0x6e, 0x63, 0x69, 0x73, 0x63, 0x6f, 0x31, 0x11, 0x30, 0x0f, + 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, + 0x74, 0x79, 0x31, 0x14, 0x30, 0x12, 0x06, 0x03, 0x55, 0x04, 0x0b, 0x13, 0x0b, + 0x53, 0x65, 0x63, 0x75, 0x72, 0x65, 0x20, 0x55, 0x6e, 0x69, 0x74, 0x31, 0x2f, + 0x30, 0x2d, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x26, 0x77, 0x77, 0x77, 0x2e, + 0x70, 0x61, 0x79, 0x70, 0x61, 0x6c, 0x2e, 0x63, 0x6f, 0x6d, 0x00, 0x73, 0x73, + 0x6c, 0x2e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x63, 0x63, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, + 0x03, 0x81, 0x8d, 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, 0xd2, 0x69, + 0xfa, 0x6f, 0x3a, 0x00, 0xb4, 0x21, 0x1b, 0xc8, 0xb1, 0x02, 0xd7, 0x3f, 0x19, + 0xb2, 0xc4, 0x6d, 0xb4, 0x54, 0xf8, 0x8b, 0x8a, 0xcc, 0xdb, 0x72, 0xc2, 0x9e, + 0x3c, 0x60, 0xb9, 0xc6, 0x91, 0x3d, 0x82, 0xb7, 0x7d, 0x99, 0xff, 0xd1, 0x29, + 0x84, 0xc1, 0x73, 0x53, 0x9c, 0x82, 0xdd, 0xfc, 0x24, 0x8c, 0x77, 0xd5, 0x41, + 0xf3, 0xe8, 0x1e, 0x42, 0xa1, 0xad, 0x2d, 0x9e, 0xff, 0x5b, 0x10, 0x26, 0xce, + 0x9d, 0x57, 0x17, 0x73, 0x16, 0x23, 0x38, 0xc8, 0xd6, 0xf1, 0xba, 0xa3, 0x96, + 0x5b, 0x16, 0x67, 0x4a, 0x4f, 0x73, 0x97, 0x3a, 0x4d, 0x14, 0xa4, 0xf4, 0xe2, + 0x3f, 0x8b, 0x05, 0x83, 0x42, 0xd1, 0xd0, 0xdc, 0x2f, 0x7a, 0xe5, 0xb6, 0x10, + 0xb2, 0x11, 0xc0, 0xdc, 0x21, 0x2a, 0x90, 0xff, 0xae, 0x97, 0x71, 0x5a, 0x49, + 0x81, 0xac, 0x40, 0xf3, 0x3b, 0xb8, 0x59, 0xb2, 0x4f, 0x02, 0x03, 0x01, 0x00, + 0x01, 0xa3, 0x82, 0x03, 0x21, 0x30, 0x82, 0x03, 0x1d, 0x30, 0x09, 0x06, 0x03, + 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x11, 0x06, 0x09, 0x60, 0x86, + 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x01, 0x04, 0x04, 0x03, 0x02, 0x06, 0x40, + 0x30, 0x0b, 0x06, 0x03, 0x55, 0x1d, 0x0f, 0x04, 0x04, 0x03, 0x02, 0x03, 0xf8, + 0x30, 0x13, 0x06, 0x03, 0x55, 0x1d, 0x25, 0x04, 0x0c, 0x30, 0x0a, 0x06, 0x08, + 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x03, 0x01, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x61, 0x8f, 0x61, 0x34, 0x43, 0x55, 0x14, + 0x7f, 0x27, 0x09, 0xce, 0x4c, 0x8b, 0xea, 0x9b, 0x7b, 0x19, 0x25, 0xbc, 0x6e, + 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, + 0x0e, 0x07, 0x60, 0xd4, 0x39, 0xc9, 0x1b, 0x5b, 0x5d, 0x90, 0x7b, 0x23, 0xc8, + 0xd2, 0x34, 0x9d, 0x4a, 0x9a, 0x46, 0x39, 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, + 0x11, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1c, 0x06, 0x03, 0x55, 0x1d, 0x12, 0x04, + 0x15, 0x30, 0x13, 0x81, 0x11, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x40, + 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x72, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x0d, 0x04, 0x65, 0x16, 0x63, + 0x4f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4e, + 0x4f, 0x54, 0x20, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x41, 0x54, 0x45, 0x44, 0x2e, + 0x20, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x20, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x20, 0x69, 0x73, 0x73, 0x75, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x68, + 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x30, 0x2f, 0x06, 0x09, 0x60, + 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x02, 0x04, 0x22, 0x16, 0x20, 0x68, + 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, + 0x32, 0x30, 0x30, 0x32, 0x2f, 0x30, 0x43, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, + 0x86, 0xf8, 0x42, 0x01, 0x04, 0x04, 0x36, 0x16, 0x34, 0x68, 0x74, 0x74, 0x70, + 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, + 0x32, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, 0x32, 0x43, 0x4c, + 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x63, 0x72, 0x6c, 0x30, 0x46, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x03, 0x04, 0x39, 0x16, 0x37, + 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, + 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, + 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, 0x72, 0x65, 0x76, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x68, 0x74, + 0x6d, 0x6c, 0x3f, 0x30, 0x43, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, + 0x42, 0x01, 0x07, 0x04, 0x36, 0x16, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, + 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, + 0x72, 0x65, 0x6e, 0x65, 0x77, 0x61, 0x6c, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, + 0x31, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x3f, 0x30, 0x41, 0x06, 0x09, 0x60, 0x86, + 0x48, 0x01, 0x86, 0xf8, 0x42, 0x01, 0x08, 0x04, 0x34, 0x16, 0x32, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x69, 0x70, 0x73, + 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, + 0x30, 0x30, 0x32, 0x2f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x4c, 0x41, + 0x53, 0x45, 0x41, 0x31, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x30, 0x81, 0x83, 0x06, + 0x03, 0x55, 0x1d, 0x1f, 0x04, 0x7c, 0x30, 0x7a, 0x30, 0x39, 0xa0, 0x37, 0xa0, + 0x35, 0x86, 0x33, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, + 0x73, 0x63, 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, + 0x32, 0x30, 0x30, 0x32, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x63, + 0x72, 0x6c, 0x30, 0x3d, 0xa0, 0x3b, 0xa0, 0x39, 0x86, 0x37, 0x68, 0x74, 0x74, + 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x62, 0x61, 0x63, 0x6b, 0x2e, 0x69, + 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x69, 0x70, 0x73, 0x63, + 0x61, 0x32, 0x30, 0x30, 0x32, 0x2f, 0x69, 0x70, 0x73, 0x63, 0x61, 0x32, 0x30, + 0x30, 0x32, 0x43, 0x4c, 0x41, 0x53, 0x45, 0x41, 0x31, 0x2e, 0x63, 0x72, 0x6c, + 0x30, 0x32, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x01, 0x01, 0x04, + 0x26, 0x30, 0x24, 0x30, 0x22, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, + 0x30, 0x01, 0x86, 0x16, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x6f, 0x63, + 0x73, 0x70, 0x2e, 0x69, 0x70, 0x73, 0x63, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, + 0x05, 0x00, 0x03, 0x81, 0x81, 0x00, 0x68, 0xee, 0x79, 0x97, 0x97, 0xdd, 0x3b, + 0xef, 0x16, 0x6a, 0x06, 0xf2, 0x14, 0x9a, 0x6e, 0xcd, 0x9e, 0x12, 0xf7, 0xaa, + 0x83, 0x10, 0xbd, 0xd1, 0x7c, 0x98, 0xfa, 0xc7, 0xae, 0xd4, 0x0e, 0x2c, 0x9e, + 0x38, 0x05, 0x9d, 0x52, 0x60, 0xa9, 0x99, 0x0a, 0x81, 0xb4, 0x98, 0x90, 0x1d, + 0xae, 0xbb, 0x4a, 0xd7, 0xb9, 0xdc, 0x88, 0x9e, 0x37, 0x78, 0x41, 0x5b, 0xf7, + 0x82, 0xa5, 0xf2, 0xba, 0x41, 0x25, 0x5a, 0x90, 0x1a, 0x1e, 0x45, 0x38, 0xa1, + 0x52, 0x58, 0x75, 0x94, 0x26, 0x44, 0xfb, 0x20, 0x07, 0xba, 0x44, 0xcc, 0xe5, + 0x4a, 0x2d, 0x72, 0x3f, 0x98, 0x47, 0xf6, 0x26, 0xdc, 0x05, 0x46, 0x05, 0x07, + 0x63, 0x21, 0xab, 0x46, 0x9b, 0x9c, 0x78, 0xd5, 0x54, 0x5b, 0x3d, 0x0c, 0x1e, + 0xc8, 0x64, 0x8c, 0xb5, 0x50, 0x23, 0x82, 0x6f, 0xdb, 0xb8, 0x22, 0x1c, 0x43, + 0x96, 0x07, 0xa8, 0xbb, +} diff --git a/src/pkg/asn1/common.go b/src/pkg/asn1/common.go new file mode 100644 index 000000000..01f4f7b6e --- /dev/null +++ b/src/pkg/asn1/common.go @@ -0,0 +1,158 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "reflect" + "strconv" + "strings" +) + +// ASN.1 objects have metadata preceding them: +// the tag: the type of the object +// a flag denoting if this object is compound or not +// the class type: the namespace of the tag +// the length of the object, in bytes + +// Here are some standard tags and classes + +const ( + tagBoolean = 1 + tagInteger = 2 + tagBitString = 3 + tagOctetString = 4 + tagOID = 6 + tagEnum = 10 + tagUTF8String = 12 + tagSequence = 16 + tagSet = 17 + tagPrintableString = 19 + tagT61String = 20 + tagIA5String = 22 + tagUTCTime = 23 + tagGeneralizedTime = 24 + tagGeneralString = 27 +) + +const ( + classUniversal = 0 + classApplication = 1 + classContextSpecific = 2 + classPrivate = 3 +) + +type tagAndLength struct { + class, tag, length int + isCompound bool +} + +// ASN.1 has IMPLICIT and EXPLICIT tags, which can be translated as "instead +// of" and "in addition to". When not specified, every primitive type has a +// default tag in the UNIVERSAL class. +// +// For example: a BIT STRING is tagged [UNIVERSAL 3] by default (although ASN.1 +// doesn't actually have a UNIVERSAL keyword). However, by saying [IMPLICIT +// CONTEXT-SPECIFIC 42], that means that the tag is replaced by another. +// +// On the other hand, if it said [EXPLICIT CONTEXT-SPECIFIC 10], then an +// /additional/ tag would wrap the default tag. This explicit tag will have the +// compound flag set. +// +// (This is used in order to remove ambiguity with optional elements.) +// +// You can layer EXPLICIT and IMPLICIT tags to an arbitrary depth, however we +// don't support that here. We support a single layer of EXPLICIT or IMPLICIT +// tagging with tag strings on the fields of a structure. + +// fieldParameters is the parsed representation of tag string from a structure field. +type fieldParameters struct { + optional bool // true iff the field is OPTIONAL + explicit bool // true iff an EXPLICIT tag is in use. + application bool // true iff an APPLICATION tag is in use. + defaultValue *int64 // a default value for INTEGER typed fields (maybe nil). + tag *int // the EXPLICIT or IMPLICIT tag (maybe nil). + stringType int // the string tag to use when marshaling. + set bool // true iff this should be encoded as a SET + + // Invariants: + // if explicit is set, tag is non-nil. +} + +// Given a tag string with the format specified in the package comment, +// parseFieldParameters will parse it into a fieldParameters structure, +// ignoring unknown parts of the string. +func parseFieldParameters(str string) (ret fieldParameters) { + for _, part := range strings.Split(str, ",") { + switch { + case part == "optional": + ret.optional = true + case part == "explicit": + ret.explicit = true + if ret.tag == nil { + ret.tag = new(int) + } + case part == "ia5": + ret.stringType = tagIA5String + case part == "printable": + ret.stringType = tagPrintableString + case strings.HasPrefix(part, "default:"): + i, err := strconv.Atoi64(part[8:]) + if err == nil { + ret.defaultValue = new(int64) + *ret.defaultValue = i + } + case strings.HasPrefix(part, "tag:"): + i, err := strconv.Atoi(part[4:]) + if err == nil { + ret.tag = new(int) + *ret.tag = i + } + case part == "set": + ret.set = true + case part == "application": + ret.application = true + if ret.tag == nil { + ret.tag = new(int) + } + } + } + return +} + +// Given a reflected Go type, getUniversalType returns the default tag number +// and expected compound flag. +func getUniversalType(t reflect.Type) (tagNumber int, isCompound, ok bool) { + switch t { + case objectIdentifierType: + return tagOID, false, true + case bitStringType: + return tagBitString, false, true + case timeType: + return tagUTCTime, false, true + case enumeratedType: + return tagEnum, false, true + case bigIntType: + return tagInteger, false, true + } + switch t.Kind() { + case reflect.Bool: + return tagBoolean, false, true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return tagInteger, false, true + case reflect.Struct: + return tagSequence, true, true + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return tagOctetString, false, true + } + if strings.HasSuffix(t.Name(), "SET") { + return tagSet, true, true + } + return tagSequence, true, true + case reflect.String: + return tagPrintableString, false, true + } + return 0, false, false +} diff --git a/src/pkg/asn1/marshal.go b/src/pkg/asn1/marshal.go new file mode 100644 index 000000000..d7eb63bf8 --- /dev/null +++ b/src/pkg/asn1/marshal.go @@ -0,0 +1,544 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "big" + "bytes" + "fmt" + "io" + "os" + "reflect" + "time" +) + +// A forkableWriter is an in-memory buffer that can be +// 'forked' to create new forkableWriters that bracket the +// original. After +// pre, post := w.fork(); +// the overall sequence of bytes represented is logically w+pre+post. +type forkableWriter struct { + *bytes.Buffer + pre, post *forkableWriter +} + +func newForkableWriter() *forkableWriter { + return &forkableWriter{bytes.NewBuffer(nil), nil, nil} +} + +func (f *forkableWriter) fork() (pre, post *forkableWriter) { + if f.pre != nil || f.post != nil { + panic("have already forked") + } + f.pre = newForkableWriter() + f.post = newForkableWriter() + return f.pre, f.post +} + +func (f *forkableWriter) Len() (l int) { + l += f.Buffer.Len() + if f.pre != nil { + l += f.pre.Len() + } + if f.post != nil { + l += f.post.Len() + } + return +} + +func (f *forkableWriter) writeTo(out io.Writer) (n int, err os.Error) { + n, err = out.Write(f.Bytes()) + if err != nil { + return + } + + var nn int + + if f.pre != nil { + nn, err = f.pre.writeTo(out) + n += nn + if err != nil { + return + } + } + + if f.post != nil { + nn, err = f.post.writeTo(out) + n += nn + } + return +} + +func marshalBase128Int(out *forkableWriter, n int64) (err os.Error) { + if n == 0 { + err = out.WriteByte(0) + return + } + + l := 0 + for i := n; i > 0; i >>= 7 { + l++ + } + + for i := l - 1; i >= 0; i-- { + o := byte(n >> uint(i*7)) + o &= 0x7f + if i != 0 { + o |= 0x80 + } + err = out.WriteByte(o) + if err != nil { + return + } + } + + return nil +} + +func marshalInt64(out *forkableWriter, i int64) (err os.Error) { + n := int64Length(i) + + for ; n > 0; n-- { + err = out.WriteByte(byte(i >> uint((n-1)*8))) + if err != nil { + return + } + } + + return nil +} + +func int64Length(i int64) (numBytes int) { + numBytes = 1 + + for i > 127 { + numBytes++ + i >>= 8 + } + + for i < -128 { + numBytes++ + i >>= 8 + } + + return +} + +func marshalBigInt(out *forkableWriter, n *big.Int) (err os.Error) { + if n.Sign() < 0 { + // A negative number has to be converted to two's-complement + // form. So we'll subtract 1 and invert. If the + // most-significant-bit isn't set then we'll need to pad the + // beginning with 0xff in order to keep the number negative. + nMinus1 := new(big.Int).Neg(n) + nMinus1.Sub(nMinus1, bigOne) + bytes := nMinus1.Bytes() + for i := range bytes { + bytes[i] ^= 0xff + } + if len(bytes) == 0 || bytes[0]&0x80 == 0 { + err = out.WriteByte(0xff) + if err != nil { + return + } + } + _, err = out.Write(bytes) + } else if n.Sign() == 0 { + // Zero is written as a single 0 zero rather than no bytes. + err = out.WriteByte(0x00) + } else { + bytes := n.Bytes() + if len(bytes) > 0 && bytes[0]&0x80 != 0 { + // We'll have to pad this with 0x00 in order to stop it + // looking like a negative number. + err = out.WriteByte(0) + if err != nil { + return + } + } + _, err = out.Write(bytes) + } + return +} + +func marshalLength(out *forkableWriter, i int) (err os.Error) { + n := lengthLength(i) + + for ; n > 0; n-- { + err = out.WriteByte(byte(i >> uint((n-1)*8))) + if err != nil { + return + } + } + + return nil +} + +func lengthLength(i int) (numBytes int) { + numBytes = 1 + for i > 255 { + numBytes++ + i >>= 8 + } + return +} + +func marshalTagAndLength(out *forkableWriter, t tagAndLength) (err os.Error) { + b := uint8(t.class) << 6 + if t.isCompound { + b |= 0x20 + } + if t.tag >= 31 { + b |= 0x1f + err = out.WriteByte(b) + if err != nil { + return + } + err = marshalBase128Int(out, int64(t.tag)) + if err != nil { + return + } + } else { + b |= uint8(t.tag) + err = out.WriteByte(b) + if err != nil { + return + } + } + + if t.length >= 128 { + l := lengthLength(t.length) + err = out.WriteByte(0x80 | byte(l)) + if err != nil { + return + } + err = marshalLength(out, t.length) + if err != nil { + return + } + } else { + err = out.WriteByte(byte(t.length)) + if err != nil { + return + } + } + + return nil +} + +func marshalBitString(out *forkableWriter, b BitString) (err os.Error) { + paddingBits := byte((8 - b.BitLength%8) % 8) + err = out.WriteByte(paddingBits) + if err != nil { + return + } + _, err = out.Write(b.Bytes) + return +} + +func marshalObjectIdentifier(out *forkableWriter, oid []int) (err os.Error) { + if len(oid) < 2 || oid[0] > 6 || oid[1] >= 40 { + return StructuralError{"invalid object identifier"} + } + + err = out.WriteByte(byte(oid[0]*40 + oid[1])) + if err != nil { + return + } + for i := 2; i < len(oid); i++ { + err = marshalBase128Int(out, int64(oid[i])) + if err != nil { + return + } + } + + return +} + +func marshalPrintableString(out *forkableWriter, s string) (err os.Error) { + b := []byte(s) + for _, c := range b { + if !isPrintable(c) { + return StructuralError{"PrintableString contains invalid character"} + } + } + + _, err = out.Write(b) + return +} + +func marshalIA5String(out *forkableWriter, s string) (err os.Error) { + b := []byte(s) + for _, c := range b { + if c > 127 { + return StructuralError{"IA5String contains invalid character"} + } + } + + _, err = out.Write(b) + return +} + +func marshalTwoDigits(out *forkableWriter, v int) (err os.Error) { + err = out.WriteByte(byte('0' + (v/10)%10)) + if err != nil { + return + } + return out.WriteByte(byte('0' + v%10)) +} + +func marshalUTCTime(out *forkableWriter, t *time.Time) (err os.Error) { + switch { + case 1950 <= t.Year && t.Year < 2000: + err = marshalTwoDigits(out, int(t.Year-1900)) + case 2000 <= t.Year && t.Year < 2050: + err = marshalTwoDigits(out, int(t.Year-2000)) + default: + return StructuralError{"Cannot represent time as UTCTime"} + } + + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Month) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Day) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Hour) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Minute) + if err != nil { + return + } + + err = marshalTwoDigits(out, t.Second) + if err != nil { + return + } + + switch { + case t.ZoneOffset/60 == 0: + err = out.WriteByte('Z') + return + case t.ZoneOffset > 0: + err = out.WriteByte('+') + case t.ZoneOffset < 0: + err = out.WriteByte('-') + } + + if err != nil { + return + } + + offsetMinutes := t.ZoneOffset / 60 + if offsetMinutes < 0 { + offsetMinutes = -offsetMinutes + } + + err = marshalTwoDigits(out, offsetMinutes/60) + if err != nil { + return + } + + err = marshalTwoDigits(out, offsetMinutes%60) + return +} + +func stripTagAndLength(in []byte) []byte { + _, offset, err := parseTagAndLength(in, 0) + if err != nil { + return in + } + return in[offset:] +} + +func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameters) (err os.Error) { + switch value.Type() { + case timeType: + return marshalUTCTime(out, value.Interface().(*time.Time)) + case bitStringType: + return marshalBitString(out, value.Interface().(BitString)) + case objectIdentifierType: + return marshalObjectIdentifier(out, value.Interface().(ObjectIdentifier)) + case bigIntType: + return marshalBigInt(out, value.Interface().(*big.Int)) + } + + switch v := value; v.Kind() { + case reflect.Bool: + if v.Bool() { + return out.WriteByte(255) + } else { + return out.WriteByte(0) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return marshalInt64(out, int64(v.Int())) + case reflect.Struct: + t := v.Type() + + startingField := 0 + + // If the first element of the structure is a non-empty + // RawContents, then we don't bother serializing the rest. + if t.NumField() > 0 && t.Field(0).Type == rawContentsType { + s := v.Field(0) + if s.Len() > 0 { + bytes := make([]byte, s.Len()) + for i := 0; i < s.Len(); i++ { + bytes[i] = uint8(s.Index(i).Uint()) + } + /* The RawContents will contain the tag and + * length fields but we'll also be writing + * those ourselves, so we strip them out of + * bytes */ + _, err = out.Write(stripTagAndLength(bytes)) + return + } else { + startingField = 1 + } + } + + for i := startingField; i < t.NumField(); i++ { + var pre *forkableWriter + pre, out = out.fork() + err = marshalField(pre, v.Field(i), parseFieldParameters(t.Field(i).Tag.Get("asn1"))) + if err != nil { + return + } + } + return + case reflect.Slice: + sliceType := v.Type() + if sliceType.Elem().Kind() == reflect.Uint8 { + bytes := make([]byte, v.Len()) + for i := 0; i < v.Len(); i++ { + bytes[i] = uint8(v.Index(i).Uint()) + } + _, err = out.Write(bytes) + return + } + + var params fieldParameters + for i := 0; i < v.Len(); i++ { + var pre *forkableWriter + pre, out = out.fork() + err = marshalField(pre, v.Index(i), params) + if err != nil { + return + } + } + return + case reflect.String: + if params.stringType == tagIA5String { + return marshalIA5String(out, v.String()) + } else { + return marshalPrintableString(out, v.String()) + } + return + } + + return StructuralError{"unknown Go type"} +} + +func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) (err os.Error) { + // If the field is an interface{} then recurse into it. + if v.Kind() == reflect.Interface && v.Type().NumMethod() == 0 { + return marshalField(out, v.Elem(), params) + } + + if params.optional && reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { + return + } + + if v.Type() == rawValueType { + rv := v.Interface().(RawValue) + err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) + if err != nil { + return + } + _, err = out.Write(rv.Bytes) + return + } + + tag, isCompound, ok := getUniversalType(v.Type()) + if !ok { + err = StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} + return + } + class := classUniversal + + if params.stringType != 0 { + if tag != tagPrintableString { + return StructuralError{"Explicit string type given to non-string member"} + } + tag = params.stringType + } + + if params.set { + if tag != tagSequence { + return StructuralError{"Non sequence tagged as set"} + } + tag = tagSet + } + + tags, body := out.fork() + + err = marshalBody(body, v, params) + if err != nil { + return + } + + bodyLen := body.Len() + + var explicitTag *forkableWriter + if params.explicit { + explicitTag, tags = tags.fork() + } + + if !params.explicit && params.tag != nil { + // implicit tag. + tag = *params.tag + class = classContextSpecific + } + + err = marshalTagAndLength(tags, tagAndLength{class, tag, bodyLen, isCompound}) + if err != nil { + return + } + + if params.explicit { + err = marshalTagAndLength(explicitTag, tagAndLength{ + class: classContextSpecific, + tag: *params.tag, + length: bodyLen + tags.Len(), + isCompound: true, + }) + } + + return nil +} + +// Marshal returns the ASN.1 encoding of val. +func Marshal(val interface{}) ([]byte, os.Error) { + var out bytes.Buffer + v := reflect.ValueOf(val) + f := newForkableWriter() + err := marshalField(f, v, fieldParameters{}) + if err != nil { + return nil, err + } + _, err = f.writeTo(&out) + return out.Bytes(), nil +} diff --git a/src/pkg/asn1/marshal_test.go b/src/pkg/asn1/marshal_test.go new file mode 100644 index 000000000..03df5f1e1 --- /dev/null +++ b/src/pkg/asn1/marshal_test.go @@ -0,0 +1,129 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asn1 + +import ( + "bytes" + "encoding/hex" + "testing" + "time" +) + +type intStruct struct { + A int +} + +type twoIntStruct struct { + A int + B int +} + +type nestedStruct struct { + A intStruct +} + +type rawContentsStruct struct { + Raw RawContent + A int +} + +type implicitTagTest struct { + A int `asn1:"implicit,tag:5"` +} + +type explicitTagTest struct { + A int `asn1:"explicit,tag:5"` +} + +type ia5StringTest struct { + A string `asn1:"ia5"` +} + +type printableStringTest struct { + A string `asn1:"printable"` +} + +type optionalRawValueTest struct { + A RawValue `asn1:"optional"` +} + +type testSET []int + +func setPST(t *time.Time) *time.Time { + t.ZoneOffset = -28800 + return t +} + +type marshalTest struct { + in interface{} + out string // hex encoded +} + +var marshalTests = []marshalTest{ + {10, "02010a"}, + {127, "02017f"}, + {128, "02020080"}, + {-128, "020180"}, + {-129, "0202ff7f"}, + {intStruct{64}, "3003020140"}, + {twoIntStruct{64, 65}, "3006020140020141"}, + {nestedStruct{intStruct{127}}, "3005300302017f"}, + {[]byte{1, 2, 3}, "0403010203"}, + {implicitTagTest{64}, "3003850140"}, + {explicitTagTest{64}, "3005a503020140"}, + {time.SecondsToUTC(0), "170d3730303130313030303030305a"}, + {time.SecondsToUTC(1258325776), "170d3039313131353232353631365a"}, + {setPST(time.SecondsToUTC(1258325776)), "17113039313131353232353631362d30383030"}, + {BitString{[]byte{0x80}, 1}, "03020780"}, + {BitString{[]byte{0x81, 0xf0}, 12}, "03030481f0"}, + {ObjectIdentifier([]int{1, 2, 3, 4}), "06032a0304"}, + {ObjectIdentifier([]int{1, 2, 840, 133549, 1, 1, 5}), "06092a864888932d010105"}, + {"test", "130474657374"}, + { + "" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", // This is 127 times 'x' + "137f" + + "7878787878787878787878787878787878787878787878787878787878787878" + + "7878787878787878787878787878787878787878787878787878787878787878" + + "7878787878787878787878787878787878787878787878787878787878787878" + + "78787878787878787878787878787878787878787878787878787878787878", + }, + { + "" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", // This is 128 times 'x' + "138180" + + "7878787878787878787878787878787878787878787878787878787878787878" + + "7878787878787878787878787878787878787878787878787878787878787878" + + "7878787878787878787878787878787878787878787878787878787878787878" + + "7878787878787878787878787878787878787878787878787878787878787878", + }, + {ia5StringTest{"test"}, "3006160474657374"}, + {optionalRawValueTest{}, "3000"}, + {printableStringTest{"test"}, "3006130474657374"}, + {printableStringTest{"test*"}, "30071305746573742a"}, + {rawContentsStruct{nil, 64}, "3003020140"}, + {rawContentsStruct{[]byte{0x30, 3, 1, 2, 3}, 64}, "3003010203"}, + {RawValue{Tag: 1, Class: 2, IsCompound: false, Bytes: []byte{1, 2, 3}}, "8103010203"}, + {testSET([]int{10}), "310302010a"}, +} + +func TestMarshal(t *testing.T) { + for i, test := range marshalTests { + data, err := Marshal(test.in) + if err != nil { + t.Errorf("#%d failed: %s", i, err) + } + out, _ := hex.DecodeString(test.out) + if bytes.Compare(out, data) != 0 { + t.Errorf("#%d got: %x want %x", i, data, out) + } + } +} diff --git a/src/pkg/big/Makefile b/src/pkg/big/Makefile new file mode 100644 index 000000000..3d4b56d78 --- /dev/null +++ b/src/pkg/big/Makefile @@ -0,0 +1,18 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=big +GOFILES=\ + arith.go\ + arith_decl.go\ + int.go\ + nat.go\ + rat.go\ + +OFILES=\ + arith_$(GOARCH).$O\ + +include ../../Make.pkg diff --git a/src/pkg/big/arith.go b/src/pkg/big/arith.go new file mode 100644 index 000000000..0a02a4ef5 --- /dev/null +++ b/src/pkg/big/arith.go @@ -0,0 +1,231 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides Go implementations of elementary multi-precision +// arithmetic operations on word vectors. Needed for platforms without +// assembly implementations of these routines. + +package big + +// TODO(gri) Decide if Word needs to remain exported. + +type Word uintptr + +const ( + // Compute the size _S of a Word in bytes. + _m = ^Word(0) + _logS = _m>>8&1 + _m>>16&1 + _m>>32&1 + _S = 1 << _logS + + _W = _S << 3 // word size in bits + _B = 1 << _W // digit base + _M = _B - 1 // digit mask + + _W2 = _W / 2 // half word size in bits + _B2 = 1 << _W2 // half digit base + _M2 = _B2 - 1 // half digit mask +) + +// ---------------------------------------------------------------------------- +// Elementary operations on words +// +// These operations are used by the vector operations below. + +// z1<<_W + z0 = x+y+c, with c == 0 or 1 +func addWW_g(x, y, c Word) (z1, z0 Word) { + yc := y + c + z0 = x + yc + if z0 < x || yc < y { + z1 = 1 + } + return +} + +// z1<<_W + z0 = x-y-c, with c == 0 or 1 +func subWW_g(x, y, c Word) (z1, z0 Word) { + yc := y + c + z0 = x - yc + if z0 > x || yc < y { + z1 = 1 + } + return +} + +// z1<<_W + z0 = x*y +// Adapted from Warren, Hacker's Delight, p. 132. +func mulWW_g(x, y Word) (z1, z0 Word) { + x0 := x & _M2 + x1 := x >> _W2 + y0 := y & _M2 + y1 := y >> _W2 + w0 := x0 * y0 + t := x1*y0 + w0>>_W2 + w1 := t & _M2 + w2 := t >> _W2 + w1 += x0 * y1 + z1 = x1*y1 + w2 + w1>>_W2 + z0 = x * y + return +} + +// z1<<_W + z0 = x*y + c +func mulAddWWW_g(x, y, c Word) (z1, z0 Word) { + z1, zz0 := mulWW(x, y) + if z0 = zz0 + c; z0 < zz0 { + z1++ + } + return +} + +// Length of x in bits. +func bitLen(x Word) (n int) { + for ; x >= 0x100; x >>= 8 { + n += 8 + } + for ; x > 0; x >>= 1 { + n++ + } + return +} + +// log2 computes the integer binary logarithm of x. +// The result is the integer n for which 2^n <= x < 2^(n+1). +// If x == 0, the result is -1. +func log2(x Word) int { + return bitLen(x) - 1 +} + +// Number of leading zeros in x. +func leadingZeros(x Word) uint { + return uint(_W - bitLen(x)) +} + +// q = (u1<<_W + u0 - r)/y +// Adapted from Warren, Hacker's Delight, p. 152. +func divWW_g(u1, u0, v Word) (q, r Word) { + if u1 >= v { + return 1<<_W - 1, 1<<_W - 1 + } + + s := leadingZeros(v) + v <<= s + + vn1 := v >> _W2 + vn0 := v & _M2 + un32 := u1<>(_W-s) + un10 := u0 << s + un1 := un10 >> _W2 + un0 := un10 & _M2 + q1 := un32 / vn1 + rhat := un32 - q1*vn1 + +again1: + if q1 >= _B2 || q1*vn0 > _B2*rhat+un1 { + q1-- + rhat += vn1 + if rhat < _B2 { + goto again1 + } + } + + un21 := un32*_B2 + un1 - q1*v + q0 := un21 / vn1 + rhat = un21 - q0*vn1 + +again2: + if q0 >= _B2 || q0*vn0 > _B2*rhat+un0 { + q0-- + rhat += vn1 + if rhat < _B2 { + goto again2 + } + } + + return q1*_B2 + q0, (un21*_B2 + un0 - q0*v) >> s +} + +func addVV_g(z, x, y []Word) (c Word) { + for i := range z { + c, z[i] = addWW_g(x[i], y[i], c) + } + return +} + +func subVV_g(z, x, y []Word) (c Word) { + for i := range z { + c, z[i] = subWW_g(x[i], y[i], c) + } + return +} + +func addVW_g(z, x []Word, y Word) (c Word) { + c = y + for i := range z { + c, z[i] = addWW_g(x[i], c, 0) + } + return +} + +func subVW_g(z, x []Word, y Word) (c Word) { + c = y + for i := range z { + c, z[i] = subWW_g(x[i], c, 0) + } + return +} + +func shlVU_g(z, x []Word, s uint) (c Word) { + if n := len(z); n > 0 { + ŝ := _W - s + w1 := x[n-1] + c = w1 >> ŝ + for i := n - 1; i > 0; i-- { + w := w1 + w1 = x[i-1] + z[i] = w<>ŝ + } + z[0] = w1 << s + } + return +} + +func shrVU_g(z, x []Word, s uint) (c Word) { + if n := len(z); n > 0 { + ŝ := _W - s + w1 := x[0] + c = w1 << ŝ + for i := 0; i < n-1; i++ { + w := w1 + w1 = x[i+1] + z[i] = w>>s | w1<<ŝ + } + z[n-1] = w1 >> s + } + return +} + +func mulAddVWW_g(z, x []Word, y, r Word) (c Word) { + c = r + for i := range z { + c, z[i] = mulAddWWW_g(x[i], y, c) + } + return +} + +func addMulVVW_g(z, x []Word, y Word) (c Word) { + for i := range z { + z1, z0 := mulAddWWW_g(x[i], y, z[i]) + c, z[i] = addWW_g(z0, c, 0) + c += z1 + } + return +} + +func divWVW_g(z []Word, xn Word, x []Word, y Word) (r Word) { + r = xn + for i := len(z) - 1; i >= 0; i-- { + z[i], r = divWW_g(r, x[i], y) + } + return +} diff --git a/src/pkg/big/arith_386.s b/src/pkg/big/arith_386.s new file mode 100644 index 000000000..07c07b02c --- /dev/null +++ b/src/pkg/big/arith_386.s @@ -0,0 +1,265 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +// func mulWW(x, y Word) (z1, z0 Word) +TEXT ·mulWW(SB),7,$0 + MOVL x+0(FP), AX + MULL y+4(FP) + MOVL DX, z1+8(FP) + MOVL AX, z0+12(FP) + RET + + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),7,$0 + MOVL x1+0(FP), DX + MOVL x0+4(FP), AX + DIVL y+8(FP) + MOVL AX, q+12(FP) + MOVL DX, r+16(FP) + RET + + +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), CX + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + MOVL $0, DX // c = 0 + JMP E1 + +L1: MOVL (SI)(BX*4), AX + RCRL $1, DX + ADCL (CX)(BX*4), AX + RCLL $1, DX + MOVL AX, (DI)(BX*4) + ADDL $1, BX // i++ + +E1: CMPL BX, BP // i < n + JL L1 + + MOVL DX, c+36(FP) + RET + + +// func subVV(z, x, y []Word) (c Word) +// (same as addVV except for SBBL instead of ADCL and label names) +TEXT ·subVV(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), CX + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + MOVL $0, DX // c = 0 + JMP E2 + +L2: MOVL (SI)(BX*4), AX + RCRL $1, DX + SBBL (CX)(BX*4), AX + RCLL $1, DX + MOVL AX, (DI)(BX*4) + ADDL $1, BX // i++ + +E2: CMPL BX, BP // i < n + JL L2 + + MOVL DX, c+36(FP) + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), AX // c = y + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + JMP E3 + +L3: ADDL (SI)(BX*4), AX + MOVL AX, (DI)(BX*4) + RCLL $1, AX + ANDL $1, AX + ADDL $1, BX // i++ + +E3: CMPL BX, BP // i < n + JL L3 + + MOVL AX, c+28(FP) + RET + + +// func subVW(z, x []Word, y Word) (c Word) +TEXT ·subVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), AX // c = y + MOVL n+4(FP), BP + MOVL $0, BX // i = 0 + JMP E4 + +L4: MOVL (SI)(BX*4), DX // TODO(gri) is there a reverse SUBL? + SUBL AX, DX + MOVL DX, (DI)(BX*4) + RCLL $1, AX + ANDL $1, AX + ADDL $1, BX // i++ + +E4: CMPL BX, BP // i < n + JL L4 + + MOVL AX, c+28(FP) + RET + + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 + MOVL n+4(FP), BX // i = n + SUBL $1, BX // i-- + JL X8b // i < 0 (n <= 0) + + // n > 0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL s+24(FP), CX + MOVL (SI)(BX*4), AX // w1 = x[n-1] + MOVL $0, DX + SHLL CX, DX:AX // w1>>ŝ + MOVL DX, c+28(FP) + + CMPL BX, $0 + JLE X8a // i <= 0 + + // i > 0 +L8: MOVL AX, DX // w = w1 + MOVL -4(SI)(BX*4), AX // w1 = x[i-1] + SHLL CX, DX:AX // w<>ŝ + MOVL DX, (DI)(BX*4) // z[i] = w<>ŝ + SUBL $1, BX // i-- + JG L8 // i > 0 + + // i <= 0 +X8a: SHLL CX, AX // w1< 0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL s+24(FP), CX + MOVL (SI), AX // w1 = x[0] + MOVL $0, DX + SHRL CX, DX:AX // w1<<ŝ + MOVL DX, c+28(FP) + + MOVL $0, BX // i = 0 + JMP E9 + + // i < n-1 +L9: MOVL AX, DX // w = w1 + MOVL 4(SI)(BX*4), AX // w1 = x[i+1] + SHRL CX, DX:AX // w>>s | w1<<ŝ + MOVL DX, (DI)(BX*4) // z[i] = w>>s | w1<<ŝ + ADDL $1, BX // i++ + +E9: CMPL BX, BP + JL L9 // i < n-1 + + // i >= n-1 +X9a: SHRL CX, AX // w1>>s + MOVL AX, (DI)(BP*4) // z[n-1] = w1>>s + RET + +X9b: MOVL $0, c+28(FP) + RET + + +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), BP + MOVL r+28(FP), CX // c = r + MOVL n+4(FP), BX + LEAL (DI)(BX*4), DI + LEAL (SI)(BX*4), SI + NEGL BX // i = -n + JMP E5 + +L5: MOVL (SI)(BX*4), AX + MULL BP + ADDL CX, AX + ADCL $0, DX + MOVL AX, (DI)(BX*4) + MOVL DX, CX + ADDL $1, BX // i++ + +E5: CMPL BX, $0 // i < 0 + JL L5 + + MOVL CX, c+32(FP) + RET + + +// func addMulVVW(z, x []Word, y Word) (c Word) +TEXT ·addMulVVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL x+12(FP), SI + MOVL y+24(FP), BP + MOVL n+4(FP), BX + LEAL (DI)(BX*4), DI + LEAL (SI)(BX*4), SI + NEGL BX // i = -n + MOVL $0, CX // c = 0 + JMP E6 + +L6: MOVL (SI)(BX*4), AX + MULL BP + ADDL CX, AX + ADCL $0, DX + ADDL AX, (DI)(BX*4) + ADCL $0, DX + MOVL DX, CX + ADDL $1, BX // i++ + +E6: CMPL BX, $0 // i < 0 + JL L6 + + MOVL CX, c+28(FP) + RET + + +// divWVW(z* Word, xn Word, x []Word, y Word) (r Word) +TEXT ·divWVW(SB),7,$0 + MOVL z+0(FP), DI + MOVL xn+12(FP), DX // r = xn + MOVL x+16(FP), SI + MOVL y+28(FP), CX + MOVL n+4(FP), BX // i = n + JMP E7 + +L7: MOVL (SI)(BX*4), AX + DIVL CX + MOVL AX, (DI)(BX*4) + +E7: SUBL $1, BX // i-- + JGE L7 // i >= 0 + + MOVL DX, r+32(FP) + RET diff --git a/src/pkg/big/arith_amd64.s b/src/pkg/big/arith_amd64.s new file mode 100644 index 000000000..89b65f38a --- /dev/null +++ b/src/pkg/big/arith_amd64.s @@ -0,0 +1,263 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +// TODO(gri) - experiment with unrolled loops for faster execution + +// func mulWW(x, y Word) (z1, z0 Word) +TEXT ·mulWW(SB),7,$0 + MOVQ x+0(FP), AX + MULQ y+8(FP) + MOVQ DX, z1+16(FP) + MOVQ AX, z0+24(FP) + RET + + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),7,$0 + MOVQ x1+0(FP), DX + MOVQ x0+8(FP), AX + DIVQ y+16(FP) + MOVQ AX, q+24(FP) + MOVQ DX, r+32(FP) + RET + + +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + MOVQ $0, DX // c = 0 + JMP E1 + +L1: MOVQ (R8)(BX*8), AX + RCRQ $1, DX + ADCQ (R9)(BX*8), AX + RCLQ $1, DX + MOVQ AX, (R10)(BX*8) + ADDL $1, BX // i++ + +E1: CMPQ BX, R11 // i < n + JL L1 + + MOVQ DX, c+48(FP) + RET + + +// func subVV(z, x, y []Word) (c Word) +// (same as addVV_s except for SBBQ instead of ADCQ and label names) +TEXT ·subVV(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + MOVQ $0, DX // c = 0 + JMP E2 + +L2: MOVQ (R8)(BX*8), AX + RCRQ $1, DX + SBBQ (R9)(BX*8), AX + RCLQ $1, DX + MOVQ AX, (R10)(BX*8) + ADDL $1, BX // i++ + +E2: CMPQ BX, R11 // i < n + JL L2 + + MOVQ DX, c+48(FP) + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), AX // c = y + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + JMP E3 + +L3: ADDQ (R8)(BX*8), AX + MOVQ AX, (R10)(BX*8) + RCLQ $1, AX + ANDQ $1, AX + ADDL $1, BX // i++ + +E3: CMPQ BX, R11 // i < n + JL L3 + + MOVQ AX, c+40(FP) + RET + + +// func subVW(z, x []Word, y Word) (c Word) +TEXT ·subVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), AX // c = y + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + JMP E4 + +L4: MOVQ (R8)(BX*8), DX // TODO(gri) is there a reverse SUBQ? + SUBQ AX, DX + MOVQ DX, (R10)(BX*8) + RCLQ $1, AX + ANDQ $1, AX + ADDL $1, BX // i++ + +E4: CMPQ BX, R11 // i < n + JL L4 + + MOVQ AX, c+40(FP) + RET + + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 + MOVL n+8(FP), BX // i = n + SUBL $1, BX // i-- + JL X8b // i < 0 (n <= 0) + + // n > 0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVL s+32(FP), CX + MOVQ (R8)(BX*8), AX // w1 = x[n-1] + MOVQ $0, DX + SHLQ CX, DX:AX // w1>>ŝ + MOVQ DX, c+40(FP) + + CMPL BX, $0 + JLE X8a // i <= 0 + + // i > 0 +L8: MOVQ AX, DX // w = w1 + MOVQ -8(R8)(BX*8), AX // w1 = x[i-1] + SHLQ CX, DX:AX // w<>ŝ + MOVQ DX, (R10)(BX*8) // z[i] = w<>ŝ + SUBL $1, BX // i-- + JG L8 // i > 0 + + // i <= 0 +X8a: SHLQ CX, AX // w1< 0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVL s+32(FP), CX + MOVQ (R8), AX // w1 = x[0] + MOVQ $0, DX + SHRQ CX, DX:AX // w1<<ŝ + MOVQ DX, c+40(FP) + + MOVQ $0, BX // i = 0 + JMP E9 + + // i < n-1 +L9: MOVQ AX, DX // w = w1 + MOVQ 8(R8)(BX*8), AX // w1 = x[i+1] + SHRQ CX, DX:AX // w>>s | w1<<ŝ + MOVQ DX, (R10)(BX*8) // z[i] = w>>s | w1<<ŝ + ADDL $1, BX // i++ + +E9: CMPQ BX, R11 + JL L9 // i < n-1 + + // i >= n-1 +X9a: SHRQ CX, AX // w1>>s + MOVQ AX, (R10)(R11*8) // z[n-1] = w1>>s + RET + +X9b: MOVQ $0, c+40(FP) + RET + + +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVQ r+40(FP), CX // c = r + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + JMP E5 + +L5: MOVQ (R8)(BX*8), AX + MULQ R9 + ADDQ CX, AX + ADCQ $0, DX + MOVQ AX, (R10)(BX*8) + MOVQ DX, CX + ADDL $1, BX // i++ + +E5: CMPQ BX, R11 // i < n + JL L5 + + MOVQ CX, c+48(FP) + RET + + +// func addMulVVW(z, x []Word, y Word) (c Word) +TEXT ·addMulVVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ x+16(FP), R8 + MOVQ y+32(FP), R9 + MOVL n+8(FP), R11 + MOVQ $0, BX // i = 0 + MOVQ $0, CX // c = 0 + JMP E6 + +L6: MOVQ (R8)(BX*8), AX + MULQ R9 + ADDQ CX, AX + ADCQ $0, DX + ADDQ AX, (R10)(BX*8) + ADCQ $0, DX + MOVQ DX, CX + ADDL $1, BX // i++ + +E6: CMPQ BX, R11 // i < n + JL L6 + + MOVQ CX, c+40(FP) + RET + + +// divWVW(z []Word, xn Word, x []Word, y Word) (r Word) +TEXT ·divWVW(SB),7,$0 + MOVQ z+0(FP), R10 + MOVQ xn+16(FP), DX // r = xn + MOVQ x+24(FP), R8 + MOVQ y+40(FP), R9 + MOVL n+8(FP), BX // i = n + JMP E7 + +L7: MOVQ (R8)(BX*8), AX + DIVQ R9 + MOVQ AX, (R10)(BX*8) + +E7: SUBL $1, BX // i-- + JGE L7 // i >= 0 + + MOVQ DX, r+48(FP) + RET diff --git a/src/pkg/big/arith_arm.s b/src/pkg/big/arith_arm.s new file mode 100644 index 000000000..60abe6eaa --- /dev/null +++ b/src/pkg/big/arith_arm.s @@ -0,0 +1,312 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +#define CFLAG 29 // bit position of carry flag + +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),7,$0 + MOVW $0, R0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R4 + MOVW R4<<2, R4 + ADD R1, R4 + B E1 +L1: + MOVW.P 4(R2), R5 + MOVW.P 4(R3), R6 + MOVW R0, CPSR + ADC.S R6, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 +E1: + CMP R1, R4 + BNE L1 + + MOVW R0>>CFLAG, R0 + AND $1, R0 + MOVW R0, c+36(FP) + RET + + +// func subVV(z, x, y []Word) (c Word) +// (same as addVV except for SBC instead of ADC and label names) +TEXT ·subVV(SB),7,$0 + MOVW $(1<>CFLAG, R0 + AND $1, R0 + EOR $1, R0 + MOVW R0, c+36(FP) + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),7,$0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R4 + MOVW R4<<2, R4 + ADD R1, R4 + CMP R1, R4 + BNE L3a + MOVW R3, c+28(FP) + RET +L3a: + MOVW.P 4(R2), R5 + ADD.S R3, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 + B E3 +L3: + MOVW.P 4(R2), R5 + MOVW R0, CPSR + ADC.S $0, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 +E3: + CMP R1, R4 + BNE L3 + + MOVW R0>>CFLAG, R0 + AND $1, R0 + MOVW R0, c+28(FP) + RET + + +// func subVW(z, x []Word, y Word) (c Word) +TEXT ·subVW(SB),7,$0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R4 + MOVW R4<<2, R4 + ADD R1, R4 + CMP R1, R4 + BNE L4a + MOVW R3, c+28(FP) + RET +L4a: + MOVW.P 4(R2), R5 + SUB.S R3, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 + B E4 +L4: + MOVW.P 4(R2), R5 + MOVW R0, CPSR + SBC.S $0, R5 + MOVW.P R5, 4(R1) + MOVW CPSR, R0 +E4: + CMP R1, R4 + BNE L4 + + MOVW R0>>CFLAG, R0 + AND $1, R0 + EOR $1, R0 + MOVW R0, c+28(FP) + RET + + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),7,$0 + MOVW n+4(FP), R5 + CMP $0, R5 + BEQ X7 + + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW R5<<2, R5 + ADD R5, R2 + ADD R1, R5 + MOVW s+24(FP), R3 + CMP $0, R3 // shift 0 is special + BEQ Y7 + ADD $4, R1 // stop one word early + MOVW $32, R4 + SUB R3, R4 + MOVW $0, R7 + + MOVW.W -4(R2), R6 + MOVW R6<>R4, R6 + MOVW R6, c+28(FP) + B E7 + +L7: + MOVW.W -4(R2), R6 + ORR R6>>R4, R7 + MOVW.W R7, -4(R5) + MOVW R6<>R3, R7 + MOVW R6<>R3, R7 +E6: + CMP R1, R5 + BNE L6 + + MOVW R7, 0(R1) + RET + +Y6: // copy loop, because shift 0 == shift 32 + MOVW.P 4(R2), R6 + MOVW.P R6, 4(R1) + CMP R1, R5 + BNE Y6 + +X6: + MOVW $0, R1 + MOVW R1, c+28(FP) + RET + + +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),7,$0 + MOVW $0, R0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW r+28(FP), R4 + MOVW n+4(FP), R5 + MOVW R5<<2, R5 + ADD R1, R5 + B E8 + + // word loop +L8: + MOVW.P 4(R2), R6 + MULLU R6, R3, (R7, R6) + ADD.S R4, R6 + ADC R0, R7 + MOVW.P R6, 4(R1) + MOVW R7, R4 +E8: + CMP R1, R5 + BNE L8 + + MOVW R4, c+32(FP) + RET + + +// func addMulVVW(z, x []Word, y Word) (c Word) +TEXT ·addMulVVW(SB),7,$0 + MOVW $0, R0 + MOVW z+0(FP), R1 + MOVW x+12(FP), R2 + MOVW y+24(FP), R3 + MOVW n+4(FP), R5 + MOVW R5<<2, R5 + ADD R1, R5 + MOVW $0, R4 + B E9 + + // word loop +L9: + MOVW.P 4(R2), R6 + MULLU R6, R3, (R7, R6) + ADD.S R4, R6 + ADC R0, R7 + MOVW 0(R1), R4 + ADD.S R4, R6 + ADC R0, R7 + MOVW.P R6, 4(R1) + MOVW R7, R4 +E9: + CMP R1, R5 + BNE L9 + + MOVW R4, c+28(FP) + RET + + +// divWVW(z* Word, xn Word, x []Word, y Word) (r Word) +TEXT ·divWVW(SB),7,$0 + // ARM has no multiword division, so use portable code. + B ·divWVW_g(SB) + + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),7,$0 + // ARM has no multiword division, so use portable code. + B ·divWW_g(SB) + + +// func mulWW(x, y Word) (z1, z0 Word) +TEXT ·mulWW(SB),7,$0 + MOVW x+0(FP), R1 + MOVW y+4(FP), R2 + MULLU R1, R2, (R4, R3) + MOVW R4, z1+8(FP) + MOVW R3, z0+12(FP) + RET diff --git a/src/pkg/big/arith_decl.go b/src/pkg/big/arith_decl.go new file mode 100644 index 000000000..95fcd8b94 --- /dev/null +++ b/src/pkg/big/arith_decl.go @@ -0,0 +1,18 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +// implemented in arith_$GOARCH.s +func mulWW(x, y Word) (z1, z0 Word) +func divWW(x1, x0, y Word) (q, r Word) +func addVV(z, x, y []Word) (c Word) +func subVV(z, x, y []Word) (c Word) +func addVW(z, x []Word, y Word) (c Word) +func subVW(z, x []Word, y Word) (c Word) +func shlVU(z, x []Word, s uint) (c Word) +func shrVU(z, x []Word, s uint) (c Word) +func mulAddVWW(z, x []Word, y, r Word) (c Word) +func addMulVVW(z, x []Word, y Word) (c Word) +func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) diff --git a/src/pkg/big/arith_test.go b/src/pkg/big/arith_test.go new file mode 100644 index 000000000..b6c56c39e --- /dev/null +++ b/src/pkg/big/arith_test.go @@ -0,0 +1,335 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import "testing" + +type funWW func(x, y, c Word) (z1, z0 Word) +type argWW struct { + x, y, c, z1, z0 Word +} + +var sumWW = []argWW{ + {0, 0, 0, 0, 0}, + {0, 1, 0, 0, 1}, + {0, 0, 1, 0, 1}, + {0, 1, 1, 0, 2}, + {12345, 67890, 0, 0, 80235}, + {12345, 67890, 1, 0, 80236}, + {_M, 1, 0, 1, 0}, + {_M, 0, 1, 1, 0}, + {_M, 1, 1, 1, 1}, + {_M, _M, 0, 1, _M - 1}, + {_M, _M, 1, 1, _M}, +} + +func testFunWW(t *testing.T, msg string, f funWW, a argWW) { + z1, z0 := f(a.x, a.y, a.c) + if z1 != a.z1 || z0 != a.z0 { + t.Errorf("%s%+v\n\tgot z1:z0 = %#x:%#x; want %#x:%#x", msg, a, z1, z0, a.z1, a.z0) + } +} + +func TestFunWW(t *testing.T) { + for _, a := range sumWW { + arg := a + testFunWW(t, "addWW_g", addWW_g, arg) + + arg = argWW{a.y, a.x, a.c, a.z1, a.z0} + testFunWW(t, "addWW_g symmetric", addWW_g, arg) + + arg = argWW{a.z0, a.x, a.c, a.z1, a.y} + testFunWW(t, "subWW_g", subWW_g, arg) + + arg = argWW{a.z0, a.y, a.c, a.z1, a.x} + testFunWW(t, "subWW_g symmetric", subWW_g, arg) + } +} + +type funVV func(z, x, y []Word) (c Word) +type argVV struct { + z, x, y nat + c Word +} + +var sumVV = []argVV{ + {}, + {nat{0}, nat{0}, nat{0}, 0}, + {nat{1}, nat{1}, nat{0}, 0}, + {nat{0}, nat{_M}, nat{1}, 1}, + {nat{80235}, nat{12345}, nat{67890}, 0}, + {nat{_M - 1}, nat{_M}, nat{_M}, 1}, + {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, nat{1, 0, 0, 0}, 1}, + {nat{0, 0, 0, _M}, nat{_M, _M, _M, _M - 1}, nat{1, 0, 0, 0}, 0}, + {nat{0, 0, 0, 0}, nat{_M, 0, _M, 0}, nat{1, _M, 0, _M}, 1}, +} + +func testFunVV(t *testing.T, msg string, f funVV, a argVV) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + +func TestFunVV(t *testing.T) { + for _, a := range sumVV { + arg := a + testFunVV(t, "addVV_g", addVV_g, arg) + testFunVV(t, "addVV", addVV, arg) + + arg = argVV{a.z, a.y, a.x, a.c} + testFunVV(t, "addVV_g symmetric", addVV_g, arg) + testFunVV(t, "addVV symmetric", addVV, arg) + + arg = argVV{a.x, a.z, a.y, a.c} + testFunVV(t, "subVV_g", subVV_g, arg) + testFunVV(t, "subVV", subVV, arg) + + arg = argVV{a.y, a.z, a.x, a.c} + testFunVV(t, "subVV_g symmetric", subVV_g, arg) + testFunVV(t, "subVV symmetric", subVV, arg) + } +} + +type funVW func(z, x []Word, y Word) (c Word) +type argVW struct { + z, x nat + y Word + c Word +} + +var sumVW = []argVW{ + {}, + {nil, nil, 2, 2}, + {nat{0}, nat{0}, 0, 0}, + {nat{1}, nat{0}, 1, 0}, + {nat{1}, nat{1}, 0, 0}, + {nat{0}, nat{_M}, 1, 1}, + {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, 1, 1}, +} + +var prodVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{_M}, 0, 0}, + {nat{0}, nat{0}, _M, 0}, + {nat{1}, nat{1}, 1, 0}, + {nat{22793}, nat{991}, 23, 0}, + {nat{0, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 0}, + {nat{0, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 0}, + {nat{0, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1 << 1, _M >> (_W - 1)}, + {nat{_M << 7 & _M}, nat{_M}, 1 << 7, _M >> (_W - 7)}, + {nat{_M << 7 & _M, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, _M >> (_W - 7)}, +} + +var lshVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{0}, 1, 0}, + {nat{0}, nat{0}, 20, 0}, + + {nat{_M}, nat{_M}, 0, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1, 1}, + {nat{_M << 20 & _M}, nat{_M}, 20, _M >> (_W - 20)}, + + {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, + {nat{_M << 1 & _M, _M, _M}, nat{_M, _M, _M}, 1, 1}, + {nat{_M << 20 & _M, _M, _M}, nat{_M, _M, _M}, 20, _M >> (_W - 20)}, +} + +var rshVW = []argVW{ + {}, + {nat{0}, nat{0}, 0, 0}, + {nat{0}, nat{0}, 1, 0}, + {nat{0}, nat{0}, 20, 0}, + + {nat{_M}, nat{_M}, 0, 0}, + {nat{_M >> 1}, nat{_M}, 1, _M << (_W - 1) & _M}, + {nat{_M >> 20}, nat{_M}, 20, _M << (_W - 20) & _M}, + + {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, + {nat{_M, _M, _M >> 1}, nat{_M, _M, _M}, 1, _M << (_W - 1) & _M}, + {nat{_M, _M, _M >> 20}, nat{_M, _M, _M}, 20, _M << (_W - 20) & _M}, +} + +func testFunVW(t *testing.T, msg string, f funVW, a argVW) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + +func makeFunVW(f func(z, x []Word, s uint) (c Word)) funVW { + return func(z, x []Word, s Word) (c Word) { + return f(z, x, uint(s)) + } +} + +func TestFunVW(t *testing.T) { + for _, a := range sumVW { + arg := a + testFunVW(t, "addVW_g", addVW_g, arg) + testFunVW(t, "addVW", addVW, arg) + + arg = argVW{a.x, a.z, a.y, a.c} + testFunVW(t, "subVW_g", subVW_g, arg) + testFunVW(t, "subVW", subVW, arg) + } + + shlVW_g := makeFunVW(shlVU_g) + shlVW := makeFunVW(shlVU) + for _, a := range lshVW { + arg := a + testFunVW(t, "shlVU_g", shlVW_g, arg) + testFunVW(t, "shlVU", shlVW, arg) + } + + shrVW_g := makeFunVW(shrVU_g) + shrVW := makeFunVW(shrVU) + for _, a := range rshVW { + arg := a + testFunVW(t, "shrVU_g", shrVW_g, arg) + testFunVW(t, "shrVU", shrVW, arg) + } +} + +type funVWW func(z, x []Word, y, r Word) (c Word) +type argVWW struct { + z, x nat + y, r Word + c Word +} + +var prodVWW = []argVWW{ + {}, + {nat{0}, nat{0}, 0, 0, 0}, + {nat{991}, nat{0}, 0, 991, 0}, + {nat{0}, nat{_M}, 0, 0, 0}, + {nat{991}, nat{_M}, 0, 991, 0}, + {nat{0}, nat{0}, _M, 0, 0}, + {nat{991}, nat{0}, _M, 991, 0}, + {nat{1}, nat{1}, 1, 0, 0}, + {nat{992}, nat{1}, 1, 991, 0}, + {nat{22793}, nat{991}, 23, 0, 0}, + {nat{22800}, nat{991}, 23, 7, 0}, + {nat{0, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 0, 0}, + {nat{7, 0, 0, 22793}, nat{0, 0, 0, 991}, 23, 7, 0}, + {nat{0, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 0, 0}, + {nat{991, 0, 0, 0}, nat{7893475, 7395495, 798547395, 68943}, 0, 991, 0}, + {nat{0, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 0, 0}, + {nat{991, 0, 0, 0}, nat{0, 0, 0, 0}, 894375984, 991, 0}, + {nat{_M << 1 & _M}, nat{_M}, 1 << 1, 0, _M >> (_W - 1)}, + {nat{_M<<1&_M + 1}, nat{_M}, 1 << 1, 1, _M >> (_W - 1)}, + {nat{_M << 7 & _M}, nat{_M}, 1 << 7, 0, _M >> (_W - 7)}, + {nat{_M<<7&_M + 1<<6}, nat{_M}, 1 << 7, 1 << 6, _M >> (_W - 7)}, + {nat{_M << 7 & _M, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, 0, _M >> (_W - 7)}, + {nat{_M<<7&_M + 1<<6, _M, _M, _M}, nat{_M, _M, _M, _M}, 1 << 7, 1 << 6, _M >> (_W - 7)}, +} + +func testFunVWW(t *testing.T, msg string, f funVWW, a argVWW) { + z := make(nat, len(a.z)) + c := f(z, a.x, a.y, a.r) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if c != a.c { + t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + } +} + +// TODO(gri) mulAddVWW and divWVW are symmetric operations but +// their signature is not symmetric. Try to unify. + +type funWVW func(z []Word, xn Word, x []Word, y Word) (r Word) +type argWVW struct { + z nat + xn Word + x nat + y Word + r Word +} + +func testFunWVW(t *testing.T, msg string, f funWVW, a argWVW) { + z := make(nat, len(a.z)) + r := f(z, a.xn, a.x, a.y) + for i, zi := range z { + if zi != a.z[i] { + t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) + break + } + } + if r != a.r { + t.Errorf("%s%+v\n\tgot r = %#x; want %#x", msg, a, r, a.r) + } +} + +func TestFunVWW(t *testing.T) { + for _, a := range prodVWW { + arg := a + testFunVWW(t, "mulAddVWW_g", mulAddVWW_g, arg) + testFunVWW(t, "mulAddVWW", mulAddVWW, arg) + + if a.y != 0 && a.r < a.y { + arg := argWVW{a.x, a.c, a.z, a.y, a.r} + testFunWVW(t, "divWVW_g", divWVW_g, arg) + testFunWVW(t, "divWVW", divWVW, arg) + } + } +} + +var mulWWTests = []struct { + x, y Word + q, r Word +}{ + {_M, _M, _M - 1, 1}, + // 32 bit only: {0xc47dfa8c, 50911, 0x98a4, 0x998587f4}, +} + +func TestMulWW(t *testing.T) { + for i, test := range mulWWTests { + q, r := mulWW_g(test.x, test.y) + if q != test.q || r != test.r { + t.Errorf("#%d got (%x, %x) want (%x, %x)", i, q, r, test.q, test.r) + } + } +} + +var mulAddWWWTests = []struct { + x, y, c Word + q, r Word +}{ + // TODO(agl): These will only work on 64-bit platforms. + // {15064310297182388543, 0xe7df04d2d35d5d80, 13537600649892366549, 13644450054494335067, 10832252001440893781}, + // {15064310297182388543, 0xdab2f18048baa68d, 13644450054494335067, 12869334219691522700, 14233854684711418382}, + {_M, _M, 0, _M - 1, 1}, + {_M, _M, _M, _M, 0}, +} + +func TestMulAddWWW(t *testing.T) { + for i, test := range mulAddWWWTests { + q, r := mulAddWWW_g(test.x, test.y, test.c) + if q != test.q || r != test.r { + t.Errorf("#%d got (%x, %x) want (%x, %x)", i, q, r, test.q, test.r) + } + } +} diff --git a/src/pkg/big/calibrate_test.go b/src/pkg/big/calibrate_test.go new file mode 100644 index 000000000..1cd93b105 --- /dev/null +++ b/src/pkg/big/calibrate_test.go @@ -0,0 +1,88 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file prints execution times for the Mul benchmark +// given different Karatsuba thresholds. The result may be +// used to manually fine-tune the threshold constant. The +// results are somewhat fragile; use repeated runs to get +// a clear picture. + +// Usage: gotest -calibrate + +package big + +import ( + "flag" + "fmt" + "testing" + "time" +) + +var calibrate = flag.Bool("calibrate", false, "run calibration test") + +// measure returns the time to run f +func measure(f func()) int64 { + const N = 100 + start := time.Nanoseconds() + for i := N; i > 0; i-- { + f() + } + stop := time.Nanoseconds() + return (stop - start) / N +} + +func computeThresholds() { + fmt.Printf("Multiplication times for varying Karatsuba thresholds\n") + fmt.Printf("(run repeatedly for good results)\n") + + // determine Tk, the work load execution time using basic multiplication + karatsubaThreshold = 1e9 // disable karatsuba + Tb := measure(benchmarkMulLoad) + fmt.Printf("Tb = %dns\n", Tb) + + // thresholds + n := 8 // any lower values for the threshold lead to very slow multiplies + th1 := -1 + th2 := -1 + + var deltaOld int64 + for count := -1; count != 0; count-- { + // determine Tk, the work load execution time using Karatsuba multiplication + karatsubaThreshold = n // enable karatsuba + Tk := measure(benchmarkMulLoad) + + // improvement over Tb + delta := (Tb - Tk) * 100 / Tb + + fmt.Printf("n = %3d Tk = %8dns %4d%%", n, Tk, delta) + + // determine break-even point + if Tk < Tb && th1 < 0 { + th1 = n + fmt.Print(" break-even point") + } + + // determine diminishing return + if 0 < delta && delta < deltaOld && th2 < 0 { + th2 = n + fmt.Print(" diminishing return") + } + deltaOld = delta + + fmt.Println() + + // trigger counter + if th1 >= 0 && th2 >= 0 && count < 0 { + count = 20 // this many extra measurements after we got both thresholds + } + + n++ + } +} + +func TestCalibrate(t *testing.T) { + if *calibrate { + computeThresholds() + } +} diff --git a/src/pkg/big/hilbert_test.go b/src/pkg/big/hilbert_test.go new file mode 100644 index 000000000..1a84341b3 --- /dev/null +++ b/src/pkg/big/hilbert_test.go @@ -0,0 +1,160 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A little test program and benchmark for rational arithmetics. +// Computes a Hilbert matrix, its inverse, multiplies them +// and verifies that the product is the identity matrix. + +package big + +import ( + "fmt" + "testing" +) + +type matrix struct { + n, m int + a []*Rat +} + +func (a *matrix) at(i, j int) *Rat { + if !(0 <= i && i < a.n && 0 <= j && j < a.m) { + panic("index out of range") + } + return a.a[i*a.m+j] +} + +func (a *matrix) set(i, j int, x *Rat) { + if !(0 <= i && i < a.n && 0 <= j && j < a.m) { + panic("index out of range") + } + a.a[i*a.m+j] = x +} + +func newMatrix(n, m int) *matrix { + if !(0 <= n && 0 <= m) { + panic("illegal matrix") + } + a := new(matrix) + a.n = n + a.m = m + a.a = make([]*Rat, n*m) + return a +} + +func newUnit(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + x := NewRat(0, 1) + if i == j { + x.SetInt64(1) + } + a.set(i, j, x) + } + } + return a +} + +func newHilbert(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + a.set(i, j, NewRat(1, int64(i+j+1))) + } + } + return a +} + +func newInverseHilbert(n int) *matrix { + a := newMatrix(n, n) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + x1 := new(Rat).SetInt64(int64(i + j + 1)) + x2 := new(Rat).SetInt(new(Int).Binomial(int64(n+i), int64(n-j-1))) + x3 := new(Rat).SetInt(new(Int).Binomial(int64(n+j), int64(n-i-1))) + x4 := new(Rat).SetInt(new(Int).Binomial(int64(i+j), int64(i))) + + x1.Mul(x1, x2) + x1.Mul(x1, x3) + x1.Mul(x1, x4) + x1.Mul(x1, x4) + + if (i+j)&1 != 0 { + x1.Neg(x1) + } + + a.set(i, j, x1) + } + } + return a +} + +func (a *matrix) mul(b *matrix) *matrix { + if a.m != b.n { + panic("illegal matrix multiply") + } + c := newMatrix(a.n, b.m) + for i := 0; i < c.n; i++ { + for j := 0; j < c.m; j++ { + x := NewRat(0, 1) + for k := 0; k < a.m; k++ { + x.Add(x, new(Rat).Mul(a.at(i, k), b.at(k, j))) + } + c.set(i, j, x) + } + } + return c +} + +func (a *matrix) eql(b *matrix) bool { + if a.n != b.n || a.m != b.m { + return false + } + for i := 0; i < a.n; i++ { + for j := 0; j < a.m; j++ { + if a.at(i, j).Cmp(b.at(i, j)) != 0 { + return false + } + } + } + return true +} + +func (a *matrix) String() string { + s := "" + for i := 0; i < a.n; i++ { + for j := 0; j < a.m; j++ { + s += fmt.Sprintf("\t%s", a.at(i, j)) + } + s += "\n" + } + return s +} + +func doHilbert(t *testing.T, n int) { + a := newHilbert(n) + b := newInverseHilbert(n) + I := newUnit(n) + ab := a.mul(b) + if !ab.eql(I) { + if t == nil { + panic("Hilbert failed") + } + t.Errorf("a = %s\n", a) + t.Errorf("b = %s\n", b) + t.Errorf("a*b = %s\n", ab) + t.Errorf("I = %s\n", I) + } +} + +func TestHilbert(t *testing.T) { + doHilbert(t, 10) +} + +func BenchmarkHilbert(b *testing.B) { + for i := 0; i < b.N; i++ { + doHilbert(nil, 10) + } +} diff --git a/src/pkg/big/int.go b/src/pkg/big/int.go new file mode 100755 index 000000000..701b69715 --- /dev/null +++ b/src/pkg/big/int.go @@ -0,0 +1,868 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements signed multi-precision integers. + +package big + +import ( + "fmt" + "io" + "os" + "rand" + "strings" +) + +// An Int represents a signed multi-precision integer. +// The zero value for an Int represents the value 0. +type Int struct { + neg bool // sign + abs nat // absolute value of the integer +} + +var intOne = &Int{false, natOne} + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Int) Sign() int { + if len(x.abs) == 0 { + return 0 + } + if x.neg { + return -1 + } + return 1 +} + +// SetInt64 sets z to x and returns z. +func (z *Int) SetInt64(x int64) *Int { + neg := false + if x < 0 { + neg = true + x = -x + } + z.abs = z.abs.setUint64(uint64(x)) + z.neg = neg + return z +} + +// NewInt allocates and returns a new Int set to x. +func NewInt(x int64) *Int { + return new(Int).SetInt64(x) +} + +// Set sets z to x and returns z. +func (z *Int) Set(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = x.neg + return z +} + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Int) Abs(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = false + return z +} + +// Neg sets z to -x and returns z. +func (z *Int) Neg(x *Int) *Int { + z.abs = z.abs.set(x.abs) + z.neg = len(z.abs) > 0 && !x.neg // 0 has no sign + return z +} + +// Add sets z to the sum x+y and returns z. +func (z *Int) Add(x, y *Int) *Int { + neg := x.neg + if x.neg == y.neg { + // x + y == x + y + // (-x) + (-y) == -(x + y) + z.abs = z.abs.add(x.abs, y.abs) + } else { + // x + (-y) == x - y == -(y - x) + // (-x) + y == y - x == -(x - y) + if x.abs.cmp(y.abs) >= 0 { + z.abs = z.abs.sub(x.abs, y.abs) + } else { + neg = !neg + z.abs = z.abs.sub(y.abs, x.abs) + } + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + return z +} + +// Sub sets z to the difference x-y and returns z. +func (z *Int) Sub(x, y *Int) *Int { + neg := x.neg + if x.neg != y.neg { + // x - (-y) == x + y + // (-x) - y == -(x + y) + z.abs = z.abs.add(x.abs, y.abs) + } else { + // x - y == x - y == -(y - x) + // (-x) - (-y) == y - x == -(x - y) + if x.abs.cmp(y.abs) >= 0 { + z.abs = z.abs.sub(x.abs, y.abs) + } else { + neg = !neg + z.abs = z.abs.sub(y.abs, x.abs) + } + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + return z +} + +// Mul sets z to the product x*y and returns z. +func (z *Int) Mul(x, y *Int) *Int { + // x * y == x * y + // x * (-y) == -(x * y) + // (-x) * y == -(x * y) + // (-x) * (-y) == x * y + z.abs = z.abs.mul(x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign + return z +} + +// MulRange sets z to the product of all integers +// in the range [a, b] inclusively and returns z. +// If a > b (empty range), the result is 1. +func (z *Int) MulRange(a, b int64) *Int { + switch { + case a > b: + return z.SetInt64(1) // empty range + case a <= 0 && b >= 0: + return z.SetInt64(0) // range includes 0 + } + // a <= b && (b < 0 || a > 0) + + neg := false + if a < 0 { + neg = (b-a)&1 == 0 + a, b = -b, -a + } + + z.abs = z.abs.mulRange(uint64(a), uint64(b)) + z.neg = neg + return z +} + +// Binomial sets z to the binomial coefficient of (n, k) and returns z. +func (z *Int) Binomial(n, k int64) *Int { + var a, b Int + a.MulRange(n-k+1, n) + b.MulRange(1, k) + return z.Quo(&a, &b) +} + +// Quo sets z to the quotient x/y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See QuoRem for more details. +func (z *Int) Quo(x, y *Int) *Int { + z.abs, _ = z.abs.div(nil, x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign + return z +} + +// Rem sets z to the remainder x%y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See QuoRem for more details. +func (z *Int) Rem(x, y *Int) *Int { + _, z.abs = nat(nil).div(z.abs, x.abs, y.abs) + z.neg = len(z.abs) > 0 && x.neg // 0 has no sign + return z +} + +// QuoRem sets z to the quotient x/y and r to the remainder x%y +// and returns the pair (z, r) for y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +// +// QuoRem implements T-division and modulus (like Go): +// +// q = x/y with the result truncated to zero +// r = x - y*q +// +// (See Daan Leijen, ``Division and Modulus for Computer Scientists''.) +// +func (z *Int) QuoRem(x, y, r *Int) (*Int, *Int) { + z.abs, r.abs = z.abs.div(r.abs, x.abs, y.abs) + z.neg, r.neg = len(z.abs) > 0 && x.neg != y.neg, len(r.abs) > 0 && x.neg // 0 has no sign + return z, r +} + +// Div sets z to the quotient x/y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See DivMod for more details. +func (z *Int) Div(x, y *Int) *Int { + y_neg := y.neg // z may be an alias for y + var r Int + z.QuoRem(x, y, &r) + if r.neg { + if y_neg { + z.Add(z, intOne) + } else { + z.Sub(z, intOne) + } + } + return z +} + +// Mod sets z to the modulus x%y for y != 0 and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +// See DivMod for more details. +func (z *Int) Mod(x, y *Int) *Int { + y0 := y // save y + if z == y || alias(z.abs, y.abs) { + y0 = new(Int).Set(y) + } + var q Int + q.QuoRem(x, y, z) + if z.neg { + if y0.neg { + z.Sub(z, y0) + } else { + z.Add(z, y0) + } + } + return z +} + +// DivMod sets z to the quotient x div y and m to the modulus x mod y +// and returns the pair (z, m) for y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +// +// DivMod implements Euclidean division and modulus (unlike Go): +// +// q = x div y such that +// m = x - y*q with 0 <= m < |q| +// +// (See Raymond T. Boute, ``The Euclidean definition of the functions +// div and mod''. ACM Transactions on Programming Languages and +// Systems (TOPLAS), 14(2):127-144, New York, NY, USA, 4/1992. +// ACM press.) +// +func (z *Int) DivMod(x, y, m *Int) (*Int, *Int) { + y0 := y // save y + if z == y || alias(z.abs, y.abs) { + y0 = new(Int).Set(y) + } + z.QuoRem(x, y, m) + if m.neg { + if y0.neg { + z.Add(z, intOne) + m.Sub(m, y0) + } else { + z.Sub(z, intOne) + m.Add(m, y0) + } + } + return z, m +} + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Int) Cmp(y *Int) (r int) { + // x cmp y == x cmp y + // x cmp (-y) == x + // (-x) cmp y == y + // (-x) cmp (-y) == -(x cmp y) + switch { + case x.neg == y.neg: + r = x.abs.cmp(y.abs) + if x.neg { + r = -r + } + case x.neg: + r = -1 + default: + r = 1 + } + return +} + +func (x *Int) String() string { + switch { + case x == nil: + return "" + case x.neg: + return "-" + x.abs.decimalString() + } + return x.abs.decimalString() +} + +func charset(ch int) string { + switch ch { + case 'b': + return lowercaseDigits[0:2] + case 'o': + return lowercaseDigits[0:8] + case 'd', 's', 'v': + return lowercaseDigits[0:10] + case 'x': + return lowercaseDigits[0:16] + case 'X': + return uppercaseDigits[0:16] + } + return "" // unknown format +} + +// write count copies of text to s +func writeMultiple(s fmt.State, text string, count int) { + if len(text) > 0 { + b := []byte(text) + for ; count > 0; count-- { + s.Write(b) + } + } +} + +// Format is a support routine for fmt.Formatter. It accepts +// the formats 'b' (binary), 'o' (octal), 'd' (decimal), 'x' +// (lowercase hexadecimal), and 'X' (uppercase hexadecimal). +// Also supported are the full suite of package fmt's format +// verbs for integral types, including '+', '-', and ' ' +// for sign control, '#' for leading zero in octal and for +// hexadecimal, a leading "0x" or "0X" for "%#x" and "%#X" +// respectively, specification of minimum digits precision, +// output field width, space or zero padding, and left or +// right justification. +// +func (x *Int) Format(s fmt.State, ch int) { + cs := charset(ch) + + // special cases + switch { + case cs == "": + // unknown format + fmt.Fprintf(s, "%%!%c(big.Int=%s)", ch, x.String()) + return + case x == nil: + fmt.Fprint(s, "") + return + } + + // determine sign character + sign := "" + switch { + case x.neg: + sign = "-" + case s.Flag('+'): // supersedes ' ' when both specified + sign = "+" + case s.Flag(' '): + sign = " " + } + + // determine prefix characters for indicating output base + prefix := "" + if s.Flag('#') { + switch ch { + case 'o': // octal + prefix = "0" + case 'x': // hexadecimal + prefix = "0x" + case 'X': + prefix = "0X" + } + } + + // determine digits with base set by len(cs) and digit characters from cs + digits := x.abs.string(cs) + + // number of characters for the three classes of number padding + var left int // space characters to left of digits for right justification ("%8d") + var zeroes int // zero characters (actually cs[0]) as left-most digits ("%.8d") + var right int // space characters to right of digits for left justification ("%-8d") + + // determine number padding from precision: the least number of digits to output + precision, precisionSet := s.Precision() + if precisionSet { + switch { + case len(digits) < precision: + zeroes = precision - len(digits) // count of zero padding + case digits == "0" && precision == 0: + return // print nothing if zero value (x == 0) and zero precision ("." or ".0") + } + } + + // determine field pad from width: the least number of characters to output + length := len(sign) + len(prefix) + zeroes + len(digits) + if width, widthSet := s.Width(); widthSet && length < width { // pad as specified + switch d := width - length; { + case s.Flag('-'): + // pad on the right with spaces; supersedes '0' when both specified + right = d + case s.Flag('0') && !precisionSet: + // pad with zeroes unless precision also specified + zeroes = d + default: + // pad on the left with spaces + left = d + } + } + + // print number as [left pad][sign][prefix][zero pad][digits][right pad] + writeMultiple(s, " ", left) + writeMultiple(s, sign, 1) + writeMultiple(s, prefix, 1) + writeMultiple(s, "0", zeroes) + writeMultiple(s, digits, 1) + writeMultiple(s, " ", right) +} + +// scan sets z to the integer value corresponding to the longest possible prefix +// read from r representing a signed integer number in a given conversion base. +// It returns z, the actual conversion base used, and an error, if any. In the +// error case, the value of z is undefined. The syntax follows the syntax of +// integer literals in Go. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z *Int) scan(r io.RuneScanner, base int) (*Int, int, os.Error) { + // determine sign + ch, _, err := r.ReadRune() + if err != nil { + return z, 0, err + } + neg := false + switch ch { + case '-': + neg = true + case '+': // nothing to do + default: + r.UnreadRune() + } + + // determine mantissa + z.abs, base, err = z.abs.scan(r, base) + if err != nil { + return z, base, err + } + z.neg = len(z.abs) > 0 && neg // 0 has no sign + + return z, base, nil +} + +// Scan is a support routine for fmt.Scanner; it sets z to the value of +// the scanned number. It accepts the formats 'b' (binary), 'o' (octal), +// 'd' (decimal), 'x' (lowercase hexadecimal), and 'X' (uppercase hexadecimal). +func (z *Int) Scan(s fmt.ScanState, ch int) os.Error { + s.SkipSpace() // skip leading space characters + base := 0 + switch ch { + case 'b': + base = 2 + case 'o': + base = 8 + case 'd': + base = 10 + case 'x', 'X': + base = 16 + case 's', 'v': + // let scan determine the base + default: + return os.NewError("Int.Scan: invalid verb") + } + _, _, err := z.scan(s, base) + return err +} + +// Int64 returns the int64 representation of x. +// If x cannot be represented in an int64, the result is undefined. +func (x *Int) Int64() int64 { + if len(x.abs) == 0 { + return 0 + } + v := int64(x.abs[0]) + if _W == 32 && len(x.abs) > 1 { + v |= int64(x.abs[1]) << 32 + } + if x.neg { + v = -v + } + return v +} + +// SetString sets z to the value of s, interpreted in the given base, +// and returns z and a boolean indicating success. If SetString fails, +// the value of z is undefined. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z *Int) SetString(s string, base int) (*Int, bool) { + r := strings.NewReader(s) + _, _, err := z.scan(r, base) + if err != nil { + return z, false + } + _, _, err = r.ReadRune() + return z, err == os.EOF // err == os.EOF => scan consumed all of s +} + +// SetBytes interprets buf as the bytes of a big-endian unsigned +// integer, sets z to that value, and returns z. +func (z *Int) SetBytes(buf []byte) *Int { + z.abs = z.abs.setBytes(buf) + z.neg = false + return z +} + +// Bytes returns the absolute value of z as a big-endian byte slice. +func (z *Int) Bytes() []byte { + buf := make([]byte, len(z.abs)*_S) + return buf[z.abs.bytes(buf):] +} + +// BitLen returns the length of the absolute value of z in bits. +// The bit length of 0 is 0. +func (z *Int) BitLen() int { + return z.abs.bitLen() +} + +// Exp sets z = x**y mod m. If m is nil, z = x**y. +// See Knuth, volume 2, section 4.6.3. +func (z *Int) Exp(x, y, m *Int) *Int { + if y.neg || len(y.abs) == 0 { + neg := x.neg + z.SetInt64(1) + z.neg = neg + return z + } + + var mWords nat + if m != nil { + mWords = m.abs + } + + z.abs = z.abs.expNN(x.abs, y.abs, mWords) + z.neg = len(z.abs) > 0 && x.neg && y.abs[0]&1 == 1 // 0 has no sign + return z +} + +// GcdInt sets d to the greatest common divisor of a and b, which must be +// positive numbers. +// If x and y are not nil, GcdInt sets x and y such that d = a*x + b*y. +// If either a or b is not positive, GcdInt sets d = x = y = 0. +func GcdInt(d, x, y, a, b *Int) { + if a.neg || b.neg { + d.SetInt64(0) + if x != nil { + x.SetInt64(0) + } + if y != nil { + y.SetInt64(0) + } + return + } + + A := new(Int).Set(a) + B := new(Int).Set(b) + + X := new(Int) + Y := new(Int).SetInt64(1) + + lastX := new(Int).SetInt64(1) + lastY := new(Int) + + q := new(Int) + temp := new(Int) + + for len(B.abs) > 0 { + r := new(Int) + q, r = q.QuoRem(A, B, r) + + A, B = B, r + + temp.Set(X) + X.Mul(X, q) + X.neg = !X.neg + X.Add(X, lastX) + lastX.Set(temp) + + temp.Set(Y) + Y.Mul(Y, q) + Y.neg = !Y.neg + Y.Add(Y, lastY) + lastY.Set(temp) + } + + if x != nil { + *x = *lastX + } + + if y != nil { + *y = *lastY + } + + *d = *A +} + +// ProbablyPrime performs n Miller-Rabin tests to check whether z is prime. +// If it returns true, z is prime with probability 1 - 1/4^n. +// If it returns false, z is not prime. +func ProbablyPrime(z *Int, n int) bool { + return !z.neg && z.abs.probablyPrime(n) +} + +// Rand sets z to a pseudo-random number in [0, n) and returns z. +func (z *Int) Rand(rnd *rand.Rand, n *Int) *Int { + z.neg = false + if n.neg == true || len(n.abs) == 0 { + z.abs = nil + return z + } + z.abs = z.abs.random(rnd, n.abs, n.abs.bitLen()) + return z +} + +// ModInverse sets z to the multiplicative inverse of g in the group ℤ/pℤ (where +// p is a prime) and returns z. +func (z *Int) ModInverse(g, p *Int) *Int { + var d Int + GcdInt(&d, z, nil, g, p) + // x and y are such that g*x + p*y = d. Since p is prime, d = 1. Taking + // that modulo p results in g*x = 1, therefore x is the inverse element. + if z.neg { + z.Add(z, p) + } + return z +} + +// Lsh sets z = x << n and returns z. +func (z *Int) Lsh(x *Int, n uint) *Int { + z.abs = z.abs.shl(x.abs, n) + z.neg = x.neg + return z +} + +// Rsh sets z = x >> n and returns z. +func (z *Int) Rsh(x *Int, n uint) *Int { + if x.neg { + // (-x) >> s == ^(x-1) >> s == ^((x-1) >> s) == -(((x-1) >> s) + 1) + t := z.abs.sub(x.abs, natOne) // no underflow because |x| > 0 + t = t.shr(t, n) + z.abs = t.add(t, natOne) + z.neg = true // z cannot be zero if x is negative + return z + } + + z.abs = z.abs.shr(x.abs, n) + z.neg = false + return z +} + +// Bit returns the value of the i'th bit of z. That is, it +// returns (z>>i)&1. The bit index i must be >= 0. +func (z *Int) Bit(i int) uint { + if i < 0 { + panic("negative bit index") + } + if z.neg { + t := nat{}.sub(z.abs, natOne) + return t.bit(uint(i)) ^ 1 + } + + return z.abs.bit(uint(i)) +} + +// SetBit sets the i'th bit of z to bit and returns z. +// That is, if bit is 1 SetBit sets z = x | (1 << i); +// if bit is 0 it sets z = x &^ (1 << i). If bit is not 0 or 1, +// SetBit will panic. +func (z *Int) SetBit(x *Int, i int, b uint) *Int { + if i < 0 { + panic("negative bit index") + } + if x.neg { + t := z.abs.sub(x.abs, natOne) + t = t.setBit(t, uint(i), b^1) + z.abs = t.add(t, natOne) + z.neg = len(z.abs) > 0 + return z + } + z.abs = z.abs.setBit(x.abs, uint(i), b) + z.neg = false + return z +} + +// And sets z = x & y and returns z. +func (z *Int) And(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) & (-y) == ^(x-1) & ^(y-1) == ^((x-1) | (y-1)) == -(((x-1) | (y-1)) + 1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.or(x1, y1), natOne) + z.neg = true // z cannot be zero if x and y are negative + return z + } + + // x & y == x & y + z.abs = z.abs.and(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // & is symmetric + } + + // x & (-y) == x & ^(y-1) == x &^ (y-1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.andNot(x.abs, y1) + z.neg = false + return z +} + +// AndNot sets z = x &^ y and returns z. +func (z *Int) AndNot(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) &^ (-y) == ^(x-1) &^ ^(y-1) == ^(x-1) & (y-1) == (y-1) &^ (x-1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.andNot(y1, x1) + z.neg = false + return z + } + + // x &^ y == x &^ y + z.abs = z.abs.andNot(x.abs, y.abs) + z.neg = false + return z + } + + if x.neg { + // (-x) &^ y == ^(x-1) &^ y == ^(x-1) & ^y == ^((x-1) | y) == -(((x-1) | y) + 1) + x1 := nat{}.sub(x.abs, natOne) + z.abs = z.abs.add(z.abs.or(x1, y.abs), natOne) + z.neg = true // z cannot be zero if x is negative and y is positive + return z + } + + // x &^ (-y) == x &^ ^(y-1) == x & (y-1) + y1 := nat{}.add(y.abs, natOne) + z.abs = z.abs.and(x.abs, y1) + z.neg = false + return z +} + +// Or sets z = x | y and returns z. +func (z *Int) Or(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) | (-y) == ^(x-1) | ^(y-1) == ^((x-1) & (y-1)) == -(((x-1) & (y-1)) + 1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.and(x1, y1), natOne) + z.neg = true // z cannot be zero if x and y are negative + return z + } + + // x | y == x | y + z.abs = z.abs.or(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // | is symmetric + } + + // x | (-y) == x | ^(y-1) == ^((y-1) &^ x) == -(^((y-1) &^ x) + 1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.andNot(y1, x.abs), natOne) + z.neg = true // z cannot be zero if one of x or y is negative + return z +} + +// Xor sets z = x ^ y and returns z. +func (z *Int) Xor(x, y *Int) *Int { + if x.neg == y.neg { + if x.neg { + // (-x) ^ (-y) == ^(x-1) ^ ^(y-1) == (x-1) ^ (y-1) + x1 := nat{}.sub(x.abs, natOne) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.xor(x1, y1) + z.neg = false + return z + } + + // x ^ y == x ^ y + z.abs = z.abs.xor(x.abs, y.abs) + z.neg = false + return z + } + + // x.neg != y.neg + if x.neg { + x, y = y, x // ^ is symmetric + } + + // x ^ (-y) == x ^ ^(y-1) == ^(x ^ (y-1)) == -((x ^ (y-1)) + 1) + y1 := nat{}.sub(y.abs, natOne) + z.abs = z.abs.add(z.abs.xor(x.abs, y1), natOne) + z.neg = true // z cannot be zero if only one of x or y is negative + return z +} + +// Not sets z = ^x and returns z. +func (z *Int) Not(x *Int) *Int { + if x.neg { + // ^(-x) == ^(^(x-1)) == x-1 + z.abs = z.abs.sub(x.abs, natOne) + z.neg = false + return z + } + + // ^x == -x-1 == -(x+1) + z.abs = z.abs.add(x.abs, natOne) + z.neg = true // z cannot be zero if x is positive + return z +} + +// Gob codec version. Permits backward-compatible changes to the encoding. +const intGobVersion byte = 1 + +// GobEncode implements the gob.GobEncoder interface. +func (z *Int) GobEncode() ([]byte, os.Error) { + buf := make([]byte, 1+len(z.abs)*_S) // extra byte for version and sign bit + i := z.abs.bytes(buf) - 1 // i >= 0 + b := intGobVersion << 1 // make space for sign bit + if z.neg { + b |= 1 + } + buf[i] = b + return buf[i:], nil +} + +// GobDecode implements the gob.GobDecoder interface. +func (z *Int) GobDecode(buf []byte) os.Error { + if len(buf) == 0 { + return os.NewError("Int.GobDecode: no data") + } + b := buf[0] + if b>>1 != intGobVersion { + return os.NewError(fmt.Sprintf("Int.GobDecode: encoding version %d not supported", b>>1)) + } + z.neg = b&1 != 0 + z.abs = z.abs.setBytes(buf[1:]) + return nil +} diff --git a/src/pkg/big/int_test.go b/src/pkg/big/int_test.go new file mode 100755 index 000000000..03446d6ae --- /dev/null +++ b/src/pkg/big/int_test.go @@ -0,0 +1,1391 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "bytes" + "encoding/hex" + "fmt" + "gob" + "testing" + "testing/quick" +) + +func isNormalized(x *Int) bool { + if len(x.abs) == 0 { + return !x.neg + } + // len(x.abs) > 0 + return x.abs[len(x.abs)-1] != 0 +} + +type funZZ func(z, x, y *Int) *Int +type argZZ struct { + z, x, y *Int +} + +var sumZZ = []argZZ{ + {NewInt(0), NewInt(0), NewInt(0)}, + {NewInt(1), NewInt(1), NewInt(0)}, + {NewInt(1111111110), NewInt(123456789), NewInt(987654321)}, + {NewInt(-1), NewInt(-1), NewInt(0)}, + {NewInt(864197532), NewInt(-123456789), NewInt(987654321)}, + {NewInt(-1111111110), NewInt(-123456789), NewInt(-987654321)}, +} + +var prodZZ = []argZZ{ + {NewInt(0), NewInt(0), NewInt(0)}, + {NewInt(0), NewInt(1), NewInt(0)}, + {NewInt(1), NewInt(1), NewInt(1)}, + {NewInt(-991 * 991), NewInt(991), NewInt(-991)}, + // TODO(gri) add larger products +} + +func TestSignZ(t *testing.T) { + var zero Int + for _, a := range sumZZ { + s := a.z.Sign() + e := a.z.Cmp(&zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, a.z) + } + } +} + +func TestSetZ(t *testing.T) { + for _, a := range sumZZ { + var z Int + z.Set(a.z) + if !isNormalized(&z) { + t.Errorf("%v is not normalized", z) + } + if (&z).Cmp(a.z) != 0 { + t.Errorf("got z = %v; want %v", z, a.z) + } + } +} + +func TestAbsZ(t *testing.T) { + var zero Int + for _, a := range sumZZ { + var z Int + z.Abs(a.z) + var e Int + e.Set(a.z) + if e.Cmp(&zero) < 0 { + e.Sub(&zero, &e) + } + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", z, e) + } + } +} + +func testFunZZ(t *testing.T, msg string, f funZZ, a argZZ) { + var z Int + f(&z, a.x, a.y) + if !isNormalized(&z) { + t.Errorf("%s%v is not normalized", z, msg) + } + if (&z).Cmp(a.z) != 0 { + t.Errorf("%s%+v\n\tgot z = %v; want %v", msg, a, &z, a.z) + } +} + +func TestSumZZ(t *testing.T) { + AddZZ := func(z, x, y *Int) *Int { return z.Add(x, y) } + SubZZ := func(z, x, y *Int) *Int { return z.Sub(x, y) } + for _, a := range sumZZ { + arg := a + testFunZZ(t, "AddZZ", AddZZ, arg) + + arg = argZZ{a.z, a.y, a.x} + testFunZZ(t, "AddZZ symmetric", AddZZ, arg) + + arg = argZZ{a.x, a.z, a.y} + testFunZZ(t, "SubZZ", SubZZ, arg) + + arg = argZZ{a.y, a.z, a.x} + testFunZZ(t, "SubZZ symmetric", SubZZ, arg) + } +} + +func TestProdZZ(t *testing.T) { + MulZZ := func(z, x, y *Int) *Int { return z.Mul(x, y) } + for _, a := range prodZZ { + arg := a + testFunZZ(t, "MulZZ", MulZZ, arg) + + arg = argZZ{a.z, a.y, a.x} + testFunZZ(t, "MulZZ symmetric", MulZZ, arg) + } +} + +// mulBytes returns x*y via grade school multiplication. Both inputs +// and the result are assumed to be in big-endian representation (to +// match the semantics of Int.Bytes and Int.SetBytes). +func mulBytes(x, y []byte) []byte { + z := make([]byte, len(x)+len(y)) + + // multiply + k0 := len(z) - 1 + for j := len(y) - 1; j >= 0; j-- { + d := int(y[j]) + if d != 0 { + k := k0 + carry := 0 + for i := len(x) - 1; i >= 0; i-- { + t := int(z[k]) + int(x[i])*d + carry + z[k], carry = byte(t), t>>8 + k-- + } + z[k] = byte(carry) + } + k0-- + } + + // normalize (remove leading 0's) + i := 0 + for i < len(z) && z[i] == 0 { + i++ + } + + return z[i:] +} + +func checkMul(a, b []byte) bool { + var x, y, z1 Int + x.SetBytes(a) + y.SetBytes(b) + z1.Mul(&x, &y) + + var z2 Int + z2.SetBytes(mulBytes(a, b)) + + return z1.Cmp(&z2) == 0 +} + +func TestMul(t *testing.T) { + if err := quick.Check(checkMul, nil); err != nil { + t.Error(err) + } +} + +var mulRangesZ = []struct { + a, b int64 + prod string +}{ + // entirely positive ranges are covered by mulRangesN + {-1, 1, "0"}, + {-2, -1, "2"}, + {-3, -2, "6"}, + {-3, -1, "-6"}, + {1, 3, "6"}, + {-10, -10, "-10"}, + {0, -1, "1"}, // empty range + {-1, -100, "1"}, // empty range + {-1, 1, "0"}, // range includes 0 + {-1e9, 0, "0"}, // range includes 0 + {-1e9, 1e9, "0"}, // range includes 0 + {-10, -1, "3628800"}, // 10! + {-20, -2, "-2432902008176640000"}, // -20! + {-99, -1, + "-933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "511852109168640000000000000000000000", // -99! + }, +} + +func TestMulRangeZ(t *testing.T) { + var tmp Int + // test entirely positive ranges + for i, r := range mulRangesN { + prod := tmp.MulRange(int64(r.a), int64(r.b)).String() + if prod != r.prod { + t.Errorf("#%da: got %s; want %s", i, prod, r.prod) + } + } + // test other ranges + for i, r := range mulRangesZ { + prod := tmp.MulRange(r.a, r.b).String() + if prod != r.prod { + t.Errorf("#%db: got %s; want %s", i, prod, r.prod) + } + } +} + +var stringTests = []struct { + in string + out string + base int + val int64 + ok bool +}{ + {in: "", ok: false}, + {in: "a", ok: false}, + {in: "z", ok: false}, + {in: "+", ok: false}, + {in: "-", ok: false}, + {in: "0b", ok: false}, + {in: "0x", ok: false}, + {in: "2", base: 2, ok: false}, + {in: "0b2", base: 0, ok: false}, + {in: "08", ok: false}, + {in: "8", base: 8, ok: false}, + {in: "0xg", base: 0, ok: false}, + {in: "g", base: 16, ok: false}, + {"0", "0", 0, 0, true}, + {"0", "0", 10, 0, true}, + {"0", "0", 16, 0, true}, + {"+0", "0", 0, 0, true}, + {"-0", "0", 0, 0, true}, + {"10", "10", 0, 10, true}, + {"10", "10", 10, 10, true}, + {"10", "10", 16, 16, true}, + {"-10", "-10", 16, -16, true}, + {"+10", "10", 16, 16, true}, + {"0x10", "16", 0, 16, true}, + {in: "0x10", base: 16, ok: false}, + {"-0x10", "-16", 0, -16, true}, + {"+0x10", "16", 0, 16, true}, + {"00", "0", 0, 0, true}, + {"0", "0", 8, 0, true}, + {"07", "7", 0, 7, true}, + {"7", "7", 8, 7, true}, + {"023", "19", 0, 19, true}, + {"23", "23", 8, 19, true}, + {"cafebabe", "cafebabe", 16, 0xcafebabe, true}, + {"0b0", "0", 0, 0, true}, + {"-111", "-111", 2, -7, true}, + {"-0b111", "-7", 0, -7, true}, + {"0b1001010111", "599", 0, 0x257, true}, + {"1001010111", "1001010111", 2, 0x257, true}, +} + +func format(base int) string { + switch base { + case 2: + return "%b" + case 8: + return "%o" + case 16: + return "%x" + } + return "%d" +} + +func TestGetString(t *testing.T) { + z := new(Int) + for i, test := range stringTests { + if !test.ok { + continue + } + z.SetInt64(test.val) + + if test.base == 10 { + s := z.String() + if s != test.out { + t.Errorf("#%da got %s; want %s", i, s, test.out) + } + } + + s := fmt.Sprintf(format(test.base), z) + if s != test.out { + t.Errorf("#%db got %s; want %s", i, s, test.out) + } + } +} + +func TestSetString(t *testing.T) { + tmp := new(Int) + for i, test := range stringTests { + n1, ok1 := new(Int).SetString(test.in, test.base) + n2, ok2 := tmp.SetString(test.in, test.base) + expected := NewInt(test.val) + if ok1 != test.ok || ok2 != test.ok { + t.Errorf("#%d (input '%s') ok incorrect (should be %t)", i, test.in, test.ok) + continue + } + if !ok1 || !ok2 { + continue + } + + if ok1 && !isNormalized(n1) { + t.Errorf("#%d (input '%s'): %v is not normalized", i, test.in, *n1) + } + if ok2 && !isNormalized(n2) { + t.Errorf("#%d (input '%s'): %v is not normalized", i, test.in, *n2) + } + + if n1.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n1, test.val) + } + if n2.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n2, test.val) + } + } +} + +var formatTests = []struct { + input string + format string + output string +}{ + {"", "%x", ""}, + {"", "%#x", ""}, + {"", "%#y", "%!y(big.Int=)"}, + + {"10", "%b", "1010"}, + {"10", "%o", "12"}, + {"10", "%d", "10"}, + {"10", "%v", "10"}, + {"10", "%x", "a"}, + {"10", "%X", "A"}, + {"-10", "%X", "-A"}, + {"10", "%y", "%!y(big.Int=10)"}, + {"-10", "%y", "%!y(big.Int=-10)"}, + + {"10", "%#b", "1010"}, + {"10", "%#o", "012"}, + {"10", "%#d", "10"}, + {"10", "%#v", "10"}, + {"10", "%#x", "0xa"}, + {"10", "%#X", "0XA"}, + {"-10", "%#X", "-0XA"}, + {"10", "%#y", "%!y(big.Int=10)"}, + {"-10", "%#y", "%!y(big.Int=-10)"}, + + {"1234", "%d", "1234"}, + {"1234", "%3d", "1234"}, + {"1234", "%4d", "1234"}, + {"-1234", "%d", "-1234"}, + {"1234", "% 5d", " 1234"}, + {"1234", "%+5d", "+1234"}, + {"1234", "%-5d", "1234 "}, + {"1234", "%x", "4d2"}, + {"1234", "%X", "4D2"}, + {"-1234", "%3x", "-4d2"}, + {"-1234", "%4x", "-4d2"}, + {"-1234", "%5x", " -4d2"}, + {"-1234", "%-5x", "-4d2 "}, + {"1234", "%03d", "1234"}, + {"1234", "%04d", "1234"}, + {"1234", "%05d", "01234"}, + {"1234", "%06d", "001234"}, + {"-1234", "%06d", "-01234"}, + {"1234", "%+06d", "+01234"}, + {"1234", "% 06d", " 01234"}, + {"1234", "%-6d", "1234 "}, + {"1234", "%-06d", "1234 "}, + {"-1234", "%-06d", "-1234 "}, + + {"1234", "%.3d", "1234"}, + {"1234", "%.4d", "1234"}, + {"1234", "%.5d", "01234"}, + {"1234", "%.6d", "001234"}, + {"-1234", "%.3d", "-1234"}, + {"-1234", "%.4d", "-1234"}, + {"-1234", "%.5d", "-01234"}, + {"-1234", "%.6d", "-001234"}, + + {"1234", "%8.3d", " 1234"}, + {"1234", "%8.4d", " 1234"}, + {"1234", "%8.5d", " 01234"}, + {"1234", "%8.6d", " 001234"}, + {"-1234", "%8.3d", " -1234"}, + {"-1234", "%8.4d", " -1234"}, + {"-1234", "%8.5d", " -01234"}, + {"-1234", "%8.6d", " -001234"}, + + {"1234", "%+8.3d", " +1234"}, + {"1234", "%+8.4d", " +1234"}, + {"1234", "%+8.5d", " +01234"}, + {"1234", "%+8.6d", " +001234"}, + {"-1234", "%+8.3d", " -1234"}, + {"-1234", "%+8.4d", " -1234"}, + {"-1234", "%+8.5d", " -01234"}, + {"-1234", "%+8.6d", " -001234"}, + + {"1234", "% 8.3d", " 1234"}, + {"1234", "% 8.4d", " 1234"}, + {"1234", "% 8.5d", " 01234"}, + {"1234", "% 8.6d", " 001234"}, + {"-1234", "% 8.3d", " -1234"}, + {"-1234", "% 8.4d", " -1234"}, + {"-1234", "% 8.5d", " -01234"}, + {"-1234", "% 8.6d", " -001234"}, + + {"1234", "%.3x", "4d2"}, + {"1234", "%.4x", "04d2"}, + {"1234", "%.5x", "004d2"}, + {"1234", "%.6x", "0004d2"}, + {"-1234", "%.3x", "-4d2"}, + {"-1234", "%.4x", "-04d2"}, + {"-1234", "%.5x", "-004d2"}, + {"-1234", "%.6x", "-0004d2"}, + + {"1234", "%8.3x", " 4d2"}, + {"1234", "%8.4x", " 04d2"}, + {"1234", "%8.5x", " 004d2"}, + {"1234", "%8.6x", " 0004d2"}, + {"-1234", "%8.3x", " -4d2"}, + {"-1234", "%8.4x", " -04d2"}, + {"-1234", "%8.5x", " -004d2"}, + {"-1234", "%8.6x", " -0004d2"}, + + {"1234", "%+8.3x", " +4d2"}, + {"1234", "%+8.4x", " +04d2"}, + {"1234", "%+8.5x", " +004d2"}, + {"1234", "%+8.6x", " +0004d2"}, + {"-1234", "%+8.3x", " -4d2"}, + {"-1234", "%+8.4x", " -04d2"}, + {"-1234", "%+8.5x", " -004d2"}, + {"-1234", "%+8.6x", " -0004d2"}, + + {"1234", "% 8.3x", " 4d2"}, + {"1234", "% 8.4x", " 04d2"}, + {"1234", "% 8.5x", " 004d2"}, + {"1234", "% 8.6x", " 0004d2"}, + {"1234", "% 8.7x", " 00004d2"}, + {"1234", "% 8.8x", " 000004d2"}, + {"-1234", "% 8.3x", " -4d2"}, + {"-1234", "% 8.4x", " -04d2"}, + {"-1234", "% 8.5x", " -004d2"}, + {"-1234", "% 8.6x", " -0004d2"}, + {"-1234", "% 8.7x", "-00004d2"}, + {"-1234", "% 8.8x", "-000004d2"}, + + {"1234", "%-8.3d", "1234 "}, + {"1234", "%-8.4d", "1234 "}, + {"1234", "%-8.5d", "01234 "}, + {"1234", "%-8.6d", "001234 "}, + {"1234", "%-8.7d", "0001234 "}, + {"1234", "%-8.8d", "00001234"}, + {"-1234", "%-8.3d", "-1234 "}, + {"-1234", "%-8.4d", "-1234 "}, + {"-1234", "%-8.5d", "-01234 "}, + {"-1234", "%-8.6d", "-001234 "}, + {"-1234", "%-8.7d", "-0001234"}, + {"-1234", "%-8.8d", "-00001234"}, + + {"16777215", "%b", "111111111111111111111111"}, // 2**24 - 1 + + {"0", "%.d", ""}, + {"0", "%.0d", ""}, + {"0", "%3.d", ""}, +} + +func TestFormat(t *testing.T) { + for i, test := range formatTests { + var x *Int + if test.input != "" { + var ok bool + x, ok = new(Int).SetString(test.input, 0) + if !ok { + t.Errorf("#%d failed reading input %s", i, test.input) + } + } + output := fmt.Sprintf(test.format, x) + if output != test.output { + t.Errorf("#%d got %q; want %q, {%q, %q, %q}", i, output, test.output, test.input, test.format, test.output) + } + } +} + +var scanTests = []struct { + input string + format string + output string + remaining int +}{ + {"1010", "%b", "10", 0}, + {"0b1010", "%v", "10", 0}, + {"12", "%o", "10", 0}, + {"012", "%v", "10", 0}, + {"10", "%d", "10", 0}, + {"10", "%v", "10", 0}, + {"a", "%x", "10", 0}, + {"0xa", "%v", "10", 0}, + {"A", "%X", "10", 0}, + {"-A", "%X", "-10", 0}, + {"+0b1011001", "%v", "89", 0}, + {"0xA", "%v", "10", 0}, + {"0 ", "%v", "0", 1}, + {"2+3", "%v", "2", 2}, + {"0XABC 12", "%v", "2748", 3}, +} + +func TestScan(t *testing.T) { + var buf bytes.Buffer + for i, test := range scanTests { + x := new(Int) + buf.Reset() + buf.WriteString(test.input) + if _, err := fmt.Fscanf(&buf, test.format, x); err != nil { + t.Errorf("#%d error: %s", i, err.String()) + } + if x.String() != test.output { + t.Errorf("#%d got %s; want %s", i, x.String(), test.output) + } + if buf.Len() != test.remaining { + t.Errorf("#%d got %d bytes remaining; want %d", i, buf.Len(), test.remaining) + } + } +} + +// Examples from the Go Language Spec, section "Arithmetic operators" +var divisionSignsTests = []struct { + x, y int64 + q, r int64 // T-division + d, m int64 // Euclidian division +}{ + {5, 3, 1, 2, 1, 2}, + {-5, 3, -1, -2, -2, 1}, + {5, -3, -1, 2, -1, 2}, + {-5, -3, 1, -2, 2, 1}, + {1, 2, 0, 1, 0, 1}, + {8, 4, 2, 0, 2, 0}, +} + +func TestDivisionSigns(t *testing.T) { + for i, test := range divisionSignsTests { + x := NewInt(test.x) + y := NewInt(test.y) + q := NewInt(test.q) + r := NewInt(test.r) + d := NewInt(test.d) + m := NewInt(test.m) + + q1 := new(Int).Quo(x, y) + r1 := new(Int).Rem(x, y) + if !isNormalized(q1) { + t.Errorf("#%d Quo: %v is not normalized", i, *q1) + } + if !isNormalized(r1) { + t.Errorf("#%d Rem: %v is not normalized", i, *r1) + } + if q1.Cmp(q) != 0 || r1.Cmp(r) != 0 { + t.Errorf("#%d QuoRem: got (%s, %s), want (%s, %s)", i, q1, r1, q, r) + } + + q2, r2 := new(Int).QuoRem(x, y, new(Int)) + if !isNormalized(q2) { + t.Errorf("#%d Quo: %v is not normalized", i, *q2) + } + if !isNormalized(r2) { + t.Errorf("#%d Rem: %v is not normalized", i, *r2) + } + if q2.Cmp(q) != 0 || r2.Cmp(r) != 0 { + t.Errorf("#%d QuoRem: got (%s, %s), want (%s, %s)", i, q2, r2, q, r) + } + + d1 := new(Int).Div(x, y) + m1 := new(Int).Mod(x, y) + if !isNormalized(d1) { + t.Errorf("#%d Div: %v is not normalized", i, *d1) + } + if !isNormalized(m1) { + t.Errorf("#%d Mod: %v is not normalized", i, *m1) + } + if d1.Cmp(d) != 0 || m1.Cmp(m) != 0 { + t.Errorf("#%d DivMod: got (%s, %s), want (%s, %s)", i, d1, m1, d, m) + } + + d2, m2 := new(Int).DivMod(x, y, new(Int)) + if !isNormalized(d2) { + t.Errorf("#%d Div: %v is not normalized", i, *d2) + } + if !isNormalized(m2) { + t.Errorf("#%d Mod: %v is not normalized", i, *m2) + } + if d2.Cmp(d) != 0 || m2.Cmp(m) != 0 { + t.Errorf("#%d DivMod: got (%s, %s), want (%s, %s)", i, d2, m2, d, m) + } + } +} + +func checkSetBytes(b []byte) bool { + hex1 := hex.EncodeToString(new(Int).SetBytes(b).Bytes()) + hex2 := hex.EncodeToString(b) + + for len(hex1) < len(hex2) { + hex1 = "0" + hex1 + } + + for len(hex1) > len(hex2) { + hex2 = "0" + hex2 + } + + return hex1 == hex2 +} + +func TestSetBytes(t *testing.T) { + if err := quick.Check(checkSetBytes, nil); err != nil { + t.Error(err) + } +} + +func checkBytes(b []byte) bool { + b2 := new(Int).SetBytes(b).Bytes() + return bytes.Compare(b, b2) == 0 +} + +func TestBytes(t *testing.T) { + if err := quick.Check(checkSetBytes, nil); err != nil { + t.Error(err) + } +} + +func checkQuo(x, y []byte) bool { + u := new(Int).SetBytes(x) + v := new(Int).SetBytes(y) + + if len(v.abs) == 0 { + return true + } + + r := new(Int) + q, r := new(Int).QuoRem(u, v, r) + + if r.Cmp(v) >= 0 { + return false + } + + uprime := new(Int).Set(q) + uprime.Mul(uprime, v) + uprime.Add(uprime, r) + + return uprime.Cmp(u) == 0 +} + +var quoTests = []struct { + x, y string + q, r string +}{ + { + "476217953993950760840509444250624797097991362735329973741718102894495832294430498335824897858659711275234906400899559094370964723884706254265559534144986498357", + "9353930466774385905609975137998169297361893554149986716853295022578535724979483772383667534691121982974895531435241089241440253066816724367338287092081996", + "50911", + "1", + }, + { + "11510768301994997771168", + "1328165573307167369775", + "8", + "885443715537658812968", + }, +} + +func TestQuo(t *testing.T) { + if err := quick.Check(checkQuo, nil); err != nil { + t.Error(err) + } + + for i, test := range quoTests { + x, _ := new(Int).SetString(test.x, 10) + y, _ := new(Int).SetString(test.y, 10) + expectedQ, _ := new(Int).SetString(test.q, 10) + expectedR, _ := new(Int).SetString(test.r, 10) + + r := new(Int) + q, r := new(Int).QuoRem(x, y, r) + + if q.Cmp(expectedQ) != 0 || r.Cmp(expectedR) != 0 { + t.Errorf("#%d got (%s, %s) want (%s, %s)", i, q, r, expectedQ, expectedR) + } + } +} + +func TestQuoStepD6(t *testing.T) { + // See Knuth, Volume 2, section 4.3.1, exercise 21. This code exercises + // a code path which only triggers 1 in 10^{-19} cases. + + u := &Int{false, nat{0, 0, 1 + 1<<(_W-1), _M ^ (1 << (_W - 1))}} + v := &Int{false, nat{5, 2 + 1<<(_W-1), 1 << (_W - 1)}} + + r := new(Int) + q, r := new(Int).QuoRem(u, v, r) + const expectedQ64 = "18446744073709551613" + const expectedR64 = "3138550867693340382088035895064302439801311770021610913807" + const expectedQ32 = "4294967293" + const expectedR32 = "39614081266355540837921718287" + if q.String() != expectedQ64 && q.String() != expectedQ32 || + r.String() != expectedR64 && r.String() != expectedR32 { + t.Errorf("got (%s, %s) want (%s, %s) or (%s, %s)", q, r, expectedQ64, expectedR64, expectedQ32, expectedR32) + } +} + +var bitLenTests = []struct { + in string + out int +}{ + {"-1", 1}, + {"0", 0}, + {"1", 1}, + {"2", 2}, + {"4", 3}, + {"0xabc", 12}, + {"0x8000", 16}, + {"0x80000000", 32}, + {"0x800000000000", 48}, + {"0x8000000000000000", 64}, + {"0x80000000000000000000", 80}, + {"-0x4000000000000000000000", 87}, +} + +func TestBitLen(t *testing.T) { + for i, test := range bitLenTests { + x, ok := new(Int).SetString(test.in, 0) + if !ok { + t.Errorf("#%d test input invalid: %s", i, test.in) + continue + } + + if n := x.BitLen(); n != test.out { + t.Errorf("#%d got %d want %d", i, n, test.out) + } + } +} + +var expTests = []struct { + x, y, m string + out string +}{ + {"5", "0", "", "1"}, + {"-5", "0", "", "-1"}, + {"5", "1", "", "5"}, + {"-5", "1", "", "-5"}, + {"-2", "3", "2", "0"}, + {"5", "2", "", "25"}, + {"1", "65537", "2", "1"}, + {"0x8000000000000000", "2", "", "0x40000000000000000000000000000000"}, + {"0x8000000000000000", "2", "6719", "4944"}, + {"0x8000000000000000", "3", "6719", "5447"}, + {"0x8000000000000000", "1000", "6719", "1603"}, + {"0x8000000000000000", "1000000", "6719", "3199"}, + { + "2938462938472983472983659726349017249287491026512746239764525612965293865296239471239874193284792387498274256129746192347", + "298472983472983471903246121093472394872319615612417471234712061", + "29834729834729834729347290846729561262544958723956495615629569234729836259263598127342374289365912465901365498236492183464", + "23537740700184054162508175125554701713153216681790245129157191391322321508055833908509185839069455749219131480588829346291", + }, +} + +func TestExp(t *testing.T) { + for i, test := range expTests { + x, ok1 := new(Int).SetString(test.x, 0) + y, ok2 := new(Int).SetString(test.y, 0) + out, ok3 := new(Int).SetString(test.out, 0) + + var ok4 bool + var m *Int + + if len(test.m) == 0 { + m, ok4 = nil, true + } else { + m, ok4 = new(Int).SetString(test.m, 0) + } + + if !ok1 || !ok2 || !ok3 || !ok4 { + t.Errorf("#%d: error in input", i) + continue + } + + z := y.Exp(x, y, m) + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, z, out) + } + } +} + +func checkGcd(aBytes, bBytes []byte) bool { + a := new(Int).SetBytes(aBytes) + b := new(Int).SetBytes(bBytes) + + x := new(Int) + y := new(Int) + d := new(Int) + + GcdInt(d, x, y, a, b) + x.Mul(x, a) + y.Mul(y, b) + x.Add(x, y) + + return x.Cmp(d) == 0 +} + +var gcdTests = []struct { + a, b int64 + d, x, y int64 +}{ + {120, 23, 1, -9, 47}, +} + +func TestGcd(t *testing.T) { + for i, test := range gcdTests { + a := NewInt(test.a) + b := NewInt(test.b) + + x := new(Int) + y := new(Int) + d := new(Int) + + expectedX := NewInt(test.x) + expectedY := NewInt(test.y) + expectedD := NewInt(test.d) + + GcdInt(d, x, y, a, b) + + if expectedX.Cmp(x) != 0 || + expectedY.Cmp(y) != 0 || + expectedD.Cmp(d) != 0 { + t.Errorf("#%d got (%s %s %s) want (%s %s %s)", i, x, y, d, expectedX, expectedY, expectedD) + } + } + + quick.Check(checkGcd, nil) +} + +var primes = []string{ + "2", + "3", + "5", + "7", + "11", + + "13756265695458089029", + "13496181268022124907", + "10953742525620032441", + "17908251027575790097", + + // http://code.google.com/p/go/issues/detail?id=638 + "18699199384836356663", + + "98920366548084643601728869055592650835572950932266967461790948584315647051443", + "94560208308847015747498523884063394671606671904944666360068158221458669711639", + + // http://primes.utm.edu/lists/small/small3.html + "449417999055441493994709297093108513015373787049558499205492347871729927573118262811508386655998299074566974373711472560655026288668094291699357843464363003144674940345912431129144354948751003607115263071543163", + "230975859993204150666423538988557839555560243929065415434980904258310530753006723857139742334640122533598517597674807096648905501653461687601339782814316124971547968912893214002992086353183070342498989426570593", + "5521712099665906221540423207019333379125265462121169655563495403888449493493629943498064604536961775110765377745550377067893607246020694972959780839151452457728855382113555867743022746090187341871655890805971735385789993", + "203956878356401977405765866929034577280193993314348263094772646453283062722701277632936616063144088173312372882677123879538709400158306567338328279154499698366071906766440037074217117805690872792848149112022286332144876183376326512083574821647933992961249917319836219304274280243803104015000563790123", +} + +var composites = []string{ + "21284175091214687912771199898307297748211672914763848041968395774954376176754", + "6084766654921918907427900243509372380954290099172559290432744450051395395951", + "84594350493221918389213352992032324280367711247940675652888030554255915464401", + "82793403787388584738507275144194252681", +} + +func TestProbablyPrime(t *testing.T) { + nreps := 20 + if testing.Short() { + nreps = 1 + } + for i, s := range primes { + p, _ := new(Int).SetString(s, 10) + if !ProbablyPrime(p, nreps) { + t.Errorf("#%d prime found to be non-prime (%s)", i, s) + } + } + + for i, s := range composites { + c, _ := new(Int).SetString(s, 10) + if ProbablyPrime(c, nreps) { + t.Errorf("#%d composite found to be prime (%s)", i, s) + } + if testing.Short() { + break + } + } +} + +type intShiftTest struct { + in string + shift uint + out string +} + +var rshTests = []intShiftTest{ + {"0", 0, "0"}, + {"-0", 0, "0"}, + {"0", 1, "0"}, + {"0", 2, "0"}, + {"1", 0, "1"}, + {"1", 1, "0"}, + {"1", 2, "0"}, + {"2", 0, "2"}, + {"2", 1, "1"}, + {"-1", 0, "-1"}, + {"-1", 1, "-1"}, + {"-1", 10, "-1"}, + {"-100", 2, "-25"}, + {"-100", 3, "-13"}, + {"-100", 100, "-1"}, + {"4294967296", 0, "4294967296"}, + {"4294967296", 1, "2147483648"}, + {"4294967296", 2, "1073741824"}, + {"18446744073709551616", 0, "18446744073709551616"}, + {"18446744073709551616", 1, "9223372036854775808"}, + {"18446744073709551616", 2, "4611686018427387904"}, + {"18446744073709551616", 64, "1"}, + {"340282366920938463463374607431768211456", 64, "18446744073709551616"}, + {"340282366920938463463374607431768211456", 128, "1"}, +} + +func TestRsh(t *testing.T) { + for i, test := range rshTests { + in, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + out := new(Int).Rsh(in, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + } +} + +func TestRshSelf(t *testing.T) { + for i, test := range rshTests { + z, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + z.Rsh(z, test.shift) + + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, z, expected) + } + } +} + +var lshTests = []intShiftTest{ + {"0", 0, "0"}, + {"0", 1, "0"}, + {"0", 2, "0"}, + {"1", 0, "1"}, + {"1", 1, "2"}, + {"1", 2, "4"}, + {"2", 0, "2"}, + {"2", 1, "4"}, + {"2", 2, "8"}, + {"-87", 1, "-174"}, + {"4294967296", 0, "4294967296"}, + {"4294967296", 1, "8589934592"}, + {"4294967296", 2, "17179869184"}, + {"18446744073709551616", 0, "18446744073709551616"}, + {"9223372036854775808", 1, "18446744073709551616"}, + {"4611686018427387904", 2, "18446744073709551616"}, + {"1", 64, "18446744073709551616"}, + {"18446744073709551616", 64, "340282366920938463463374607431768211456"}, + {"1", 128, "340282366920938463463374607431768211456"}, +} + +func TestLsh(t *testing.T) { + for i, test := range lshTests { + in, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + out := new(Int).Lsh(in, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + } +} + +func TestLshSelf(t *testing.T) { + for i, test := range lshTests { + z, _ := new(Int).SetString(test.in, 10) + expected, _ := new(Int).SetString(test.out, 10) + z.Lsh(z, test.shift) + + if !isNormalized(z) { + t.Errorf("#%d: %v is not normalized", i, *z) + } + if z.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, z, expected) + } + } +} + +func TestLshRsh(t *testing.T) { + for i, test := range rshTests { + in, _ := new(Int).SetString(test.in, 10) + out := new(Int).Lsh(in, test.shift) + out = out.Rsh(out, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if in.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } + for i, test := range lshTests { + in, _ := new(Int).SetString(test.in, 10) + out := new(Int).Lsh(in, test.shift) + out.Rsh(out, test.shift) + + if !isNormalized(out) { + t.Errorf("#%d: %v is not normalized", i, *out) + } + if in.Cmp(out) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } +} + +var int64Tests = []int64{ + 0, + 1, + -1, + 4294967295, + -4294967295, + 4294967296, + -4294967296, + 9223372036854775807, + -9223372036854775807, + -9223372036854775808, +} + +func TestInt64(t *testing.T) { + for i, testVal := range int64Tests { + in := NewInt(testVal) + out := in.Int64() + + if out != testVal { + t.Errorf("#%d got %d want %d", i, out, testVal) + } + } +} + +var bitwiseTests = []struct { + x, y string + and, or, xor, andNot string +}{ + {"0x00", "0x00", "0x00", "0x00", "0x00", "0x00"}, + {"0x00", "0x01", "0x00", "0x01", "0x01", "0x00"}, + {"0x01", "0x00", "0x00", "0x01", "0x01", "0x01"}, + {"-0x01", "0x00", "0x00", "-0x01", "-0x01", "-0x01"}, + {"-0xaf", "-0x50", "-0xf0", "-0x0f", "0xe1", "0x41"}, + {"0x00", "-0x01", "0x00", "-0x01", "-0x01", "0x00"}, + {"0x01", "0x01", "0x01", "0x01", "0x00", "0x00"}, + {"-0x01", "-0x01", "-0x01", "-0x01", "0x00", "0x00"}, + {"0x07", "0x08", "0x00", "0x0f", "0x0f", "0x07"}, + {"0x05", "0x0f", "0x05", "0x0f", "0x0a", "0x00"}, + {"0x013ff6", "0x9a4e", "0x1a46", "0x01bffe", "0x01a5b8", "0x0125b0"}, + {"-0x013ff6", "0x9a4e", "0x800a", "-0x0125b2", "-0x01a5bc", "-0x01c000"}, + {"-0x013ff6", "-0x9a4e", "-0x01bffe", "-0x1a46", "0x01a5b8", "0x8008"}, + { + "0x1000009dc6e3d9822cba04129bcbe3401", + "0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "0x1000001186210100001000009048c2001", + "0xb9bd7d543685789d57cb918e8bfeff7fddb2ebe87dfbbdfe35fd", + "0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fc", + "0x8c40c2d8822caa04120b8321400", + }, + { + "0x1000009dc6e3d9822cba04129bcbe3401", + "-0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "0x8c40c2d8822caa04120b8321401", + "-0xb9bd7d543685789d57ca918e82229142459020483cd2014001fd", + "-0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fe", + "0x1000001186210100001000009048c2000", + }, + { + "-0x1000009dc6e3d9822cba04129bcbe3401", + "-0xb9bd7d543685789d57cb918e833af352559021483cdb05cc21fd", + "-0xb9bd7d543685789d57cb918e8bfeff7fddb2ebe87dfbbdfe35fd", + "-0x1000001186210100001000009048c2001", + "0xb9bd7d543685789d57ca918e8ae69d6fcdb2eae87df2b97215fc", + "0xb9bd7d543685789d57ca918e82229142459020483cd2014001fc", + }, +} + +type bitFun func(z, x, y *Int) *Int + +func testBitFun(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { + expected := new(Int) + expected.SetString(exp, 0) + + out := f(new(Int), x, y) + if out.Cmp(expected) != 0 { + t.Errorf("%s: got %s want %s", msg, out, expected) + } +} + +func testBitFunSelf(t *testing.T, msg string, f bitFun, x, y *Int, exp string) { + self := new(Int) + self.Set(x) + expected := new(Int) + expected.SetString(exp, 0) + + self = f(self, self, y) + if self.Cmp(expected) != 0 { + t.Errorf("%s: got %s want %s", msg, self, expected) + } +} + +func altBit(x *Int, i int) uint { + z := new(Int).Rsh(x, uint(i)) + z = z.And(z, NewInt(1)) + if z.Cmp(new(Int)) != 0 { + return 1 + } + return 0 +} + +func altSetBit(z *Int, x *Int, i int, b uint) *Int { + one := NewInt(1) + m := one.Lsh(one, uint(i)) + switch b { + case 1: + return z.Or(x, m) + case 0: + return z.AndNot(x, m) + } + panic("set bit is not 0 or 1") +} + +func testBitset(t *testing.T, x *Int) { + n := x.BitLen() + z := new(Int).Set(x) + z1 := new(Int).Set(x) + for i := 0; i < n+10; i++ { + old := z.Bit(i) + old1 := altBit(z1, i) + if old != old1 { + t.Errorf("bitset: inconsistent value for Bit(%s, %d), got %v want %v", z1, i, old, old1) + } + z := new(Int).SetBit(z, i, 1) + z1 := altSetBit(new(Int), z1, i, 1) + if z.Bit(i) == 0 { + t.Errorf("bitset: bit %d of %s got 0 want 1", i, x) + } + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit 1, got %s want %s", z, z1) + } + z.SetBit(z, i, 0) + altSetBit(z1, z1, i, 0) + if z.Bit(i) != 0 { + t.Errorf("bitset: bit %d of %s got 1 want 0", i, x) + } + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit 0, got %s want %s", z, z1) + } + altSetBit(z1, z1, i, old) + z.SetBit(z, i, old) + if z.Cmp(z1) != 0 { + t.Errorf("bitset: inconsistent value after SetBit old, got %s want %s", z, z1) + } + } + if z.Cmp(x) != 0 { + t.Errorf("bitset: got %s want %s", z, x) + } +} + +var bitsetTests = []struct { + x string + i int + b uint +}{ + {"0", 0, 0}, + {"0", 200, 0}, + {"1", 0, 1}, + {"1", 1, 0}, + {"-1", 0, 1}, + {"-1", 200, 1}, + {"0x2000000000000000000000000000", 108, 0}, + {"0x2000000000000000000000000000", 109, 1}, + {"0x2000000000000000000000000000", 110, 0}, + {"-0x2000000000000000000000000001", 108, 1}, + {"-0x2000000000000000000000000001", 109, 0}, + {"-0x2000000000000000000000000001", 110, 1}, +} + +func TestBitSet(t *testing.T) { + for _, test := range bitwiseTests { + x := new(Int) + x.SetString(test.x, 0) + testBitset(t, x) + x = new(Int) + x.SetString(test.y, 0) + testBitset(t, x) + } + for i, test := range bitsetTests { + x := new(Int) + x.SetString(test.x, 0) + b := x.Bit(test.i) + if b != test.b { + + t.Errorf("#%d want %v got %v", i, test.b, b) + } + } +} + +func BenchmarkBitset(b *testing.B) { + z := new(Int) + z.SetBit(z, 512, 1) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + z.SetBit(z, i&512, 1) + } +} + +func BenchmarkBitsetNeg(b *testing.B) { + z := NewInt(-1) + z.SetBit(z, 512, 0) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + z.SetBit(z, i&512, 0) + } +} + +func BenchmarkBitsetOrig(b *testing.B) { + z := new(Int) + altSetBit(z, z, 512, 1) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + altSetBit(z, z, i&512, 1) + } +} + +func BenchmarkBitsetNegOrig(b *testing.B) { + z := NewInt(-1) + altSetBit(z, z, 512, 0) + b.ResetTimer() + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + altSetBit(z, z, i&512, 0) + } +} + +func TestBitwise(t *testing.T) { + x := new(Int) + y := new(Int) + for _, test := range bitwiseTests { + x.SetString(test.x, 0) + y.SetString(test.y, 0) + + testBitFun(t, "and", (*Int).And, x, y, test.and) + testBitFunSelf(t, "and", (*Int).And, x, y, test.and) + testBitFun(t, "andNot", (*Int).AndNot, x, y, test.andNot) + testBitFunSelf(t, "andNot", (*Int).AndNot, x, y, test.andNot) + testBitFun(t, "or", (*Int).Or, x, y, test.or) + testBitFunSelf(t, "or", (*Int).Or, x, y, test.or) + testBitFun(t, "xor", (*Int).Xor, x, y, test.xor) + testBitFunSelf(t, "xor", (*Int).Xor, x, y, test.xor) + } +} + +var notTests = []struct { + in string + out string +}{ + {"0", "-1"}, + {"1", "-2"}, + {"7", "-8"}, + {"0", "-1"}, + {"-81910", "81909"}, + { + "298472983472983471903246121093472394872319615612417471234712061", + "-298472983472983471903246121093472394872319615612417471234712062", + }, +} + +func TestNot(t *testing.T) { + in := new(Int) + out := new(Int) + expected := new(Int) + for i, test := range notTests { + in.SetString(test.in, 10) + expected.SetString(test.out, 10) + out = out.Not(in) + if out.Cmp(expected) != 0 { + t.Errorf("#%d: got %s want %s", i, out, expected) + } + out = out.Not(out) + if out.Cmp(in) != 0 { + t.Errorf("#%d: got %s want %s", i, out, in) + } + } +} + +var modInverseTests = []struct { + element string + prime string +}{ + {"1", "7"}, + {"1", "13"}, + {"239487239847", "2410312426921032588552076022197566074856950548502459942654116941958108831682612228890093858261341614673227141477904012196503648957050582631942730706805009223062734745341073406696246014589361659774041027169249453200378729434170325843778659198143763193776859869524088940195577346119843545301547043747207749969763750084308926339295559968882457872412993810129130294592999947926365264059284647209730384947211681434464714438488520940127459844288859336526896320919633919"}, +} + +func TestModInverse(t *testing.T) { + var element, prime Int + one := NewInt(1) + for i, test := range modInverseTests { + (&element).SetString(test.element, 10) + (&prime).SetString(test.prime, 10) + inverse := new(Int).ModInverse(&element, &prime) + inverse.Mul(inverse, &element) + inverse.Mod(inverse, &prime) + if inverse.Cmp(one) != 0 { + t.Errorf("#%d: failed (e·e^(-1)=%s)", i, inverse) + } + } +} + +// used by TestIntGobEncoding and TestRatGobEncoding +var gobEncodingTests = []string{ + "0", + "1", + "2", + "10", + "42", + "1234567890", + "298472983472983471903246121093472394872319615612417471234712061", +} + +func TestIntGobEncoding(t *testing.T) { + var medium bytes.Buffer + enc := gob.NewEncoder(&medium) + dec := gob.NewDecoder(&medium) + for i, test := range gobEncodingTests { + for j := 0; j < 2; j++ { + medium.Reset() // empty buffer for each test case (in case of failures) + stest := test + if j != 0 { + // negative numbers + stest = "-" + test + } + var tx Int + tx.SetString(stest, 10) + if err := enc.Encode(&tx); err != nil { + t.Errorf("#%d%c: encoding failed: %s", i, 'a'+j, err) + } + var rx Int + if err := dec.Decode(&rx); err != nil { + t.Errorf("#%d%c: decoding failed: %s", i, 'a'+j, err) + } + if rx.Cmp(&tx) != 0 { + t.Errorf("#%d%c: transmission failed: got %s want %s", i, 'a'+j, &rx, &tx) + } + } + } +} diff --git a/src/pkg/big/nat.go b/src/pkg/big/nat.go new file mode 100755 index 000000000..be3aff29d --- /dev/null +++ b/src/pkg/big/nat.go @@ -0,0 +1,1272 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package big implements multi-precision arithmetic (big numbers). +// The following numeric types are supported: +// +// - Int signed integers +// - Rat rational numbers +// +// All methods on Int take the result as the receiver; if it is one +// of the operands it may be overwritten (and its memory reused). +// To enable chaining of operations, the result is also returned. +// +package big + +// This file contains operations on unsigned multi-precision integers. +// These are the building blocks for the operations on signed integers +// and rationals. + +import ( + "io" + "os" + "rand" +) + +// An unsigned integer x of the form +// +// x = x[n-1]*_B^(n-1) + x[n-2]*_B^(n-2) + ... + x[1]*_B + x[0] +// +// with 0 <= x[i] < _B and 0 <= i < n is stored in a slice of length n, +// with the digits x[i] as the slice elements. +// +// A number is normalized if the slice contains no leading 0 digits. +// During arithmetic operations, denormalized values may occur but are +// always normalized before returning the final result. The normalized +// representation of 0 is the empty or nil slice (length = 0). + +type nat []Word + +var ( + natOne = nat{1} + natTwo = nat{2} + natTen = nat{10} +) + +func (z nat) clear() { + for i := range z { + z[i] = 0 + } +} + +func (z nat) norm() nat { + i := len(z) + for i > 0 && z[i-1] == 0 { + i-- + } + return z[0:i] +} + +func (z nat) make(n int) nat { + if n <= cap(z) { + return z[0:n] // reuse z + } + // Choosing a good value for e has significant performance impact + // because it increases the chance that a value can be reused. + const e = 4 // extra capacity + return make(nat, n, n+e) +} + +func (z nat) setWord(x Word) nat { + if x == 0 { + return z.make(0) + } + z = z.make(1) + z[0] = x + return z +} + +func (z nat) setUint64(x uint64) nat { + // single-digit values + if w := Word(x); uint64(w) == x { + return z.setWord(w) + } + + // compute number of words n required to represent x + n := 0 + for t := x; t > 0; t >>= _W { + n++ + } + + // split x into n words + z = z.make(n) + for i := range z { + z[i] = Word(x & _M) + x >>= _W + } + + return z +} + +func (z nat) set(x nat) nat { + z = z.make(len(x)) + copy(z, x) + return z +} + +func (z nat) add(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.add(y, x) + case m == 0: + // n == 0 because m >= n; result is 0 + return z.make(0) + case n == 0: + // result is x + return z.set(x) + } + // m > 0 + + z = z.make(m + 1) + c := addVV(z[0:n], x, y) + if m > n { + c = addVW(z[n:m], x[n:], c) + } + z[m] = c + + return z.norm() +} + +func (z nat) sub(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + panic("underflow") + case m == 0: + // n == 0 because m >= n; result is 0 + return z.make(0) + case n == 0: + // result is x + return z.set(x) + } + // m > 0 + + z = z.make(m) + c := subVV(z[0:n], x, y) + if m > n { + c = subVW(z[n:], x[n:], c) + } + if c != 0 { + panic("underflow") + } + + return z.norm() +} + +func (x nat) cmp(y nat) (r int) { + m := len(x) + n := len(y) + if m != n || m == 0 { + switch { + case m < n: + r = -1 + case m > n: + r = 1 + } + return + } + + i := m - 1 + for i > 0 && x[i] == y[i] { + i-- + } + + switch { + case x[i] < y[i]: + r = -1 + case x[i] > y[i]: + r = 1 + } + return +} + +func (z nat) mulAddWW(x nat, y, r Word) nat { + m := len(x) + if m == 0 || y == 0 { + return z.setWord(r) // result is r + } + // m > 0 + + z = z.make(m + 1) + z[m] = mulAddVWW(z[0:m], x, y, r) + + return z.norm() +} + +// basicMul multiplies x and y and leaves the result in z. +// The (non-normalized) result is placed in z[0 : len(x) + len(y)]. +func basicMul(z, x, y nat) { + z[0 : len(x)+len(y)].clear() // initialize z + for i, d := range y { + if d != 0 { + z[len(x)+i] = addMulVVW(z[i:i+len(x)], x, d) + } + } +} + +// Fast version of z[0:n+n>>1].add(z[0:n+n>>1], x[0:n]) w/o bounds checks. +// Factored out for readability - do not use outside karatsuba. +func karatsubaAdd(z, x nat, n int) { + if c := addVV(z[0:n], z, x); c != 0 { + addVW(z[n:n+n>>1], z[n:], c) + } +} + +// Like karatsubaAdd, but does subtract. +func karatsubaSub(z, x nat, n int) { + if c := subVV(z[0:n], z, x); c != 0 { + subVW(z[n:n+n>>1], z[n:], c) + } +} + +// Operands that are shorter than karatsubaThreshold are multiplied using +// "grade school" multiplication; for longer operands the Karatsuba algorithm +// is used. +var karatsubaThreshold int = 32 // computed by calibrate.go + +// karatsuba multiplies x and y and leaves the result in z. +// Both x and y must have the same length n and n must be a +// power of 2. The result vector z must have len(z) >= 6*n. +// The (non-normalized) result is placed in z[0 : 2*n]. +func karatsuba(z, x, y nat) { + n := len(y) + + // Switch to basic multiplication if numbers are odd or small. + // (n is always even if karatsubaThreshold is even, but be + // conservative) + if n&1 != 0 || n < karatsubaThreshold || n < 2 { + basicMul(z, x, y) + return + } + // n&1 == 0 && n >= karatsubaThreshold && n >= 2 + + // Karatsuba multiplication is based on the observation that + // for two numbers x and y with: + // + // x = x1*b + x0 + // y = y1*b + y0 + // + // the product x*y can be obtained with 3 products z2, z1, z0 + // instead of 4: + // + // x*y = x1*y1*b*b + (x1*y0 + x0*y1)*b + x0*y0 + // = z2*b*b + z1*b + z0 + // + // with: + // + // xd = x1 - x0 + // yd = y0 - y1 + // + // z1 = xd*yd + z1 + z0 + // = (x1-x0)*(y0 - y1) + z1 + z0 + // = x1*y0 - x1*y1 - x0*y0 + x0*y1 + z1 + z0 + // = x1*y0 - z1 - z0 + x0*y1 + z1 + z0 + // = x1*y0 + x0*y1 + + // split x, y into "digits" + n2 := n >> 1 // n2 >= 1 + x1, x0 := x[n2:], x[0:n2] // x = x1*b + y0 + y1, y0 := y[n2:], y[0:n2] // y = y1*b + y0 + + // z is used for the result and temporary storage: + // + // 6*n 5*n 4*n 3*n 2*n 1*n 0*n + // z = [z2 copy|z0 copy| xd*yd | yd:xd | x1*y1 | x0*y0 ] + // + // For each recursive call of karatsuba, an unused slice of + // z is passed in that has (at least) half the length of the + // caller's z. + + // compute z0 and z2 with the result "in place" in z + karatsuba(z, x0, y0) // z0 = x0*y0 + karatsuba(z[n:], x1, y1) // z2 = x1*y1 + + // compute xd (or the negative value if underflow occurs) + s := 1 // sign of product xd*yd + xd := z[2*n : 2*n+n2] + if subVV(xd, x1, x0) != 0 { // x1-x0 + s = -s + subVV(xd, x0, x1) // x0-x1 + } + + // compute yd (or the negative value if underflow occurs) + yd := z[2*n+n2 : 3*n] + if subVV(yd, y0, y1) != 0 { // y0-y1 + s = -s + subVV(yd, y1, y0) // y1-y0 + } + + // p = (x1-x0)*(y0-y1) == x1*y0 - x1*y1 - x0*y0 + x0*y1 for s > 0 + // p = (x0-x1)*(y0-y1) == x0*y0 - x0*y1 - x1*y0 + x1*y1 for s < 0 + p := z[n*3:] + karatsuba(p, xd, yd) + + // save original z2:z0 + // (ok to use upper half of z since we're done recursing) + r := z[n*4:] + copy(r, z) + + // add up all partial products + // + // 2*n n 0 + // z = [ z2 | z0 ] + // + [ z0 ] + // + [ z2 ] + // + [ p ] + // + karatsubaAdd(z[n2:], r, n) + karatsubaAdd(z[n2:], r[n:], n) + if s > 0 { + karatsubaAdd(z[n2:], p, n) + } else { + karatsubaSub(z[n2:], p, n) + } +} + +// alias returns true if x and y share the same base array. +func alias(x, y nat) bool { + return cap(x) > 0 && cap(y) > 0 && &x[0:cap(x)][cap(x)-1] == &y[0:cap(y)][cap(y)-1] +} + +// addAt implements z += x*(1<<(_W*i)); z must be long enough. +// (we don't use nat.add because we need z to stay the same +// slice, and we don't need to normalize z after each addition) +func addAt(z, x nat, i int) { + if n := len(x); n > 0 { + if c := addVV(z[i:i+n], z[i:], x); c != 0 { + j := i + n + if j < len(z) { + addVW(z[j:], z[j:], c) + } + } + } +} + +func max(x, y int) int { + if x > y { + return x + } + return y +} + +// karatsubaLen computes an approximation to the maximum k <= n such that +// k = p<= 0. Thus, the +// result is the largest number that can be divided repeatedly by 2 before +// becoming about the value of karatsubaThreshold. +func karatsubaLen(n int) int { + i := uint(0) + for n > karatsubaThreshold { + n >>= 1 + i++ + } + return n << i +} + +func (z nat) mul(x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.mul(y, x) + case m == 0 || n == 0: + return z.make(0) + case n == 1: + return z.mulAddWW(x, y[0], 0) + } + // m >= n > 1 + + // determine if z can be reused + if alias(z, x) || alias(z, y) { + z = nil // z is an alias for x or y - cannot reuse + } + + // use basic multiplication if the numbers are small + if n < karatsubaThreshold || n < 2 { + z = z.make(m + n) + basicMul(z, x, y) + return z.norm() + } + // m >= n && n >= karatsubaThreshold && n >= 2 + + // determine Karatsuba length k such that + // + // x = x1*b + x0 + // y = y1*b + y0 (and k <= len(y), which implies k <= len(x)) + // b = 1<<(_W*k) ("base" of digits xi, yi) + // + k := karatsubaLen(n) + // k <= n + + // multiply x0 and y0 via Karatsuba + x0 := x[0:k] // x0 is not normalized + y0 := y[0:k] // y0 is not normalized + z = z.make(max(6*k, m+n)) // enough space for karatsuba of x0*y0 and full result of x*y + karatsuba(z, x0, y0) + z = z[0 : m+n] // z has final length but may be incomplete, upper portion is garbage + + // If x1 and/or y1 are not 0, add missing terms to z explicitly: + // + // m+n 2*k 0 + // z = [ ... | x0*y0 ] + // + [ x1*y1 ] + // + [ x1*y0 ] + // + [ x0*y1 ] + // + if k < n || m != n { + x1 := x[k:] // x1 is normalized because x is + y1 := y[k:] // y1 is normalized because y is + var t nat + t = t.mul(x1, y1) + copy(z[2*k:], t) + z[2*k+len(t):].clear() // upper portion of z is garbage + t = t.mul(x1, y0.norm()) + addAt(z, t, k) + t = t.mul(x0.norm(), y1) + addAt(z, t, k) + } + + return z.norm() +} + +// mulRange computes the product of all the unsigned integers in the +// range [a, b] inclusively. If a > b (empty range), the result is 1. +func (z nat) mulRange(a, b uint64) nat { + switch { + case a == 0: + // cut long ranges short (optimization) + return z.setUint64(0) + case a > b: + return z.setUint64(1) + case a == b: + return z.setUint64(a) + case a+1 == b: + return z.mul(nat(nil).setUint64(a), nat(nil).setUint64(b)) + } + m := (a + b) / 2 + return z.mul(nat(nil).mulRange(a, m), nat(nil).mulRange(m+1, b)) +} + +// q = (x-r)/y, with 0 <= r < y +func (z nat) divW(x nat, y Word) (q nat, r Word) { + m := len(x) + switch { + case y == 0: + panic("division by zero") + case y == 1: + q = z.set(x) // result is x + return + case m == 0: + q = z.make(0) // result is 0 + return + } + // m > 0 + z = z.make(m) + r = divWVW(z, 0, x, y) + q = z.norm() + return +} + +func (z nat) div(z2, u, v nat) (q, r nat) { + if len(v) == 0 { + panic("division by zero") + } + + if u.cmp(v) < 0 { + q = z.make(0) + r = z2.set(u) + return + } + + if len(v) == 1 { + var rprime Word + q, rprime = z.divW(u, v[0]) + if rprime > 0 { + r = z2.make(1) + r[0] = rprime + } else { + r = z2.make(0) + } + return + } + + q, r = z.divLarge(z2, u, v) + return +} + +// q = (uIn-r)/v, with 0 <= r < y +// Uses z as storage for q, and u as storage for r if possible. +// See Knuth, Volume 2, section 4.3.1, Algorithm D. +// Preconditions: +// len(v) >= 2 +// len(uIn) >= len(v) +func (z nat) divLarge(u, uIn, v nat) (q, r nat) { + n := len(v) + m := len(uIn) - n + + // determine if z can be reused + // TODO(gri) should find a better solution - this if statement + // is very costly (see e.g. time pidigits -s -n 10000) + if alias(z, uIn) || alias(z, v) { + z = nil // z is an alias for uIn or v - cannot reuse + } + q = z.make(m + 1) + + qhatv := make(nat, n+1) + if alias(u, uIn) || alias(u, v) { + u = nil // u is an alias for uIn or v - cannot reuse + } + u = u.make(len(uIn) + 1) + u.clear() + + // D1. + shift := leadingZeros(v[n-1]) + if shift > 0 { + // do not modify v, it may be used by another goroutine simultaneously + v1 := make(nat, n) + shlVU(v1, v, shift) + v = v1 + } + u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) + + // D2. + for j := m; j >= 0; j-- { + // D3. + qhat := Word(_M) + if u[j+n] != v[n-1] { + var rhat Word + qhat, rhat = divWW(u[j+n], u[j+n-1], v[n-1]) + + // x1 | x2 = q̂v_{n-2} + x1, x2 := mulWW(qhat, v[n-2]) + // test if q̂v_{n-2} > br̂ + u_{j+n-2} + for greaterThan(x1, x2, rhat, u[j+n-2]) { + qhat-- + prevRhat := rhat + rhat += v[n-1] + // v[n-1] >= 0, so this tests for overflow. + if rhat < prevRhat { + break + } + x1, x2 = mulWW(qhat, v[n-2]) + } + } + + // D4. + qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) + + c := subVV(u[j:j+len(qhatv)], u[j:], qhatv) + if c != 0 { + c := addVV(u[j:j+n], u[j:], v) + u[j+n] += c + qhat-- + } + + q[j] = qhat + } + + q = q.norm() + shrVU(u, u, shift) + r = u.norm() + + return q, r +} + +// Length of x in bits. x must be normalized. +func (x nat) bitLen() int { + if i := len(x) - 1; i >= 0 { + return i*_W + bitLen(x[i]) + } + return 0 +} + +// MaxBase is the largest number base accepted for string conversions. +const MaxBase = 'z' - 'a' + 10 + 1 // = hexValue('z') + 1 + + +func hexValue(ch int) Word { + d := MaxBase + 1 // illegal base + switch { + case '0' <= ch && ch <= '9': + d = ch - '0' + case 'a' <= ch && ch <= 'z': + d = ch - 'a' + 10 + case 'A' <= ch && ch <= 'Z': + d = ch - 'A' + 10 + } + return Word(d) +} + +// scan sets z to the natural number corresponding to the longest possible prefix +// read from r representing an unsigned integer in a given conversion base. +// It returns z, the actual conversion base used, and an error, if any. In the +// error case, the value of z is undefined. The syntax follows the syntax of +// unsigned integer literals in Go. +// +// The base argument must be 0 or a value from 2 through MaxBase. If the base +// is 0, the string prefix determines the actual conversion base. A prefix of +// ``0x'' or ``0X'' selects base 16; the ``0'' prefix selects base 8, and a +// ``0b'' or ``0B'' prefix selects base 2. Otherwise the selected base is 10. +// +func (z nat) scan(r io.RuneScanner, base int) (nat, int, os.Error) { + // reject illegal bases + if base < 0 || base == 1 || MaxBase < base { + return z, 0, os.NewError("illegal number base") + } + + // one char look-ahead + ch, _, err := r.ReadRune() + if err != nil { + return z, 0, err + } + + // determine base if necessary + b := Word(base) + if base == 0 { + b = 10 + if ch == '0' { + switch ch, _, err = r.ReadRune(); err { + case nil: + b = 8 + switch ch { + case 'x', 'X': + b = 16 + case 'b', 'B': + b = 2 + } + if b == 2 || b == 16 { + if ch, _, err = r.ReadRune(); err != nil { + return z, 0, err + } + } + case os.EOF: + return z, 10, nil + default: + return z, 10, err + } + } + } + + // convert string + // - group as many digits d as possible together into a "super-digit" dd with "super-base" bb + // - only when bb does not fit into a word anymore, do a full number mulAddWW using bb and dd + z = z.make(0) + bb := Word(1) + dd := Word(0) + for max := _M / b; ; { + d := hexValue(ch) + if d >= b { + r.UnreadRune() // ch does not belong to number anymore + break + } + + if bb <= max { + bb *= b + dd = dd*b + d + } else { + // bb * b would overflow + z = z.mulAddWW(z, bb, dd) + bb = b + dd = d + } + + if ch, _, err = r.ReadRune(); err != nil { + if err != os.EOF { + return z, int(b), err + } + break + } + } + + switch { + case bb > 1: + // there was at least one mantissa digit + z = z.mulAddWW(z, bb, dd) + case base == 0 && b == 8: + // there was only the octal prefix 0 (possibly followed by digits > 7); + // return base 10, not 8 + return z, 10, nil + case base != 0 || b != 8: + // there was neither a mantissa digit nor the octal prefix 0 + return z, int(b), os.NewError("syntax error scanning number") + } + + return z.norm(), int(b), nil +} + +// Character sets for string conversion. +const ( + lowercaseDigits = "0123456789abcdefghijklmnopqrstuvwxyz" + uppercaseDigits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +// decimalString returns a decimal representation of x. +// It calls x.string with the charset "0123456789". +func (x nat) decimalString() string { + return x.string(lowercaseDigits[0:10]) +} + +// string converts x to a string using digits from a charset; a digit with +// value d is represented by charset[d]. The conversion base is determined +// by len(charset), which must be >= 2. +func (x nat) string(charset string) string { + b := Word(len(charset)) + + // special cases + switch { + case b < 2 || b > 256: + panic("illegal base") + case len(x) == 0: + return string(charset[0]) + } + + // allocate buffer for conversion + i := x.bitLen()/log2(b) + 1 // +1: round up + s := make([]byte, i) + + // special case: power of two bases can avoid divisions completely + if b == b&-b { + // shift is base-b digit size in bits + shift := uint(trailingZeroBits(b)) // shift > 0 because b >= 2 + mask := Word(1)<= shift { + i-- + s[i] = charset[w&mask] + w >>= shift + nbits -= shift + } + + // convert any partial leading digit and advance to next word + if nbits == 0 { + // no partial digit remaining, just advance + w = x[k] + nbits = _W + } else { + // partial digit in current (k-1) and next (k) word + w |= x[k] << nbits + i-- + s[i] = charset[w&mask] + + // advance + w = x[k] >> (shift - nbits) + nbits = _W - (shift - nbits) + } + } + + // convert digits of most-significant word (omit leading zeros) + for nbits >= 0 && w != 0 { + i-- + s[i] = charset[w&mask] + w >>= shift + nbits -= shift + } + + return string(s[i:]) + } + + // general case: extract groups of digits by multiprecision division + + // maximize ndigits where b**ndigits < 2^_W; bb (big base) is b**ndigits + bb := Word(1) + ndigits := 0 + for max := Word(_M / b); bb <= max; bb *= b { + ndigits++ + } + + // preserve x, create local copy for use in repeated divisions + q := nat(nil).set(x) + var r Word + + // convert + if b == 10 { // hard-coding for 10 here speeds this up by 1.25x + for len(q) > 0 { + // extract least significant, base bb "digit" + q, r = q.divW(q, bb) // N.B. >82% of time is here. Optimize divW + if len(q) == 0 { + // skip leading zeros in most-significant group of digits + for j := 0; j < ndigits && r != 0; j++ { + i-- + s[i] = charset[r%10] + r /= 10 + } + } else { + for j := 0; j < ndigits; j++ { + i-- + s[i] = charset[r%10] + r /= 10 + } + } + } + } else { + for len(q) > 0 { + // extract least significant group of digits + q, r = q.divW(q, bb) // N.B. >82% of time is here. Optimize divW + if len(q) == 0 { + // skip leading zeros in most-significant group of digits + for j := 0; j < ndigits && r != 0; j++ { + i-- + s[i] = charset[r%b] + r /= b + } + } else { + for j := 0; j < ndigits; j++ { + i-- + s[i] = charset[r%b] + r /= b + } + } + } + } + + return string(s[i:]) +} + +const deBruijn32 = 0x077CB531 + +var deBruijn32Lookup = []byte{ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, +} + +const deBruijn64 = 0x03f79d71b4ca8b09 + +var deBruijn64Lookup = []byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +// trailingZeroBits returns the number of consecutive zero bits on the right +// side of the given Word. +// See Knuth, volume 4, section 7.3.1 +func trailingZeroBits(x Word) int { + // x & -x leaves only the right-most bit set in the word. Let k be the + // index of that bit. Since only a single bit is set, the value is two + // to the power of k. Multiplying by a power of two is equivalent to + // left shifting, in this case by k bits. The de Bruijn constant is + // such that all six bit, consecutive substrings are distinct. + // Therefore, if we have a left shifted version of this constant we can + // find by how many bits it was shifted by looking at which six bit + // substring ended up at the top of the word. + switch _W { + case 32: + return int(deBruijn32Lookup[((x&-x)*deBruijn32)>>27]) + case 64: + return int(deBruijn64Lookup[((x&-x)*(deBruijn64&_M))>>58]) + default: + panic("Unknown word size") + } + + return 0 +} + +// z = x << s +func (z nat) shl(x nat, s uint) nat { + m := len(x) + if m == 0 { + return z.make(0) + } + // m > 0 + + n := m + int(s/_W) + z = z.make(n + 1) + z[n] = shlVU(z[n-m:n], x, s%_W) + z[0 : n-m].clear() + + return z.norm() +} + +// z = x >> s +func (z nat) shr(x nat, s uint) nat { + m := len(x) + n := m - int(s/_W) + if n <= 0 { + return z.make(0) + } + // n > 0 + + z = z.make(n) + shrVU(z, x[m-n:], s%_W) + + return z.norm() +} + +func (z nat) setBit(x nat, i uint, b uint) nat { + j := int(i / _W) + m := Word(1) << (i % _W) + n := len(x) + switch b { + case 0: + z = z.make(n) + copy(z, x) + if j >= n { + // no need to grow + return z + } + z[j] &^= m + return z.norm() + case 1: + if j >= n { + n = j + 1 + } + z = z.make(n) + copy(z, x) + z[j] |= m + // no need to normalize + return z + } + panic("set bit is not 0 or 1") +} + +func (z nat) bit(i uint) uint { + j := int(i / _W) + if j >= len(z) { + return 0 + } + return uint(z[j] >> (i % _W) & 1) +} + +func (z nat) and(x, y nat) nat { + m := len(x) + n := len(y) + if m > n { + m = n + } + // m <= n + + z = z.make(m) + for i := 0; i < m; i++ { + z[i] = x[i] & y[i] + } + + return z.norm() +} + +func (z nat) andNot(x, y nat) nat { + m := len(x) + n := len(y) + if n > m { + n = m + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] &^ y[i] + } + copy(z[n:m], x[n:m]) + + return z.norm() +} + +func (z nat) or(x, y nat) nat { + m := len(x) + n := len(y) + s := x + if m < n { + n, m = m, n + s = y + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] | y[i] + } + copy(z[n:m], s[n:m]) + + return z.norm() +} + +func (z nat) xor(x, y nat) nat { + m := len(x) + n := len(y) + s := x + if m < n { + n, m = m, n + s = y + } + // m >= n + + z = z.make(m) + for i := 0; i < n; i++ { + z[i] = x[i] ^ y[i] + } + copy(z[n:m], s[n:m]) + + return z.norm() +} + +// greaterThan returns true iff (x1<<_W + x2) > (y1<<_W + y2) +func greaterThan(x1, x2, y1, y2 Word) bool { + return x1 > y1 || x1 == y1 && x2 > y2 +} + +// modW returns x % d. +func (x nat) modW(d Word) (r Word) { + // TODO(agl): we don't actually need to store the q value. + var q nat + q = q.make(len(x)) + return divWVW(q, 0, x, d) +} + +// powersOfTwoDecompose finds q and k with x = q * 1<= 0; i-- { + v = y[i] + + for j := 0; j < _W; j++ { + z = z.mul(z, z) + + if v&mask != 0 { + z = z.mul(z, x) + } + + if m != nil { + q, z = q.div(z, z, m) + } + + v <<= 1 + } + } + + return z +} + +// probablyPrime performs reps Miller-Rabin tests to check whether n is prime. +// If it returns true, n is prime with probability 1 - 1/4^reps. +// If it returns false, n is not prime. +func (n nat) probablyPrime(reps int) bool { + if len(n) == 0 { + return false + } + + if len(n) == 1 { + if n[0] < 2 { + return false + } + + if n[0]%2 == 0 { + return n[0] == 2 + } + + // We have to exclude these cases because we reject all + // multiples of these numbers below. + switch n[0] { + case 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53: + return true + } + } + + const primesProduct32 = 0xC0CFD797 // Π {p ∈ primes, 2 < p <= 29} + const primesProduct64 = 0xE221F97C30E94E1D // Π {p ∈ primes, 2 < p <= 53} + + var r Word + switch _W { + case 32: + r = n.modW(primesProduct32) + case 64: + r = n.modW(primesProduct64 & _M) + default: + panic("Unknown word size") + } + + if r%3 == 0 || r%5 == 0 || r%7 == 0 || r%11 == 0 || + r%13 == 0 || r%17 == 0 || r%19 == 0 || r%23 == 0 || r%29 == 0 { + return false + } + + if _W == 64 && (r%31 == 0 || r%37 == 0 || r%41 == 0 || + r%43 == 0 || r%47 == 0 || r%53 == 0) { + return false + } + + nm1 := nat(nil).sub(n, natOne) + // 1<= len(z)*_S. The value of z is encoded in the +// slice buf[i:]. The number i of unused bytes at the beginning of +// buf is returned as result. +func (z nat) bytes(buf []byte) (i int) { + i = len(buf) + for _, d := range z { + for j := 0; j < _S; j++ { + i-- + buf[i] = byte(d) + d >>= 8 + } + } + + for i < len(buf) && buf[i] == 0 { + i++ + } + + return +} + +// setBytes interprets buf as the bytes of a big-endian unsigned +// integer, sets z to that value, and returns z. +func (z nat) setBytes(buf []byte) nat { + z = z.make((len(buf) + _S - 1) / _S) + + k := 0 + s := uint(0) + var d Word + for i := len(buf); i > 0; i-- { + d |= Word(buf[i-1]) << s + if s += 8; s == _S*8 { + z[k] = d + k++ + s = 0 + d = 0 + } + } + if k < len(z) { + z[k] = d + } + + return z.norm() +} diff --git a/src/pkg/big/nat_test.go b/src/pkg/big/nat_test.go new file mode 100755 index 000000000..71d086087 --- /dev/null +++ b/src/pkg/big/nat_test.go @@ -0,0 +1,669 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "fmt" + "os" + "strings" + "testing" +) + +var cmpTests = []struct { + x, y nat + r int +}{ + {nil, nil, 0}, + {nil, nat{}, 0}, + {nat{}, nil, 0}, + {nat{}, nat{}, 0}, + {nat{0}, nat{0}, 0}, + {nat{0}, nat{1}, -1}, + {nat{1}, nat{0}, 1}, + {nat{1}, nat{1}, 0}, + {nat{0, _M}, nat{1}, 1}, + {nat{1}, nat{0, _M}, -1}, + {nat{1, _M}, nat{0, _M}, 1}, + {nat{0, _M}, nat{1, _M}, -1}, + {nat{16, 571956, 8794, 68}, nat{837, 9146, 1, 754489}, -1}, + {nat{34986, 41, 105, 1957}, nat{56, 7458, 104, 1957}, 1}, +} + +func TestCmp(t *testing.T) { + for i, a := range cmpTests { + r := a.x.cmp(a.y) + if r != a.r { + t.Errorf("#%d got r = %v; want %v", i, r, a.r) + } + } +} + +type funNN func(z, x, y nat) nat +type argNN struct { + z, x, y nat +} + +var sumNN = []argNN{ + {}, + {nat{1}, nil, nat{1}}, + {nat{1111111110}, nat{123456789}, nat{987654321}}, + {nat{0, 0, 0, 1}, nil, nat{0, 0, 0, 1}}, + {nat{0, 0, 0, 1111111110}, nat{0, 0, 0, 123456789}, nat{0, 0, 0, 987654321}}, + {nat{0, 0, 0, 1}, nat{0, 0, _M}, nat{0, 0, 1}}, +} + +var prodNN = []argNN{ + {}, + {nil, nil, nil}, + {nil, nat{991}, nil}, + {nat{991}, nat{991}, nat{1}}, + {nat{991 * 991}, nat{991}, nat{991}}, + {nat{0, 0, 991 * 991}, nat{0, 991}, nat{0, 991}}, + {nat{1 * 991, 2 * 991, 3 * 991, 4 * 991}, nat{1, 2, 3, 4}, nat{991}}, + {nat{4, 11, 20, 30, 20, 11, 4}, nat{1, 2, 3, 4}, nat{4, 3, 2, 1}}, +} + +func TestSet(t *testing.T) { + for _, a := range sumNN { + z := nat(nil).set(a.z) + if z.cmp(a.z) != 0 { + t.Errorf("got z = %v; want %v", z, a.z) + } + } +} + +func testFunNN(t *testing.T, msg string, f funNN, a argNN) { + z := f(nil, a.x, a.y) + if z.cmp(a.z) != 0 { + t.Errorf("%s%+v\n\tgot z = %v; want %v", msg, a, z, a.z) + } +} + +func TestFunNN(t *testing.T) { + for _, a := range sumNN { + arg := a + testFunNN(t, "add", nat.add, arg) + + arg = argNN{a.z, a.y, a.x} + testFunNN(t, "add symmetric", nat.add, arg) + + arg = argNN{a.x, a.z, a.y} + testFunNN(t, "sub", nat.sub, arg) + + arg = argNN{a.y, a.z, a.x} + testFunNN(t, "sub symmetric", nat.sub, arg) + } + + for _, a := range prodNN { + arg := a + testFunNN(t, "mul", nat.mul, arg) + + arg = argNN{a.z, a.y, a.x} + testFunNN(t, "mul symmetric", nat.mul, arg) + } +} + +var mulRangesN = []struct { + a, b uint64 + prod string +}{ + {0, 0, "0"}, + {1, 1, "1"}, + {1, 2, "2"}, + {1, 3, "6"}, + {10, 10, "10"}, + {0, 100, "0"}, + {0, 1e9, "0"}, + {1, 0, "1"}, // empty range + {100, 1, "1"}, // empty range + {1, 10, "3628800"}, // 10! + {1, 20, "2432902008176640000"}, // 20! + {1, 100, + "933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "51185210916864000000000000000000000000", // 100! + }, +} + +func TestMulRangeN(t *testing.T) { + for i, r := range mulRangesN { + prod := nat(nil).mulRange(r.a, r.b).decimalString() + if prod != r.prod { + t.Errorf("#%d: got %s; want %s", i, prod, r.prod) + } + } +} + +var mulArg, mulTmp nat + +func init() { + const n = 1000 + mulArg = make(nat, n) + for i := 0; i < n; i++ { + mulArg[i] = _M + } +} + +func benchmarkMulLoad() { + for j := 1; j <= 10; j++ { + x := mulArg[0 : j*100] + mulTmp.mul(x, x) + } +} + +func BenchmarkMul(b *testing.B) { + for i := 0; i < b.N; i++ { + benchmarkMulLoad() + } +} + +func toString(x nat, charset string) string { + base := len(charset) + + // special cases + switch { + case base < 2: + panic("illegal base") + case len(x) == 0: + return string(charset[0]) + } + + // allocate buffer for conversion + i := x.bitLen()/log2(Word(base)) + 1 // +1: round up + s := make([]byte, i) + + // don't destroy x + q := nat(nil).set(x) + + // convert + for len(q) > 0 { + i-- + var r Word + q, r = q.divW(q, Word(base)) + s[i] = charset[r] + } + + return string(s[i:]) +} + +var strTests = []struct { + x nat // nat value to be converted + c string // conversion charset + s string // expected result +}{ + {nil, "01", "0"}, + {nat{1}, "01", "1"}, + {nat{0xc5}, "01", "11000101"}, + {nat{03271}, lowercaseDigits[0:8], "3271"}, + {nat{10}, lowercaseDigits[0:10], "10"}, + {nat{1234567890}, uppercaseDigits[0:10], "1234567890"}, + {nat{0xdeadbeef}, lowercaseDigits[0:16], "deadbeef"}, + {nat{0xdeadbeef}, uppercaseDigits[0:16], "DEADBEEF"}, + {nat{0x229be7}, lowercaseDigits[0:17], "1a2b3c"}, + {nat{0x309663e6}, uppercaseDigits[0:32], "O9COV6"}, +} + +func TestString(t *testing.T) { + for _, a := range strTests { + s := a.x.string(a.c) + if s != a.s { + t.Errorf("string%+v\n\tgot s = %s; want %s", a, s, a.s) + } + + x, b, err := nat(nil).scan(strings.NewReader(a.s), len(a.c)) + if x.cmp(a.x) != 0 { + t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x) + } + if b != len(a.c) { + t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, len(a.c)) + } + if err != nil { + t.Errorf("scan%+v\n\tgot error = %s", a, err) + } + } +} + +var natScanTests = []struct { + s string // string to be scanned + base int // input base + x nat // expected nat + b int // expected base + ok bool // expected success + next int // next character (or 0, if at EOF) +}{ + // error: illegal base + {base: -1}, + {base: 1}, + {base: 37}, + + // error: no mantissa + {}, + {s: "?"}, + {base: 10}, + {base: 36}, + {s: "?", base: 10}, + {s: "0x"}, + {s: "345", base: 2}, + + // no errors + {"0", 0, nil, 10, true, 0}, + {"0", 10, nil, 10, true, 0}, + {"0", 36, nil, 36, true, 0}, + {"1", 0, nat{1}, 10, true, 0}, + {"1", 10, nat{1}, 10, true, 0}, + {"0 ", 0, nil, 10, true, ' '}, + {"08", 0, nil, 10, true, '8'}, + {"018", 0, nat{1}, 8, true, '8'}, + {"0b1", 0, nat{1}, 2, true, 0}, + {"0b11000101", 0, nat{0xc5}, 2, true, 0}, + {"03271", 0, nat{03271}, 8, true, 0}, + {"10ab", 0, nat{10}, 10, true, 'a'}, + {"1234567890", 0, nat{1234567890}, 10, true, 0}, + {"xyz", 36, nat{(33*36+34)*36 + 35}, 36, true, 0}, + {"xyz?", 36, nat{(33*36+34)*36 + 35}, 36, true, '?'}, + {"0x", 16, nil, 16, true, 'x'}, + {"0xdeadbeef", 0, nat{0xdeadbeef}, 16, true, 0}, + {"0XDEADBEEF", 0, nat{0xdeadbeef}, 16, true, 0}, +} + +func TestScanBase(t *testing.T) { + for _, a := range natScanTests { + r := strings.NewReader(a.s) + x, b, err := nat(nil).scan(r, a.base) + if err == nil && !a.ok { + t.Errorf("scan%+v\n\texpected error", a) + } + if err != nil { + if a.ok { + t.Errorf("scan%+v\n\tgot error = %s", a, err) + } + continue + } + if x.cmp(a.x) != 0 { + t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x) + } + if b != a.b { + t.Errorf("scan%+v\n\tgot b = %d; want %d", a, b, a.base) + } + next, _, err := r.ReadRune() + if err == os.EOF { + next = 0 + err = nil + } + if err == nil && next != a.next { + t.Errorf("scan%+v\n\tgot next = %q; want %q", a, next, a.next) + } + } +} + +var pi = "3" + + "14159265358979323846264338327950288419716939937510582097494459230781640628620899862803482534211706798214808651" + + "32823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461" + + "28475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920" + + "96282925409171536436789259036001133053054882046652138414695194151160943305727036575959195309218611738193261179" + + "31051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798" + + "60943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901" + + "22495343014654958537105079227968925892354201995611212902196086403441815981362977477130996051870721134999999837" + + "29780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083" + + "81420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909" + + "21642019893809525720106548586327886593615338182796823030195203530185296899577362259941389124972177528347913151" + + "55748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035" + + "63707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104" + + "75216205696602405803815019351125338243003558764024749647326391419927260426992279678235478163600934172164121992" + + "45863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818" + + "34797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548" + + "16136115735255213347574184946843852332390739414333454776241686251898356948556209921922218427255025425688767179" + + "04946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886" + + "26945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645" + + "99581339047802759009946576407895126946839835259570982582262052248940772671947826848260147699090264013639443745" + + "53050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382" + + "68683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244" + + "13654976278079771569143599770012961608944169486855584840635342207222582848864815845602850601684273945226746767" + + "88952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288" + + "79710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821" + + "68299894872265880485756401427047755513237964145152374623436454285844479526586782105114135473573952311342716610" + + "21359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435" + + "06430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675" + + "14269123974894090718649423196156794520809514655022523160388193014209376213785595663893778708303906979207734672" + + "21825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539" + + "05796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007" + + "23055876317635942187312514712053292819182618612586732157919841484882916447060957527069572209175671167229109816" + + "90915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398" + + "31501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064" + + "20467525907091548141654985946163718027098199430992448895757128289059232332609729971208443357326548938239119325" + + "97463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100" + + "44929321516084244485963766983895228684783123552658213144957685726243344189303968642624341077322697802807318915" + + "44110104468232527162010526522721116603966655730925471105578537634668206531098965269186205647693125705863566201" + + "85581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318" + + "58676975145661406800700237877659134401712749470420562230538994561314071127000407854733269939081454664645880797" + + "27082668306343285878569830523580893306575740679545716377525420211495576158140025012622859413021647155097925923" + + "09907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111" + + "79042978285647503203198691514028708085990480109412147221317947647772622414254854540332157185306142288137585043" + + "06332175182979866223717215916077166925474873898665494945011465406284336639379003976926567214638530673609657120" + + "91807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862" + + "94726547364252308177036751590673502350728354056704038674351362222477158915049530984448933309634087807693259939" + + "78054193414473774418426312986080998886874132604721569516239658645730216315981931951673538129741677294786724229" + + "24654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001" + + "59377647165122893578601588161755782973523344604281512627203734314653197777416031990665541876397929334419521541" + + "34189948544473456738316249934191318148092777710386387734317720754565453220777092120190516609628049092636019759" + + "88281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267" + + "94561275318134078330336254232783944975382437205835311477119926063813346776879695970309833913077109870408591337" + +// Test case for BenchmarkScanPi. +func TestScanPi(t *testing.T) { + var x nat + z, _, err := x.scan(strings.NewReader(pi), 10) + if err != nil { + t.Errorf("scanning pi: %s", err) + } + if s := z.decimalString(); s != pi { + t.Errorf("scanning pi: got %s", s) + } +} + +func BenchmarkScanPi(b *testing.B) { + for i := 0; i < b.N; i++ { + var x nat + x.scan(strings.NewReader(pi), 10) + } +} + +const ( + // 314**271 + // base 2: 2249 digits + // base 8: 751 digits + // base 10: 678 digits + // base 16: 563 digits + shortBase = 314 + shortExponent = 271 + + // 3141**2178 + // base 2: 31577 digits + // base 8: 10527 digits + // base 10: 9507 digits + // base 16: 7895 digits + mediumBase = 3141 + mediumExponent = 2718 + + // 3141**2178 + // base 2: 406078 digits + // base 8: 135360 digits + // base 10: 122243 digits + // base 16: 101521 digits + longBase = 31415 + longExponent = 27182 +) + +func BenchmarkScanShort2(b *testing.B) { + ScanHelper(b, 2, shortBase, shortExponent) +} + +func BenchmarkScanShort8(b *testing.B) { + ScanHelper(b, 8, shortBase, shortExponent) +} + +func BenchmarkScanSort10(b *testing.B) { + ScanHelper(b, 10, shortBase, shortExponent) +} + +func BenchmarkScanShort16(b *testing.B) { + ScanHelper(b, 16, shortBase, shortExponent) +} + +func BenchmarkScanMedium2(b *testing.B) { + ScanHelper(b, 2, mediumBase, mediumExponent) +} + +func BenchmarkScanMedium8(b *testing.B) { + ScanHelper(b, 8, mediumBase, mediumExponent) +} + +func BenchmarkScanMedium10(b *testing.B) { + ScanHelper(b, 10, mediumBase, mediumExponent) +} + +func BenchmarkScanMedium16(b *testing.B) { + ScanHelper(b, 16, mediumBase, mediumExponent) +} + +func BenchmarkScanLong2(b *testing.B) { + ScanHelper(b, 2, longBase, longExponent) +} + +func BenchmarkScanLong8(b *testing.B) { + ScanHelper(b, 8, longBase, longExponent) +} + +func BenchmarkScanLong10(b *testing.B) { + ScanHelper(b, 10, longBase, longExponent) +} + +func BenchmarkScanLong16(b *testing.B) { + ScanHelper(b, 16, longBase, longExponent) +} + +func ScanHelper(b *testing.B, base int, xv, yv Word) { + b.StopTimer() + var x, y, z nat + x = x.setWord(xv) + y = y.setWord(yv) + z = z.expNN(x, y, nil) + + var s string + s = z.string(lowercaseDigits[0:base]) + if t := toString(z, lowercaseDigits[0:base]); t != s { + panic(fmt.Sprintf("scanning: got %s; want %s", s, t)) + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + x.scan(strings.NewReader(s), base) + } +} + +func BenchmarkStringShort2(b *testing.B) { + StringHelper(b, 2, shortBase, shortExponent) +} + +func BenchmarkStringShort8(b *testing.B) { + StringHelper(b, 8, shortBase, shortExponent) +} + +func BenchmarkStringShort10(b *testing.B) { + StringHelper(b, 10, shortBase, shortExponent) +} + +func BenchmarkStringShort16(b *testing.B) { + StringHelper(b, 16, shortBase, shortExponent) +} + +func BenchmarkStringMedium2(b *testing.B) { + StringHelper(b, 2, mediumBase, mediumExponent) +} + +func BenchmarkStringMedium8(b *testing.B) { + StringHelper(b, 8, mediumBase, mediumExponent) +} + +func BenchmarkStringMedium10(b *testing.B) { + StringHelper(b, 10, mediumBase, mediumExponent) +} + +func BenchmarkStringMedium16(b *testing.B) { + StringHelper(b, 16, mediumBase, mediumExponent) +} + +func BenchmarkStringLong2(b *testing.B) { + StringHelper(b, 2, longBase, longExponent) +} + +func BenchmarkStringLong8(b *testing.B) { + StringHelper(b, 8, longBase, longExponent) +} + +func BenchmarkStringLong10(b *testing.B) { + StringHelper(b, 10, longBase, longExponent) +} + +func BenchmarkStringLong16(b *testing.B) { + StringHelper(b, 16, longBase, longExponent) +} + +func StringHelper(b *testing.B, base int, xv, yv Word) { + b.StopTimer() + var x, y, z nat + x = x.setWord(xv) + y = y.setWord(yv) + z = z.expNN(x, y, nil) + b.StartTimer() + + for i := 0; i < b.N; i++ { + z.string(lowercaseDigits[0:base]) + } +} + +func TestLeadingZeros(t *testing.T) { + var x Word = _B >> 1 + for i := 0; i <= _W; i++ { + if int(leadingZeros(x)) != i { + t.Errorf("failed at %x: got %d want %d", x, leadingZeros(x), i) + } + x >>= 1 + } +} + +type shiftTest struct { + in nat + shift uint + out nat +} + +var leftShiftTests = []shiftTest{ + {nil, 0, nil}, + {nil, 1, nil}, + {natOne, 0, natOne}, + {natOne, 1, natTwo}, + {nat{1 << (_W - 1)}, 1, nat{0}}, + {nat{1 << (_W - 1), 0}, 1, nat{0, 1}}, +} + +func TestShiftLeft(t *testing.T) { + for i, test := range leftShiftTests { + var z nat + z = z.shl(test.in, test.shift) + for j, d := range test.out { + if j >= len(z) || z[j] != d { + t.Errorf("#%d: got: %v want: %v", i, z, test.out) + break + } + } + } +} + +var rightShiftTests = []shiftTest{ + {nil, 0, nil}, + {nil, 1, nil}, + {natOne, 0, natOne}, + {natOne, 1, nil}, + {natTwo, 1, natOne}, + {nat{0, 1}, 1, nat{1 << (_W - 1)}}, + {nat{2, 1, 1}, 1, nat{1<<(_W-1) + 1, 1 << (_W - 1)}}, +} + +func TestShiftRight(t *testing.T) { + for i, test := range rightShiftTests { + var z nat + z = z.shr(test.in, test.shift) + for j, d := range test.out { + if j >= len(z) || z[j] != d { + t.Errorf("#%d: got: %v want: %v", i, z, test.out) + break + } + } + } +} + +type modWTest struct { + in string + dividend string + out string +} + +var modWTests32 = []modWTest{ + {"23492635982634928349238759823742", "252341", "220170"}, +} + +var modWTests64 = []modWTest{ + {"6527895462947293856291561095690465243862946", "524326975699234", "375066989628668"}, +} + +func runModWTests(t *testing.T, tests []modWTest) { + for i, test := range tests { + in, _ := new(Int).SetString(test.in, 10) + d, _ := new(Int).SetString(test.dividend, 10) + out, _ := new(Int).SetString(test.out, 10) + + r := in.abs.modW(d.abs[0]) + if r != out.abs[0] { + t.Errorf("#%d failed: got %s want %s", i, r, out) + } + } +} + +func TestModW(t *testing.T) { + if _W >= 32 { + runModWTests(t, modWTests32) + } + if _W >= 64 { + runModWTests(t, modWTests64) + } +} + +func TestTrailingZeroBits(t *testing.T) { + var x Word + x-- + for i := 0; i < _W; i++ { + if trailingZeroBits(x) != i { + t.Errorf("Failed at step %d: x: %x got: %d", i, x, trailingZeroBits(x)) + } + x <<= 1 + } +} + +var expNNTests = []struct { + x, y, m string + out string +}{ + {"0x8000000000000000", "2", "", "0x40000000000000000000000000000000"}, + {"0x8000000000000000", "2", "6719", "4944"}, + {"0x8000000000000000", "3", "6719", "5447"}, + {"0x8000000000000000", "1000", "6719", "1603"}, + {"0x8000000000000000", "1000000", "6719", "3199"}, + { + "2938462938472983472983659726349017249287491026512746239764525612965293865296239471239874193284792387498274256129746192347", + "298472983472983471903246121093472394872319615612417471234712061", + "29834729834729834729347290846729561262544958723956495615629569234729836259263598127342374289365912465901365498236492183464", + "23537740700184054162508175125554701713153216681790245129157191391322321508055833908509185839069455749219131480588829346291", + }, +} + +func TestExpNN(t *testing.T) { + for i, test := range expNNTests { + x, _, _ := nat(nil).scan(strings.NewReader(test.x), 0) + y, _, _ := nat(nil).scan(strings.NewReader(test.y), 0) + out, _, _ := nat(nil).scan(strings.NewReader(test.out), 0) + + var m nat + + if len(test.m) > 0 { + m, _, _ = nat(nil).scan(strings.NewReader(test.m), 0) + } + + z := nat(nil).expNN(x, y, m) + if z.cmp(out) != 0 { + t.Errorf("#%d got %v want %v", i, z, out) + } + } +} diff --git a/src/pkg/big/rat.go b/src/pkg/big/rat.go new file mode 100644 index 000000000..327b9bd9c --- /dev/null +++ b/src/pkg/big/rat.go @@ -0,0 +1,371 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements multi-precision rational numbers. + +package big + +import ( + "encoding/binary" + "fmt" + "os" + "strings" +) + +// A Rat represents a quotient a/b of arbitrary precision. The zero value for +// a Rat, 0/0, is not a legal Rat. +type Rat struct { + a Int + b nat +} + +// NewRat creates a new Rat with numerator a and denominator b. +func NewRat(a, b int64) *Rat { + return new(Rat).SetFrac64(a, b) +} + +// SetFrac sets z to a/b and returns z. +func (z *Rat) SetFrac(a, b *Int) *Rat { + z.a.Set(a) + z.a.neg = a.neg != b.neg + z.b = z.b.set(b.abs) + return z.norm() +} + +// SetFrac64 sets z to a/b and returns z. +func (z *Rat) SetFrac64(a, b int64) *Rat { + z.a.SetInt64(a) + if b < 0 { + b = -b + z.a.neg = !z.a.neg + } + z.b = z.b.setUint64(uint64(b)) + return z.norm() +} + +// SetInt sets z to x (by making a copy of x) and returns z. +func (z *Rat) SetInt(x *Int) *Rat { + z.a.Set(x) + z.b = z.b.setWord(1) + return z +} + +// SetInt64 sets z to x and returns z. +func (z *Rat) SetInt64(x int64) *Rat { + z.a.SetInt64(x) + z.b = z.b.setWord(1) + return z +} + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Rat) Sign() int { + return x.a.Sign() +} + +// IsInt returns true if the denominator of x is 1. +func (x *Rat) IsInt() bool { + return len(x.b) == 1 && x.b[0] == 1 +} + +// Num returns the numerator of z; it may be <= 0. +// The result is a reference to z's numerator; it +// may change if a new value is assigned to z. +func (z *Rat) Num() *Int { + return &z.a +} + +// Denom returns the denominator of z; it is always > 0. +// The result is a reference to z's denominator; it +// may change if a new value is assigned to z. +func (z *Rat) Denom() *Int { + return &Int{false, z.b} +} + +func gcd(x, y nat) nat { + // Euclidean algorithm. + var a, b nat + a = a.set(x) + b = b.set(y) + for len(b) != 0 { + var q, r nat + _, r = q.div(r, a, b) + a = b + b = r + } + return a +} + +func (z *Rat) norm() *Rat { + f := gcd(z.a.abs, z.b) + if len(z.a.abs) == 0 { + // z == 0 + z.a.neg = false // normalize sign + z.b = z.b.setWord(1) + return z + } + if f.cmp(natOne) != 0 { + z.a.abs, _ = z.a.abs.div(nil, z.a.abs, f) + z.b, _ = z.b.div(nil, z.b, f) + } + return z +} + +func mulNat(x *Int, y nat) *Int { + var z Int + z.abs = z.abs.mul(x.abs, y) + z.neg = len(z.abs) > 0 && x.neg + return &z +} + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Rat) Cmp(y *Rat) (r int) { + return mulNat(&x.a, y.b).Cmp(mulNat(&y.a, x.b)) +} + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Rat) Abs(x *Rat) *Rat { + z.a.Abs(&x.a) + z.b = z.b.set(x.b) + return z +} + +// Add sets z to the sum x+y and returns z. +func (z *Rat) Add(x, y *Rat) *Rat { + a1 := mulNat(&x.a, y.b) + a2 := mulNat(&y.a, x.b) + z.a.Add(a1, a2) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + +// Sub sets z to the difference x-y and returns z. +func (z *Rat) Sub(x, y *Rat) *Rat { + a1 := mulNat(&x.a, y.b) + a2 := mulNat(&y.a, x.b) + z.a.Sub(a1, a2) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + +// Mul sets z to the product x*y and returns z. +func (z *Rat) Mul(x, y *Rat) *Rat { + z.a.Mul(&x.a, &y.a) + z.b = z.b.mul(x.b, y.b) + return z.norm() +} + +// Quo sets z to the quotient x/y and returns z. +// If y == 0, a division-by-zero run-time panic occurs. +func (z *Rat) Quo(x, y *Rat) *Rat { + if len(y.a.abs) == 0 { + panic("division by zero") + } + a := mulNat(&x.a, y.b) + b := mulNat(&y.a, x.b) + z.a.abs = a.abs + z.b = b.abs + z.a.neg = a.neg != b.neg + return z.norm() +} + +// Neg sets z to -x (by making a copy of x if necessary) and returns z. +func (z *Rat) Neg(x *Rat) *Rat { + z.a.Neg(&x.a) + z.b = z.b.set(x.b) + return z +} + +// Set sets z to x (by making a copy of x if necessary) and returns z. +func (z *Rat) Set(x *Rat) *Rat { + z.a.Set(&x.a) + z.b = z.b.set(x.b) + return z +} + +func ratTok(ch int) bool { + return strings.IndexRune("+-/0123456789.eE", ch) >= 0 +} + +// Scan is a support routine for fmt.Scanner. It accepts the formats +// 'e', 'E', 'f', 'F', 'g', 'G', and 'v'. All formats are equivalent. +func (z *Rat) Scan(s fmt.ScanState, ch int) os.Error { + tok, err := s.Token(true, ratTok) + if err != nil { + return err + } + if strings.IndexRune("efgEFGv", ch) < 0 { + return os.NewError("Rat.Scan: invalid verb") + } + if _, ok := z.SetString(string(tok)); !ok { + return os.NewError("Rat.Scan: invalid syntax") + } + return nil +} + +// SetString sets z to the value of s and returns z and a boolean indicating +// success. s can be given as a fraction "a/b" or as a floating-point number +// optionally followed by an exponent. If the operation failed, the value of z +// is undefined. +func (z *Rat) SetString(s string) (*Rat, bool) { + if len(s) == 0 { + return z, false + } + + // check for a quotient + sep := strings.Index(s, "/") + if sep >= 0 { + if _, ok := z.a.SetString(s[0:sep], 10); !ok { + return z, false + } + s = s[sep+1:] + var err os.Error + if z.b, _, err = z.b.scan(strings.NewReader(s), 10); err != nil { + return z, false + } + return z.norm(), true + } + + // check for a decimal point + sep = strings.Index(s, ".") + // check for an exponent + e := strings.IndexAny(s, "eE") + var exp Int + if e >= 0 { + if e < sep { + // The E must come after the decimal point. + return z, false + } + if _, ok := exp.SetString(s[e+1:], 10); !ok { + return z, false + } + s = s[0:e] + } + if sep >= 0 { + s = s[0:sep] + s[sep+1:] + exp.Sub(&exp, NewInt(int64(len(s)-sep))) + } + + if _, ok := z.a.SetString(s, 10); !ok { + return z, false + } + powTen := nat{}.expNN(natTen, exp.abs, nil) + if exp.neg { + z.b = powTen + z.norm() + } else { + z.a.abs = z.a.abs.mul(z.a.abs, powTen) + z.b = z.b.setWord(1) + } + + return z, true +} + +// String returns a string representation of z in the form "a/b" (even if b == 1). +func (z *Rat) String() string { + return z.a.String() + "/" + z.b.decimalString() +} + +// RatString returns a string representation of z in the form "a/b" if b != 1, +// and in the form "a" if b == 1. +func (z *Rat) RatString() string { + if z.IsInt() { + return z.a.String() + } + return z.String() +} + +// FloatString returns a string representation of z in decimal form with prec +// digits of precision after the decimal point and the last digit rounded. +func (z *Rat) FloatString(prec int) string { + if z.IsInt() { + s := z.a.String() + if prec > 0 { + s += "." + strings.Repeat("0", prec) + } + return s + } + + q, r := nat{}.div(nat{}, z.a.abs, z.b) + + p := natOne + if prec > 0 { + p = nat{}.expNN(natTen, nat{}.setUint64(uint64(prec)), nil) + } + + r = r.mul(r, p) + r, r2 := r.div(nat{}, r, z.b) + + // see if we need to round up + r2 = r2.add(r2, r2) + if z.b.cmp(r2) <= 0 { + r = r.add(r, natOne) + if r.cmp(p) >= 0 { + q = nat{}.add(q, natOne) + r = nat{}.sub(r, p) + } + } + + s := q.decimalString() + if z.a.neg { + s = "-" + s + } + + if prec > 0 { + rs := r.decimalString() + leadingZeros := prec - len(rs) + s += "." + strings.Repeat("0", leadingZeros) + rs + } + + return s +} + +// Gob codec version. Permits backward-compatible changes to the encoding. +const ratGobVersion byte = 1 + +// GobEncode implements the gob.GobEncoder interface. +func (z *Rat) GobEncode() ([]byte, os.Error) { + buf := make([]byte, 1+4+(len(z.a.abs)+len(z.b))*_S) // extra bytes for version and sign bit (1), and numerator length (4) + i := z.b.bytes(buf) + j := z.a.abs.bytes(buf[0:i]) + n := i - j + if int(uint32(n)) != n { + // this should never happen + return nil, os.NewError("Rat.GobEncode: numerator too large") + } + binary.BigEndian.PutUint32(buf[j-4:j], uint32(n)) + j -= 1 + 4 + b := ratGobVersion << 1 // make space for sign bit + if z.a.neg { + b |= 1 + } + buf[j] = b + return buf[j:], nil +} + +// GobDecode implements the gob.GobDecoder interface. +func (z *Rat) GobDecode(buf []byte) os.Error { + if len(buf) == 0 { + return os.NewError("Rat.GobDecode: no data") + } + b := buf[0] + if b>>1 != ratGobVersion { + return os.NewError(fmt.Sprintf("Rat.GobDecode: encoding version %d not supported", b>>1)) + } + const j = 1 + 4 + i := j + binary.BigEndian.Uint32(buf[j-4:j]) + z.a.neg = b&1 != 0 + z.a.abs = z.a.abs.setBytes(buf[j:i]) + z.b = z.b.setBytes(buf[i:]) + return nil +} diff --git a/src/pkg/big/rat_test.go b/src/pkg/big/rat_test.go new file mode 100644 index 000000000..dbc5bb6cc --- /dev/null +++ b/src/pkg/big/rat_test.go @@ -0,0 +1,332 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import ( + "bytes" + "fmt" + "gob" + "testing" +) + +var setStringTests = []struct { + in, out string + ok bool +}{ + {"0", "0", true}, + {"-0", "0", true}, + {"1", "1", true}, + {"-1", "-1", true}, + {"1.", "1", true}, + {"1e0", "1", true}, + {"1.e1", "10", true}, + {in: "1e", ok: false}, + {in: "1.e", ok: false}, + {in: "1e+14e-5", ok: false}, + {in: "1e4.5", ok: false}, + {in: "r", ok: false}, + {in: "a/b", ok: false}, + {in: "a.b", ok: false}, + {"-0.1", "-1/10", true}, + {"-.1", "-1/10", true}, + {"2/4", "1/2", true}, + {".25", "1/4", true}, + {"-1/5", "-1/5", true}, + {"8129567.7690E14", "812956776900000000000", true}, + {"78189e+4", "781890000", true}, + {"553019.8935e+8", "55301989350000", true}, + {"98765432109876543210987654321e-10", "98765432109876543210987654321/10000000000", true}, + {"9877861857500000E-7", "3951144743/4", true}, + {"2169378.417e-3", "2169378417/1000000", true}, + {"884243222337379604041632732738665534", "884243222337379604041632732738665534", true}, + {"53/70893980658822810696", "53/70893980658822810696", true}, + {"106/141787961317645621392", "53/70893980658822810696", true}, + {"204211327800791583.81095", "4084226556015831676219/20000", true}, +} + +func TestRatSetString(t *testing.T) { + for i, test := range setStringTests { + x, ok := new(Rat).SetString(test.in) + + if ok != test.ok || ok && x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + +func TestRatScan(t *testing.T) { + var buf bytes.Buffer + for i, test := range setStringTests { + x := new(Rat) + buf.Reset() + buf.WriteString(test.in) + + _, err := fmt.Fscanf(&buf, "%v", x) + if err == nil != test.ok { + if test.ok { + t.Errorf("#%d error: %s", i, err.String()) + } else { + t.Errorf("#%d expected error", i) + } + continue + } + if err == nil && x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + +var floatStringTests = []struct { + in string + prec int + out string +}{ + {"0", 0, "0"}, + {"0", 4, "0.0000"}, + {"1", 0, "1"}, + {"1", 2, "1.00"}, + {"-1", 0, "-1"}, + {".25", 2, "0.25"}, + {".25", 1, "0.3"}, + {".25", 3, "0.250"}, + {"-1/3", 3, "-0.333"}, + {"-2/3", 4, "-0.6667"}, + {"0.96", 1, "1.0"}, + {"0.999", 2, "1.00"}, + {"0.9", 0, "1"}, + {".25", -1, "0"}, + {".55", -1, "1"}, +} + +func TestFloatString(t *testing.T) { + for i, test := range floatStringTests { + x, _ := new(Rat).SetString(test.in) + + if x.FloatString(test.prec) != test.out { + t.Errorf("#%d got %s want %s", i, x.FloatString(test.prec), test.out) + } + } +} + +func TestRatSign(t *testing.T) { + zero := NewRat(0, 1) + for _, a := range setStringTests { + var x Rat + x.SetString(a.in) + s := x.Sign() + e := x.Cmp(zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, &x) + } + } +} + +var ratCmpTests = []struct { + rat1, rat2 string + out int +}{ + {"0", "0/1", 0}, + {"1/1", "1", 0}, + {"-1", "-2/2", 0}, + {"1", "0", 1}, + {"0/1", "1/1", -1}, + {"-5/1434770811533343057144", "-5/1434770811533343057145", -1}, + {"49832350382626108453/8964749413", "49832350382626108454/8964749413", -1}, + {"-37414950961700930/7204075375675961", "37414950961700930/7204075375675961", -1}, + {"37414950961700930/7204075375675961", "74829901923401860/14408150751351922", 0}, +} + +func TestRatCmp(t *testing.T) { + for i, test := range ratCmpTests { + x, _ := new(Rat).SetString(test.rat1) + y, _ := new(Rat).SetString(test.rat2) + + out := x.Cmp(y) + if out != test.out { + t.Errorf("#%d got out = %v; want %v", i, out, test.out) + } + } +} + +func TestIsInt(t *testing.T) { + one := NewInt(1) + for _, a := range setStringTests { + var x Rat + x.SetString(a.in) + i := x.IsInt() + e := x.Denom().Cmp(one) == 0 + if i != e { + t.Errorf("got %v; want %v for z = %v", i, e, &x) + } + } +} + +func TestRatAbs(t *testing.T) { + zero := NewRat(0, 1) + for _, a := range setStringTests { + var z Rat + z.SetString(a.in) + var e Rat + e.Set(&z) + if e.Cmp(zero) < 0 { + e.Sub(zero, &e) + } + z.Abs(&z) + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", &z, &e) + } + } +} + +type ratBinFun func(z, x, y *Rat) *Rat +type ratBinArg struct { + x, y, z string +} + +func testRatBin(t *testing.T, i int, name string, f ratBinFun, a ratBinArg) { + x, _ := NewRat(0, 1).SetString(a.x) + y, _ := NewRat(0, 1).SetString(a.y) + z, _ := NewRat(0, 1).SetString(a.z) + out := f(NewRat(0, 1), x, y) + + if out.Cmp(z) != 0 { + t.Errorf("%s #%d got %s want %s", name, i, out, z) + } +} + +var ratBinTests = []struct { + x, y string + sum, prod string +}{ + {"0", "0", "0", "0"}, + {"0", "1", "1", "0"}, + {"-1", "0", "-1", "0"}, + {"-1", "1", "0", "-1"}, + {"1", "1", "2", "1"}, + {"1/2", "1/2", "1", "1/4"}, + {"1/4", "1/3", "7/12", "1/12"}, + {"2/5", "-14/3", "-64/15", "-28/15"}, + {"4707/49292519774798173060", "-3367/70976135186689855734", "84058377121001851123459/1749296273614329067191168098769082663020", "-1760941/388732505247628681598037355282018369560"}, + {"-61204110018146728334/3", "-31052192278051565633/2", "-215564796870448153567/6", "950260896245257153059642991192710872711/3"}, + {"-854857841473707320655/4237645934602118692642972629634714039", "-18/31750379913563777419", "-27/133467566250814981", "15387441146526731771790/134546868362786310073779084329032722548987800600710485341"}, + {"618575745270541348005638912139/19198433543745179392300736", "-19948846211000086/637313996471", "27674141753240653/30123979153216", "-6169936206128396568797607742807090270137721977/6117715203873571641674006593837351328"}, + {"-3/26206484091896184128", "5/2848423294177090248", "15310893822118706237/9330894968229805033368778458685147968", "-5/24882386581946146755650075889827061248"}, + {"26946729/330400702820", "41563965/225583428284", "1238218672302860271/4658307703098666660055", "224002580204097/14906584649915733312176"}, + {"-8259900599013409474/7", "-84829337473700364773/56707961321161574960", "-468402123685491748914621885145127724451/396955729248131024720", "350340947706464153265156004876107029701/198477864624065512360"}, + {"575775209696864/1320203974639986246357", "29/712593081308", "410331716733912717985762465/940768218243776489278275419794956", "808/45524274987585732633"}, + {"1786597389946320496771/2066653520653241", "6269770/1992362624741777", "3559549865190272133656109052308126637/4117523232840525481453983149257", "8967230/3296219033"}, + {"-36459180403360509753/32150500941194292113930", "9381566963714/9633539", "301622077145533298008420642898530153/309723104686531919656937098270", "-3784609207827/3426986245"}, +} + +func TestRatBin(t *testing.T) { + for i, test := range ratBinTests { + arg := ratBinArg{test.x, test.y, test.sum} + testRatBin(t, i, "Add", (*Rat).Add, arg) + + arg = ratBinArg{test.y, test.x, test.sum} + testRatBin(t, i, "Add symmetric", (*Rat).Add, arg) + + arg = ratBinArg{test.sum, test.x, test.y} + testRatBin(t, i, "Sub", (*Rat).Sub, arg) + + arg = ratBinArg{test.sum, test.y, test.x} + testRatBin(t, i, "Sub symmetric", (*Rat).Sub, arg) + + arg = ratBinArg{test.x, test.y, test.prod} + testRatBin(t, i, "Mul", (*Rat).Mul, arg) + + arg = ratBinArg{test.y, test.x, test.prod} + testRatBin(t, i, "Mul symmetric", (*Rat).Mul, arg) + + if test.x != "0" { + arg = ratBinArg{test.prod, test.x, test.y} + testRatBin(t, i, "Quo", (*Rat).Quo, arg) + } + + if test.y != "0" { + arg = ratBinArg{test.prod, test.y, test.x} + testRatBin(t, i, "Quo symmetric", (*Rat).Quo, arg) + } + } +} + +func TestIssue820(t *testing.T) { + x := NewRat(3, 1) + y := NewRat(2, 1) + z := y.Quo(x, y) + q := NewRat(3, 2) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } + + y = NewRat(3, 1) + x = NewRat(2, 1) + z = y.Quo(x, y) + q = NewRat(2, 3) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } + + x = NewRat(3, 1) + z = x.Quo(x, x) + q = NewRat(3, 3) + if z.Cmp(q) != 0 { + t.Errorf("got %s want %s", z, q) + } +} + +var setFrac64Tests = []struct { + a, b int64 + out string +}{ + {0, 1, "0"}, + {0, -1, "0"}, + {1, 1, "1"}, + {-1, 1, "-1"}, + {1, -1, "-1"}, + {-1, -1, "1"}, + {-9223372036854775808, -9223372036854775808, "1"}, +} + +func TestRatSetFrac64Rat(t *testing.T) { + for i, test := range setFrac64Tests { + x := new(Rat).SetFrac64(test.a, test.b) + if x.RatString() != test.out { + t.Errorf("#%d got %s want %s", i, x.RatString(), test.out) + } + } +} + +func TestRatGobEncoding(t *testing.T) { + var medium bytes.Buffer + enc := gob.NewEncoder(&medium) + dec := gob.NewDecoder(&medium) + for i, test := range gobEncodingTests { + for j := 0; j < 4; j++ { + medium.Reset() // empty buffer for each test case (in case of failures) + stest := test + if j&1 != 0 { + // negative numbers + stest = "-" + test + } + if j%2 != 0 { + // fractions + stest = stest + "." + test + } + var tx Rat + tx.SetString(stest) + if err := enc.Encode(&tx); err != nil { + t.Errorf("#%d%c: encoding failed: %s", i, 'a'+j, err) + } + var rx Rat + if err := dec.Decode(&rx); err != nil { + t.Errorf("#%d%c: decoding failed: %s", i, 'a'+j, err) + } + if rx.Cmp(&tx) != 0 { + t.Errorf("#%d%c: transmission failed: got %s want %s", i, 'a'+j, &rx, &tx) + } + } + } +} diff --git a/src/pkg/bufio/Makefile b/src/pkg/bufio/Makefile new file mode 100644 index 000000000..85430e8e8 --- /dev/null +++ b/src/pkg/bufio/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=bufio +GOFILES=\ + bufio.go\ + +include ../../Make.pkg diff --git a/src/pkg/bufio/bufio.go b/src/pkg/bufio/bufio.go new file mode 100644 index 000000000..727ebfdbb --- /dev/null +++ b/src/pkg/bufio/bufio.go @@ -0,0 +1,549 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer +// object, creating another object (Reader or Writer) that also implements +// the interface but provides buffering and some help for textual I/O. +package bufio + +import ( + "bytes" + "io" + "os" + "strconv" + "utf8" +) + +const ( + defaultBufSize = 4096 +) + +// Errors introduced by this package. +type Error struct { + ErrorString string +} + +func (err *Error) String() string { return err.ErrorString } + +var ( + ErrInvalidUnreadByte os.Error = &Error{"bufio: invalid use of UnreadByte"} + ErrInvalidUnreadRune os.Error = &Error{"bufio: invalid use of UnreadRune"} + ErrBufferFull os.Error = &Error{"bufio: buffer full"} + ErrNegativeCount os.Error = &Error{"bufio: negative count"} + errInternal os.Error = &Error{"bufio: internal error"} +) + +// BufSizeError is the error representing an invalid buffer size. +type BufSizeError int + +func (b BufSizeError) String() string { + return "bufio: bad buffer size " + strconv.Itoa(int(b)) +} + +// Buffered input. + +// Reader implements buffering for an io.Reader object. +type Reader struct { + buf []byte + rd io.Reader + r, w int + err os.Error + lastByte int + lastRuneSize int +} + +// NewReaderSize creates a new Reader whose buffer has the specified size, +// which must be greater than zero. If the argument io.Reader is already a +// Reader with large enough size, it returns the underlying Reader. +// It returns the Reader and any error. +func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) { + if size <= 0 { + return nil, BufSizeError(size) + } + // Is it already a Reader? + b, ok := rd.(*Reader) + if ok && len(b.buf) >= size { + return b, nil + } + b = new(Reader) + b.buf = make([]byte, size) + b.rd = rd + b.lastByte = -1 + b.lastRuneSize = -1 + return b, nil +} + +// NewReader returns a new Reader whose buffer has the default size. +func NewReader(rd io.Reader) *Reader { + b, err := NewReaderSize(rd, defaultBufSize) + if err != nil { + // cannot happen - defaultBufSize is a valid size + panic(err) + } + return b +} + +// fill reads a new chunk into the buffer. +func (b *Reader) fill() { + // Slide existing data to beginning. + if b.r > 0 { + copy(b.buf, b.buf[b.r:b.w]) + b.w -= b.r + b.r = 0 + } + + // Read new data. + n, e := b.rd.Read(b.buf[b.w:]) + b.w += n + if e != nil { + b.err = e + } +} + +func (b *Reader) readErr() os.Error { + err := b.err + b.err = nil + return err +} + +// Peek returns the next n bytes without advancing the reader. The bytes stop +// being valid at the next read call. If Peek returns fewer than n bytes, it +// also returns an error explaining why the read is short. The error is +// ErrBufferFull if n is larger than b's buffer size. +func (b *Reader) Peek(n int) ([]byte, os.Error) { + if n < 0 { + return nil, ErrNegativeCount + } + if n > len(b.buf) { + return nil, ErrBufferFull + } + for b.w-b.r < n && b.err == nil { + b.fill() + } + m := b.w - b.r + if m > n { + m = n + } + err := b.readErr() + if m < n && err == nil { + err = ErrBufferFull + } + return b.buf[b.r : b.r+m], err +} + +// Read reads data into p. +// It returns the number of bytes read into p. +// It calls Read at most once on the underlying Reader, +// hence n may be less than len(p). +// At EOF, the count will be zero and err will be os.EOF. +func (b *Reader) Read(p []byte) (n int, err os.Error) { + n = len(p) + if n == 0 { + return 0, b.readErr() + } + if b.w == b.r { + if b.err != nil { + return 0, b.readErr() + } + if len(p) >= len(b.buf) { + // Large read, empty buffer. + // Read directly into p to avoid copy. + n, b.err = b.rd.Read(p) + if n > 0 { + b.lastByte = int(p[n-1]) + b.lastRuneSize = -1 + } + return n, b.readErr() + } + b.fill() + if b.w == b.r { + return 0, b.readErr() + } + } + + if n > b.w-b.r { + n = b.w - b.r + } + copy(p[0:n], b.buf[b.r:]) + b.r += n + b.lastByte = int(b.buf[b.r-1]) + b.lastRuneSize = -1 + return n, nil +} + +// ReadByte reads and returns a single byte. +// If no byte is available, returns an error. +func (b *Reader) ReadByte() (c byte, err os.Error) { + b.lastRuneSize = -1 + for b.w == b.r { + if b.err != nil { + return 0, b.readErr() + } + b.fill() + } + c = b.buf[b.r] + b.r++ + b.lastByte = int(c) + return c, nil +} + +// UnreadByte unreads the last byte. Only the most recently read byte can be unread. +func (b *Reader) UnreadByte() os.Error { + b.lastRuneSize = -1 + if b.r == b.w && b.lastByte >= 0 { + b.w = 1 + b.r = 0 + b.buf[0] = byte(b.lastByte) + b.lastByte = -1 + return nil + } + if b.r <= 0 { + return ErrInvalidUnreadByte + } + b.r-- + b.lastByte = -1 + return nil +} + +// ReadRune reads a single UTF-8 encoded Unicode character and returns the +// rune and its size in bytes. +func (b *Reader) ReadRune() (rune int, size int, err os.Error) { + for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil { + b.fill() + } + b.lastRuneSize = -1 + if b.r == b.w { + return 0, 0, b.readErr() + } + rune, size = int(b.buf[b.r]), 1 + if rune >= 0x80 { + rune, size = utf8.DecodeRune(b.buf[b.r:b.w]) + } + b.r += size + b.lastByte = int(b.buf[b.r-1]) + b.lastRuneSize = size + return rune, size, nil +} + +// UnreadRune unreads the last rune. If the most recent read operation on +// the buffer was not a ReadRune, UnreadRune returns an error. (In this +// regard it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Reader) UnreadRune() os.Error { + if b.lastRuneSize < 0 || b.r == 0 { + return ErrInvalidUnreadRune + } + b.r -= b.lastRuneSize + b.lastByte = -1 + b.lastRuneSize = -1 + return nil +} + +// Buffered returns the number of bytes that can be read from the current buffer. +func (b *Reader) Buffered() int { return b.w - b.r } + +// ReadSlice reads until the first occurrence of delim in the input, +// returning a slice pointing at the bytes in the buffer. +// The bytes stop being valid at the next read call. +// If ReadSlice encounters an error before finding a delimiter, +// it returns all the data in the buffer and the error itself (often os.EOF). +// ReadSlice fails with error ErrBufferFull if the buffer fills without a delim. +// Because the data returned from ReadSlice will be overwritten +// by the next I/O operation, most clients should use +// ReadBytes or ReadString instead. +// ReadSlice returns err != nil if and only if line does not end in delim. +func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) { + // Look in buffer. + if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 { + line1 := b.buf[b.r : b.r+i+1] + b.r += i + 1 + return line1, nil + } + + // Read more into buffer, until buffer fills or we find delim. + for { + if b.err != nil { + line := b.buf[b.r:b.w] + b.r = b.w + return line, b.readErr() + } + + n := b.Buffered() + b.fill() + + // Search new part of buffer + if i := bytes.IndexByte(b.buf[n:b.w], delim); i >= 0 { + line := b.buf[0 : n+i+1] + b.r = n + i + 1 + return line, nil + } + + // Buffer is full? + if b.Buffered() >= len(b.buf) { + b.r = b.w + return b.buf, ErrBufferFull + } + } + panic("not reached") +} + +// ReadLine tries to return a single line, not including the end-of-line bytes. +// If the line was too long for the buffer then isPrefix is set and the +// beginning of the line is returned. The rest of the line will be returned +// from future calls. isPrefix will be false when returning the last fragment +// of the line. The returned buffer is only valid until the next call to +// ReadLine. ReadLine either returns a non-nil line or it returns an error, +// never both. +func (b *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { + line, err = b.ReadSlice('\n') + if err == ErrBufferFull { + return line, true, nil + } + + if len(line) == 0 { + return + } + err = nil + + if line[len(line)-1] == '\n' { + line = line[:len(line)-1] + } + if len(line) > 0 && line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + return +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) { + // Use ReadSlice to look for array, + // accumulating full buffers. + var frag []byte + var full [][]byte + err = nil + + for { + var e os.Error + frag, e = b.ReadSlice(delim) + if e == nil { // got final fragment + break + } + if e != ErrBufferFull { // unexpected error + err = e + break + } + + // Make a copy of the buffer. + buf := make([]byte, len(frag)) + copy(buf, frag) + full = append(full, buf) + } + + // Allocate new buffer to hold the full pieces and the fragment. + n := 0 + for i := range full { + n += len(full[i]) + } + n += len(frag) + + // Copy full pieces and fragment in. + buf := make([]byte, n) + n = 0 + for i := range full { + n += copy(buf[n:], full[i]) + } + copy(buf[n:], frag) + return buf, err +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadString returns err != nil if and only if the returned data does not end in +// delim. +func (b *Reader) ReadString(delim byte) (line string, err os.Error) { + bytes, e := b.ReadBytes(delim) + return string(bytes), e +} + +// buffered output + +// Writer implements buffering for an io.Writer object. +type Writer struct { + err os.Error + buf []byte + n int + wr io.Writer +} + +// NewWriterSize creates a new Writer whose buffer has the specified size, +// which must be greater than zero. If the argument io.Writer is already a +// Writer with large enough size, it returns the underlying Writer. +// It returns the Writer and any error. +func NewWriterSize(wr io.Writer, size int) (*Writer, os.Error) { + if size <= 0 { + return nil, BufSizeError(size) + } + // Is it already a Writer? + b, ok := wr.(*Writer) + if ok && len(b.buf) >= size { + return b, nil + } + b = new(Writer) + b.buf = make([]byte, size) + b.wr = wr + return b, nil +} + +// NewWriter returns a new Writer whose buffer has the default size. +func NewWriter(wr io.Writer) *Writer { + b, err := NewWriterSize(wr, defaultBufSize) + if err != nil { + // cannot happen - defaultBufSize is valid size + panic(err) + } + return b +} + +// Flush writes any buffered data to the underlying io.Writer. +func (b *Writer) Flush() os.Error { + if b.err != nil { + return b.err + } + if b.n == 0 { + return nil + } + n, e := b.wr.Write(b.buf[0:b.n]) + if n < b.n && e == nil { + e = io.ErrShortWrite + } + if e != nil { + if n > 0 && n < b.n { + copy(b.buf[0:b.n-n], b.buf[n:b.n]) + } + b.n -= n + b.err = e + return e + } + b.n = 0 + return nil +} + +// Available returns how many bytes are unused in the buffer. +func (b *Writer) Available() int { return len(b.buf) - b.n } + +// Buffered returns the number of bytes that have been written into the current buffer. +func (b *Writer) Buffered() int { return b.n } + +// Write writes the contents of p into the buffer. +// It returns the number of bytes written. +// If nn < len(p), it also returns an error explaining +// why the write is short. +func (b *Writer) Write(p []byte) (nn int, err os.Error) { + for len(p) > b.Available() && b.err == nil { + var n int + if b.Buffered() == 0 { + // Large write, empty buffer. + // Write directly from p to avoid copy. + n, b.err = b.wr.Write(p) + } else { + n = copy(b.buf[b.n:], p) + b.n += n + b.Flush() + } + nn += n + p = p[n:] + } + if b.err != nil { + return nn, b.err + } + n := copy(b.buf[b.n:], p) + b.n += n + nn += n + return nn, nil +} + +// WriteByte writes a single byte. +func (b *Writer) WriteByte(c byte) os.Error { + if b.err != nil { + return b.err + } + if b.Available() <= 0 && b.Flush() != nil { + return b.err + } + b.buf[b.n] = c + b.n++ + return nil +} + +// WriteRune writes a single Unicode code point, returning +// the number of bytes written and any error. +func (b *Writer) WriteRune(rune int) (size int, err os.Error) { + if rune < utf8.RuneSelf { + err = b.WriteByte(byte(rune)) + if err != nil { + return 0, err + } + return 1, nil + } + if b.err != nil { + return 0, b.err + } + n := b.Available() + if n < utf8.UTFMax { + if b.Flush(); b.err != nil { + return 0, b.err + } + n = b.Available() + if n < utf8.UTFMax { + // Can only happen if buffer is silly small. + return b.WriteString(string(rune)) + } + } + size = utf8.EncodeRune(b.buf[b.n:], rune) + b.n += size + return size, nil +} + +// WriteString writes a string. +// It returns the number of bytes written. +// If the count is less than len(s), it also returns an error explaining +// why the write is short. +func (b *Writer) WriteString(s string) (int, os.Error) { + nn := 0 + for len(s) > b.Available() && b.err == nil { + n := copy(b.buf[b.n:], s) + b.n += n + nn += n + s = s[n:] + b.Flush() + } + if b.err != nil { + return nn, b.err + } + n := copy(b.buf[b.n:], s) + b.n += n + nn += n + return nn, nil +} + +// buffered input and output + +// ReadWriter stores pointers to a Reader and a Writer. +// It implements io.ReadWriter. +type ReadWriter struct { + *Reader + *Writer +} + +// NewReadWriter allocates a new ReadWriter that dispatches to r and w. +func NewReadWriter(r *Reader, w *Writer) *ReadWriter { + return &ReadWriter{r, w} +} diff --git a/src/pkg/bufio/bufio_test.go b/src/pkg/bufio/bufio_test.go new file mode 100644 index 000000000..82c73d36a --- /dev/null +++ b/src/pkg/bufio/bufio_test.go @@ -0,0 +1,699 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bufio_test + +import ( + . "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" + "testing/iotest" + "utf8" +) + +// Reads from a reader and rot13s the result. +type rot13Reader struct { + r io.Reader +} + +func newRot13Reader(r io.Reader) *rot13Reader { + r13 := new(rot13Reader) + r13.r = r + return r13 +} + +func (r13 *rot13Reader) Read(p []byte) (int, os.Error) { + n, e := r13.r.Read(p) + if e != nil { + return n, e + } + for i := 0; i < n; i++ { + c := p[i] | 0x20 // lowercase byte + if 'a' <= c && c <= 'm' { + p[i] += 13 + } else if 'n' <= c && c <= 'z' { + p[i] -= 13 + } + } + return n, nil +} + +// Call ReadByte to accumulate the text of a file +func readBytes(buf *Reader) string { + var b [1000]byte + nb := 0 + for { + c, e := buf.ReadByte() + if e == os.EOF { + break + } + if e == nil { + b[nb] = c + nb++ + } else if e != iotest.ErrTimeout { + panic("Data: " + e.String()) + } + } + return string(b[0:nb]) +} + +func TestReaderSimple(t *testing.T) { + data := "hello world" + b := NewReader(bytes.NewBufferString(data)) + if s := readBytes(b); s != "hello world" { + t.Errorf("simple hello world test failed: got %q", s) + } + + b = NewReader(newRot13Reader(bytes.NewBufferString(data))) + if s := readBytes(b); s != "uryyb jbeyq" { + t.Errorf("rot13 hello world test failed: got %q", s) + } +} + +type readMaker struct { + name string + fn func(io.Reader) io.Reader +} + +var readMakers = []readMaker{ + {"full", func(r io.Reader) io.Reader { return r }}, + {"byte", iotest.OneByteReader}, + {"half", iotest.HalfReader}, + {"data+err", iotest.DataErrReader}, + {"timeout", iotest.TimeoutReader}, +} + +// Call ReadString (which ends up calling everything else) +// to accumulate the text of a file. +func readLines(b *Reader) string { + s := "" + for { + s1, e := b.ReadString('\n') + if e == os.EOF { + break + } + if e != nil && e != iotest.ErrTimeout { + panic("GetLines: " + e.String()) + } + s += s1 + } + return s +} + +// Call Read to accumulate the text of a file +func reads(buf *Reader, m int) string { + var b [1000]byte + nb := 0 + for { + n, e := buf.Read(b[nb : nb+m]) + nb += n + if e == os.EOF { + break + } + } + return string(b[0:nb]) +} + +type bufReader struct { + name string + fn func(*Reader) string +} + +var bufreaders = []bufReader{ + {"1", func(b *Reader) string { return reads(b, 1) }}, + {"2", func(b *Reader) string { return reads(b, 2) }}, + {"3", func(b *Reader) string { return reads(b, 3) }}, + {"4", func(b *Reader) string { return reads(b, 4) }}, + {"5", func(b *Reader) string { return reads(b, 5) }}, + {"7", func(b *Reader) string { return reads(b, 7) }}, + {"bytes", readBytes}, + {"lines", readLines}, +} + +var bufsizes = []int{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 23, 32, 46, 64, 93, 128, 1024, 4096, +} + +func TestReader(t *testing.T) { + var texts [31]string + str := "" + all := "" + for i := 0; i < len(texts)-1; i++ { + texts[i] = str + "\n" + all += texts[i] + str += string(i%26 + 'a') + } + texts[len(texts)-1] = all + + for h := 0; h < len(texts); h++ { + text := texts[h] + for i := 0; i < len(readMakers); i++ { + for j := 0; j < len(bufreaders); j++ { + for k := 0; k < len(bufsizes); k++ { + readmaker := readMakers[i] + bufreader := bufreaders[j] + bufsize := bufsizes[k] + read := readmaker.fn(bytes.NewBufferString(text)) + buf, _ := NewReaderSize(read, bufsize) + s := bufreader.fn(buf) + if s != text { + t.Errorf("reader=%s fn=%s bufsize=%d want=%q got=%q", + readmaker.name, bufreader.name, bufsize, text, s) + } + } + } + } + } +} + +// A StringReader delivers its data one string segment at a time via Read. +type StringReader struct { + data []string + step int +} + +func (r *StringReader) Read(p []byte) (n int, err os.Error) { + if r.step < len(r.data) { + s := r.data[r.step] + n = copy(p, s) + r.step++ + } else { + err = os.EOF + } + return +} + +func readRuneSegments(t *testing.T, segments []string) { + got := "" + want := strings.Join(segments, "") + r := NewReader(&StringReader{data: segments}) + for { + rune, _, err := r.ReadRune() + if err != nil { + if err != os.EOF { + return + } + break + } + got += string(rune) + } + if got != want { + t.Errorf("segments=%v got=%s want=%s", segments, got, want) + } +} + +var segmentList = [][]string{ + {}, + {""}, + {"日", "本語"}, + {"\u65e5", "\u672c", "\u8a9e"}, + {"\U000065e5", "\U0000672c", "\U00008a9e"}, + {"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"}, + {"Hello", ", ", "World", "!"}, + {"Hello", ", ", "", "World", "!"}, +} + +func TestReadRune(t *testing.T) { + for _, s := range segmentList { + readRuneSegments(t, s) + } +} + +func TestUnreadRune(t *testing.T) { + got := "" + segments := []string{"Hello, world:", "日本語"} + data := strings.Join(segments, "") + r := NewReader(&StringReader{data: segments}) + // Normal execution. + for { + rune, _, err := r.ReadRune() + if err != nil { + if err != os.EOF { + t.Error("unexpected EOF") + } + break + } + got += string(rune) + // Put it back and read it again + if err = r.UnreadRune(); err != nil { + t.Error("unexpected error on UnreadRune:", err) + } + rune1, _, err := r.ReadRune() + if err != nil { + t.Error("unexpected error reading after unreading:", err) + } + if rune != rune1 { + t.Errorf("incorrect rune after unread: got %c wanted %c", rune1, rune) + } + } + if got != data { + t.Errorf("want=%q got=%q", data, got) + } +} + +// Test that UnreadRune fails if the preceding operation was not a ReadRune. +func TestUnreadRuneError(t *testing.T) { + buf := make([]byte, 3) // All runes in this test are 3 bytes long + r := NewReader(&StringReader{data: []string{"日本語日本語日本語"}}) + if r.UnreadRune() == nil { + t.Error("expected error on UnreadRune from fresh buffer") + } + _, _, err := r.ReadRune() + if err != nil { + t.Error("unexpected error on ReadRune (1):", err) + } + if err = r.UnreadRune(); err != nil { + t.Error("unexpected error on UnreadRune (1):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after UnreadRune (1)") + } + // Test error after Read. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (2):", err) + } + _, err = r.Read(buf) + if err != nil { + t.Error("unexpected error on Read (2):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after Read (2)") + } + // Test error after ReadByte. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (2):", err) + } + for _ = range buf { + _, err = r.ReadByte() + if err != nil { + t.Error("unexpected error on ReadByte (2):", err) + } + } + if r.UnreadRune() == nil { + t.Error("expected error after ReadByte") + } + // Test error after UnreadByte. + _, _, err = r.ReadRune() // reset state + if err != nil { + t.Error("unexpected error on ReadRune (3):", err) + } + _, err = r.ReadByte() + if err != nil { + t.Error("unexpected error on ReadByte (3):", err) + } + err = r.UnreadByte() + if err != nil { + t.Error("unexpected error on UnreadByte (3):", err) + } + if r.UnreadRune() == nil { + t.Error("expected error after UnreadByte (3)") + } +} + +func TestUnreadRuneAtEOF(t *testing.T) { + // UnreadRune/ReadRune should error at EOF (was a bug; used to panic) + r := NewReader(strings.NewReader("x")) + r.ReadRune() + r.ReadRune() + r.UnreadRune() + _, _, err := r.ReadRune() + if err == nil { + t.Error("expected error at EOF") + } else if err != os.EOF { + t.Error("expected EOF; got", err) + } +} + +func TestReadWriteRune(t *testing.T) { + const NRune = 1000 + byteBuf := new(bytes.Buffer) + w := NewWriter(byteBuf) + // Write the runes out using WriteRune + buf := make([]byte, utf8.UTFMax) + for rune := 0; rune < NRune; rune++ { + size := utf8.EncodeRune(buf, rune) + nbytes, err := w.WriteRune(rune) + if err != nil { + t.Fatalf("WriteRune(0x%x) error: %s", rune, err) + } + if nbytes != size { + t.Fatalf("WriteRune(0x%x) expected %d, got %d", rune, size, nbytes) + } + } + w.Flush() + + r := NewReader(byteBuf) + // Read them back with ReadRune + for rune := 0; rune < NRune; rune++ { + size := utf8.EncodeRune(buf, rune) + nr, nbytes, err := r.ReadRune() + if nr != rune || nbytes != size || err != nil { + t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } +} + +func TestWriter(t *testing.T) { + var data [8192]byte + + for i := 0; i < len(data); i++ { + data[i] = byte(' ' + i%('~'-' ')) + } + w := new(bytes.Buffer) + for i := 0; i < len(bufsizes); i++ { + for j := 0; j < len(bufsizes); j++ { + nwrite := bufsizes[i] + bs := bufsizes[j] + + // Write nwrite bytes using buffer size bs. + // Check that the right amount makes it out + // and that the data is correct. + + w.Reset() + buf, e := NewWriterSize(w, bs) + context := fmt.Sprintf("nwrite=%d bufsize=%d", nwrite, bs) + if e != nil { + t.Errorf("%s: NewWriterSize %d: %v", context, bs, e) + continue + } + n, e1 := buf.Write(data[0:nwrite]) + if e1 != nil || n != nwrite { + t.Errorf("%s: buf.Write %d = %d, %v", context, nwrite, n, e1) + continue + } + if e = buf.Flush(); e != nil { + t.Errorf("%s: buf.Flush = %v", context, e) + } + + written := w.Bytes() + if len(written) != nwrite { + t.Errorf("%s: %d bytes written", context, len(written)) + } + for l := 0; l < len(written); l++ { + if written[i] != data[i] { + t.Errorf("wrong bytes written") + t.Errorf("want=%q", data[0:len(written)]) + t.Errorf("have=%q", written) + } + } + } + } +} + +// Check that write errors are returned properly. + +type errorWriterTest struct { + n, m int + err os.Error + expect os.Error +} + +func (w errorWriterTest) Write(p []byte) (int, os.Error) { + return len(p) * w.n / w.m, w.err +} + +var errorWriterTests = []errorWriterTest{ + {0, 1, nil, io.ErrShortWrite}, + {1, 2, nil, io.ErrShortWrite}, + {1, 1, nil, nil}, + {0, 1, os.EPIPE, os.EPIPE}, + {1, 2, os.EPIPE, os.EPIPE}, + {1, 1, os.EPIPE, os.EPIPE}, +} + +func TestWriteErrors(t *testing.T) { + for _, w := range errorWriterTests { + buf := NewWriter(w) + _, e := buf.Write([]byte("hello world")) + if e != nil { + t.Errorf("Write hello to %v: %v", w, e) + continue + } + e = buf.Flush() + if e != w.expect { + t.Errorf("Flush %v: got %v, wanted %v", w, e, w.expect) + } + } +} + +func TestNewReaderSizeIdempotent(t *testing.T) { + const BufSize = 1000 + b, err := NewReaderSize(bytes.NewBufferString("hello world"), BufSize) + if err != nil { + t.Error("NewReaderSize create fail", err) + } + // Does it recognize itself? + b1, err2 := NewReaderSize(b, BufSize) + if err2 != nil { + t.Error("NewReaderSize #2 create fail", err2) + } + if b1 != b { + t.Error("NewReaderSize did not detect underlying Reader") + } + // Does it wrap if existing buffer is too small? + b2, err3 := NewReaderSize(b, 2*BufSize) + if err3 != nil { + t.Error("NewReaderSize #3 create fail", err3) + } + if b2 == b { + t.Error("NewReaderSize did not enlarge buffer") + } +} + +func TestNewWriterSizeIdempotent(t *testing.T) { + const BufSize = 1000 + b, err := NewWriterSize(new(bytes.Buffer), BufSize) + if err != nil { + t.Error("NewWriterSize create fail", err) + } + // Does it recognize itself? + b1, err2 := NewWriterSize(b, BufSize) + if err2 != nil { + t.Error("NewWriterSize #2 create fail", err2) + } + if b1 != b { + t.Error("NewWriterSize did not detect underlying Writer") + } + // Does it wrap if existing buffer is too small? + b2, err3 := NewWriterSize(b, 2*BufSize) + if err3 != nil { + t.Error("NewWriterSize #3 create fail", err3) + } + if b2 == b { + t.Error("NewWriterSize did not enlarge buffer") + } +} + +func TestWriteString(t *testing.T) { + const BufSize = 8 + buf := new(bytes.Buffer) + b, err := NewWriterSize(buf, BufSize) + if err != nil { + t.Error("NewWriterSize create fail", err) + } + b.WriteString("0") // easy + b.WriteString("123456") // still easy + b.WriteString("7890") // easy after flush + b.WriteString("abcdefghijklmnopqrstuvwxy") // hard + b.WriteString("z") + if err := b.Flush(); err != nil { + t.Error("WriteString", err) + } + s := "01234567890abcdefghijklmnopqrstuvwxyz" + if string(buf.Bytes()) != s { + t.Errorf("WriteString wants %q gets %q", s, string(buf.Bytes())) + } +} + +func TestBufferFull(t *testing.T) { + buf, _ := NewReaderSize(strings.NewReader("hello, world"), 5) + line, err := buf.ReadSlice(',') + if string(line) != "hello" || err != ErrBufferFull { + t.Errorf("first ReadSlice(,) = %q, %v", line, err) + } + line, err = buf.ReadSlice(',') + if string(line) != "," || err != nil { + t.Errorf("second ReadSlice(,) = %q, %v", line, err) + } +} + +func TestPeek(t *testing.T) { + p := make([]byte, 10) + buf, _ := NewReaderSize(strings.NewReader("abcdefghij"), 4) + if s, err := buf.Peek(1); string(s) != "a" || err != nil { + t.Fatalf("want %q got %q, err=%v", "a", string(s), err) + } + if s, err := buf.Peek(4); string(s) != "abcd" || err != nil { + t.Fatalf("want %q got %q, err=%v", "abcd", string(s), err) + } + if _, err := buf.Peek(5); err != ErrBufferFull { + t.Fatalf("want ErrBufFull got %v", err) + } + if _, err := buf.Read(p[0:3]); string(p[0:3]) != "abc" || err != nil { + t.Fatalf("want %q got %q, err=%v", "abc", string(p[0:3]), err) + } + if s, err := buf.Peek(1); string(s) != "d" || err != nil { + t.Fatalf("want %q got %q, err=%v", "d", string(s), err) + } + if s, err := buf.Peek(2); string(s) != "de" || err != nil { + t.Fatalf("want %q got %q, err=%v", "de", string(s), err) + } + if _, err := buf.Read(p[0:3]); string(p[0:3]) != "def" || err != nil { + t.Fatalf("want %q got %q, err=%v", "def", string(p[0:3]), err) + } + if s, err := buf.Peek(4); string(s) != "ghij" || err != nil { + t.Fatalf("want %q got %q, err=%v", "ghij", string(s), err) + } + if _, err := buf.Read(p[0:4]); string(p[0:4]) != "ghij" || err != nil { + t.Fatalf("want %q got %q, err=%v", "ghij", string(p[0:3]), err) + } + if s, err := buf.Peek(0); string(s) != "" || err != nil { + t.Fatalf("want %q got %q, err=%v", "", string(s), err) + } + if _, err := buf.Peek(1); err != os.EOF { + t.Fatalf("want EOF got %v", err) + } +} + +func TestPeekThenUnreadRune(t *testing.T) { + // This sequence used to cause a crash. + r := NewReader(strings.NewReader("x")) + r.ReadRune() + r.Peek(1) + r.UnreadRune() + r.ReadRune() // Used to panic here +} + +var testOutput = []byte("0123456789abcdefghijklmnopqrstuvwxy") +var testInput = []byte("012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy") +var testInputrn = []byte("012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\nuvw\r\nxy\r\n\n\r\n") + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err os.Error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = os.EOF + } + return +} + +func testReadLine(t *testing.T, input []byte) { + //for stride := 1; stride < len(input); stride++ { + for stride := 1; stride < 2; stride++ { + done := 0 + reader := testReader{input, stride} + l, _ := NewReaderSize(&reader, len(input)+1) + for { + line, isPrefix, err := l.ReadLine() + if len(line) > 0 && err != nil { + t.Errorf("ReadLine returned both data and error: %s", err) + } + if isPrefix { + t.Errorf("ReadLine returned prefix") + } + if err != nil { + if err != os.EOF { + t.Fatalf("Got unknown error: %s", err) + } + break + } + if want := testOutput[done : done+len(line)]; !bytes.Equal(want, line) { + t.Errorf("Bad line at stride %d: want: %x got: %x", stride, want, line) + } + done += len(line) + } + if done != len(testOutput) { + t.Errorf("ReadLine didn't return everything: got: %d, want: %d (stride: %d)", done, len(testOutput), stride) + } + } +} + +func TestReadLine(t *testing.T) { + testReadLine(t, testInput) + testReadLine(t, testInputrn) +} + +func TestLineTooLong(t *testing.T) { + buf := bytes.NewBuffer([]byte("aaabbbcc\n")) + l, _ := NewReaderSize(buf, 3) + line, isPrefix, err := l.ReadLine() + if !isPrefix || !bytes.Equal(line, []byte("aaa")) || err != nil { + t.Errorf("bad result for first line: %x %s", line, err) + } + line, isPrefix, err = l.ReadLine() + if !isPrefix || !bytes.Equal(line, []byte("bbb")) || err != nil { + t.Errorf("bad result for second line: %x", line) + } + line, isPrefix, err = l.ReadLine() + if isPrefix || !bytes.Equal(line, []byte("cc")) || err != nil { + t.Errorf("bad result for third line: %x", line) + } + line, isPrefix, err = l.ReadLine() + if isPrefix || err == nil { + t.Errorf("expected no more lines: %x %s", line, err) + } +} + +func TestReadAfterLines(t *testing.T) { + line1 := "line1" + restData := "line2\nline 3\n" + inbuf := bytes.NewBuffer([]byte(line1 + "\n" + restData)) + outbuf := new(bytes.Buffer) + maxLineLength := len(line1) + len(restData)/2 + l, _ := NewReaderSize(inbuf, maxLineLength) + line, isPrefix, err := l.ReadLine() + if isPrefix || err != nil || string(line) != line1 { + t.Errorf("bad result for first line: isPrefix=%v err=%v line=%q", isPrefix, err, string(line)) + } + n, err := io.Copy(outbuf, l) + if int(n) != len(restData) || err != nil { + t.Errorf("bad result for Read: n=%d err=%v", n, err) + } + if outbuf.String() != restData { + t.Errorf("bad result for Read: got %q; expected %q", outbuf.String(), restData) + } +} + +func TestReadEmptyBuffer(t *testing.T) { + l, _ := NewReaderSize(bytes.NewBuffer(nil), 10) + line, isPrefix, err := l.ReadLine() + if err != os.EOF { + t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) + } +} + +func TestLinesAfterRead(t *testing.T) { + l, _ := NewReaderSize(bytes.NewBuffer([]byte("foo")), 10) + _, err := ioutil.ReadAll(l) + if err != nil { + t.Error(err) + return + } + + line, isPrefix, err := l.ReadLine() + if err != os.EOF { + t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) + } +} diff --git a/src/pkg/builtin/builtin.go b/src/pkg/builtin/builtin.go new file mode 100644 index 000000000..07acce4f7 --- /dev/null +++ b/src/pkg/builtin/builtin.go @@ -0,0 +1,135 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package builtin provides documentation for Go's built-in functions. + The functions documented here are not actually in package builtin + but their descriptions here allow godoc to present documentation + for the language's special functions. +*/ +package builtin + +// Type is here for the purposes of documentation only. It is a stand-in +// for any Go type, but represents the same type for any given function +// invocation. +type Type int + +// IntegerType is here for the purposes of documentation only. It is a stand-in +// for any integer type: int, uint, int8 etc. +type IntegerType int + +// FloatType is here for the purposes of documentation only. It is a stand-in +// for either float type: float32 or float64. +type FloatType int + +// ComplexType is here for the purposes of documentation only. It is a +// stand-in for either complex type: complex64 or complex128. +type ComplexType int + +// The append built-in function appends elements to the end of a slice. If +// it has sufficient capacity, the destination is resliced to accommodate the +// new elements. If it does not, a new underlying array will be allocated. +// Append returns the updated slice. It is therefore necessary to store the +// result of append, often in the variable holding the slice itself: +// slice = append(slice, elem1, elem2) +// slice = append(slice, anotherSlice...) +func append(slice []Type, elems ...Type) []Type + +// The copy built-in function copies elements from a source slice into a +// destination slice. (As a special case, it also will copy bytes from a +// string to a slice of bytes.) The source and destination may overlap. Copy +// returns the number of elements copied, which will be the minimum of +// len(src) and len(dst). +func copy(dst, src []Type) int + +// The len built-in function returns the length of v, according to its type: +// Array: the number of elements in v. +// Pointer to array: the number of elements in *v (even if v is nil). +// Slice, or map: the number of elements in v; if v is nil, len(v) is zero. +// String: the number of bytes in v. +// Channel: the number of elements queued (unread) in the channel buffer; +// if v is nil, len(v) is zero. +func len(v Type) int + +// The cap built-in function returns the capacity of v, according to its type: +// Array: the number of elements in v (same as len(v)). +// Pointer to array: the number of elements in *v (same as len(v)). +// Slice: the maximum length the slice can reach when resliced; +// if v is nil, cap(v) is zero. +// Channel: the channel buffer capacity, in units of elements; +// if v is nil, cap(v) is zero. +func cap(v Type) int + +// The make built-in function allocates and initializes an object of type +// slice, map, or chan (only). Like new, the first argument is a type, not a +// value. Unlike new, make's return type is the same as the type of its +// argument, not a pointer to it. The specification of the result depends on +// the type: +// Slice: The size specifies the length. The capacity of the slice is +// equal to its length. A second integer argument may be provided to +// specify a different capacity; it must be no smaller than the +// length, so make([]int, 0, 10) allocates a slice of length 0 and +// capacity 10. +// Map: An initial allocation is made according to the size but the +// resulting map has length 0. The size may be omitted, in which case +// a small starting size is allocated. +// Channel: The channel's buffer is initialized with the specified +// buffer capacity. If zero, or the size is omitted, the channel is +// unbuffered. +func make(Type, size IntegerType) Type + +// The new built-in function allocates memory. The first argument is a type, +// not a value, and the value returned is a pointer to a newly +// allocated zero value of that type. +func new(Type) *Type + +// The complex built-in function constructs a complex value from two +// floating-point values. The real and imaginary parts must be of the same +// size, either float32 or float64 (or assignable to them), and the return +// value will be the corresponding complex type (complex64 for float32, +// complex128 for float64). +func complex(r, i FloatType) ComplexType + +// The real built-in function returns the real part of the complex number c. +// The return value will be floating point type corresponding to the type of c. +func real(c ComplexType) FloatType + +// The imaginary built-in function returns the imaginary part of the complex +// number c. The return value will be floating point type corresponding to +// the type of c. +func imag(c ComplexType) FloatType + +// The close built-in function closes a channel, which must be either +// bidirectional or send-only. It should be executed only by the sender, +// never the receiver, and has the effect of shutting down the channel after +// the last sent value is received. After the last value has been received +// from a closed channel c, any receive from c will succeed without +// blocking, returning the zero value for the channel element. The form +// x, ok := <-c +// will also set ok to false for a closed channel. +func close(c chan<- Type) + +// The panic built-in function stops normal execution of the current +// goroutine. When a function F calls panic, normal execution of F stops +// immediately. Any functions whose execution was deferred by F are run in +// the usual way, and then F returns to its caller. To the caller G, the +// invocation of F then behaves like a call to panic, terminating G's +// execution and running any deferred functions. This continues until all +// functions in the executing goroutine have stopped, in reverse order. At +// that point, the program is terminated and the error condition is reported, +// including the value of the argument to panic. This termination sequence +// is called panicking and can be controlled by the built-in function +// recover. +func panic(v interface{}) + +// The recover built-in function allows a program to manage behavior of a +// panicking goroutine. Executing a call to recover inside a deferred +// function (but not any function called by it) stops the panicking sequence +// by restoring normal execution and retrieves the error value passed to the +// call of panic. If recover is called outside the deferred function it will +// not stop a panicking sequence. In this case, or when the goroutine is not +// panicking, or if the argument supplied to panic was nil, recover returns +// nil. Thus the return value from recover reports whether the goroutine is +// panicking. +func recover() interface{} diff --git a/src/pkg/bytes/Makefile b/src/pkg/bytes/Makefile new file mode 100644 index 000000000..03395c7a4 --- /dev/null +++ b/src/pkg/bytes/Makefile @@ -0,0 +1,16 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=bytes +GOFILES=\ + buffer.go\ + bytes.go\ + bytes_decl.go\ + +OFILES=\ + asm_$(GOARCH).$O\ + +include ../../Make.pkg diff --git a/src/pkg/bytes/asm_386.s b/src/pkg/bytes/asm_386.s new file mode 100644 index 000000000..f3391740b --- /dev/null +++ b/src/pkg/bytes/asm_386.s @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·IndexByte(SB),7,$0 + MOVL p+0(FP), SI + MOVL len+4(FP), CX + MOVB b+12(FP), AL + MOVL SI, DI + CLD; REPN; SCASB + JZ 3(PC) + MOVL $-1, ret+16(FP) + RET + SUBL SI, DI + SUBL $1, DI + MOVL DI, ret+16(FP) + RET diff --git a/src/pkg/bytes/asm_amd64.s b/src/pkg/bytes/asm_amd64.s new file mode 100644 index 000000000..c6793cbdc --- /dev/null +++ b/src/pkg/bytes/asm_amd64.s @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·IndexByte(SB),7,$0 + MOVQ p+0(FP), SI + MOVL len+8(FP), BX + MOVB b+16(FP), AL + MOVQ SI, DI + + CMPL BX, $16 + JLT small + + // round up to first 16-byte boundary + TESTQ $15, SI + JZ aligned + MOVQ SI, CX + ANDQ $~15, CX + ADDQ $16, CX + + // search the beginning + SUBQ SI, CX + REPN; SCASB + JZ success + +// DI is 16-byte aligned; get ready to search using SSE instructions +aligned: + // round down to last 16-byte boundary + MOVQ BX, R11 + ADDQ SI, R11 + ANDQ $~15, R11 + + // shuffle X0 around so that each byte contains c + MOVD AX, X0 + PUNPCKLBW X0, X0 + PUNPCKLBW X0, X0 + PSHUFL $0, X0, X0 + JMP condition + +sse: + // move the next 16-byte chunk of the buffer into X1 + MOVO (DI), X1 + // compare bytes in X0 to X1 + PCMPEQB X0, X1 + // take the top bit of each byte in X1 and put the result in DX + PMOVMSKB X1, DX + TESTL DX, DX + JNZ ssesuccess + ADDQ $16, DI + +condition: + CMPQ DI, R11 + JLT sse + + // search the end + MOVQ SI, CX + ADDQ BX, CX + SUBQ R11, CX + // if CX == 0, the zero flag will be set and we'll end up + // returning a false success + JZ failure + REPN; SCASB + JZ success + +failure: + MOVL $-1, ret+24(FP) + RET + +// handle for lengths < 16 +small: + MOVL BX, CX + REPN; SCASB + JZ success + MOVL $-1, ret+24(FP) + RET + +// we've found the chunk containing the byte +// now just figure out which specific byte it is +ssesuccess: + // get the index of the least significant set bit + BSFW DX, DX + SUBQ SI, DI + ADDQ DI, DX + MOVL DX, ret+24(FP) + RET + +success: + SUBQ SI, DI + SUBL $1, DI + MOVL DI, ret+24(FP) + RET + diff --git a/src/pkg/bytes/asm_arm.s b/src/pkg/bytes/asm_arm.s new file mode 100644 index 000000000..f32fca136 --- /dev/null +++ b/src/pkg/bytes/asm_arm.s @@ -0,0 +1,8 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// no memchr implementation on arm yet +TEXT ·IndexByte(SB),7,$0 + B ·indexBytePortable(SB) + diff --git a/src/pkg/bytes/buffer.go b/src/pkg/bytes/buffer.go new file mode 100644 index 000000000..5de86105d --- /dev/null +++ b/src/pkg/bytes/buffer.go @@ -0,0 +1,348 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Simple byte buffer for marshaling data. + +import ( + "io" + "os" + "utf8" +) + +// A Buffer is a variable-sized buffer of bytes with Read and Write methods. +// The zero value for Buffer is an empty buffer ready to use. +type Buffer struct { + buf []byte // contents are the bytes buf[off : len(buf)] + off int // read at &buf[off], write at &buf[len(buf)] + runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune + bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation. + lastRead readOp // last read operation, so that Unread* can work correctly. +} + +// The readOp constants describe the last action performed on +// the buffer, so that UnreadRune and UnreadByte can +// check for invalid usage. +type readOp int + +const ( + opInvalid readOp = iota // Non-read operation. + opReadRune // Read rune. + opRead // Any other read operation. +) + +// Bytes returns a slice of the contents of the unread portion of the buffer; +// len(b.Bytes()) == b.Len(). If the caller changes the contents of the +// returned slice, the contents of the buffer will change provided there +// are no intervening method calls on the Buffer. +func (b *Buffer) Bytes() []byte { return b.buf[b.off:] } + +// String returns the contents of the unread portion of the buffer +// as a string. If the Buffer is a nil pointer, it returns "". +func (b *Buffer) String() string { + if b == nil { + // Special case, useful in debugging. + return "" + } + return string(b.buf[b.off:]) +} + +// Len returns the number of bytes of the unread portion of the buffer; +// b.Len() == len(b.Bytes()). +func (b *Buffer) Len() int { return len(b.buf) - b.off } + +// Truncate discards all but the first n unread bytes from the buffer. +// It is an error to call b.Truncate(n) with n > b.Len(). +func (b *Buffer) Truncate(n int) { + b.lastRead = opInvalid + if n == 0 { + // Reuse buffer space. + b.off = 0 + } + b.buf = b.buf[0 : b.off+n] +} + +// Reset resets the buffer so it has no content. +// b.Reset() is the same as b.Truncate(0). +func (b *Buffer) Reset() { b.Truncate(0) } + +// Grow buffer to guarantee space for n more bytes. +// Return index where bytes should be written. +func (b *Buffer) grow(n int) int { + m := b.Len() + // If buffer is empty, reset to recover space. + if m == 0 && b.off != 0 { + b.Truncate(0) + } + if len(b.buf)+n > cap(b.buf) { + var buf []byte + if b.buf == nil && n <= len(b.bootstrap) { + buf = b.bootstrap[0:] + } else { + // not enough space anywhere + buf = make([]byte, 2*cap(b.buf)+n) + copy(buf, b.buf[b.off:]) + } + b.buf = buf + b.off = 0 + } + b.buf = b.buf[0 : b.off+m+n] + return b.off + m +} + +// Write appends the contents of p to the buffer. The return +// value n is the length of p; err is always nil. +func (b *Buffer) Write(p []byte) (n int, err os.Error) { + b.lastRead = opInvalid + m := b.grow(len(p)) + copy(b.buf[m:], p) + return len(p), nil +} + +// WriteString appends the contents of s to the buffer. The return +// value n is the length of s; err is always nil. +func (b *Buffer) WriteString(s string) (n int, err os.Error) { + b.lastRead = opInvalid + m := b.grow(len(s)) + return copy(b.buf[m:], s), nil +} + +// MinRead is the minimum slice size passed to a Read call by +// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond +// what is required to hold the contents of r, ReadFrom will not grow the +// underlying buffer. +const MinRead = 512 + +// ReadFrom reads data from r until EOF and appends it to the buffer. +// The return value n is the number of bytes read. +// Any error except os.EOF encountered during the read +// is also returned. +func (b *Buffer) ReadFrom(r io.Reader) (n int64, err os.Error) { + b.lastRead = opInvalid + // If buffer is empty, reset to recover space. + if b.off >= len(b.buf) { + b.Truncate(0) + } + for { + if cap(b.buf)-len(b.buf) < MinRead { + var newBuf []byte + // can we get space without allocation? + if b.off+cap(b.buf)-len(b.buf) >= MinRead { + // reuse beginning of buffer + newBuf = b.buf[0 : len(b.buf)-b.off] + } else { + // not enough space at end; put space on end + newBuf = make([]byte, len(b.buf)-b.off, 2*(cap(b.buf)-b.off)+MinRead) + } + copy(newBuf, b.buf[b.off:]) + b.buf = newBuf + b.off = 0 + } + m, e := r.Read(b.buf[len(b.buf):cap(b.buf)]) + b.buf = b.buf[0 : len(b.buf)+m] + n += int64(m) + if e == os.EOF { + break + } + if e != nil { + return n, e + } + } + return n, nil // err is EOF, so return nil explicitly +} + +// WriteTo writes data to w until the buffer is drained or an error +// occurs. The return value n is the number of bytes written; it always +// fits into an int, but it is int64 to match the io.WriterTo interface. +// Any error encountered during the write is also returned. +func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) { + b.lastRead = opInvalid + if b.off < len(b.buf) { + m, e := w.Write(b.buf[b.off:]) + b.off += m + n = int64(m) + if e != nil { + return n, e + } + // otherwise all bytes were written, by definition of + // Write method in io.Writer + } + // Buffer is now empty; reset. + b.Truncate(0) + return +} + +// WriteByte appends the byte c to the buffer. +// The returned error is always nil, but is included +// to match bufio.Writer's WriteByte. +func (b *Buffer) WriteByte(c byte) os.Error { + b.lastRead = opInvalid + m := b.grow(1) + b.buf[m] = c + return nil +} + +// WriteRune appends the UTF-8 encoding of Unicode +// code point r to the buffer, returning its length and +// an error, which is always nil but is included +// to match bufio.Writer's WriteRune. +func (b *Buffer) WriteRune(r int) (n int, err os.Error) { + if r < utf8.RuneSelf { + b.WriteByte(byte(r)) + return 1, nil + } + n = utf8.EncodeRune(b.runeBytes[0:], r) + b.Write(b.runeBytes[0:n]) + return n, nil +} + +// Read reads the next len(p) bytes from the buffer or until the buffer +// is drained. The return value n is the number of bytes read. If the +// buffer has no data to return, err is os.EOF even if len(p) is zero; +// otherwise it is nil. +func (b *Buffer) Read(p []byte) (n int, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, os.EOF + } + n = copy(p, b.buf[b.off:]) + b.off += n + if n > 0 { + b.lastRead = opRead + } + return +} + +// Next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +// If there are fewer than n bytes in the buffer, Next returns the entire buffer. +// The slice is only valid until the next call to a read or write method. +func (b *Buffer) Next(n int) []byte { + b.lastRead = opInvalid + m := b.Len() + if n > m { + n = m + } + data := b.buf[b.off : b.off+n] + b.off += n + if n > 0 { + b.lastRead = opRead + } + return data +} + +// ReadByte reads and returns the next byte from the buffer. +// If no byte is available, it returns error os.EOF. +func (b *Buffer) ReadByte() (c byte, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, os.EOF + } + c = b.buf[b.off] + b.off++ + b.lastRead = opRead + return c, nil +} + +// ReadRune reads and returns the next UTF-8-encoded +// Unicode code point from the buffer. +// If no bytes are available, the error returned is os.EOF. +// If the bytes are an erroneous UTF-8 encoding, it +// consumes one byte and returns U+FFFD, 1. +func (b *Buffer) ReadRune() (r int, size int, err os.Error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, 0, os.EOF + } + b.lastRead = opReadRune + c := b.buf[b.off] + if c < utf8.RuneSelf { + b.off++ + return int(c), 1, nil + } + r, n := utf8.DecodeRune(b.buf[b.off:]) + b.off += n + return r, n, nil +} + +// UnreadRune unreads the last rune returned by ReadRune. +// If the most recent read or write operation on the buffer was +// not a ReadRune, UnreadRune returns an error. (In this regard +// it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Buffer) UnreadRune() os.Error { + if b.lastRead != opReadRune { + return os.NewError("bytes.Buffer: UnreadRune: previous operation was not ReadRune") + } + b.lastRead = opInvalid + if b.off > 0 { + _, n := utf8.DecodeLastRune(b.buf[0:b.off]) + b.off -= n + } + return nil +} + +// UnreadByte unreads the last byte returned by the most recent +// read operation. If write has happened since the last read, UnreadByte +// returns an error. +func (b *Buffer) UnreadByte() os.Error { + if b.lastRead != opReadRune && b.lastRead != opRead { + return os.NewError("bytes.Buffer: UnreadByte: previous operation was not a read") + } + b.lastRead = opInvalid + if b.off > 0 { + b.off-- + } + return nil +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Buffer) ReadBytes(delim byte) (line []byte, err os.Error) { + i := IndexByte(b.buf[b.off:], delim) + size := i + 1 + if i < 0 { + size = len(b.buf) - b.off + err = os.EOF + } + line = make([]byte, size) + copy(line, b.buf[b.off:]) + b.off += size + return +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadString returns err != nil if and only if the returned data does not end +// in delim. +func (b *Buffer) ReadString(delim byte) (line string, err os.Error) { + bytes, err := b.ReadBytes(delim) + return string(bytes), err +} + +// NewBuffer creates and initializes a new Buffer using buf as its initial +// contents. It is intended to prepare a Buffer to read existing data. It +// can also be used to size the internal buffer for writing. To do that, +// buf should have the desired capacity but a length of zero. +func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } + +// NewBufferString creates and initializes a new Buffer using string s as its +// initial contents. It is intended to prepare a buffer to read an existing +// string. +func NewBufferString(s string) *Buffer { + return &Buffer{buf: []byte(s)} +} diff --git a/src/pkg/bytes/buffer_test.go b/src/pkg/bytes/buffer_test.go new file mode 100644 index 000000000..06d2a65c6 --- /dev/null +++ b/src/pkg/bytes/buffer_test.go @@ -0,0 +1,376 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "os" + "rand" + "testing" + "utf8" +) + +const N = 10000 // make this bigger for a larger (and slower) test +var data string // test data for write tests +var bytes []byte // test data; same as data but as a slice. + + +func init() { + bytes = make([]byte, N) + for i := 0; i < N; i++ { + bytes[i] = 'a' + byte(i%26) + } + data = string(bytes) +} + +// Verify that contents of buf match the string s. +func check(t *testing.T, testname string, buf *Buffer, s string) { + bytes := buf.Bytes() + str := buf.String() + if buf.Len() != len(bytes) { + t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes)) + } + + if buf.Len() != len(str) { + t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str)) + } + + if buf.Len() != len(s) { + t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s)) + } + + if string(bytes) != s { + t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s) + } +} + +// Fill buf through n writes of string fus. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillString(t *testing.T, testname string, buf *Buffer, s string, n int, fus string) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.WriteString(fus) + if m != len(fus) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fus)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += fus + check(t, testname+" (fill 4)", buf, s) + } + return s +} + +// Fill buf through n writes of byte slice fub. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.Write(fub) + if m != len(fub) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += string(fub) + check(t, testname+" (fill 4)", buf, s) + } + return s +} + +func TestNewBuffer(t *testing.T) { + buf := NewBuffer(bytes) + check(t, "NewBuffer", buf, data) +} + +func TestNewBufferString(t *testing.T) { + buf := NewBufferString(data) + check(t, "NewBufferString", buf, data) +} + +// Empty buf through repeated reads into fub. +// The initial contents of buf corresponds to the string s. +func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) { + check(t, testname+" (empty 1)", buf, s) + + for { + n, err := buf.Read(fub) + if n == 0 { + break + } + if err != nil { + t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err) + } + s = s[n:] + check(t, testname+" (empty 3)", buf, s) + } + + check(t, testname+" (empty 4)", buf, "") +} + +func TestBasicOperations(t *testing.T) { + var buf Buffer + + for i := 0; i < 5; i++ { + check(t, "TestBasicOperations (1)", &buf, "") + + buf.Reset() + check(t, "TestBasicOperations (2)", &buf, "") + + buf.Truncate(0) + check(t, "TestBasicOperations (3)", &buf, "") + + n, err := buf.Write([]byte(data[0:1])) + if n != 1 { + t.Errorf("wrote 1 byte, but n == %d", n) + } + if err != nil { + t.Errorf("err should always be nil, but err == %s", err) + } + check(t, "TestBasicOperations (4)", &buf, "a") + + buf.WriteByte(data[1]) + check(t, "TestBasicOperations (5)", &buf, "ab") + + n, err = buf.Write([]byte(data[2:26])) + if n != 24 { + t.Errorf("wrote 25 bytes, but n == %d", n) + } + check(t, "TestBasicOperations (6)", &buf, string(data[0:26])) + + buf.Truncate(26) + check(t, "TestBasicOperations (7)", &buf, string(data[0:26])) + + buf.Truncate(20) + check(t, "TestBasicOperations (8)", &buf, string(data[0:20])) + + empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5)) + empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100)) + + buf.WriteByte(data[1]) + c, err := buf.ReadByte() + if err != nil { + t.Error("ReadByte unexpected eof") + } + if c != data[1] { + t.Errorf("ReadByte wrong value c=%v", c) + } + c, err = buf.ReadByte() + if err == nil { + t.Error("ReadByte unexpected not eof") + } + } +} + +func TestLargeStringWrites(t *testing.T) { + var buf Buffer + limit := 30 + if testing.Short() { + limit = 9 + } + for i := 3; i < limit; i += 3 { + s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data) + empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeStringWrites (3)", &buf, "") +} + +func TestLargeByteWrites(t *testing.T) { + var buf Buffer + limit := 30 + if testing.Short() { + limit = 9 + } + for i := 3; i < limit; i += 3 { + s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, bytes) + empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeByteWrites (3)", &buf, "") +} + +func TestLargeStringReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeStringReads (3)", &buf, "") +} + +func TestLargeByteReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeByteReads (3)", &buf, "") +} + +func TestMixedReadsAndWrites(t *testing.T) { + var buf Buffer + s := "" + for i := 0; i < 50; i++ { + wlen := rand.Intn(len(data)) + if i%2 == 0 { + s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen]) + } else { + s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, bytes[0:wlen]) + } + + rlen := rand.Intn(len(data)) + fub := make([]byte, rlen) + n, _ := buf.Read(fub) + s = s[n:] + } + empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len())) +} + +func TestNil(t *testing.T) { + var b *Buffer + if b.String() != "" { + t.Errorf("expected ; got %q", b.String()) + } +} + +func TestReadFrom(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + var b Buffer + b.ReadFrom(&buf) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + +func TestWriteTo(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, bytes[0:len(bytes)/i]) + var b Buffer + buf.WriteTo(&b) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + +func TestRuneIO(t *testing.T) { + const NRune = 1000 + // Built a test array while we write the data + b := make([]byte, utf8.UTFMax*NRune) + var buf Buffer + n := 0 + for r := 0; r < NRune; r++ { + size := utf8.EncodeRune(b[n:], r) + nbytes, err := buf.WriteRune(r) + if err != nil { + t.Fatalf("WriteRune(%U) error: %s", r, err) + } + if nbytes != size { + t.Fatalf("WriteRune(%U) expected %d, got %d", r, size, nbytes) + } + n += size + } + b = b[0:n] + + // Check the resulting bytes + if !Equal(buf.Bytes(), b) { + t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b) + } + + p := make([]byte, utf8.UTFMax) + // Read it back with ReadRune + for r := 0; r < NRune; r++ { + size := utf8.EncodeRune(p, r) + nr, nbytes, err := buf.ReadRune() + if nr != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) got %U,%d not %U,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } + + // Check that UnreadRune works + buf.Reset() + buf.Write(b) + for r := 0; r < NRune; r++ { + r1, size, _ := buf.ReadRune() + if err := buf.UnreadRune(); err != nil { + t.Fatalf("UnreadRune(%U) got error %q", r, err) + } + r2, nbytes, err := buf.ReadRune() + if r1 != r2 || r1 != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) after UnreadRune got %U,%d not %U,%d (err=%s)", r, r2, nbytes, r, size, err) + } + } +} + +func TestNext(t *testing.T) { + b := []byte{0, 1, 2, 3, 4} + tmp := make([]byte, 5) + for i := 0; i <= 5; i++ { + for j := i; j <= 5; j++ { + for k := 0; k <= 6; k++ { + // 0 <= i <= j <= 5; 0 <= k <= 6 + // Check that if we start with a buffer + // of length j at offset i and ask for + // Next(k), we get the right bytes. + buf := NewBuffer(b[0:j]) + n, _ := buf.Read(tmp[0:i]) + if n != i { + t.Fatalf("Read %d returned %d", i, n) + } + bb := buf.Next(k) + want := k + if want > j-i { + want = j - i + } + if len(bb) != want { + t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb)) + } + for l, v := range bb { + if v != byte(l+i) { + t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i) + } + } + } + } + } +} + +var readBytesTests = []struct { + buffer string + delim byte + expected []string + err os.Error +}{ + {"", 0, []string{""}, os.EOF}, + {"a\x00", 0, []string{"a\x00"}, nil}, + {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil}, + {"hello\x01world", 1, []string{"hello\x01"}, nil}, + {"foo\nbar", 0, []string{"foo\nbar"}, os.EOF}, + {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil}, + {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, os.EOF}, +} + +func TestReadBytes(t *testing.T) { + for _, test := range readBytesTests { + buf := NewBufferString(test.buffer) + var err os.Error + for _, expected := range test.expected { + var bytes []byte + bytes, err = buf.ReadBytes(test.delim) + if string(bytes) != expected { + t.Errorf("expected %q, got %q", expected, bytes) + } + if err != nil { + break + } + } + if err != test.err { + t.Errorf("expected error %v, got %v", test.err, err) + } + } +} diff --git a/src/pkg/bytes/bytes.go b/src/pkg/bytes/bytes.go new file mode 100644 index 000000000..5119fce94 --- /dev/null +++ b/src/pkg/bytes/bytes.go @@ -0,0 +1,605 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bytes implements functions for the manipulation of byte slices. +// It is analogous to the facilities of the strings package. +package bytes + +import ( + "unicode" + "utf8" +) + +// Compare returns an integer comparing the two byte arrays lexicographically. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b +func Compare(a, b []byte) int { + m := len(a) + if m > len(b) { + m = len(b) + } + for i, ac := range a[0:m] { + bc := b[i] + switch { + case ac > bc: + return 1 + case ac < bc: + return -1 + } + } + switch { + case len(a) < len(b): + return -1 + case len(a) > len(b): + return 1 + } + return 0 +} + +// Equal returns a boolean reporting whether a == b. +func Equal(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i, c := range a { + if c != b[i] { + return false + } + } + return true +} + +// explode splits s into an array of UTF-8 sequences, one per Unicode character (still arrays of bytes), +// up to a maximum of n byte arrays. Invalid UTF-8 sequences are chopped into individual bytes. +func explode(s []byte, n int) [][]byte { + if n <= 0 { + n = len(s) + } + a := make([][]byte, n) + var size int + na := 0 + for len(s) > 0 { + if na+1 >= n { + a[na] = s + na++ + break + } + _, size = utf8.DecodeRune(s) + a[na] = s[0:size] + s = s[size:] + na++ + } + return a[0:na] +} + +// Count counts the number of non-overlapping instances of sep in s. +func Count(s, sep []byte) int { + if len(sep) == 0 { + return utf8.RuneCount(s) + 1 + } + c := sep[0] + n := 0 + for i := 0; i+len(sep) <= len(s); i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + n++ + i += len(sep) - 1 + } + } + return n +} + +// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. +func Index(s, sep []byte) int { + n := len(sep) + if n == 0 { + return 0 + } + c := sep[0] + for i := 0; i+n <= len(s); i++ { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +func indexBytePortable(s []byte, c byte) int { + for i, b := range s { + if b == c { + return i + } + } + return -1 +} + +// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s. +func LastIndex(s, sep []byte) int { + n := len(sep) + if n == 0 { + return len(s) + } + c := sep[0] + for i := len(s) - n; i >= 0; i-- { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of the given rune. +// It returns -1 if rune is not present in s. +func IndexRune(s []byte, rune int) int { + for i := 0; i < len(s); { + r, size := utf8.DecodeRune(s[i:]) + if r == rune { + return i + } + i += size + } + return -1 +} + +// IndexAny interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of any of the Unicode +// code points in chars. It returns -1 if chars is empty or if there is no code +// point in common. +func IndexAny(s []byte, chars string) int { + if len(chars) > 0 { + var rune, width int + for i := 0; i < len(s); i += width { + rune = int(s[i]) + if rune < utf8.RuneSelf { + width = 1 + } else { + rune, width = utf8.DecodeRune(s[i:]) + } + for _, r := range chars { + if rune == r { + return i + } + } + } + } + return -1 +} + +// LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code +// points. It returns the byte index of the last occurrence in s of any of +// the Unicode code points in chars. It returns -1 if chars is empty or if +// there is no code point in common. +func LastIndexAny(s []byte, chars string) int { + if len(chars) > 0 { + for i := len(s); i > 0; { + rune, size := utf8.DecodeLastRune(s[0:i]) + i -= size + for _, m := range chars { + if rune == m { + return i + } + } + } + } + return -1 +} + +// Generic split: splits after each instance of sep, +// including sepSave bytes of sep in the subarrays. +func genSplit(s, sep []byte, sepSave, n int) [][]byte { + if n == 0 { + return nil + } + if len(sep) == 0 { + return explode(s, n) + } + if n < 0 { + n = Count(s, sep) + 1 + } + c := sep[0] + start := 0 + a := make([][]byte, n) + na := 0 + for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + a[na] = s[start : i+sepSave] + na++ + start = i + len(sep) + i += len(sep) - 1 + } + } + a[na] = s[start:] + return a[0 : na+1] +} + +// SplitN slices s into subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, SplitN splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitN(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } + +// SplitAfterN slices s into subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfterN splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitAfterN(s, sep []byte, n int) [][]byte { + return genSplit(s, sep, len(sep), n) +} + +// Split slices s into all subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, Split splits after each UTF-8 sequence. +// It is equivalent to SplitN with a count of -1. +func Split(s, sep []byte) [][]byte { return genSplit(s, sep, 0, -1) } + +// SplitAfter slices s into all subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfter splits after each UTF-8 sequence. +// It is equivalent to SplitAfterN with a count of -1. +func SplitAfter(s, sep []byte) [][]byte { + return genSplit(s, sep, len(sep), -1) +} + +// Fields splits the array s around each instance of one or more consecutive white space +// characters, returning a slice of subarrays of s or an empty list if s contains only white space. +func Fields(s []byte) [][]byte { + return FieldsFunc(s, unicode.IsSpace) +} + +// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It splits the array s at each run of code points c satisfying f(c) and +// returns a slice of subarrays of s. If no code points in s satisfy f(c), an +// empty slice is returned. +func FieldsFunc(s []byte, f func(int) bool) [][]byte { + n := 0 + inField := false + for i := 0; i < len(s); { + rune, size := utf8.DecodeRune(s[i:]) + wasInField := inField + inField = !f(rune) + if inField && !wasInField { + n++ + } + i += size + } + + a := make([][]byte, n) + na := 0 + fieldStart := -1 + for i := 0; i <= len(s) && na < n; { + rune, size := utf8.DecodeRune(s[i:]) + if fieldStart < 0 && size > 0 && !f(rune) { + fieldStart = i + i += size + continue + } + if fieldStart >= 0 && (size == 0 || f(rune)) { + a[na] = s[fieldStart:i] + na++ + fieldStart = -1 + } + if size == 0 { + break + } + i += size + } + return a[0:na] +} + +// Join concatenates the elements of a to create a single byte array. The separator +// sep is placed between elements in the resulting array. +func Join(a [][]byte, sep []byte) []byte { + if len(a) == 0 { + return []byte{} + } + if len(a) == 1 { + return a[0] + } + n := len(sep) * (len(a) - 1) + for i := 0; i < len(a); i++ { + n += len(a[i]) + } + + b := make([]byte, n) + bp := copy(b, a[0]) + for _, s := range a[1:] { + bp += copy(b[bp:], sep) + bp += copy(b[bp:], s) + } + return b +} + +// HasPrefix tests whether the byte array s begins with prefix. +func HasPrefix(s, prefix []byte) bool { + return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) +} + +// HasSuffix tests whether the byte array s ends with suffix. +func HasSuffix(s, suffix []byte) bool { + return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) +} + +// Map returns a copy of the byte array s with all its characters modified +// according to the mapping function. If mapping returns a negative value, the character is +// dropped from the string with no replacement. The characters in s and the +// output are interpreted as UTF-8-encoded Unicode code points. +func Map(mapping func(rune int) int, s []byte) []byte { + // In the worst case, the array can grow when mapped, making + // things unpleasant. But it's so rare we barge in assuming it's + // fine. It could also shrink but that falls out naturally. + maxbytes := len(s) // length of b + nbytes := 0 // number of bytes encoded in b + b := make([]byte, maxbytes) + for i := 0; i < len(s); { + wid := 1 + rune := int(s[i]) + if rune >= utf8.RuneSelf { + rune, wid = utf8.DecodeRune(s[i:]) + } + rune = mapping(rune) + if rune >= 0 { + if nbytes+utf8.RuneLen(rune) > maxbytes { + // Grow the buffer. + maxbytes = maxbytes*2 + utf8.UTFMax + nb := make([]byte, maxbytes) + copy(nb, b[0:nbytes]) + b = nb + } + nbytes += utf8.EncodeRune(b[nbytes:maxbytes], rune) + } + i += wid + } + return b[0:nbytes] +} + +// Repeat returns a new byte slice consisting of count copies of b. +func Repeat(b []byte, count int) []byte { + nb := make([]byte, len(b)*count) + bp := 0 + for i := 0; i < count; i++ { + for j := 0; j < len(b); j++ { + nb[bp] = b[j] + bp++ + } + } + return nb +} + +// ToUpper returns a copy of the byte array s with all Unicode letters mapped to their upper case. +func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) } + +// ToUpper returns a copy of the byte array s with all Unicode letters mapped to their lower case. +func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) } + +// ToTitle returns a copy of the byte array s with all Unicode letters mapped to their title case. +func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) } + +// ToUpperSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// upper case, giving priority to the special casing rules. +func ToUpperSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToUpper(r) }, s) +} + +// ToLowerSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// lower case, giving priority to the special casing rules. +func ToLowerSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToLower(r) }, s) +} + +// ToTitleSpecial returns a copy of the byte array s with all Unicode letters mapped to their +// title case, giving priority to the special casing rules. +func ToTitleSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r int) int { return _case.ToTitle(r) }, s) +} + +// isSeparator reports whether the rune could mark a word boundary. +// TODO: update when package unicode captures more of the properties. +func isSeparator(rune int) bool { + // ASCII alphanumerics and underscore are not separators + if rune <= 0x7F { + switch { + case '0' <= rune && rune <= '9': + return false + case 'a' <= rune && rune <= 'z': + return false + case 'A' <= rune && rune <= 'Z': + return false + case rune == '_': + return false + } + return true + } + // Letters and digits are not separators + if unicode.IsLetter(rune) || unicode.IsDigit(rune) { + return false + } + // Otherwise, all we can do for now is treat spaces as separators. + return unicode.IsSpace(rune) +} + +// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly. + +// Title returns a copy of s with all Unicode letters that begin words +// mapped to their title case. +func Title(s []byte) []byte { + // Use a closure here to remember state. + // Hackish but effective. Depends on Map scanning in order and calling + // the closure once per rune. + prev := ' ' + return Map( + func(r int) int { + if isSeparator(prev) { + prev = r + return unicode.ToTitle(r) + } + prev = r + return r + }, + s) +} + +// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded +// Unicode code points c that satisfy f(c). +func TrimLeftFunc(s []byte, f func(r int) bool) []byte { + i := indexFunc(s, f, false) + if i == -1 { + return nil + } + return s[i:] +} + +// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8 +// encoded Unicode code points c that satisfy f(c). +func TrimRightFunc(s []byte, f func(r int) bool) []byte { + i := lastIndexFunc(s, f, false) + if i >= 0 && s[i] >= utf8.RuneSelf { + _, wid := utf8.DecodeRune(s[i:]) + i += wid + } else { + i++ + } + return s[0:i] +} + +// TrimFunc returns a subslice of s by slicing off all leading and trailing +// UTF-8-encoded Unicode code points c that satisfy f(c). +func TrimFunc(s []byte, f func(r int) bool) []byte { + return TrimRightFunc(TrimLeftFunc(s, f), f) +} + +// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the first Unicode +// code point satisfying f(c), or -1 if none do. +func IndexFunc(s []byte, f func(r int) bool) int { + return indexFunc(s, f, true) +} + +// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the last Unicode +// code point satisfying f(c), or -1 if none do. +func LastIndexFunc(s []byte, f func(r int) bool) int { + return lastIndexFunc(s, f, true) +} + +// indexFunc is the same as IndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func indexFunc(s []byte, f func(r int) bool, truth bool) int { + start := 0 + for start < len(s) { + wid := 1 + rune := int(s[start]) + if rune >= utf8.RuneSelf { + rune, wid = utf8.DecodeRune(s[start:]) + } + if f(rune) == truth { + return start + } + start += wid + } + return -1 +} + +// lastIndexFunc is the same as LastIndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func lastIndexFunc(s []byte, f func(r int) bool, truth bool) int { + for i := len(s); i > 0; { + rune, size := utf8.DecodeLastRune(s[0:i]) + i -= size + if f(rune) == truth { + return i + } + } + return -1 +} + +func makeCutsetFunc(cutset string) func(rune int) bool { + return func(rune int) bool { + for _, c := range cutset { + if c == rune { + return true + } + } + return false + } +} + +// Trim returns a subslice of s by slicing off all leading and +// trailing UTF-8-encoded Unicode code points contained in cutset. +func Trim(s []byte, cutset string) []byte { + return TrimFunc(s, makeCutsetFunc(cutset)) +} + +// TrimLeft returns a subslice of s by slicing off all leading +// UTF-8-encoded Unicode code points contained in cutset. +func TrimLeft(s []byte, cutset string) []byte { + return TrimLeftFunc(s, makeCutsetFunc(cutset)) +} + +// TrimRight returns a subslice of s by slicing off all trailing +// UTF-8-encoded Unicode code points that are contained in cutset. +func TrimRight(s []byte, cutset string) []byte { + return TrimRightFunc(s, makeCutsetFunc(cutset)) +} + +// TrimSpace returns a subslice of s by slicing off all leading and +// trailing white space, as defined by Unicode. +func TrimSpace(s []byte) []byte { + return TrimFunc(s, unicode.IsSpace) +} + +// Runes returns a slice of runes (Unicode code points) equivalent to s. +func Runes(s []byte) []int { + t := make([]int, utf8.RuneCount(s)) + i := 0 + for len(s) > 0 { + r, l := utf8.DecodeRune(s) + t[i] = r + i++ + s = s[l:] + } + return t +} + +// Replace returns a copy of the slice s with the first n +// non-overlapping instances of old replaced by new. +// If n < 0, there is no limit on the number of replacements. +func Replace(s, old, new []byte, n int) []byte { + if n == 0 { + return s // avoid allocation + } + // Compute number of replacements. + if m := Count(s, old); m == 0 { + return s // avoid allocation + } else if n <= 0 || m < n { + n = m + } + + // Apply replacements to buffer. + t := make([]byte, len(s)+n*(len(new)-len(old))) + w := 0 + start := 0 + for i := 0; i < n; i++ { + j := start + if len(old) == 0 { + if i > 0 { + _, wid := utf8.DecodeRune(s[start:]) + j += wid + } + } else { + j += Index(s[start:], old) + } + w += copy(t[w:], s[start:j]) + w += copy(t[w:], new) + start = j + len(old) + } + w += copy(t[w:], s[start:]) + return t[0:w] +} diff --git a/src/pkg/bytes/bytes_decl.go b/src/pkg/bytes/bytes_decl.go new file mode 100644 index 000000000..5d2b9e639 --- /dev/null +++ b/src/pkg/bytes/bytes_decl.go @@ -0,0 +1,8 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// IndexByte returns the index of the first instance of c in s, or -1 if c is not present in s. +func IndexByte(s []byte, c byte) int // asm_$GOARCH.s diff --git a/src/pkg/bytes/bytes_test.go b/src/pkg/bytes/bytes_test.go new file mode 100644 index 000000000..9444358a8 --- /dev/null +++ b/src/pkg/bytes/bytes_test.go @@ -0,0 +1,858 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "reflect" + "testing" + "unicode" + "utf8" +) + +func eq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func arrayOfString(a [][]byte) []string { + result := make([]string, len(a)) + for j := 0; j < len(a); j++ { + result[j] = string(a[j]) + } + return result +} + +// For ease of reading, the test cases use strings that are converted to byte +// arrays before invoking the functions. + +var abcd = "abcd" +var faces = "☺☻☹" +var commas = "1,2,3,4" +var dots = "1....2....3....4" + +type BinOpTest struct { + a string + b string + i int +} + +var comparetests = []BinOpTest{ + {"", "", 0}, + {"a", "", 1}, + {"", "a", -1}, + {"abc", "abc", 0}, + {"ab", "abc", -1}, + {"abc", "ab", 1}, + {"x", "ab", 1}, + {"ab", "x", -1}, + {"x", "a", 1}, + {"b", "x", -1}, +} + +func TestCompare(t *testing.T) { + for _, tt := range comparetests { + a := []byte(tt.a) + b := []byte(tt.b) + cmp := Compare(a, b) + eql := Equal(a, b) + if cmp != tt.i { + t.Errorf(`Compare(%q, %q) = %v`, tt.a, tt.b, cmp) + } + if eql != (tt.i == 0) { + t.Errorf(`Equal(%q, %q) = %v`, tt.a, tt.b, eql) + } + } +} + +var indexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"oofofoofooo", "f", 2}, + {"oofofoofooo", "foo", 4}, + {"barfoobarfoo", "foo", 3}, + {"foo", "", 0}, + {"foo", "o", 1}, + {"abcABCabc", "A", 3}, + // cases with one byte strings - test IndexByte and special case in Index() + {"", "a", -1}, + {"x", "a", -1}, + {"x", "x", 0}, + {"abc", "a", 0}, + {"abc", "b", 1}, + {"abc", "c", 2}, + {"abc", "x", -1}, + {"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33}, + {"foofyfoobarfoobar", "y", 4}, + {"oooooooooooooooooooooo", "r", -1}, +} + +var lastIndexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"foo", "f", 0}, + {"oofofoofooo", "f", 7}, + {"oofofoofooo", "foo", 7}, + {"barfoobarfoo", "foo", 9}, + {"foo", "", 3}, + {"foo", "o", 2}, + {"abcABCabc", "A", 3}, + {"abcABCabc", "a", 6}, +} + +var indexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 0}, + {"abc", "xyz", -1}, + {"abc", "xcz", 2}, + {"ab☺c", "x☺yz", 2}, + {"aRegExp*", ".(|)*+?^$[]", 7}, + {dots + dots + dots, " ", -1}, +} + +var lastIndexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 2}, + {"abc", "xyz", -1}, + {"abc", "ab", 1}, + {"a☺b☻c☹d", "uvw☻xyz", 2 + len("☺")}, + {"a.RegExp*", ".(|)*+?^$[]", 8}, + {dots + dots + dots, " ", -1}, +} + +var indexRuneTests = []BinOpTest{ + {"", "a", -1}, + {"", "☺", -1}, + {"foo", "☹", -1}, + {"foo", "o", 1}, + {"foo☺bar", "☺", 3}, + {"foo☺☻☹bar", "☹", 9}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runIndexTests(t *testing.T, f func(s, sep []byte) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + b := []byte(test.b) + actual := f(a, b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, b, actual, test.i) + } + } +} + +func runIndexAnyTests(t *testing.T, f func(s []byte, chars string) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + actual := f(a, test.b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, test.b, actual, test.i) + } + } +} + +func TestIndex(t *testing.T) { runIndexTests(t, Index, "Index", indexTests) } +func TestLastIndex(t *testing.T) { runIndexTests(t, LastIndex, "LastIndex", lastIndexTests) } +func TestIndexAny(t *testing.T) { runIndexAnyTests(t, IndexAny, "IndexAny", indexAnyTests) } +func TestLastIndexAny(t *testing.T) { + runIndexAnyTests(t, LastIndexAny, "LastIndexAny", lastIndexAnyTests) +} + +func TestIndexByte(t *testing.T) { + for _, tt := range indexTests { + if len(tt.b) != 1 { + continue + } + a := []byte(tt.a) + b := tt.b[0] + pos := IndexByte(a, b) + if pos != tt.i { + t.Errorf(`IndexByte(%q, '%c') = %v`, tt.a, b, pos) + } + posp := IndexBytePortable(a, b) + if posp != tt.i { + t.Errorf(`indexBytePortable(%q, '%c') = %v`, tt.a, b, posp) + } + } +} + +// test a larger buffer with different sizes and alignments +func TestIndexByteBig(t *testing.T) { + var n = 1024 + if testing.Short() { + n = 128 + } + b := make([]byte, n) + for i := 0; i < n; i++ { + // different start alignments + b1 := b[i:] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different end alignments + b1 = b[:i] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different start and end alignments + b1 = b[i/2 : n-(i+1)/2] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + } +} + +func TestIndexRune(t *testing.T) { + for _, tt := range indexRuneTests { + a := []byte(tt.a) + r, _ := utf8.DecodeRuneInString(tt.b) + pos := IndexRune(a, r) + if pos != tt.i { + t.Errorf(`IndexRune(%q, '%c') = %v`, tt.a, r, pos) + } + } +} + +func BenchmarkIndexByte4K(b *testing.B) { bmIndex(b, IndexByte, 4<<10) } + +func BenchmarkIndexByte4M(b *testing.B) { bmIndex(b, IndexByte, 4<<20) } + +func BenchmarkIndexByte64M(b *testing.B) { bmIndex(b, IndexByte, 64<<20) } + +func BenchmarkIndexBytePortable4K(b *testing.B) { + bmIndex(b, IndexBytePortable, 4<<10) +} + +func BenchmarkIndexBytePortable4M(b *testing.B) { + bmIndex(b, IndexBytePortable, 4<<20) +} + +func BenchmarkIndexBytePortable64M(b *testing.B) { + bmIndex(b, IndexBytePortable, 64<<20) +} + +var bmbuf []byte + +func bmIndex(b *testing.B, index func([]byte, byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + for i := 0; i < b.N; i++ { + j := index(buf, 'x') + if j != n-1 { + println("bad index", j) + panic("bad index") + } + } + buf[n-1] = '0' +} + +type ExplodeTest struct { + s string + n int + a []string +} + +var explodetests = []ExplodeTest{ + {"", -1, []string{}}, + {abcd, -1, []string{"a", "b", "c", "d"}}, + {faces, -1, []string{"☺", "☻", "☹"}}, + {abcd, 2, []string{"a", "bcd"}}, +} + +func TestExplode(t *testing.T) { + for _, tt := range explodetests { + a := SplitN([]byte(tt.s), nil, tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Explode("%s", %d) = %v; want %v`, tt.s, tt.n, result, tt.a) + continue + } + s := Join(a, []byte{}) + if string(s) != tt.s { + t.Errorf(`Join(Explode("%s", %d), "") = "%s"`, tt.s, tt.n, s) + } + } +} + +type SplitTest struct { + s string + sep string + n int + a []string +} + +var splittests = []SplitTest{ + {abcd, "a", 0, nil}, + {abcd, "a", -1, []string{"", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1", "2", "3", "4"}}, + {dots, "...", -1, []string{"1", ".2", ".3", ".4"}}, + {faces, "☹", -1, []string{"☺☻", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1", "2", "3 4"}}, + {"1 2", " ", 3, []string{"1", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplit(t *testing.T) { + for _, tt := range splittests { + a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + if tt.n == 0 { + continue + } + s := Join(a, []byte(tt.sep)) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + if tt.n < 0 { + b := Split([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("Split disagrees withSplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } + } +} + +var splitaftertests = []SplitTest{ + {abcd, "a", -1, []string{"a", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1,", "2,", "3,", "4"}}, + {dots, "...", -1, []string{"1...", ".2...", ".3...", ".4"}}, + {faces, "☹", -1, []string{"☺☻☹", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1 ", "2 ", "3 4"}}, + {"1 2 3", " ", 3, []string{"1 ", "2 ", "3"}}, + {"1 2", " ", 3, []string{"1 ", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplitAfter(t *testing.T) { + for _, tt := range splitaftertests { + a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + s := Join(a, nil) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + if tt.n < 0 { + b := SplitAfter([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("SplitAfter disagrees withSplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } + } +} + +type FieldsTest struct { + s string + a []string +} + +var fieldstests = []FieldsTest{ + {"", []string{}}, + {" ", []string{}}, + {" \t ", []string{}}, + {" abc ", []string{"abc"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1\t\t2\t\t3\t4", []string{"1", "2", "3", "4"}}, + {"1\u20002\u20013\u20024", []string{"1", "2", "3", "4"}}, + {"\u2000\u2001\u2002", []string{}}, + {"\n™\t™\n", []string{"™", "™"}}, + {faces, []string{faces}}, +} + +func TestFields(t *testing.T) { + for _, tt := range fieldstests { + a := Fields([]byte(tt.s)) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) + continue + } + } +} + +func TestFieldsFunc(t *testing.T) { + pred := func(c int) bool { return c == 'X' } + var fieldsFuncTests = []FieldsTest{ + {"", []string{}}, + {"XX", []string{}}, + {"XXhiXXX", []string{"hi"}}, + {"aXXbXXXcX", []string{"a", "b", "c"}}, + } + for _, tt := range fieldsFuncTests { + a := FieldsFunc([]byte(tt.s), pred) + result := arrayOfString(a) + if !eq(result, tt.a) { + t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) + } + } +} + +// Test case for any function which accepts and returns a byte array. +// For ease of creation, we write the byte arrays as strings. +type StringTest struct { + in, out string +} + +var upperTests = []StringTest{ + {"", ""}, + {"abc", "ABC"}, + {"AbC123", "ABC123"}, + {"azAZ09_", "AZAZ09_"}, + {"\u0250\u0250\u0250\u0250\u0250", "\u2C6F\u2C6F\u2C6F\u2C6F\u2C6F"}, // grows one byte per char +} + +var lowerTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {"AbC123", "abc123"}, + {"azAZ09_", "azaz09_"}, + {"\u2C6D\u2C6D\u2C6D\u2C6D\u2C6D", "\u0251\u0251\u0251\u0251\u0251"}, // shrinks one byte per char +} + +const space = "\t\v\r\f\n\u0085\u00a0\u2000\u3000" + +var trimSpaceTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {space + "abc" + space, "abc"}, + {" ", ""}, + {" \t\r\n \t\t\r\r\n\n ", ""}, + {" \t\r\n x\t\t\r\r\n\n ", "x"}, + {" \u2000\t\r\n x\t\t\r\r\ny\n \u3000", "x\t\t\r\r\ny"}, + {"1 \t\r\n2", "1 \t\r\n2"}, + {" x\x80", "x\x80"}, + {" x\xc0", "x\xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x \xc0", "x \xc0"}, + {"x \xc0 ", "x \xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x ☺\xc0\xc0 ", "x ☺\xc0\xc0"}, + {"x ☺ ", "x ☺"}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runStringTests(t *testing.T, f func([]byte) []byte, funcName string, testCases []StringTest) { + for _, tc := range testCases { + actual := string(f([]byte(tc.in))) + if actual != tc.out { + t.Errorf("%s(%q) = %q; want %q", funcName, tc.in, actual, tc.out) + } + } +} + +func tenRunes(rune int) string { + r := make([]int, 10) + for i := range r { + r[i] = rune + } + return string(r) +} + +// User-defined self-inverse mapping function +func rot13(rune int) int { + step := 13 + if rune >= 'a' && rune <= 'z' { + return ((rune - 'a' + step) % 26) + 'a' + } + if rune >= 'A' && rune <= 'Z' { + return ((rune - 'A' + step) % 26) + 'A' + } + return rune +} + +func TestMap(t *testing.T) { + // Run a couple of awful growth/shrinkage tests + a := tenRunes('a') + + // 1. Grow. This triggers two reallocations in Map. + maxRune := func(rune int) int { return unicode.MaxRune } + m := Map(maxRune, []byte(a)) + expect := tenRunes(unicode.MaxRune) + if string(m) != expect { + t.Errorf("growing: expected %q got %q", expect, m) + } + + // 2. Shrink + minRune := func(rune int) int { return 'a' } + m = Map(minRune, []byte(tenRunes(unicode.MaxRune))) + expect = a + if string(m) != expect { + t.Errorf("shrinking: expected %q got %q", expect, m) + } + + // 3. Rot13 + m = Map(rot13, []byte("a to zed")) + expect = "n gb mrq" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 4. Rot13^2 + m = Map(rot13, Map(rot13, []byte("a to zed"))) + expect = "a to zed" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 5. Drop + dropNotLatin := func(rune int) int { + if unicode.Is(unicode.Latin, rune) { + return rune + } + return -1 + } + m = Map(dropNotLatin, []byte("Hello, 세계")) + expect = "Hello" + if string(m) != expect { + t.Errorf("drop: expected %q got %q", expect, m) + } +} + +func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } + +func TestToLower(t *testing.T) { runStringTests(t, ToLower, "ToLower", lowerTests) } + +func TestTrimSpace(t *testing.T) { runStringTests(t, TrimSpace, "TrimSpace", trimSpaceTests) } + +type RepeatTest struct { + in, out string + count int +} + +var RepeatTests = []RepeatTest{ + {"", "", 0}, + {"", "", 1}, + {"", "", 2}, + {"-", "", 0}, + {"-", "-", 1}, + {"-", "----------", 10}, + {"abc ", "abc abc abc ", 3}, +} + +func TestRepeat(t *testing.T) { + for _, tt := range RepeatTests { + tin := []byte(tt.in) + tout := []byte(tt.out) + a := Repeat(tin, tt.count) + if !Equal(a, tout) { + t.Errorf("Repeat(%q, %d) = %q; want %q", tin, tt.count, a, tout) + continue + } + } +} + +func runesEqual(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i, r := range a { + if r != b[i] { + return false + } + } + return true +} + +type RunesTest struct { + in string + out []int + lossy bool +} + +var RunesTests = []RunesTest{ + {"", []int{}, false}, + {" ", []int{32}, false}, + {"ABC", []int{65, 66, 67}, false}, + {"abc", []int{97, 98, 99}, false}, + {"\u65e5\u672c\u8a9e", []int{26085, 26412, 35486}, false}, + {"ab\x80c", []int{97, 98, 0xFFFD, 99}, true}, + {"ab\xc0c", []int{97, 98, 0xFFFD, 99}, true}, +} + +func TestRunes(t *testing.T) { + for _, tt := range RunesTests { + tin := []byte(tt.in) + a := Runes(tin) + if !runesEqual(a, tt.out) { + t.Errorf("Runes(%q) = %v; want %v", tin, a, tt.out) + continue + } + if !tt.lossy { + // can only test reassembly if we didn't lose information + s := string(a) + if s != tt.in { + t.Errorf("string(Runes(%q)) = %x; want %x", tin, s, tin) + } + } + } +} + +type TrimTest struct { + f func([]byte, string) []byte + in, cutset, out string +} + +var trimTests = []TrimTest{ + {Trim, "abba", "a", "bb"}, + {Trim, "abba", "ab", ""}, + {TrimLeft, "abba", "ab", ""}, + {TrimRight, "abba", "ab", ""}, + {TrimLeft, "abba", "a", "bba"}, + {TrimRight, "abba", "a", "abb"}, + {Trim, "", "<>", "tag"}, + {Trim, "* listitem", " *", "listitem"}, + {Trim, `"quote"`, `"`, "quote"}, + {Trim, "\u2C6F\u2C6F\u0250\u0250\u2C6F\u2C6F", "\u2C6F", "\u0250\u0250"}, + //empty string tests + {Trim, "abba", "", "abba"}, + {Trim, "", "123", ""}, + {Trim, "", "", ""}, + {TrimLeft, "abba", "", "abba"}, + {TrimLeft, "", "123", ""}, + {TrimLeft, "", "", ""}, + {TrimRight, "abba", "", "abba"}, + {TrimRight, "", "123", ""}, + {TrimRight, "", "", ""}, + {TrimRight, "☺\xc0", "☺", "☺\xc0"}, +} + +func TestTrim(t *testing.T) { + for _, tc := range trimTests { + actual := string(tc.f([]byte(tc.in), tc.cutset)) + var name string + switch tc.f { + case Trim: + name = "Trim" + case TrimLeft: + name = "TrimLeft" + case TrimRight: + name = "TrimRight" + default: + t.Error("Undefined trim function") + } + if actual != tc.out { + t.Errorf("%s(%q, %q) = %q; want %q", name, tc.in, tc.cutset, actual, tc.out) + } + } +} + +type predicate struct { + f func(r int) bool + name string +} + +var isSpace = predicate{unicode.IsSpace, "IsSpace"} +var isDigit = predicate{unicode.IsDigit, "IsDigit"} +var isUpper = predicate{unicode.IsUpper, "IsUpper"} +var isValidRune = predicate{ + func(r int) bool { + return r != utf8.RuneError + }, + "IsValidRune", +} + +type TrimFuncTest struct { + f predicate + in, out string +} + +func not(p predicate) predicate { + return predicate{ + func(r int) bool { + return !p.f(r) + }, + "not " + p.name, + } +} + +var trimFuncTests = []TrimFuncTest{ + {isSpace, space + " hello " + space, "hello"}, + {isDigit, "\u0e50\u0e5212hello34\u0e50\u0e51", "hello"}, + {isUpper, "\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", "hello"}, + {not(isSpace), "hello" + space + "hello", space}, + {not(isDigit), "hello\u0e50\u0e521234\u0e50\u0e51helo", "\u0e50\u0e521234\u0e50\u0e51"}, + {isValidRune, "ab\xc0a\xc0cd", "\xc0a\xc0"}, + {not(isValidRune), "\xc0a\xc0", "a"}, +} + +func TestTrimFunc(t *testing.T) { + for _, tc := range trimFuncTests { + actual := string(TrimFunc([]byte(tc.in), tc.f.f)) + if actual != tc.out { + t.Errorf("TrimFunc(%q, %q) = %q; want %q", tc.in, tc.f.name, actual, tc.out) + } + } +} + +type IndexFuncTest struct { + in string + f predicate + first, last int +} + +var indexFuncTests = []IndexFuncTest{ + {"", isValidRune, -1, -1}, + {"abc", isDigit, -1, -1}, + {"0123", isDigit, 0, 3}, + {"a1b", isDigit, 1, 1}, + {space, isSpace, 0, len(space) - 3}, // last rune in space is 3 bytes + {"\u0e50\u0e5212hello34\u0e50\u0e51", isDigit, 0, 18}, + {"\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", isUpper, 0, 34}, + {"12\u0e50\u0e52hello34\u0e50\u0e51", not(isDigit), 8, 12}, + + // tests of invalid UTF-8 + {"\x801", isDigit, 1, 1}, + {"\x80abc", isDigit, -1, -1}, + {"\xc0a\xc0", isValidRune, 1, 1}, + {"\xc0a\xc0", not(isValidRune), 0, 2}, + {"\xc0☺\xc0", not(isValidRune), 0, 4}, + {"\xc0☺\xc0\xc0", not(isValidRune), 0, 5}, + {"ab\xc0a\xc0cd", not(isValidRune), 2, 4}, + {"a\xe0\x80cd", not(isValidRune), 1, 2}, +} + +func TestIndexFunc(t *testing.T) { + for _, tc := range indexFuncTests { + first := IndexFunc([]byte(tc.in), tc.f.f) + if first != tc.first { + t.Errorf("IndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, first, tc.first) + } + last := LastIndexFunc([]byte(tc.in), tc.f.f) + if last != tc.last { + t.Errorf("LastIndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, last, tc.last) + } + } +} + +type ReplaceTest struct { + in string + old, new string + n int + out string +} + +var ReplaceTests = []ReplaceTest{ + {"hello", "l", "L", 0, "hello"}, + {"hello", "l", "L", -1, "heLLo"}, + {"hello", "x", "X", -1, "hello"}, + {"", "x", "X", -1, ""}, + {"radar", "r", "", -1, "ada"}, + {"", "", "<>", -1, "<>"}, + {"banana", "a", "<>", -1, "b<>n<>n<>"}, + {"banana", "a", "<>", 1, "b<>nana"}, + {"banana", "a", "<>", 1000, "b<>n<>n<>"}, + {"banana", "an", "<>", -1, "b<><>a"}, + {"banana", "ana", "<>", -1, "b<>na"}, + {"banana", "", "<>", -1, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 10, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 6, "<>b<>a<>n<>a<>n<>a"}, + {"banana", "", "<>", 5, "<>b<>a<>n<>a<>na"}, + {"banana", "", "<>", 1, "<>banana"}, + {"banana", "a", "a", -1, "banana"}, + {"banana", "a", "a", 1, "banana"}, + {"☺☻☹", "", "<>", -1, "<>☺<>☻<>☹<>"}, +} + +func TestReplace(t *testing.T) { + for _, tt := range ReplaceTests { + if s := string(Replace([]byte(tt.in), []byte(tt.old), []byte(tt.new), tt.n)); s != tt.out { + t.Errorf("Replace(%q, %q, %q, %d) = %q, want %q", tt.in, tt.old, tt.new, tt.n, s, tt.out) + } + } +} + +type TitleTest struct { + in, out string +} + +var TitleTests = []TitleTest{ + {"", ""}, + {"a", "A"}, + {" aaa aaa aaa ", " Aaa Aaa Aaa "}, + {" Aaa Aaa Aaa ", " Aaa Aaa Aaa "}, + {"123a456", "123a456"}, + {"double-blind", "Double-Blind"}, + {"ÿøû", "Ÿøû"}, +} + +func TestTitle(t *testing.T) { + for _, tt := range TitleTests { + if s := string(Title([]byte(tt.in))); s != tt.out { + t.Errorf("Title(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} diff --git a/src/pkg/bytes/export_test.go b/src/pkg/bytes/export_test.go new file mode 100644 index 000000000..b65428d9c --- /dev/null +++ b/src/pkg/bytes/export_test.go @@ -0,0 +1,8 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Export func for testing +var IndexBytePortable = indexBytePortable diff --git a/src/pkg/cmath/Makefile b/src/pkg/cmath/Makefile new file mode 100644 index 000000000..486caace4 --- /dev/null +++ b/src/pkg/cmath/Makefile @@ -0,0 +1,25 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=cmath + +GOFILES=\ + abs.go\ + asin.go\ + conj.go\ + exp.go\ + isinf.go\ + isnan.go\ + log.go\ + phase.go\ + polar.go\ + pow.go\ + rect.go\ + sin.go\ + sqrt.go\ + tan.go\ + +include ../../Make.pkg diff --git a/src/pkg/cmath/abs.go b/src/pkg/cmath/abs.go new file mode 100644 index 000000000..f3199cad5 --- /dev/null +++ b/src/pkg/cmath/abs.go @@ -0,0 +1,12 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cmath provides basic constants and mathematical functions for +// complex numbers. +package cmath + +import "math" + +// Abs returns the absolute value (also called the modulus) of x. +func Abs(x complex128) float64 { return math.Hypot(real(x), imag(x)) } diff --git a/src/pkg/cmath/asin.go b/src/pkg/cmath/asin.go new file mode 100644 index 000000000..d6a3ca480 --- /dev/null +++ b/src/pkg/cmath/asin.go @@ -0,0 +1,170 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular arc sine +// +// DESCRIPTION: +// +// Inverse complex sine: +// 2 +// w = -i clog( iz + csqrt( 1 - z ) ). +// +// casin(z) = -i casinh(iz) +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 10100 2.1e-15 3.4e-16 +// IEEE -10,+10 30000 2.2e-14 2.7e-15 +// Larger relative error can be observed for z near zero. +// Also tested by csin(casin(z)) = z. + +// Asin returns the inverse sine of x. +func Asin(x complex128) complex128 { + if imag(x) == 0 { + if math.Fabs(real(x)) > 1 { + return complex(math.Pi/2, 0) // DOMAIN error + } + return complex(math.Asin(real(x)), 0) + } + ct := complex(-imag(x), real(x)) // i * x + xx := x * x + x1 := complex(1-real(xx), -imag(xx)) // 1 - x*x + x2 := Sqrt(x1) // x2 = sqrt(1 - x*x) + w := Log(ct + x2) + return complex(imag(w), -real(w)) // -i * w +} + +// Asinh returns the inverse hyperbolic sine of x. +func Asinh(x complex128) complex128 { + // TODO check range + if imag(x) == 0 { + if math.Fabs(real(x)) > 1 { + return complex(math.Pi/2, 0) // DOMAIN error + } + return complex(math.Asinh(real(x)), 0) + } + xx := x * x + x1 := complex(1+real(xx), imag(xx)) // 1 + x*x + return Log(x + Sqrt(x1)) // log(x + sqrt(1 + x*x)) +} + +// Complex circular arc cosine +// +// DESCRIPTION: +// +// w = arccos z = PI/2 - arcsin z. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5200 1.6e-15 2.8e-16 +// IEEE -10,+10 30000 1.8e-14 2.2e-15 + +// Acos returns the inverse cosine of x. +func Acos(x complex128) complex128 { + w := Asin(x) + return complex(math.Pi/2-real(w), -imag(w)) +} + +// Acosh returns the inverse hyperbolic cosine of x. +func Acosh(x complex128) complex128 { + w := Acos(x) + if imag(w) <= 0 { + return complex(-imag(w), real(w)) // i * w + } + return complex(imag(w), -real(w)) // -i * w +} + +// Complex circular arc tangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// 1 ( 2x ) +// Re w = - arctan(-----------) + k PI +// 2 ( 2 2) +// (1 - x - y ) +// +// ( 2 2) +// 1 (x + (y+1) ) +// Im w = - log(------------) +// 4 ( 2 2) +// (x + (y-1) ) +// +// Where k is an arbitrary integer. +// +// catan(z) = -i catanh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5900 1.3e-16 7.8e-18 +// IEEE -10,+10 30000 2.3e-15 8.5e-17 +// The check catan( ctan(z) ) = z, with |x| and |y| < PI/2, +// had peak relative error 1.5e-16, rms relative error +// 2.9e-17. See also clog(). + +// Atan returns the inverse tangent of x. +func Atan(x complex128) complex128 { + if real(x) == 0 && imag(x) > 1 { + return NaN() + } + + x2 := real(x) * real(x) + a := 1 - x2 - imag(x)*imag(x) + if a == 0 { + return NaN() + } + t := 0.5 * math.Atan2(2*real(x), a) + w := reducePi(t) + + t = imag(x) - 1 + b := x2 + t*t + if b == 0 { + return NaN() + } + t = imag(x) + 1 + c := (x2 + t*t) / b + return complex(w, 0.25*math.Log(c)) +} + +// Atanh returns the inverse hyperbolic tangent of x. +func Atanh(x complex128) complex128 { + z := complex(-imag(x), real(x)) // z = i * x + z = Atan(z) + return complex(imag(z), -real(z)) // z = -i * z +} diff --git a/src/pkg/cmath/cmath_test.go b/src/pkg/cmath/cmath_test.go new file mode 100644 index 000000000..6a595b0a6 --- /dev/null +++ b/src/pkg/cmath/cmath_test.go @@ -0,0 +1,853 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import ( + "math" + "testing" +) + +var vc26 = []complex128{ + (4.97901192488367350108546816 + 7.73887247457810456552351752i), + (7.73887247457810456552351752 - 0.27688005719200159404635997i), + (-0.27688005719200159404635997 - 5.01060361827107492160848778i), + (-5.01060361827107492160848778 + 9.63629370719841737980004837i), + (9.63629370719841737980004837 + 2.92637723924396464525443662i), + (2.92637723924396464525443662 + 5.22908343145930665230025625i), + (5.22908343145930665230025625 + 2.72793991043601025126008608i), + (2.72793991043601025126008608 + 1.82530809168085506044576505i), + (1.82530809168085506044576505 - 8.68592476857560136238589621i), + (-8.68592476857560136238589621 + 4.97901192488367350108546816i), +} +var vc = []complex128{ + (4.9790119248836735e+00 + 7.7388724745781045e+00i), + (7.7388724745781045e+00 - 2.7688005719200159e-01i), + (-2.7688005719200159e-01 - 5.0106036182710749e+00i), + (-5.0106036182710749e+00 + 9.6362937071984173e+00i), + (9.6362937071984173e+00 + 2.9263772392439646e+00i), + (2.9263772392439646e+00 + 5.2290834314593066e+00i), + (5.2290834314593066e+00 + 2.7279399104360102e+00i), + (2.7279399104360102e+00 + 1.8253080916808550e+00i), + (1.8253080916808550e+00 - 8.6859247685756013e+00i), + (-8.6859247685756013e+00 + 4.9790119248836735e+00i), +} + +// The expected results below were computed by the high precision calculators +// at http://keisan.casio.com/. More exact input values (array vc[], above) +// were obtained by printing them with "%.26f". The answers were calculated +// to 26 digits (by using the "Digit number" drop-down control of each +// calculator). + +var abs = []float64{ + 9.2022120669932650313380972e+00, + 7.7438239742296106616261394e+00, + 5.0182478202557746902556648e+00, + 1.0861137372799545160704002e+01, + 1.0070841084922199607011905e+01, + 5.9922447613166942183705192e+00, + 5.8978784056736762299945176e+00, + 3.2822866700678709020367184e+00, + 8.8756430028990417290744307e+00, + 1.0011785496777731986390856e+01, +} + +var acos = []complex128{ + (1.0017679804707456328694569 - 2.9138232718554953784519807i), + (0.03606427612041407369636057 + 2.7358584434576260925091256i), + (1.6249365462333796703711823 + 2.3159537454335901187730929i), + (2.0485650849650740120660391 - 3.0795576791204117911123886i), + (0.29621132089073067282488147 - 3.0007392508200622519398814i), + (1.0664555914934156601503632 - 2.4872865024796011364747111i), + (0.48681307452231387690013905 - 2.463655912283054555225301i), + (0.6116977071277574248407752 - 1.8734458851737055262693056i), + (1.3649311280370181331184214 + 2.8793528632328795424123832i), + (2.6189310485682988308904501 - 2.9956543302898767795858704i), +} +var acosh = []complex128{ + (2.9138232718554953784519807 + 1.0017679804707456328694569i), + (2.7358584434576260925091256 - 0.03606427612041407369636057i), + (2.3159537454335901187730929 - 1.6249365462333796703711823i), + (3.0795576791204117911123886 + 2.0485650849650740120660391i), + (3.0007392508200622519398814 + 0.29621132089073067282488147i), + (2.4872865024796011364747111 + 1.0664555914934156601503632i), + (2.463655912283054555225301 + 0.48681307452231387690013905i), + (1.8734458851737055262693056 + 0.6116977071277574248407752i), + (2.8793528632328795424123832 - 1.3649311280370181331184214i), + (2.9956543302898767795858704 + 2.6189310485682988308904501i), +} +var asin = []complex128{ + (0.56902834632415098636186476 + 2.9138232718554953784519807i), + (1.5347320506744825455349611 - 2.7358584434576260925091256i), + (-0.054140219438483051139860579 - 2.3159537454335901187730929i), + (-0.47776875817017739283471738 + 3.0795576791204117911123886i), + (1.2745850059041659464064402 + 3.0007392508200622519398814i), + (0.50434073530148095908095852 + 2.4872865024796011364747111i), + (1.0839832522725827423311826 + 2.463655912283054555225301i), + (0.9590986196671391943905465 + 1.8734458851737055262693056i), + (0.20586519875787848611290031 - 2.8793528632328795424123832i), + (-1.0481347217734022116591284 + 2.9956543302898767795858704i), +} +var asinh = []complex128{ + (2.9113760469415295679342185 + 0.99639459545704326759805893i), + (2.7441755423994259061579029 - 0.035468308789000500601119392i), + (-2.2962136462520690506126678 - 1.5144663565690151885726707i), + (-3.0771233459295725965402455 + 1.0895577967194013849422294i), + (3.0048366100923647417557027 + 0.29346979169819220036454168i), + (2.4800059370795363157364643 + 1.0545868606049165710424232i), + (2.4718773838309585611141821 + 0.47502344364250803363708842i), + (1.8910743588080159144378396 + 0.56882925572563602341139174i), + (2.8735426423367341878069406 - 1.362376149648891420997548i), + (-2.9981750586172477217567878 + 0.5183571985225367505624207i), +} +var atan = []complex128{ + (1.5115747079332741358607654 + 0.091324403603954494382276776i), + (1.4424504323482602560806727 - 0.0045416132642803911503770933i), + (-1.5593488703630532674484026 - 0.20163295409248362456446431i), + (-1.5280619472445889867794105 + 0.081721556230672003746956324i), + (1.4759909163240799678221039 + 0.028602969320691644358773586i), + (1.4877353772046548932715555 + 0.14566877153207281663773599i), + (1.4206983927779191889826 + 0.076830486127880702249439993i), + (1.3162236060498933364869556 + 0.16031313000467530644933363i), + (1.5473450684303703578810093 - 0.11064907507939082484935782i), + (-1.4841462340185253987375812 + 0.049341850305024399493142411i), +} +var atanh = []complex128{ + (0.058375027938968509064640438 + 1.4793488495105334458167782i), + (0.12977343497790381229915667 - 1.5661009410463561327262499i), + (-0.010576456067347252072200088 - 1.3743698658402284549750563i), + (-0.042218595678688358882784918 + 1.4891433968166405606692604i), + (0.095218997991316722061828397 + 1.5416884098777110330499698i), + (0.079965459366890323857556487 + 1.4252510353873192700350435i), + (0.15051245471980726221708301 + 1.4907432533016303804884461i), + (0.25082072933993987714470373 + 1.392057665392187516442986i), + (0.022896108815797135846276662 - 1.4609224989282864208963021i), + (-0.08665624101841876130537396 + 1.5207902036935093480142159i), +} +var conj = []complex128{ + (4.9790119248836735e+00 - 7.7388724745781045e+00i), + (7.7388724745781045e+00 + 2.7688005719200159e-01i), + (-2.7688005719200159e-01 + 5.0106036182710749e+00i), + (-5.0106036182710749e+00 - 9.6362937071984173e+00i), + (9.6362937071984173e+00 - 2.9263772392439646e+00i), + (2.9263772392439646e+00 - 5.2290834314593066e+00i), + (5.2290834314593066e+00 - 2.7279399104360102e+00i), + (2.7279399104360102e+00 - 1.8253080916808550e+00i), + (1.8253080916808550e+00 + 8.6859247685756013e+00i), + (-8.6859247685756013e+00 - 4.9790119248836735e+00i), +} +var cos = []complex128{ + (3.024540920601483938336569e+02 + 1.1073797572517071650045357e+03i), + (1.192858682649064973252758e-01 + 2.7857554122333065540970207e-01i), + (7.2144394304528306603857962e+01 - 2.0500129667076044169954205e+01i), + (2.24921952538403984190541e+03 - 7.317363745602773587049329e+03i), + (-9.148222970032421760015498e+00 + 1.953124661113563541862227e+00i), + (-9.116081175857732248227078e+01 - 1.992669213569952232487371e+01i), + (3.795639179042704640002918e+00 + 6.623513350981458399309662e+00i), + (-2.9144840732498869560679084e+00 - 1.214620271628002917638748e+00i), + (-7.45123482501299743872481e+02 + 2.8641692314488080814066734e+03i), + (-5.371977967039319076416747e+01 + 4.893348341339375830564624e+01i), +} +var cosh = []complex128{ + (8.34638383523018249366948e+00 + 7.2181057886425846415112064e+01i), + (1.10421967379919366952251e+03 - 3.1379638689277575379469861e+02i), + (3.051485206773701584738512e-01 - 2.6805384730105297848044485e-01i), + (-7.33294728684187933370938e+01 + 1.574445942284918251038144e+01i), + (-7.478643293945957535757355e+03 + 1.6348382209913353929473321e+03i), + (4.622316522966235701630926e+00 - 8.088695185566375256093098e+00i), + (-8.544333183278877406197712e+01 + 3.7505836120128166455231717e+01i), + (-1.934457815021493925115198e+00 + 7.3725859611767228178358673e+00i), + (-2.352958770061749348353548e+00 - 2.034982010440878358915409e+00i), + (7.79756457532134748165069e+02 + 2.8549350716819176560377717e+03i), +} +var exp = []complex128{ + (1.669197736864670815125146e+01 + 1.4436895109507663689174096e+02i), + (2.2084389286252583447276212e+03 - 6.2759289284909211238261917e+02i), + (2.227538273122775173434327e-01 + 7.2468284028334191250470034e-01i), + (-6.5182985958153548997881627e-03 - 1.39965837915193860879044e-03i), + (-1.4957286524084015746110777e+04 + 3.269676455931135688988042e+03i), + (9.218158701983105935659273e+00 - 1.6223985291084956009304582e+01i), + (-1.7088175716853040841444505e+02 + 7.501382609870410713795546e+01i), + (-3.852461315830959613132505e+00 + 1.4808420423156073221970892e+01i), + (-4.586775503301407379786695e+00 - 4.178501081246873415144744e+00i), + (4.451337963005453491095747e-05 - 1.62977574205442915935263e-04i), +} +var log = []complex128{ + (2.2194438972179194425697051e+00 + 9.9909115046919291062461269e-01i), + (2.0468956191154167256337289e+00 - 3.5762575021856971295156489e-02i), + (1.6130808329853860438751244e+00 - 1.6259990074019058442232221e+00i), + (2.3851910394823008710032651e+00 + 2.0502936359659111755031062e+00i), + (2.3096442270679923004800651e+00 + 2.9483213155446756211881774e-01i), + (1.7904660933974656106951860e+00 + 1.0605860367252556281902109e+00i), + (1.7745926939841751666177512e+00 + 4.8084556083358307819310911e-01i), + (1.1885403350045342425648780e+00 + 5.8969634164776659423195222e-01i), + (2.1833107837679082586772505e+00 - 1.3636647724582455028314573e+00i), + (2.3037629487273259170991671e+00 + 2.6210913895386013290915234e+00i), +} +var log10 = []complex128{ + (9.6389223745559042474184943e-01 + 4.338997735671419492599631e-01i), + (8.8895547241376579493490892e-01 - 1.5531488990643548254864806e-02i), + (7.0055210462945412305244578e-01 - 7.0616239649481243222248404e-01i), + (1.0358753067322445311676952e+00 + 8.9043121238134980156490909e-01i), + (1.003065742975330237172029e+00 + 1.2804396782187887479857811e-01i), + (7.7758954439739162532085157e-01 + 4.6060666333341810869055108e-01i), + (7.7069581462315327037689152e-01 + 2.0882857371769952195512475e-01i), + (5.1617650901191156135137239e-01 + 2.5610186717615977620363299e-01i), + (9.4819982567026639742663212e-01 - 5.9223208584446952284914289e-01i), + (1.0005115362454417135973429e+00 + 1.1383255270407412817250921e+00i), +} + +type ff struct { + r, theta float64 +} + +var polar = []ff{ + {9.2022120669932650313380972e+00, 9.9909115046919291062461269e-01}, + {7.7438239742296106616261394e+00, -3.5762575021856971295156489e-02}, + {5.0182478202557746902556648e+00, -1.6259990074019058442232221e+00}, + {1.0861137372799545160704002e+01, 2.0502936359659111755031062e+00}, + {1.0070841084922199607011905e+01, 2.9483213155446756211881774e-01}, + {5.9922447613166942183705192e+00, 1.0605860367252556281902109e+00}, + {5.8978784056736762299945176e+00, 4.8084556083358307819310911e-01}, + {3.2822866700678709020367184e+00, 5.8969634164776659423195222e-01}, + {8.8756430028990417290744307e+00, -1.3636647724582455028314573e+00}, + {1.0011785496777731986390856e+01, 2.6210913895386013290915234e+00}, +} +var pow = []complex128{ + (-2.499956739197529585028819e+00 + 1.759751724335650228957144e+00i), + (7.357094338218116311191939e+04 - 5.089973412479151648145882e+04i), + (1.320777296067768517259592e+01 - 3.165621914333901498921986e+01i), + (-3.123287828297300934072149e-07 - 1.9849567521490553032502223E-7i), + (8.0622651468477229614813e+04 - 7.80028727944573092944363e+04i), + (-1.0268824572103165858577141e+00 - 4.716844738244989776610672e-01i), + (-4.35953819012244175753187e+01 + 2.2036445974645306917648585e+02i), + (8.3556092283250594950239e-01 - 1.2261571947167240272593282e+01i), + (1.582292972120769306069625e+03 + 1.273564263524278244782512e+04i), + (6.592208301642122149025369e-08 + 2.584887236651661903526389e-08i), +} +var sin = []complex128{ + (-1.1073801774240233539648544e+03 + 3.024539773002502192425231e+02i), + (1.0317037521400759359744682e+00 - 3.2208979799929570242818e-02i), + (-2.0501952097271429804261058e+01 - 7.2137981348240798841800967e+01i), + (7.3173638080346338642193078e+03 + 2.249219506193664342566248e+03i), + (-1.964375633631808177565226e+00 - 9.0958264713870404464159683e+00i), + (1.992783647158514838337674e+01 - 9.11555769410191350416942e+01i), + (-6.680335650741921444300349e+00 + 3.763353833142432513086117e+00i), + (1.2794028166657459148245993e+00 - 2.7669092099795781155109602e+00i), + (2.8641693949535259594188879e+03 + 7.451234399649871202841615e+02i), + (-4.893811726244659135553033e+01 - 5.371469305562194635957655e+01i), +} +var sinh = []complex128{ + (8.34559353341652565758198e+00 + 7.2187893208650790476628899e+01i), + (1.1042192548260646752051112e+03 - 3.1379650595631635858792056e+02i), + (-8.239469336509264113041849e-02 + 9.9273668758439489098514519e-01i), + (7.332295456982297798219401e+01 - 1.574585908122833444899023e+01i), + (-7.4786432301380582103534216e+03 + 1.63483823493980029604071e+03i), + (4.595842179016870234028347e+00 - 8.135290105518580753211484e+00i), + (-8.543842533574163435246793e+01 + 3.750798997857594068272375e+01i), + (-1.918003500809465688017307e+00 + 7.4358344619793504041350251e+00i), + (-2.233816733239658031433147e+00 - 2.143519070805995056229335e+00i), + (-7.797564130187551181105341e+02 - 2.8549352346594918614806877e+03i), +} +var sqrt = []complex128{ + (2.6628203086086130543813948e+00 + 1.4531345674282185229796902e+00i), + (2.7823278427251986247149295e+00 - 4.9756907317005224529115567e-02i), + (1.5397025302089642757361015e+00 - 1.6271336573016637535695727e+00i), + (1.7103411581506875260277898e+00 + 2.8170677122737589676157029e+00i), + (3.1390392472953103383607947e+00 + 4.6612625849858653248980849e-01i), + (2.1117080764822417640789287e+00 + 1.2381170223514273234967850e+00i), + (2.3587032281672256703926939e+00 + 5.7827111903257349935720172e-01i), + (1.7335262588873410476661577e+00 + 5.2647258220721269141550382e-01i), + (2.3131094974708716531499282e+00 - 1.8775429304303785570775490e+00i), + (8.1420535745048086240947359e-01 + 3.0575897587277248522656113e+00i), +} +var tan = []complex128{ + (-1.928757919086441129134525e-07 + 1.0000003267499169073251826e+00i), + (1.242412685364183792138948e+00 - 3.17149693883133370106696e+00i), + (-4.6745126251587795225571826e-05 - 9.9992439225263959286114298e-01i), + (4.792363401193648192887116e-09 + 1.0000000070589333451557723e+00i), + (2.345740824080089140287315e-03 + 9.947733046570988661022763e-01i), + (-2.396030789494815566088809e-05 + 9.9994781345418591429826779e-01i), + (-7.370204836644931340905303e-03 + 1.0043553413417138987717748e+00i), + (-3.691803847992048527007457e-02 + 9.6475071993469548066328894e-01i), + (-2.781955256713729368401878e-08 - 1.000000049848910609006646e+00i), + (9.4281590064030478879791249e-05 + 9.9999119340863718183758545e-01i), +} +var tanh = []complex128{ + (1.0000921981225144748819918e+00 + 2.160986245871518020231507e-05i), + (9.9999967727531993209562591e-01 - 1.9953763222959658873657676e-07i), + (-1.765485739548037260789686e+00 + 1.7024216325552852445168471e+00i), + (-9.999189442732736452807108e-01 + 3.64906070494473701938098e-05i), + (9.9999999224622333738729767e-01 - 3.560088949517914774813046e-09i), + (1.0029324933367326862499343e+00 - 4.948790309797102353137528e-03i), + (9.9996113064788012488693567e-01 - 4.226995742097032481451259e-05i), + (1.0074784189316340029873945e+00 - 4.194050814891697808029407e-03i), + (9.9385534229718327109131502e-01 + 5.144217985914355502713437e-02i), + (-1.0000000491604982429364892e+00 - 2.901873195374433112227349e-08i), +} + +// special cases +var vcAbsSC = []complex128{ + NaN(), +} +var absSC = []float64{ + math.NaN(), +} +var vcAcosSC = []complex128{ + NaN(), +} +var acosSC = []complex128{ + NaN(), +} +var vcAcoshSC = []complex128{ + NaN(), +} +var acoshSC = []complex128{ + NaN(), +} +var vcAsinSC = []complex128{ + NaN(), +} +var asinSC = []complex128{ + NaN(), +} +var vcAsinhSC = []complex128{ + NaN(), +} +var asinhSC = []complex128{ + NaN(), +} +var vcAtanSC = []complex128{ + NaN(), +} +var atanSC = []complex128{ + NaN(), +} +var vcAtanhSC = []complex128{ + NaN(), +} +var atanhSC = []complex128{ + NaN(), +} +var vcConjSC = []complex128{ + NaN(), +} +var conjSC = []complex128{ + NaN(), +} +var vcCosSC = []complex128{ + NaN(), +} +var cosSC = []complex128{ + NaN(), +} +var vcCoshSC = []complex128{ + NaN(), +} +var coshSC = []complex128{ + NaN(), +} +var vcExpSC = []complex128{ + NaN(), +} +var expSC = []complex128{ + NaN(), +} +var vcIsNaNSC = []complex128{ + complex(math.Inf(-1), math.Inf(-1)), + complex(math.Inf(-1), math.NaN()), + complex(math.NaN(), math.Inf(-1)), + complex(0, math.NaN()), + complex(math.NaN(), 0), + complex(math.Inf(1), math.Inf(1)), + complex(math.Inf(1), math.NaN()), + complex(math.NaN(), math.Inf(1)), + complex(math.NaN(), math.NaN()), +} +var isNaNSC = []bool{ + false, + false, + false, + true, + true, + false, + false, + false, + true, +} +var vcLogSC = []complex128{ + NaN(), +} +var logSC = []complex128{ + NaN(), +} +var vcLog10SC = []complex128{ + NaN(), +} +var log10SC = []complex128{ + NaN(), +} +var vcPolarSC = []complex128{ + NaN(), +} +var polarSC = []ff{ + {math.NaN(), math.NaN()}, +} +var vcPowSC = [][2]complex128{ + {NaN(), NaN()}, +} +var powSC = []complex128{ + NaN(), +} +var vcSinSC = []complex128{ + NaN(), +} +var sinSC = []complex128{ + NaN(), +} +var vcSinhSC = []complex128{ + NaN(), +} +var sinhSC = []complex128{ + NaN(), +} +var vcSqrtSC = []complex128{ + NaN(), +} +var sqrtSC = []complex128{ + NaN(), +} +var vcTanSC = []complex128{ + NaN(), +} +var tanSC = []complex128{ + NaN(), +} +var vcTanhSC = []complex128{ + NaN(), +} +var tanhSC = []complex128{ + NaN(), +} + +// functions borrowed from pkg/math/all_test.go +func tolerance(a, b, e float64) bool { + d := a - b + if d < 0 { + d = -d + } + + if a != 0 { + e = e * a + if e < 0 { + e = -e + } + } + return d < e +} +func soclose(a, b, e float64) bool { return tolerance(a, b, e) } +func veryclose(a, b float64) bool { return tolerance(a, b, 4e-16) } +func alike(a, b float64) bool { + switch { + case a != a && b != b: // math.IsNaN(a) && math.IsNaN(b): + return true + case a == b: + return math.Signbit(a) == math.Signbit(b) + } + return false +} + +func cTolerance(a, b complex128, e float64) bool { + d := Abs(a - b) + if a != 0 { + e = e * Abs(a) + if e < 0 { + e = -e + } + } + return d < e +} +func cSoclose(a, b complex128, e float64) bool { return cTolerance(a, b, e) } +func cVeryclose(a, b complex128) bool { return cTolerance(a, b, 4e-16) } +func cAlike(a, b complex128) bool { + switch { + case IsNaN(a) && IsNaN(b): + return true + case a == b: + return math.Signbit(real(a)) == math.Signbit(real(b)) && math.Signbit(imag(a)) == math.Signbit(imag(b)) + } + return false +} + +func TestAbs(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Abs(vc[i]); !veryclose(abs[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vc[i], f, abs[i]) + } + } + for i := 0; i < len(vcAbsSC); i++ { + if f := Abs(vcAbsSC[i]); !alike(absSC[i], f) { + t.Errorf("Abs(%g) = %g, want %g", vcAbsSC[i], f, absSC[i]) + } + } +} +func TestAcos(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Acos(vc[i]); !cSoclose(acos[i], f, 1e-14) { + t.Errorf("Acos(%g) = %g, want %g", vc[i], f, acos[i]) + } + } + for i := 0; i < len(vcAcosSC); i++ { + if f := Acos(vcAcosSC[i]); !cAlike(acosSC[i], f) { + t.Errorf("Acos(%g) = %g, want %g", vcAcosSC[i], f, acosSC[i]) + } + } +} +func TestAcosh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Acosh(vc[i]); !cSoclose(acosh[i], f, 1e-14) { + t.Errorf("Acosh(%g) = %g, want %g", vc[i], f, acosh[i]) + } + } + for i := 0; i < len(vcAcoshSC); i++ { + if f := Acosh(vcAcoshSC[i]); !cAlike(acoshSC[i], f) { + t.Errorf("Acosh(%g) = %g, want %g", vcAcoshSC[i], f, acoshSC[i]) + } + } +} +func TestAsin(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Asin(vc[i]); !cSoclose(asin[i], f, 1e-14) { + t.Errorf("Asin(%g) = %g, want %g", vc[i], f, asin[i]) + } + } + for i := 0; i < len(vcAsinSC); i++ { + if f := Asin(vcAsinSC[i]); !cAlike(asinSC[i], f) { + t.Errorf("Asin(%g) = %g, want %g", vcAsinSC[i], f, asinSC[i]) + } + } +} +func TestAsinh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Asinh(vc[i]); !cSoclose(asinh[i], f, 4e-15) { + t.Errorf("Asinh(%g) = %g, want %g", vc[i], f, asinh[i]) + } + } + for i := 0; i < len(vcAsinhSC); i++ { + if f := Asinh(vcAsinhSC[i]); !cAlike(asinhSC[i], f) { + t.Errorf("Asinh(%g) = %g, want %g", vcAsinhSC[i], f, asinhSC[i]) + } + } +} +func TestAtan(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Atan(vc[i]); !cVeryclose(atan[i], f) { + t.Errorf("Atan(%g) = %g, want %g", vc[i], f, atan[i]) + } + } + for i := 0; i < len(vcAtanSC); i++ { + if f := Atan(vcAtanSC[i]); !cAlike(atanSC[i], f) { + t.Errorf("Atan(%g) = %g, want %g", vcAtanSC[i], f, atanSC[i]) + } + } +} +func TestAtanh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Atanh(vc[i]); !cVeryclose(atanh[i], f) { + t.Errorf("Atanh(%g) = %g, want %g", vc[i], f, atanh[i]) + } + } + for i := 0; i < len(vcAtanhSC); i++ { + if f := Atanh(vcAtanhSC[i]); !cAlike(atanhSC[i], f) { + t.Errorf("Atanh(%g) = %g, want %g", vcAtanhSC[i], f, atanhSC[i]) + } + } +} +func TestConj(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Conj(vc[i]); !cVeryclose(conj[i], f) { + t.Errorf("Conj(%g) = %g, want %g", vc[i], f, conj[i]) + } + } + for i := 0; i < len(vcConjSC); i++ { + if f := Conj(vcConjSC[i]); !cAlike(conjSC[i], f) { + t.Errorf("Conj(%g) = %g, want %g", vcConjSC[i], f, conjSC[i]) + } + } +} +func TestCos(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Cos(vc[i]); !cSoclose(cos[i], f, 3e-15) { + t.Errorf("Cos(%g) = %g, want %g", vc[i], f, cos[i]) + } + } + for i := 0; i < len(vcCosSC); i++ { + if f := Cos(vcCosSC[i]); !cAlike(cosSC[i], f) { + t.Errorf("Cos(%g) = %g, want %g", vcCosSC[i], f, cosSC[i]) + } + } +} +func TestCosh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Cosh(vc[i]); !cSoclose(cosh[i], f, 2e-15) { + t.Errorf("Cosh(%g) = %g, want %g", vc[i], f, cosh[i]) + } + } + for i := 0; i < len(vcCoshSC); i++ { + if f := Cosh(vcCoshSC[i]); !cAlike(coshSC[i], f) { + t.Errorf("Cosh(%g) = %g, want %g", vcCoshSC[i], f, coshSC[i]) + } + } +} +func TestExp(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Exp(vc[i]); !cSoclose(exp[i], f, 1e-15) { + t.Errorf("Exp(%g) = %g, want %g", vc[i], f, exp[i]) + } + } + for i := 0; i < len(vcExpSC); i++ { + if f := Exp(vcExpSC[i]); !cAlike(expSC[i], f) { + t.Errorf("Exp(%g) = %g, want %g", vcExpSC[i], f, expSC[i]) + } + } +} +func TestIsNaN(t *testing.T) { + for i := 0; i < len(vcIsNaNSC); i++ { + if f := IsNaN(vcIsNaNSC[i]); isNaNSC[i] != f { + t.Errorf("IsNaN(%v) = %v, want %v", vcIsNaNSC[i], f, isNaNSC[i]) + } + } +} +func TestLog(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Log(vc[i]); !cVeryclose(log[i], f) { + t.Errorf("Log(%g) = %g, want %g", vc[i], f, log[i]) + } + } + for i := 0; i < len(vcLogSC); i++ { + if f := Log(vcLogSC[i]); !cAlike(logSC[i], f) { + t.Errorf("Log(%g) = %g, want %g", vcLogSC[i], f, logSC[i]) + } + } +} +func TestLog10(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Log10(vc[i]); !cVeryclose(log10[i], f) { + t.Errorf("Log10(%g) = %g, want %g", vc[i], f, log10[i]) + } + } + for i := 0; i < len(vcLog10SC); i++ { + if f := Log10(vcLog10SC[i]); !cAlike(log10SC[i], f) { + t.Errorf("Log10(%g) = %g, want %g", vcLog10SC[i], f, log10SC[i]) + } + } +} +func TestPolar(t *testing.T) { + for i := 0; i < len(vc); i++ { + if r, theta := Polar(vc[i]); !veryclose(polar[i].r, r) && !veryclose(polar[i].theta, theta) { + t.Errorf("Polar(%g) = %g, %g want %g, %g", vc[i], r, theta, polar[i].r, polar[i].theta) + } + } + for i := 0; i < len(vcPolarSC); i++ { + if r, theta := Polar(vcPolarSC[i]); !alike(polarSC[i].r, r) && !alike(polarSC[i].theta, theta) { + t.Errorf("Polar(%g) = %g, %g, want %g, %g", vcPolarSC[i], r, theta, polarSC[i].r, polarSC[i].theta) + } + } +} +func TestPow(t *testing.T) { + var a = complex(3.0, 3.0) + for i := 0; i < len(vc); i++ { + if f := Pow(a, vc[i]); !cSoclose(pow[i], f, 4e-15) { + t.Errorf("Pow(%g, %g) = %g, want %g", a, vc[i], f, pow[i]) + } + } + for i := 0; i < len(vcPowSC); i++ { + if f := Pow(vcPowSC[i][0], vcPowSC[i][0]); !cAlike(powSC[i], f) { + t.Errorf("Pow(%g, %g) = %g, want %g", vcPowSC[i][0], vcPowSC[i][0], f, powSC[i]) + } + } +} +func TestRect(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Rect(polar[i].r, polar[i].theta); !cVeryclose(vc[i], f) { + t.Errorf("Rect(%g, %g) = %g want %g", polar[i].r, polar[i].theta, f, vc[i]) + } + } + for i := 0; i < len(vcPolarSC); i++ { + if f := Rect(polarSC[i].r, polarSC[i].theta); !cAlike(vcPolarSC[i], f) { + t.Errorf("Rect(%g, %g) = %g, want %g", polarSC[i].r, polarSC[i].theta, f, vcPolarSC[i]) + } + } +} +func TestSin(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sin(vc[i]); !cSoclose(sin[i], f, 2e-15) { + t.Errorf("Sin(%g) = %g, want %g", vc[i], f, sin[i]) + } + } + for i := 0; i < len(vcSinSC); i++ { + if f := Sin(vcSinSC[i]); !cAlike(sinSC[i], f) { + t.Errorf("Sin(%g) = %g, want %g", vcSinSC[i], f, sinSC[i]) + } + } +} +func TestSinh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sinh(vc[i]); !cSoclose(sinh[i], f, 2e-15) { + t.Errorf("Sinh(%g) = %g, want %g", vc[i], f, sinh[i]) + } + } + for i := 0; i < len(vcSinhSC); i++ { + if f := Sinh(vcSinhSC[i]); !cAlike(sinhSC[i], f) { + t.Errorf("Sinh(%g) = %g, want %g", vcSinhSC[i], f, sinhSC[i]) + } + } +} +func TestSqrt(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Sqrt(vc[i]); !cVeryclose(sqrt[i], f) { + t.Errorf("Sqrt(%g) = %g, want %g", vc[i], f, sqrt[i]) + } + } + for i := 0; i < len(vcSqrtSC); i++ { + if f := Sqrt(vcSqrtSC[i]); !cAlike(sqrtSC[i], f) { + t.Errorf("Sqrt(%g) = %g, want %g", vcSqrtSC[i], f, sqrtSC[i]) + } + } +} +func TestTan(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Tan(vc[i]); !cSoclose(tan[i], f, 3e-15) { + t.Errorf("Tan(%g) = %g, want %g", vc[i], f, tan[i]) + } + } + for i := 0; i < len(vcTanSC); i++ { + if f := Tan(vcTanSC[i]); !cAlike(tanSC[i], f) { + t.Errorf("Tan(%g) = %g, want %g", vcTanSC[i], f, tanSC[i]) + } + } +} +func TestTanh(t *testing.T) { + for i := 0; i < len(vc); i++ { + if f := Tanh(vc[i]); !cSoclose(tanh[i], f, 2e-15) { + t.Errorf("Tanh(%g) = %g, want %g", vc[i], f, tanh[i]) + } + } + for i := 0; i < len(vcTanhSC); i++ { + if f := Tanh(vcTanhSC[i]); !cAlike(tanhSC[i], f) { + t.Errorf("Tanh(%g) = %g, want %g", vcTanhSC[i], f, tanhSC[i]) + } + } +} + +func BenchmarkAbs(b *testing.B) { + for i := 0; i < b.N; i++ { + Abs(complex(2.5, 3.5)) + } +} +func BenchmarkAcos(b *testing.B) { + for i := 0; i < b.N; i++ { + Acos(complex(2.5, 3.5)) + } +} +func BenchmarkAcosh(b *testing.B) { + for i := 0; i < b.N; i++ { + Acosh(complex(2.5, 3.5)) + } +} +func BenchmarkAsin(b *testing.B) { + for i := 0; i < b.N; i++ { + Asin(complex(2.5, 3.5)) + } +} +func BenchmarkAsinh(b *testing.B) { + for i := 0; i < b.N; i++ { + Asinh(complex(2.5, 3.5)) + } +} +func BenchmarkAtan(b *testing.B) { + for i := 0; i < b.N; i++ { + Atan(complex(2.5, 3.5)) + } +} +func BenchmarkAtanh(b *testing.B) { + for i := 0; i < b.N; i++ { + Atanh(complex(2.5, 3.5)) + } +} +func BenchmarkConj(b *testing.B) { + for i := 0; i < b.N; i++ { + Conj(complex(2.5, 3.5)) + } +} +func BenchmarkCos(b *testing.B) { + for i := 0; i < b.N; i++ { + Cos(complex(2.5, 3.5)) + } +} +func BenchmarkCosh(b *testing.B) { + for i := 0; i < b.N; i++ { + Cosh(complex(2.5, 3.5)) + } +} +func BenchmarkExp(b *testing.B) { + for i := 0; i < b.N; i++ { + Exp(complex(2.5, 3.5)) + } +} +func BenchmarkLog(b *testing.B) { + for i := 0; i < b.N; i++ { + Log(complex(2.5, 3.5)) + } +} +func BenchmarkLog10(b *testing.B) { + for i := 0; i < b.N; i++ { + Log10(complex(2.5, 3.5)) + } +} +func BenchmarkPhase(b *testing.B) { + for i := 0; i < b.N; i++ { + Phase(complex(2.5, 3.5)) + } +} +func BenchmarkPolar(b *testing.B) { + for i := 0; i < b.N; i++ { + Polar(complex(2.5, 3.5)) + } +} +func BenchmarkPow(b *testing.B) { + for i := 0; i < b.N; i++ { + Pow(complex(2.5, 3.5), complex(2.5, 3.5)) + } +} +func BenchmarkRect(b *testing.B) { + for i := 0; i < b.N; i++ { + Rect(2.5, 1.5) + } +} +func BenchmarkSin(b *testing.B) { + for i := 0; i < b.N; i++ { + Sin(complex(2.5, 3.5)) + } +} +func BenchmarkSinh(b *testing.B) { + for i := 0; i < b.N; i++ { + Sinh(complex(2.5, 3.5)) + } +} +func BenchmarkSqrt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sqrt(complex(2.5, 3.5)) + } +} +func BenchmarkTan(b *testing.B) { + for i := 0; i < b.N; i++ { + Tan(complex(2.5, 3.5)) + } +} +func BenchmarkTanh(b *testing.B) { + for i := 0; i < b.N; i++ { + Tanh(complex(2.5, 3.5)) + } +} diff --git a/src/pkg/cmath/conj.go b/src/pkg/cmath/conj.go new file mode 100644 index 000000000..776b57da7 --- /dev/null +++ b/src/pkg/cmath/conj.go @@ -0,0 +1,8 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +// Conj returns the complex conjugate of x. +func Conj(x complex128) complex128 { return complex(real(x), -imag(x)) } diff --git a/src/pkg/cmath/exp.go b/src/pkg/cmath/exp.go new file mode 100644 index 000000000..64c1ef409 --- /dev/null +++ b/src/pkg/cmath/exp.go @@ -0,0 +1,55 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex exponential function +// +// DESCRIPTION: +// +// Returns the complex exponential of the complex argument z. +// +// If +// z = x + iy, +// r = exp(x), +// then +// w = r cos y + i r sin y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8700 3.7e-17 1.1e-17 +// IEEE -10,+10 30000 3.0e-16 8.7e-17 + +// Exp returns e**x, the base-e exponential of x. +func Exp(x complex128) complex128 { + r := math.Exp(real(x)) + s, c := math.Sincos(imag(x)) + return complex(r*c, r*s) +} diff --git a/src/pkg/cmath/isinf.go b/src/pkg/cmath/isinf.go new file mode 100644 index 000000000..f23d2dea7 --- /dev/null +++ b/src/pkg/cmath/isinf.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// IsInf returns true if either real(x) or imag(x) is an infinity. +func IsInf(x complex128) bool { + if math.IsInf(real(x), 0) || math.IsInf(imag(x), 0) { + return true + } + return false +} + +// Inf returns a complex infinity, complex(+Inf, +Inf). +func Inf() complex128 { + inf := math.Inf(1) + return complex(inf, inf) +} diff --git a/src/pkg/cmath/isnan.go b/src/pkg/cmath/isnan.go new file mode 100644 index 000000000..2063bb835 --- /dev/null +++ b/src/pkg/cmath/isnan.go @@ -0,0 +1,25 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// IsNaN returns true if either real(x) or imag(x) is NaN +// and neither is an infinity. +func IsNaN(x complex128) bool { + switch { + case math.IsInf(real(x), 0) || math.IsInf(imag(x), 0): + return false + case math.IsNaN(real(x)) || math.IsNaN(imag(x)): + return true + } + return false +} + +// NaN returns a complex ``not-a-number'' value. +func NaN() complex128 { + nan := math.NaN() + return complex(nan, nan) +} diff --git a/src/pkg/cmath/log.go b/src/pkg/cmath/log.go new file mode 100644 index 000000000..8e6964fee --- /dev/null +++ b/src/pkg/cmath/log.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex natural logarithm +// +// DESCRIPTION: +// +// Returns complex logarithm to the base e (2.718...) of +// the complex argument z. +// +// If +// z = x + iy, r = sqrt( x**2 + y**2 ), +// then +// w = log(r) + i arctan(y/x). +// +// The arctangent ranges from -PI to +PI. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 7000 8.5e-17 1.9e-17 +// IEEE -10,+10 30000 5.0e-15 1.1e-16 +// +// Larger relative error can be observed for z near 1 +i0. +// In IEEE arithmetic the peak absolute error is 5.2e-16, rms +// absolute error 1.0e-16. + +// Log returns the natural logarithm of x. +func Log(x complex128) complex128 { + return complex(math.Log(Abs(x)), Phase(x)) +} + +// Log10 returns the decimal logarithm of x. +func Log10(x complex128) complex128 { + return math.Log10E * Log(x) +} diff --git a/src/pkg/cmath/phase.go b/src/pkg/cmath/phase.go new file mode 100644 index 000000000..2d67aa34c --- /dev/null +++ b/src/pkg/cmath/phase.go @@ -0,0 +1,11 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// Phase returns the phase (also called the argument) of x. +// The returned value is in the range [-Pi, Pi]. +func Phase(x complex128) float64 { return math.Atan2(imag(x), real(x)) } diff --git a/src/pkg/cmath/polar.go b/src/pkg/cmath/polar.go new file mode 100644 index 000000000..033676acc --- /dev/null +++ b/src/pkg/cmath/polar.go @@ -0,0 +1,12 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +// Polar returns the absolute value r and phase θ of x, +// such that x = r * e**θi. +// The phase is in the range [-Pi, Pi]. +func Polar(x complex128) (r, θ float64) { + return Abs(x), Phase(x) +} diff --git a/src/pkg/cmath/pow.go b/src/pkg/cmath/pow.go new file mode 100644 index 000000000..68e1207c6 --- /dev/null +++ b/src/pkg/cmath/pow.go @@ -0,0 +1,60 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex power function +// +// DESCRIPTION: +// +// Raises complex A to the complex Zth power. +// Definition is per AMS55 # 4.2.8, +// analytically equivalent to cpow(a,z) = cexp(z clog(a)). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 9.4e-15 1.5e-15 + +// Pow returns x**y, the base-x exponential of y. +func Pow(x, y complex128) complex128 { + modulus := Abs(x) + if modulus == 0 { + return complex(0, 0) + } + r := math.Pow(modulus, real(y)) + arg := Phase(x) + theta := real(y) * arg + if imag(y) != 0 { + r *= math.Exp(-imag(y) * arg) + theta += imag(y) * math.Log(modulus) + } + s, c := math.Sincos(theta) + return complex(r*c, r*s) +} diff --git a/src/pkg/cmath/rect.go b/src/pkg/cmath/rect.go new file mode 100644 index 000000000..b955f0bf7 --- /dev/null +++ b/src/pkg/cmath/rect.go @@ -0,0 +1,13 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// Rect returns the complex number x with polar coordinates r, θ. +func Rect(r, θ float64) complex128 { + s, c := math.Sincos(θ) + return complex(r*c, r*s) +} diff --git a/src/pkg/cmath/sin.go b/src/pkg/cmath/sin.go new file mode 100644 index 000000000..8900ecdde --- /dev/null +++ b/src/pkg/cmath/sin.go @@ -0,0 +1,132 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular sine +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// w = sin x cosh y + i cos x sinh y. +// +// csin(z) = -i csinh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8400 5.3e-17 1.3e-17 +// IEEE -10,+10 30000 3.8e-16 1.0e-16 +// Also tested by csin(casin(z)) = z. + +// Sin returns the sine of x. +func Sin(x complex128) complex128 { + s, c := math.Sincos(real(x)) + sh, ch := sinhcosh(imag(x)) + return complex(s*ch, c*sh) +} + +// Complex hyperbolic sine +// +// DESCRIPTION: +// +// csinh z = (cexp(z) - cexp(-z))/2 +// = sinh x * cos y + i cosh x * sin y . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 3.1e-16 8.2e-17 + +// Sinh returns the hyperbolic sine of x. +func Sinh(x complex128) complex128 { + s, c := math.Sincos(imag(x)) + sh, ch := sinhcosh(real(x)) + return complex(c*sh, s*ch) +} + +// Complex circular cosine +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// w = cos x cosh y - i sin x sinh y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 8400 4.5e-17 1.3e-17 +// IEEE -10,+10 30000 3.8e-16 1.0e-16 + +// Cos returns the cosine of x. +func Cos(x complex128) complex128 { + s, c := math.Sincos(real(x)) + sh, ch := sinhcosh(imag(x)) + return complex(c*ch, -s*sh) +} + +// Complex hyperbolic cosine +// +// DESCRIPTION: +// +// ccosh(z) = cosh x cos y + i sinh x sin y . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 2.9e-16 8.1e-17 + +// Cosh returns the hyperbolic cosine of x. +func Cosh(x complex128) complex128 { + s, c := math.Sincos(imag(x)) + sh, ch := sinhcosh(real(x)) + return complex(c*ch, s*sh) +} + +// calculate sinh and cosh +func sinhcosh(x float64) (sh, ch float64) { + if math.Fabs(x) <= 0.5 { + return math.Sinh(x), math.Cosh(x) + } + e := math.Exp(x) + ei := 0.5 / e + e *= 0.5 + return e - ei, e + ei +} diff --git a/src/pkg/cmath/sqrt.go b/src/pkg/cmath/sqrt.go new file mode 100644 index 000000000..e77a9b9df --- /dev/null +++ b/src/pkg/cmath/sqrt.go @@ -0,0 +1,103 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex square root +// +// DESCRIPTION: +// +// If z = x + iy, r = |z|, then +// +// 1/2 +// Re w = [ (r + x)/2 ] , +// +// 1/2 +// Im w = [ (r - x)/2 ] . +// +// Cancellation error in r-x or r+x is avoided by using the +// identity 2 Re w Im w = y. +// +// Note that -w is also a square root of z. The root chosen +// is always in the right half plane and Im w has the same sign as y. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 25000 3.2e-17 9.6e-18 +// IEEE -10,+10 1,000,000 2.9e-16 6.1e-17 + +// Sqrt returns the square root of x. +func Sqrt(x complex128) complex128 { + if imag(x) == 0 { + if real(x) == 0 { + return complex(0, 0) + } + if real(x) < 0 { + return complex(0, math.Sqrt(-real(x))) + } + return complex(math.Sqrt(real(x)), 0) + } + if real(x) == 0 { + if imag(x) < 0 { + r := math.Sqrt(-0.5 * imag(x)) + return complex(r, -r) + } + r := math.Sqrt(0.5 * imag(x)) + return complex(r, r) + } + a := real(x) + b := imag(x) + var scale float64 + // Rescale to avoid internal overflow or underflow. + if math.Fabs(a) > 4 || math.Fabs(b) > 4 { + a *= 0.25 + b *= 0.25 + scale = 2 + } else { + a *= 1.8014398509481984e16 // 2**54 + b *= 1.8014398509481984e16 + scale = 7.450580596923828125e-9 // 2**-27 + } + r := math.Hypot(a, b) + var t float64 + if a > 0 { + t = math.Sqrt(0.5*r + 0.5*a) + r = scale * math.Fabs((0.5*b)/t) + t *= scale + } else { + r = math.Sqrt(0.5*r - 0.5*a) + t = scale * math.Fabs((0.5*b)/r) + r *= scale + } + if b < 0 { + return complex(t, -r) + } + return complex(t, r) +} diff --git a/src/pkg/cmath/tan.go b/src/pkg/cmath/tan.go new file mode 100644 index 000000000..94b517521 --- /dev/null +++ b/src/pkg/cmath/tan.go @@ -0,0 +1,184 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmath + +import "math" + +// The original C code, the long comment, and the constants +// below are from http://netlib.sandia.gov/cephes/c9x-complex/clog.c. +// The go code is a simplified version of the original C. +// +// Cephes Math Library Release 2.8: June, 2000 +// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier +// +// The readme file at http://netlib.sandia.gov/cephes/ says: +// Some software in this archive may be from the book _Methods and +// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster +// International, 1989) or from the Cephes Mathematical Library, a +// commercial product. In either event, it is copyrighted by the author. +// What you see here may be used freely but it comes with no support or +// guarantee. +// +// The two known misprints in the book are repaired here in the +// source listings for the gamma function and the incomplete beta +// integral. +// +// Stephen L. Moshier +// moshier@na-net.ornl.gov + +// Complex circular tangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// sin 2x + i sinh 2y +// w = --------------------. +// cos 2x + cosh 2y +// +// On the real axis the denominator is zero at odd multiples +// of PI/2. The denominator is evaluated by its Taylor +// series near these points. +// +// ctan(z) = -i ctanh(iz). +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 5200 7.1e-17 1.6e-17 +// IEEE -10,+10 30000 7.2e-16 1.2e-16 +// Also tested by ctan * ccot = 1 and catan(ctan(z)) = z. + +// Tan returns the tangent of x. +func Tan(x complex128) complex128 { + d := math.Cos(2*real(x)) + math.Cosh(2*imag(x)) + if math.Fabs(d) < 0.25 { + d = tanSeries(x) + } + if d == 0 { + return Inf() + } + return complex(math.Sin(2*real(x))/d, math.Sinh(2*imag(x))/d) +} + +// Complex hyperbolic tangent +// +// DESCRIPTION: +// +// tanh z = (sinh 2x + i sin 2y) / (cosh 2x + cos 2y) . +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// IEEE -10,+10 30000 1.7e-14 2.4e-16 + +// Tanh returns the hyperbolic tangent of x. +func Tanh(x complex128) complex128 { + d := math.Cosh(2*real(x)) + math.Cos(2*imag(x)) + if d == 0 { + return Inf() + } + return complex(math.Sinh(2*real(x))/d, math.Sin(2*imag(x))/d) +} + +// Program to subtract nearest integer multiple of PI +func reducePi(x float64) float64 { + const ( + // extended precision value of PI: + DP1 = 3.14159265160560607910E0 // ?? 0x400921fb54000000 + DP2 = 1.98418714791870343106E-9 // ?? 0x3e210b4610000000 + DP3 = 1.14423774522196636802E-17 // ?? 0x3c6a62633145c06e + ) + t := x / math.Pi + if t >= 0 { + t += 0.5 + } else { + t -= 0.5 + } + t = float64(int64(t)) // int64(t) = the multiple + return ((x - t*DP1) - t*DP2) - t*DP3 +} + +// Taylor series expansion for cosh(2y) - cos(2x) +func tanSeries(z complex128) float64 { + const MACHEP = 1.0 / (1 << 53) + x := math.Fabs(2 * real(z)) + y := math.Fabs(2 * imag(z)) + x = reducePi(x) + x = x * x + y = y * y + x2 := 1.0 + y2 := 1.0 + f := 1.0 + rn := 0.0 + d := 0.0 + for { + rn += 1 + f *= rn + rn += 1 + f *= rn + x2 *= x + y2 *= y + t := y2 + x2 + t /= f + d += t + + rn += 1 + f *= rn + rn += 1 + f *= rn + x2 *= x + y2 *= y + t = y2 - x2 + t /= f + d += t + if math.Fabs(t/d) <= MACHEP { + break + } + } + return d +} + +// Complex circular cotangent +// +// DESCRIPTION: +// +// If +// z = x + iy, +// +// then +// +// sin 2x - i sinh 2y +// w = --------------------. +// cosh 2y - cos 2x +// +// On the real axis, the denominator has zeros at even +// multiples of PI/2. Near these points it is evaluated +// by a Taylor series. +// +// ACCURACY: +// +// Relative error: +// arithmetic domain # trials peak rms +// DEC -10,+10 3000 6.5e-17 1.6e-17 +// IEEE -10,+10 30000 9.2e-16 1.2e-16 +// Also tested by ctan * ccot = 1 + i0. + +// Cot returns the cotangent of x. +func Cot(x complex128) complex128 { + d := math.Cosh(2*imag(x)) - math.Cos(2*real(x)) + if math.Fabs(d) < 0.25 { + d = tanSeries(x) + } + if d == 0 { + return Inf() + } + return complex(math.Sin(2*real(x))/d, -math.Sinh(2*imag(x))/d) +} diff --git a/src/pkg/compress/bzip2/Makefile b/src/pkg/compress/bzip2/Makefile new file mode 100644 index 000000000..a4bceef16 --- /dev/null +++ b/src/pkg/compress/bzip2/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/bzip2 +GOFILES=\ + bit_reader.go\ + bzip2.go\ + huffman.go\ + move_to_front.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/bzip2/bit_reader.go b/src/pkg/compress/bzip2/bit_reader.go new file mode 100644 index 000000000..50f0ec836 --- /dev/null +++ b/src/pkg/compress/bzip2/bit_reader.go @@ -0,0 +1,88 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bufio" + "io" + "os" +) + +// bitReader wraps an io.Reader and provides the ability to read values, +// bit-by-bit, from it. Its Read* methods don't return the usual os.Error +// because the error handling was verbose. Instead, any error is kept and can +// be checked afterwards. +type bitReader struct { + r byteReader + n uint64 + bits uint + err os.Error +} + +// bitReader needs to read bytes from an io.Reader. We attempt to cast the +// given io.Reader to this interface and, if it doesn't already fit, we wrap in +// a bufio.Reader. +type byteReader interface { + ReadByte() (byte, os.Error) +} + +func newBitReader(r io.Reader) bitReader { + byter, ok := r.(byteReader) + if !ok { + byter = bufio.NewReader(r) + } + return bitReader{r: byter} +} + +// ReadBits64 reads the given number of bits and returns them in the +// least-significant part of a uint64. In the event of an error, it returns 0 +// and the error can be obtained by calling Error(). +func (br *bitReader) ReadBits64(bits uint) (n uint64) { + for bits > br.bits { + b, err := br.r.ReadByte() + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + br.err = err + return 0 + } + br.n <<= 8 + br.n |= uint64(b) + br.bits += 8 + } + + // br.n looks like this (assuming that br.bits = 14 and bits = 6): + // Bit: 111111 + // 5432109876543210 + // + // (6 bits, the desired output) + // |-----| + // V V + // 0101101101001110 + // ^ ^ + // |------------| + // br.bits (num valid bits) + // + // This the next line right shifts the desired bits into the + // least-significant places and masks off anything above. + n = (br.n >> (br.bits - bits)) & ((1 << bits) - 1) + br.bits -= bits + return +} + +func (br *bitReader) ReadBits(bits uint) (n int) { + n64 := br.ReadBits64(bits) + return int(n64) +} + +func (br *bitReader) ReadBit() bool { + n := br.ReadBits(1) + return n != 0 +} + +func (br *bitReader) Error() os.Error { + return br.err +} diff --git a/src/pkg/compress/bzip2/bzip2.go b/src/pkg/compress/bzip2/bzip2.go new file mode 100644 index 000000000..8b4572306 --- /dev/null +++ b/src/pkg/compress/bzip2/bzip2.go @@ -0,0 +1,390 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bzip2 implements bzip2 decompression. +package bzip2 + +import ( + "io" + "os" +) + +// There's no RFC for bzip2. I used the Wikipedia page for reference and a lot +// of guessing: http://en.wikipedia.org/wiki/Bzip2 +// The source code to pyflate was useful for debugging: +// http://www.paul.sladen.org/projects/pyflate + +// A StructuralError is returned when the bzip2 data is found to be +// syntactically invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "bzip2 data invalid: " + string(s) +} + +// A reader decompresses bzip2 compressed data. +type reader struct { + br bitReader + setupDone bool // true if we have parsed the bzip2 header. + blockSize int // blockSize in bytes, i.e. 900 * 1024. + eof bool + buf []byte // stores Burrows-Wheeler transformed data. + c [256]uint // the `C' array for the inverse BWT. + tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits. + tPos uint32 // Index of the next output byte in tt. + + preRLE []uint32 // contains the RLE data still to be processed. + preRLEUsed int // number of entries of preRLE used. + lastByte int // the last byte value seen. + byteRepeats uint // the number of repeats of lastByte seen. + repeats uint // the number of copies of lastByte to output. +} + +// NewReader returns an io.Reader which decompresses bzip2 data from r. +func NewReader(r io.Reader) io.Reader { + bz2 := new(reader) + bz2.br = newBitReader(r) + return bz2 +} + +const bzip2FileMagic = 0x425a // "BZ" +const bzip2BlockMagic = 0x314159265359 +const bzip2FinalMagic = 0x177245385090 + +// setup parses the bzip2 header. +func (bz2 *reader) setup() os.Error { + br := &bz2.br + + magic := br.ReadBits(16) + if magic != bzip2FileMagic { + return StructuralError("bad magic value") + } + + t := br.ReadBits(8) + if t != 'h' { + return StructuralError("non-Huffman entropy encoding") + } + + level := br.ReadBits(8) + if level < '1' || level > '9' { + return StructuralError("invalid compression level") + } + + bz2.blockSize = 100 * 1024 * (int(level) - '0') + bz2.tt = make([]uint32, bz2.blockSize) + return nil +} + +func (bz2 *reader) Read(buf []byte) (n int, err os.Error) { + if bz2.eof { + return 0, os.EOF + } + + if !bz2.setupDone { + err = bz2.setup() + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + if err != nil { + return 0, err + } + bz2.setupDone = true + } + + n, err = bz2.read(buf) + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + return +} + +func (bz2 *reader) read(buf []byte) (n int, err os.Error) { + // bzip2 is a block based compressor, except that it has a run-length + // preprocessing step. The block based nature means that we can + // preallocate fixed-size buffers and reuse them. However, the RLE + // preprocessing would require allocating huge buffers to store the + // maximum expansion. Thus we process blocks all at once, except for + // the RLE which we decompress as required. + + for (bz2.repeats > 0 || bz2.preRLEUsed < len(bz2.preRLE)) && n < len(buf) { + // We have RLE data pending. + + // The run-length encoding works like this: + // Any sequence of four equal bytes is followed by a length + // byte which contains the number of repeats of that byte to + // include. (The number of repeats can be zero.) Because we are + // decompressing on-demand our state is kept in the reader + // object. + + if bz2.repeats > 0 { + buf[n] = byte(bz2.lastByte) + n++ + bz2.repeats-- + if bz2.repeats == 0 { + bz2.lastByte = -1 + } + continue + } + + bz2.tPos = bz2.preRLE[bz2.tPos] + b := byte(bz2.tPos) + bz2.tPos >>= 8 + bz2.preRLEUsed++ + + if bz2.byteRepeats == 3 { + bz2.repeats = uint(b) + bz2.byteRepeats = 0 + continue + } + + if bz2.lastByte == int(b) { + bz2.byteRepeats++ + } else { + bz2.byteRepeats = 0 + } + bz2.lastByte = int(b) + + buf[n] = b + n++ + } + + if n > 0 { + return + } + + // No RLE data is pending so we need to read a block. + + br := &bz2.br + magic := br.ReadBits64(48) + if magic == bzip2FinalMagic { + br.ReadBits64(32) // ignored CRC + bz2.eof = true + return 0, os.EOF + } else if magic != bzip2BlockMagic { + return 0, StructuralError("bad magic value found") + } + + err = bz2.readBlock() + if err != nil { + return 0, err + } + + return bz2.read(buf) +} + +// readBlock reads a bzip2 block. The magic number should already have been consumed. +func (bz2 *reader) readBlock() (err os.Error) { + br := &bz2.br + br.ReadBits64(32) // skip checksum. TODO: check it if we can figure out what it is. + randomized := br.ReadBits(1) + if randomized != 0 { + return StructuralError("deprecated randomized files") + } + origPtr := uint(br.ReadBits(24)) + + // If not every byte value is used in the block (i.e., it's text) then + // the symbol set is reduced. The symbols used are stored as a + // two-level, 16x16 bitmap. + symbolRangeUsedBitmap := br.ReadBits(16) + symbolPresent := make([]bool, 256) + numSymbols := 0 + for symRange := uint(0); symRange < 16; symRange++ { + if symbolRangeUsedBitmap&(1<<(15-symRange)) != 0 { + bits := br.ReadBits(16) + for symbol := uint(0); symbol < 16; symbol++ { + if bits&(1<<(15-symbol)) != 0 { + symbolPresent[16*symRange+symbol] = true + numSymbols++ + } + } + } + } + + // A block uses between two and six different Huffman trees. + numHuffmanTrees := br.ReadBits(3) + if numHuffmanTrees < 2 || numHuffmanTrees > 6 { + return StructuralError("invalid number of Huffman trees") + } + + // The Huffman tree can switch every 50 symbols so there's a list of + // tree indexes telling us which tree to use for each 50 symbol block. + numSelectors := br.ReadBits(15) + treeIndexes := make([]uint8, numSelectors) + + // The tree indexes are move-to-front transformed and stored as unary + // numbers. + mtfTreeDecoder := newMTFDecoderWithRange(numHuffmanTrees) + for i := range treeIndexes { + c := 0 + for { + inc := br.ReadBits(1) + if inc == 0 { + break + } + c++ + } + if c >= numHuffmanTrees { + return StructuralError("tree index too large") + } + treeIndexes[i] = uint8(mtfTreeDecoder.Decode(c)) + } + + // The list of symbols for the move-to-front transform is taken from + // the previously decoded symbol bitmap. + symbols := make([]byte, numSymbols) + nextSymbol := 0 + for i := 0; i < 256; i++ { + if symbolPresent[i] { + symbols[nextSymbol] = byte(i) + nextSymbol++ + } + } + mtf := newMTFDecoder(symbols) + + numSymbols += 2 // to account for RUNA and RUNB symbols + huffmanTrees := make([]huffmanTree, numHuffmanTrees) + + // Now we decode the arrays of code-lengths for each tree. + lengths := make([]uint8, numSymbols) + for i := 0; i < numHuffmanTrees; i++ { + // The code lengths are delta encoded from a 5-bit base value. + length := br.ReadBits(5) + for j := 0; j < numSymbols; j++ { + for { + if !br.ReadBit() { + break + } + if br.ReadBit() { + length-- + } else { + length++ + } + } + if length < 0 || length > 20 { + return StructuralError("Huffman length out of range") + } + lengths[j] = uint8(length) + } + huffmanTrees[i], err = newHuffmanTree(lengths) + if err != nil { + return err + } + } + + selectorIndex := 1 // the next tree index to use + currentHuffmanTree := huffmanTrees[treeIndexes[0]] + bufIndex := 0 // indexes bz2.buf, the output buffer. + // The output of the move-to-front transform is run-length encoded and + // we merge the decoding into the Huffman parsing loop. These two + // variables accumulate the repeat count. See the Wikipedia page for + // details. + repeat := 0 + repeat_power := 0 + + // The `C' array (used by the inverse BWT) needs to be zero initialized. + for i := range bz2.c { + bz2.c[i] = 0 + } + + decoded := 0 // counts the number of symbols decoded by the current tree. + for { + if decoded == 50 { + currentHuffmanTree = huffmanTrees[treeIndexes[selectorIndex]] + selectorIndex++ + decoded = 0 + } + + v := currentHuffmanTree.Decode(br) + decoded++ + + if v < 2 { + // This is either the RUNA or RUNB symbol. + if repeat == 0 { + repeat_power = 1 + } + repeat += repeat_power << v + repeat_power <<= 1 + + // This limit of 2 million comes from the bzip2 source + // code. It prevents repeat from overflowing. + if repeat > 2*1024*1024 { + return StructuralError("repeat count too large") + } + continue + } + + if repeat > 0 { + // We have decoded a complete run-length so we need to + // replicate the last output symbol. + for i := 0; i < repeat; i++ { + b := byte(mtf.First()) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + repeat = 0 + } + + if int(v) == numSymbols-1 { + // This is the EOF symbol. Because it's always at the + // end of the move-to-front list, and never gets moved + // to the front, it has this unique value. + break + } + + // Since two metasymbols (RUNA and RUNB) have values 0 and 1, + // one would expect |v-2| to be passed to the MTF decoder. + // However, the front of the MTF list is never referenced as 0, + // it's always referenced with a run-length of 1. Thus 0 + // doesn't need to be encoded and we have |v-1| in the next + // line. + b := byte(mtf.Decode(int(v - 1))) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + + if origPtr >= uint(bufIndex) { + return StructuralError("origPtr out of bounds") + } + + // We have completed the entropy decoding. Now we can perform the + // inverse BWT and setup the RLE buffer. + bz2.preRLE = bz2.tt[:bufIndex] + bz2.preRLEUsed = 0 + bz2.tPos = inverseBWT(bz2.preRLE, origPtr, bz2.c[:]) + bz2.lastByte = -1 + bz2.byteRepeats = 0 + bz2.repeats = 0 + + return nil +} + +// inverseBWT implements the inverse Burrows-Wheeler transform as described in +// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf, section 4.2. +// In that document, origPtr is called `I' and c is the `C' array after the +// first pass over the data. It's an argument here because we merge the first +// pass with the Huffman decoding. +// +// This also implements the `single array' method from the bzip2 source code +// which leaves the output, still shuffled, in the bottom 8 bits of tt with the +// index of the next byte in the top 24-bits. The index of the first byte is +// returned. +func inverseBWT(tt []uint32, origPtr uint, c []uint) uint32 { + sum := uint(0) + for i := 0; i < 256; i++ { + sum += c[i] + c[i] = sum - c[i] + } + + for i := range tt { + b := tt[i] & 0xff + tt[c[b]] |= uint32(i) << 8 + c[b]++ + } + + return tt[origPtr] >> 8 +} diff --git a/src/pkg/compress/bzip2/bzip2_test.go b/src/pkg/compress/bzip2/bzip2_test.go new file mode 100644 index 000000000..156eea83f --- /dev/null +++ b/src/pkg/compress/bzip2/bzip2_test.go @@ -0,0 +1,158 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestBitReader(t *testing.T) { + buf := bytes.NewBuffer([]byte{0xaa}) + br := newBitReader(buf) + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 1 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 2 wrong") + } + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 3 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 4 wrong") + } +} + +func TestBitReaderLarge(t *testing.T) { + buf := bytes.NewBuffer([]byte{0x12, 0x34, 0x56, 0x78}) + br := newBitReader(buf) + if n := br.ReadBits(32); n != 0x12345678 { + t.Errorf("got: %x want: %x", n, 0x12345678) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func decompressHex(s string) (out []byte, err os.Error) { + r := NewReader(readerFromHex(s)) + return ioutil.ReadAll(r) +} + +func TestHelloWorldBZ2(t *testing.T) { + out, err := decompressHex(helloWorldBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + if !bytes.Equal(helloWorld, out) { + t.Errorf("got %x, want %x", out, helloWorld) + } +} + +func testZeros(t *testing.T, inHex string, n int) { + out, err := decompressHex(inHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, n) + + if !bytes.Equal(expected, out) { + allZeros := true + for _, b := range out { + if b != 0 { + allZeros = false + break + } + } + t.Errorf("incorrect result, got %d bytes (allZeros: %t)", len(out), allZeros) + } +} + +func Test32Zeros(t *testing.T) { + testZeros(t, thirtyTwoZerosBZ2Hex, 32) +} + +func Test1MBZeros(t *testing.T) { + testZeros(t, oneMBZerosBZ2Hex, 1024*1024) +} + +func testRandomData(t *testing.T, compressedHex, uncompressedHex string) { + out, err := decompressHex(compressedHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected, _ := hex.DecodeString(uncompressedHex) + + if !bytes.Equal(out, expected) { + t.Errorf("incorrect result\ngot: %x\nwant: %x", out, expected) + } +} + +func TestRandomData1(t *testing.T) { + testRandomData(t, randBZ2Hex, randHex) +} + +func TestRandomData2(t *testing.T) { + // This test involves several repeated bytes in the output, but they + // should trigger RLE decoding. + testRandomData(t, rand2BZ2Hex, rand2Hex) +} + +func TestRandomData3(t *testing.T) { + // This test uses the full range of symbols. + testRandomData(t, rand3BZ2Hex, rand3Hex) +} + +func Test1MBSawtooth(t *testing.T) { + out, err := decompressHex(oneMBSawtoothBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, 1024*1024) + + for i := range expected { + expected[i] = byte(i) + } + + if !bytes.Equal(out, expected) { + t.Error("incorrect result") + } +} + +const helloWorldBZ2Hex = "425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0" + +var helloWorld = []byte("hello world\n") + +const thirtyTwoZerosBZ2Hex = "425a6839314159265359b5aa5098000000600040000004200021008283177245385090b5aa5098" +const oneMBZerosBZ2Hex = "425a683931415926535938571ce50008084000c0040008200030cc0529a60806c4201e2ee48a70a12070ae39ca" + +const randBZ2Hex = "425a6839314159265359905d990d0001957fffffffffffafffffffffffffffffbfff6fffdfffffffffffffffffffffffffffffc002b6dd75676ed5b77720098320d11a64626981323d4da47a83131a13d09e8040f534cd4f4d27a464d193008cd09804601347a980026350c9886234d36864193d1351b44c136919e90340d26127a4cd264c32023009898981310c0344c340027a8303427a99a04c00003534c230d034f5006468d268cf54d36a3009a69a62626261311b40026013d34201a6934c9a604c98ca6c8460989fa9346234d30d3469a2604fd4131a7aa6d0046043d4c62098479269e89e835190d018d4c046001a11e801a0264792321932308c43a130688c260d46686804cd01a9e80981193684c6a68c00000004c4c20c04627a4c0000260003400d04c0681a01334026009a6f48041466132581ec5212b081d96b0effc16543e2228b052fcd30f2567ee8d970e0f10aabca68dd8270591c376cfc1baae0dba00aaff2d6caf6b211322c997cc18eaee5927f75185336bf907021324c71626c1dd20e22b9b0977f05d0f901eaa51db9fbaf7c603b4c87bc82890e6dd7e61d0079e27ec050dd788fd958152061cd01e222f9547cb9efc465d775b6fc98bac7d387bffd151ae09dadf19494f7a638e2eae58e550faba5fe6820ea520eb986096de4e527d80def3ba625e71fbefdcf7e7844e0a25d29b52dcd1344fca083737d42692aab38d230485f3c8ed54c2ed31f15cf0270c8143765b10b92157233fa1dfe0d7ce8ffe70b8b8f7250071701dfe9f1c94de362c9031455951c93eb098a6b50ee45c6131fefc3b6f9643e21f4adc59497138e246f5c57d834aa67c4f10d8bd8b3908d8130dd7388409c299a268eab3664fa4907c5c31574874bd8d388a4ab22b339660804e53e1b8d05867d40e3082560608d35d5d2c6054e8bab23da28f61f83efd41d25529ad6ea15fb50505cacfabb0902166427354ca3830a2c8415f21b19e592690fbe447020d685a4bcd16ecc4ff1a1c0e572627d0ef6265c008a43fc243240541061ed7840606be466d1c0dac2c53250ed567507d926c844154560d631960c65e15157829b2c7f16859f111a3a8cb72bf24ffa57a680c3be67b1be67c8dd8aea73ac2437a78df5b686d427080ebc01bd30b71a49f6ea31dc0f08e4849e38face96717690239538bc08b6cc5aa8d467cb9c36aa83d40ac7e58bddbfa185b22065e89a86c0145569d9e23726651aec49e31588d70f40fe9a4449dcf4f89eac220171e9c938e803dc195679651004b79ad33cc0c13aeeba5941b33ffeeb8fbe16e76c7811445c67b4269c90479433ddf9e8ed1d00c166b6c17217fb22c3ef1b0c1c7e28e185446a111c37f1ea6c07a59fbcc6546ecc6968d36ba58bc5489a5640647e426b0c39350cb6f07d5dc7a717648c4ec7f841467597ae1f65f408fd2d9940a4b1b860b3c9ae351dcae0b4425f7e8538710f2e40b7f70d13b51ac05ccc6ecda8264a88cad2d721d18132a9b9110a9e759c2483c77dcefc7e464ec88588174cb0c9abff93230ea0bed8decdd8ed8bfe2b5df0a253803678df04fab44c03b9ab7cc97d6e6d6fd0c4c840ce0efc498436f453bbb181603459471f2b588724592b222ec990614db530e10cadd84705621cfdd9261fa44a5f5806a2d74b575056b3c915255c65678f9c16e6dc00a99180fef1a840aff0e842ac02731080cc92782538360a60a727991013984da4fad95f79d5030677b7528d076b2483685fca4429edf804682fdc110dfc2f7c30e23e20a72e039108a0ad6fdee2f76985a4b4be4f5afc6101bf9d5042b657a05dc914e1424241766434" +const randHex = "c95138082bdf2b9bfa5b1072b23f729735d42c785eeb94320fb14c265b9c2ca421d01a3db986df1ac2acde5a0e6bf955d6f95e61261540905928e195f1a66644cc7f37281744fff4dc6df35566a494c41a8167151950eb74f5fc45f85ad0e5ed28b49adfe218aa7ec1707e8e1d55825f61f72beda3b4c006b8c9188d7336a5d875329b1b58c27cc4e89ecbae02c7712400c39dd131d2c6de82e2863da51d472bdfb21ecce62cc9cf769ed28aedc7583d755da45a0d90874bda269dd53283a9bdfd05f95fc8e9a304bb338ea1a2111894678c18134f17d31a15d9bfc1237894650f3e715e2548639ecbddb845cfe4a46a7b3a3c540f48629488e8c869f1e9f3f4c552243a8105b20eb8e264994214349dae83b165fd6c2a5b8e83fce09fc0a80d3281c8d53a9a08095bd19cbc1388df23975646ed259e003d39261ee68cbece8bcf32971f7fe7e588e8ba8f5e8597909abaea693836a79a1964050ed910a45a0f13a58cd2d3ae18992c5b23082407fd920d0bf01e33118a017bb5e39f44931346845af52128f7965206759433a346034ea481671f501280067567619f5ecef6cded077f92ed7f3b3ce8e308c80f34ba06939e9303f91b4318c8c1dd4cc223c1f057ac0c91211c629cd30e46ee9ec1d9fd493086b7bc2bc83e33f08749a5d430b0ed4f79d70f481940c9b0930b16321886a0df4fa5a1465d5208c7d3494a7987d9a5e42aa256f0c9523947f8318d0ef0af3d59a45cfc2418d0785c9a548b32b81e7de18be7d55a69a4c156bbb3d7579c0ac8e9c72b24646e54b0d0e8725f8f49fb44ae3c6b9d0287be118586255a90a4a83483ed0328518037e52aa959c5748ed83e13023e532306be98b8288da306bbb040bcf5d92176f84a9306dc6b274b040370b61d71fde58dd6d20e6fee348eae0c54bd0a5a487b2d005f329794f2a902c296af0a4c1f638f63292a1fa18e006c1b1838636f4de71c73635b25660d32e88a0917e1a5677f6a02ca65585b82cbd99fb4badbfa97a585da1e6cadf6737b4ec6ca33f245d66ee6a9fae6785d69b003c17b9fc6ec34fe5824ab8caae5e8e14dc6f9e116e7bf4a60c04388783c8ae929e1b46b3ef3bbe81b38f2fa6da771bf39dfba2374d3d2ed356b8e2c42081d885a91a3afb2f31986d2f9873354c48cf5448492c32e62385af423aa4f83db6d1b2669650379a1134b0a04cbca0862d6f9743c791cbb527d36cd5d1f0fc7f503831c8bd1b7a0ef8ae1a5ed1155dfdd9e32b6bb33138112d3d476b802179cb85a2a6c354ccfed2f31604fbd8d6ec4baf9f1c8454f72c6588c06a7df3178c43a6970bfa02dd6f74cb5ec3b63f9eddaa17db5cbf27fac6de8e57c384afd0954179f7b5690c3bee42abc4fa79b4b12101a9cf5f0b9aecdda945def0bd04163237247d3539850e123fe18139f316fa0256d5bd2faa8" + +const oneMBSawtoothBZ2Hex = "425a683931415926535971931ea00006ddffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe007de00000000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000009aaaaa0000000000000000000000000000000000000000000000000000000000000000498002600026000000000000000000000000000000000000000000000000000000007fc42271980d044c0a822607411304a08982d044c1a82260f411308a08984d044c2a82261741130ca08986d044c3a82261f411310a08988d044c4a822627411314a0898ad044c5a82262f411318a0898cd044c6a82263741131ca0898ed044c7a82263f411320a08990d044c8a822647411324a08992d044c9a82264f411328a08994d044caa82265741132ca08996d044cba82265f411330a08998d044cca822667411334a0899ad044cda82266f411338a0899cd044cea82267741133ca0899ed044cfa82267f411340a089a0d044d0a822687411344a089a2d044d1a82268f411348a089a4d044d2a82269741134ca089a6d044d3a82269f411350a089a8d044d4a8226a7411354a089aad044d5a8226af411358a089acd044d6a8226b741135ca089aed044d7a8226bf411360a089b0d044d8a8226c7411364a089b2d044d9a8226cf411368a089b4d044daa8226d741136ca089b6d044dba8226df411370a089b8d044dca8226e7411374a089bad044dda8226ef411378a089bcd044dea8226f741137ca089bed044dfa8226ff411380a089c0d044e0a822707411384a089c2d044e1a82270f411388a089c4d044e2a82271741138ca089c59089c69089c71089c79089c81089c89089c91089c99089ca1089ca9089cb1089cb9089cc1089cc9089cd1089cd9089ce1089ce9089cf1089cf9089d01089d09089d11089d19089d21089d29089d31089d39089d41089d49089d51089d59089d61089d69089d71089d79089d81089d89089d91089d99089da1089da9089db1089db9089dc1089dc9089dd1089dd9089de1089de9089df1089df9089e01089e09089e11089e19089e21089e29089e31089e39089e41089e49089e51089e59089e61089e69089e71089e79089e81089e89089e91089e99089ea1089ea9089eb1089eb9089ec1089ec9089ed1089ed9089ee1089ee9089ef1089ef9089f01089f09089f11089f19089f21089f29089f31089f39089f41089f49089f51089f59089f61089f69089f71089f79089f81089f89089f91089f99089fa1089fa9089fb1089fb9089fc1089fc9089fd1089fd9089fe1089fe9089ff1089ff98a0ac9329acf23ba884804fdd3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0034f800000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000024c0013000130000000000000000000000000000000000000000000000000000000002955540000000000000000000000000000000000000000000000000000000000000001ff108c00846024230221181908c108460a4230621183908c20846124230a21185908c308461a4230e21187908c40846224231221189908c508462a423162118b908c60846324231a2118d908c708463a4231e2118f908c80846424232221191908c908464a4232621193908ca0846524232a21195908cb08465a4232e21197908cc0846624233221199908cd08466a423362119b908ce0846724233a2119d908cf08467a4233e2119f908d008468242342211a1908d108468a42346211a3908d20846924234a211a5908d308469a4234e211a7908d40846a242352211a9908d50846aa42356211ab908d60846b24235a211ad908d70846ba4235e211af908d80846c242362211b1908d90846ca42366211b3908da0846d24236a211b5908db0846da4236e211b7908dc0846e242372211b9908dd0846ea42376211bb908de0846f24237a211bd908df0846fa4237e211bf908e008470242382211c1908e108470a42386211c3908e20847124238a211c5908e2f8c211c6c8471d211c7c84721211c8c84725211c9c84729211cac8472d211cbc84731211ccc84735211cdc84739211cec8473d211cfc84741211d0c84745211d1c84749211d2c8474d211d3c84751211d4c84755211d5c84759211d6c8475d211d7c84761211d8c84765211d9c84769211dac8476d211dbc84771211dcc84775211ddc84779211dec8477d211dfc84781211e0c84785211e1c84789211e2c8478d211e3c84791211e4c84795211e5c84799211e6c8479d211e7c847a1211e8c847a5211e9c847a9211eac847ad211ebc847b1211ecc847b5211edc847b9211eec847bd211efc847c1211f0c847c5211f1c847c9211f2c847cd211f3c847d1211f4c847d5211f5c847d9211f6c847dd211f7c847e1211f8c847e5211f9c847e9211fac847ed211fbc847f1211fcc847f5211fdc847f9211fec847fd211ff8bb9229c284803a8b6248" + +const rand2BZ2Hex = "425a6839314159265359d992d0f60000137dfe84020310091c1e280e100e042801099210094806c0110002e70806402000546034000034000000f2830000032000d3403264049270eb7a9280d308ca06ad28f6981bee1bf8160727c7364510d73a1e123083421b63f031f63993a0f40051fbf177245385090d992d0f60" +const rand2Hex = "92d5652616ac444a4a04af1a8a3964aca0450d43d6cf233bd03233f4ba92f8719e6c2a2bd4f5f88db07ecd0da3a33b263483db9b2c158786ad6363be35d17335ba" + +const rand3BZ2Hex = "425a68393141592653593be669d00000327ffffffffffffffffffffffffffffffffffff7ffffffffffffffffffffffffffffffc002b3b2b1b6e2bae400004c00132300004c0d268c004c08c0130026001a008683234c0684c34008c230261a04c0260064d07a8d00034000d27a1268c9931a8d327a3427a41faa69ea0da264c1a34219326869b51b49a6469a3268c689fa53269a62794687a9a68f5189994c9e487a8f534fd49a3d34043629e8c93d04da4f4648d30d4f44d3234c4d3023d0840680984d309934c234d3131a000640984f536a6132601300130130c8d00d04d1841ea7a8d31a02609b40023460010c01a34d4c1a0d04d3069306810034d0d0d4c0046130d034d0131a9a64d321804c68003400098344c13000991808c0001a00000000098004d3d4da4604c47a13012140aadf8d673c922c607ef6212a8c0403adea4b28aee578900e653b9cdeb8d11e6b838815f3ebaad5a01c5408d84a332170aff8734d4e06612d3c2889f31925fb89e33561f5100ae89b1f7047102e729373d3667e58d73aaa80fa7be368a1cc2dadd81d81ec8e1b504bd772ca31d03649269b01ceddaca07bf3d4eba24de141be3f86f93601e03714c0f64654671684f9f9528626fd4e1b76753dc0c54b842486b8d59d8ab314e86ca818e7a1f079463cbbd70d9b79b283c7edc419406311022e4be98c2c1374df9cdde2d008ce1d00e5f06ad1024baf555631f70831fc1023034e62be7c4bcb648caf276963ffa20e96bb50377fe1c113da0db4625b50741c35a058edb009c6ee5dbf93b8a6b060eec568180e8db791b82aab96cbf4326ca98361461379425ba8dcc347be670bdba7641883e5526ae3d833f6e9cb9bac9557747c79e206151072f7f0071dff3880411846f66bf4075c7462f302b53cb3400a74cf35652ad5641ed33572fd54e7ed7f85f58a0acba89327e7c6be5c58cb71528b99df2431f1d0358f8d28d81d95292da631fb06701decabb205fac59ff0fb1df536afc681eece6ea658c4d9eaa45f1342aa1ff70bdaff2ddaf25ec88c22f12829a0553db1ec2505554cb17d7b282e213a5a2aa30431ded2bce665bb199d023840832fedb2c0c350a27291407ff77440792872137df281592e82076a05c64c345ffb058c64f7f7c207ef78420b7010520610f17e302cc4dfcfaef72a0ed091aab4b541eb0531bbe941ca2f792bf7b31ca6162882b68054a8470115bc2c19f2df2023f7800432b39b04d3a304e8085ba3f1f0ca5b1ba4d38d339e6084de979cdea6d0e244c6c9fa0366bd890621e3d30846f5e8497e21597b8f29bbf52c961a485dfbea647600da0fc1f25ce4d203a8352ece310c39073525044e7ac46acf2ed9120bae1b4f6f02364abfe343f80b290983160c103557af1c68416480d024cc31b6c06cfec011456f1e95c420a12b48b1c3fe220c2879a982fb099948ac440db844b9a112a5188c7783fd3b19593290785f908d95c9db4b280bafe89c1313aeec24772046d9bc089645f0d182a21184e143823c5f52de50e5d7e98d3d7ab56f5413bbccd1415c9bcff707def475b643fb7f29842582104d4cc1dbaaca8f10a2f44273c339e0984f2b1e06ab2f0771db01fafa8142298345f3196f23e5847bda024034b6f59b11c29e981c881456e40d211929fd4f766200258aad8212016322bd5c605790dcfdf1bd2a93d99c9b8f498722d311d7eae7ff420496a31804c55f4759a7b13aaaf5f7ce006c3a8a998897d5e0a504398c2b627852545baf440798bcc5cc049357cf3f17d9771e4528a1af3d77dc794a11346e1bdf5efe37a405b127b4c43b616d61fbc5dc914e14240ef99a7400" +const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c593553f33bd786e3d0ce31626f511bc985f59d1a88aa38ba8ad6218d306abee60dd9172540232b95be1af146c69e72e5fde667a090dc3f93bdc5c5af0ab80acdbaa7a505f628c59dc0247b31a439cacf5010a94376d71521df08c178b02fb96fdb1809144ea38c68536187c53201fea8631fb0a880b4451ccdca7cc61f6aafca21cc7449d920599db61789ac3b1e164b3390124f95022aeea39ccca3ec1053f4fa10de2978e2861ea58e477085c2220021a0927aa94c5d0006b5055abba340e4f9eba22e969978dfd18e278a8b89d877328ae34268bc0174cfe211954c0036f078025217d1269fac1932a03b05a0b616012271bbe1fb554171c7a59b196d8a4479f45a77931b5d97aaf6c0c673cbe597b79b96e2a0c1eae2e66e46ccc8c85798e23ffe972ebdaa3f6caea243c004e60321eb47cd79137d78fd0613be606feacc5b3637bdc96a89c13746db8cad886f3ccf912b2178c823bcac395f06d28080269bdca2debf3419c66c690fd1adcfbd53e32e79443d7a42511a84cb22ca94fffad9149275a075b2f8ae0b021dcde9bf62b102db920733b897560518b06e1ad7f4b03458493ddaa7f4fa2c1609f7a1735aeeb1b3e2cea3ab45fc376323cc91873b7e9c90d07c192e38d3f5dfc9bfab1fd821c854da9e607ea596c391c7ec4161c6c4493929a8176badaa5a5af7211c623f29643a937677d3df0da9266181b7c4da5dd40376db677fe8f4a1dc456adf6f33c1e37cec471dd318c2647644fe52f93707a77da7d1702380a80e14cc0fdce7bf2eed48a529090bae0388ee277ce6c7018c5fb00b88362554362205c641f0d0fab94fd5b8357b5ff08b207fee023709bc126ec90cfb17c006754638f8186aaeb1265e80be0c1189ec07d01d5f6f96cb9ce82744147d18490de7dc72862f42f024a16968891a356f5e7e0e695d8c933ba5b5e43ad4c4ade5399bc2cae9bb6189b7870d7f22956194d277f28b10e01c10c6ffe3e065f7e2d6d056aa790db5649ca84dc64c35566c0af1b68c32b5b7874aaa66467afa44f40e9a0846a07ae75360a641dd2acc69d93219b2891f190621511e62a27f5e4fbe641ece1fa234fc7e9a74f48d2a760d82160d9540f649256b169d1fed6fbefdc491126530f3cbad7913e19fbd7aa53b1e243fbf28d5f38c10ebd77c8b986775975cc1d619efb27cdcd733fa1ca36cffe9c0a33cc9f02463c91a886601fd349efee85ef1462065ef9bd2c8f533220ad93138b8382d5938103ab25b2d9af8ae106e1211eb9b18793fba033900c809c02cd6d17e2f3e6fc84dae873411f8e87c3f0a8f1765b7825d185ce3730f299c3028d4a62da9ee95c2b870fb70c79370d485f9d5d9acb78926d20444033d960524d2776dc31988ec7c0dbf23b9905d" diff --git a/src/pkg/compress/bzip2/huffman.go b/src/pkg/compress/bzip2/huffman.go new file mode 100644 index 000000000..dc05739c7 --- /dev/null +++ b/src/pkg/compress/bzip2/huffman.go @@ -0,0 +1,223 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "os" + "sort" +) + +// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a +// symbol. +type huffmanTree struct { + // nodes contains all the non-leaf nodes in the tree. nodes[0] is the + // root of the tree and nextNode contains the index of the next element + // of nodes to use when the tree is being constructed. + nodes []huffmanNode + nextNode int +} + +// A huffmanNode is a node in the tree. left and right contain indexes into the +// nodes slice of the tree. If left or right is invalidNodeValue then the child +// is a left node and its value is in leftValue/rightValue. +// +// The symbols are uint16s because bzip2 encodes not only MTF indexes in the +// tree, but also two magic values for run-length encoding and an EOF symbol. +// Thus there are more than 256 possible symbols. +type huffmanNode struct { + left, right uint16 + leftValue, rightValue uint16 +} + +// invalidNodeValue is an invalid index which marks a leaf node in the tree. +const invalidNodeValue = 0xffff + +// Decode reads bits from the given bitReader and navigates the tree until a +// symbol is found. +func (t huffmanTree) Decode(br *bitReader) (v uint16) { + nodeIndex := uint16(0) // node 0 is the root of the tree. + + for { + node := &t.nodes[nodeIndex] + bit := br.ReadBit() + // bzip2 encodes left as a true bit. + if bit { + // left + if node.left == invalidNodeValue { + return node.leftValue + } + nodeIndex = node.left + } else { + // right + if node.right == invalidNodeValue { + return node.rightValue + } + nodeIndex = node.right + } + } + + panic("unreachable") +} + +// newHuffmanTree builds a Huffman tree from a slice containing the code +// lengths of each symbol. The maximum code length is 32 bits. +func newHuffmanTree(lengths []uint8) (huffmanTree, os.Error) { + // There are many possible trees that assign the same code length to + // each symbol (consider reflecting a tree down the middle, for + // example). Since the code length assignments determine the + // efficiency of the tree, each of these trees is equally good. In + // order to minimize the amount of information needed to build a tree + // bzip2 uses a canonical tree so that it can be reconstructed given + // only the code length assignments. + + if len(lengths) < 2 { + panic("newHuffmanTree: too few symbols") + } + + var t huffmanTree + + // First we sort the code length assignments by ascending code length, + // using the symbol value to break ties. + pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths))) + for i, length := range lengths { + pairs[i].value = uint16(i) + pairs[i].length = length + } + + sort.Sort(pairs) + + // Now we assign codes to the symbols, starting with the longest code. + // We keep the codes packed into a uint32, at the most-significant end. + // So branches are taken from the MSB downwards. This makes it easy to + // sort them later. + code := uint32(0) + length := uint8(32) + + codes := huffmanCodes(make([]huffmanCode, len(lengths))) + for i := len(pairs) - 1; i >= 0; i-- { + if length > pairs[i].length { + // If the code length decreases we shift in order to + // zero any bits beyond the end of the code. + length >>= 32 - pairs[i].length + length <<= 32 - pairs[i].length + length = pairs[i].length + } + codes[i].code = code + codes[i].codeLen = length + codes[i].value = pairs[i].value + // We need to 'increment' the code, which means treating |code| + // like a |length| bit number. + code += 1 << (32 - length) + } + + // Now we can sort by the code so that the left half of each branch are + // grouped together, recursively. + sort.Sort(codes) + + t.nodes = make([]huffmanNode, len(codes)) + _, err := buildHuffmanNode(&t, codes, 0) + return t, err +} + +// huffmanSymbolLengthPair contains a symbol and its code length. +type huffmanSymbolLengthPair struct { + value uint16 + length uint8 +} + +// huffmanSymbolLengthPair is used to provide an interface for sorting. +type huffmanSymbolLengthPairs []huffmanSymbolLengthPair + +func (h huffmanSymbolLengthPairs) Len() int { + return len(h) +} + +func (h huffmanSymbolLengthPairs) Less(i, j int) bool { + if h[i].length < h[j].length { + return true + } + if h[i].length > h[j].length { + return false + } + if h[i].value < h[j].value { + return true + } + return false +} + +func (h huffmanSymbolLengthPairs) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +// huffmanCode contains a symbol, its code and code length. +type huffmanCode struct { + code uint32 + codeLen uint8 + value uint16 +} + +// huffmanCodes is used to provide an interface for sorting. +type huffmanCodes []huffmanCode + +func (n huffmanCodes) Len() int { + return len(n) +} + +func (n huffmanCodes) Less(i, j int) bool { + return n[i].code < n[j].code +} + +func (n huffmanCodes) Swap(i, j int) { + n[i], n[j] = n[j], n[i] +} + +// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in +// the Huffman tree at the given level. It returns the index of the newly +// constructed node. +func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err os.Error) { + test := uint32(1) << (31 - level) + + // We have to search the list of codes to find the divide between the left and right sides. + firstRightIndex := len(codes) + for i, code := range codes { + if code.code&test != 0 { + firstRightIndex = i + break + } + } + + left := codes[:firstRightIndex] + right := codes[firstRightIndex:] + + if len(left) == 0 || len(right) == 0 { + return 0, StructuralError("superfluous level in Huffman tree") + } + + nodeIndex = uint16(t.nextNode) + node := &t.nodes[t.nextNode] + t.nextNode++ + + if len(left) == 1 { + // leaf node + node.left = invalidNodeValue + node.leftValue = left[0].value + } else { + node.left, err = buildHuffmanNode(t, left, level+1) + } + + if err != nil { + return + } + + if len(right) == 1 { + // leaf node + node.right = invalidNodeValue + node.rightValue = right[0].value + } else { + node.right, err = buildHuffmanNode(t, right, level+1) + } + + return +} diff --git a/src/pkg/compress/bzip2/move_to_front.go b/src/pkg/compress/bzip2/move_to_front.go new file mode 100644 index 000000000..0ed19dec3 --- /dev/null +++ b/src/pkg/compress/bzip2/move_to_front.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +// moveToFrontDecoder implements a move-to-front list. Such a list is an +// efficient way to transform a string with repeating elements into one with +// many small valued numbers, which is suitable for entropy encoding. It works +// by starting with an initial list of symbols and references symbols by their +// index into that list. When a symbol is referenced, it's moved to the front +// of the list. Thus, a repeated symbol ends up being encoded with many zeros, +// as the symbol will be at the front of the list after the first access. +type moveToFrontDecoder struct { + // Rather than actually keep the list in memory, the symbols are stored + // as a circular, double linked list with the symbol indexed by head + // at the front of the list. + symbols []byte + next []uint8 + prev []uint8 + head uint8 +} + +// newMTFDecoder creates a move-to-front decoder with an explicit initial list +// of symbols. +func newMTFDecoder(symbols []byte) *moveToFrontDecoder { + if len(symbols) > 256 { + panic("too many symbols") + } + + m := &moveToFrontDecoder{ + symbols: symbols, + next: make([]uint8, len(symbols)), + prev: make([]uint8, len(symbols)), + } + + m.threadLinkedList() + return m +} + +// newMTFDecoderWithRange creates a move-to-front decoder with an initial +// symbol list of 0...n-1. +func newMTFDecoderWithRange(n int) *moveToFrontDecoder { + if n > 256 { + panic("newMTFDecoderWithRange: cannot have > 256 symbols") + } + + m := &moveToFrontDecoder{ + symbols: make([]uint8, n), + next: make([]uint8, n), + prev: make([]uint8, n), + } + + for i := 0; i < n; i++ { + m.symbols[i] = byte(i) + } + + m.threadLinkedList() + return m +} + +// threadLinkedList creates the initial linked-list pointers. +func (m *moveToFrontDecoder) threadLinkedList() { + if len(m.symbols) == 0 { + return + } + + m.prev[0] = uint8(len(m.symbols) - 1) + + for i := 0; i < len(m.symbols)-1; i++ { + m.next[i] = uint8(i + 1) + m.prev[i+1] = uint8(i) + } + + m.next[len(m.symbols)-1] = 0 +} + +func (m *moveToFrontDecoder) Decode(n int) (b byte) { + // Most of the time, n will be zero so it's worth dealing with this + // simple case. + if n == 0 { + return m.symbols[m.head] + } + + i := m.head + for j := 0; j < n; j++ { + i = m.next[i] + } + b = m.symbols[i] + + m.next[m.prev[i]] = m.next[i] + m.prev[m.next[i]] = m.prev[i] + m.next[i] = m.head + m.prev[i] = m.prev[m.head] + m.next[m.prev[m.head]] = i + m.prev[m.head] = i + m.head = i + + return +} + +// First returns the symbol at the front of the list. +func (m *moveToFrontDecoder) First() byte { + return m.symbols[m.head] +} diff --git a/src/pkg/compress/flate/Makefile b/src/pkg/compress/flate/Makefile new file mode 100644 index 000000000..197828a92 --- /dev/null +++ b/src/pkg/compress/flate/Makefile @@ -0,0 +1,17 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/flate +GOFILES=\ + deflate.go\ + huffman_bit_writer.go\ + huffman_code.go\ + inflate.go\ + reverse_bits.go\ + token.go\ + util.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/flate/deflate.go b/src/pkg/compress/flate/deflate.go new file mode 100644 index 000000000..b1cee0b2f --- /dev/null +++ b/src/pkg/compress/flate/deflate.go @@ -0,0 +1,493 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" + "os" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + fastCompression = 3 + BestCompression = 9 + DefaultCompression = -1 + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + logMaxOffsetSize = 15 // Standard DEFLATE + minMatchLength = 3 // The smallest match that the compressor looks for + maxMatchLength = 258 // The longest match for the compressor + minOffsetSize = 1 // The shortest offset that makes any sence + + // The maximum number of tokens we put into a single flat block, just too + // stop things from getting too large. + maxFlateBlockTokens = 1 << 14 + maxStoreBlockSize = 65535 + hashBits = 15 + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + hashShift = (hashBits + minMatchLength - 1) / minMatchLength +) + +type compressionLevel struct { + good, lazy, nice, chain, fastSkipHashing int +} + +var levels = []compressionLevel{ + {}, // 0 + // For levels 1-3 we don't bother trying with lazy matches + {3, 0, 8, 4, 4}, + {3, 0, 16, 8, 5}, + {3, 0, 32, 32, 6}, + // Levels 4-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {4, 4, 16, 16, math.MaxInt32}, + {8, 16, 32, 32, math.MaxInt32}, + {8, 16, 128, 128, math.MaxInt32}, + {8, 32, 128, 256, math.MaxInt32}, + {32, 128, 258, 1024, math.MaxInt32}, + {32, 258, 258, 4096, math.MaxInt32}, +} + +type compressor struct { + compressionLevel + + w *huffmanBitWriter + + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + sync bool // requesting flush + + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + chainHead int + hashHead []int + hashPrev []int + + // input window: unprocessed data is window[index:windowEnd] + index int + window []byte + windowEnd int + blockStart int // window index where current tokens start + byteAvailable bool // if true, still need to process window[index-1]. + + // queued output tokens: tokens[:ti] + tokens []token + ti int + + // deflate state + length int + offset int + hash int + maxInsertIndex int + err os.Error +} + +func (d *compressor) fillDeflate(b []byte) int { + if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + copy(d.window, d.window[windowSize:2*windowSize]) + d.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize + } else { + d.blockStart = math.MaxInt32 + } + for i, h := range d.hashHead { + v := h - windowSize + if v < -1 { + v = -1 + } + d.hashHead[i] = v + } + for i, h := range d.hashPrev { + v := -h - windowSize + if v < -1 { + v = -1 + } + d.hashPrev[i] = v + } + } + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) writeBlock(tokens []token, index int, eof bool) os.Error { + if index > 0 || eof { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + d.w.writeBlock(tokens, eof, window) + return d.w.err + } + return nil +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + minMatchLook := maxMatchLength + if lookahead < minMatchLook { + minMatchLook = lookahead + } + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := len(win) - pos + if d.nice < nice { + nice = d.nice + } + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = prevLength + if length >= d.good { + tries >>= 2 + } + + w0 := win[pos] + w1 := win[pos+1] + wEnd := win[pos+length] + minIndex := pos - windowSize + + for i := prevHead; tries > 0; tries-- { + if w0 == win[i] && w1 == win[i+1] && wEnd == win[i+length] { + // The hash function ensures that if win[i] and win[i+1] match, win[i+2] matches + + n := 3 + for pos+n < len(win) && win[i+n] == win[pos+n] { + n++ + } + if n > length && (n > 3 || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + if i = d.hashPrev[i&windowMask]; i < minIndex || i < 0 { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) os.Error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +func (d *compressor) initDeflate() { + d.hashHead = make([]int, hashSize) + d.hashPrev = make([]int, windowSize) + d.window = make([]byte, 2*windowSize) + fillInts(d.hashHead, -1) + d.tokens = make([]token, maxFlateBlockTokens, maxFlateBlockTokens+1) + d.length = minMatchLength - 1 + d.offset = 0 + d.byteAvailable = false + d.index = 0 + d.ti = 0 + d.hash = 0 + d.chainHead = -1 +} + +func (d *compressor) deflate() { + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + if d.index < d.maxInsertIndex { + d.hash = int(d.window[d.index])< d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + break Loop + } + if d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens[d.ti] = literalToken(uint32(d.window[d.index-1])) + d.ti++ + d.byteAvailable = false + } + if d.ti > 0 { + if d.err = d.writeBlock(d.tokens[0:d.ti], d.index, false); d.err != nil { + return + } + d.ti = 0 + } + break Loop + } + } + if d.index < d.maxInsertIndex { + // Update the hash + d.hash = (d.hash<= minIndex && + (d.fastSkipHashing != 0 && lookahead > minMatchLength-1 || + d.fastSkipHashing == 0 && lookahead > prevLength && prevLength < d.lazy) { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if d.fastSkipHashing != 0 && d.length >= minMatchLength || + d.fastSkipHashing == 0 && prevLength >= minMatchLength && d.length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + if d.fastSkipHashing != 0 { + d.tokens[d.ti] = matchToken(uint32(d.length-minMatchLength), uint32(d.offset-minOffsetSize)) + } else { + d.tokens[d.ti] = matchToken(uint32(prevLength-minMatchLength), uint32(prevOffset-minOffsetSize)) + } + d.ti++ + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if d.length <= d.fastSkipHashing { + var newIndex int + if d.fastSkipHashing != 0 { + newIndex = d.index + d.length + } else { + newIndex = prevLength - 1 + } + for d.index++; d.index < newIndex; d.index++ { + if d.index < d.maxInsertIndex { + d.hash = (d.hash< 0 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + } + d.windowEnd = 0 +} + +func (d *compressor) write(b []byte) (n int, err os.Error) { + n = len(b) + b = b[d.fill(d, b):] + for len(b) > 0 { + d.step(d) + b = b[d.fill(d, b):] + } + return n, d.err +} + +func (d *compressor) syncFlush() os.Error { + d.sync = true + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err os.Error) { + d.w = newHuffmanBitWriter(w) + + switch { + case level == NoCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).store + case level == DefaultCompression: + level = 6 + fallthrough + case 1 <= level && level <= 9: + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + d.step = (*compressor).deflate + default: + return WrongValueError{"level", 0, 9, int32(level)} + } + return nil +} + +func (d *compressor) close() os.Error { + d.sync = true + d.step(d) + if d.err != nil { + return d.err + } + if d.w.writeStoredHeader(0, true); d.w.err != nil { + return d.w.err + } + d.w.flush() + return d.w.err +} + +// NewWriter returns a new Writer compressing +// data at the given level. Following zlib, levels +// range from 1 (BestSpeed) to 9 (BestCompression); +// higher levels typically run slower but compress more. +// Level 0 (NoCompression) does not attempt any +// compression; it only adds the necessary DEFLATE framing. +func NewWriter(w io.Writer, level int) *Writer { + const logWindowSize = logMaxOffsetSize + var dw Writer + dw.d.init(w, level) + return &dw +} + +// NewWriterDict is like NewWriter but initializes the new +// Writer with a preset dictionary. The returned Writer behaves +// as if the dictionary had been written to it without producing +// any compressed output. The compressed data written to w +// can only be decompressed by a Reader initialized with the +// same dictionary. +func NewWriterDict(w io.Writer, level int, dict []byte) *Writer { + dw := &dictWriter{w, false} + zw := NewWriter(dw, level) + zw.Write(dict) + zw.Flush() + dw.enabled = true + return zw +} + +type dictWriter struct { + w io.Writer + enabled bool +} + +func (w *dictWriter) Write(b []byte) (n int, err os.Error) { + if w.enabled { + return w.w.Write(b) + } + return len(b), nil +} + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see NewWriter). +type Writer struct { + d compressor +} + +// Write writes data to w, which will eventually write the +// compressed form of data to its underlying writer. +func (w *Writer) Write(data []byte) (n int, err os.Error) { + return w.d.write(data) +} + +// Flush flushes any pending compressed data to the underlying writer. +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. +// Flush does not return until the data has been written. +// If the underlying writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (w *Writer) Flush() os.Error { + // For more about flushing: + // http://www.bolet.org/~pornin/deflate-flush.html + return w.d.syncFlush() +} + +// Close flushes and closes the writer. +func (w *Writer) Close() os.Error { + return w.d.close() +} diff --git a/src/pkg/compress/flate/deflate_test.go b/src/pkg/compress/flate/deflate_test.go new file mode 100644 index 000000000..930823685 --- /dev/null +++ b/src/pkg/compress/flate/deflate_test.go @@ -0,0 +1,321 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "sync" + "testing" +) + +type deflateTest struct { + in []byte + level int + out []byte +} + +type deflateInflateTest struct { + in []byte +} + +type reverseBitsTest struct { + in uint16 + bitCount uint8 + out uint16 +} + +var deflateTests = []*deflateTest{ + &deflateTest{[]byte{}, 0, []byte{1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, -1, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, DefaultCompression, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, 4, []byte{18, 4, 4, 0, 0, 255, 255}}, + + &deflateTest{[]byte{0x11}, 0, []byte{0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x12}, 0, []byte{0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0, + []byte{0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255}, + }, + &deflateTest{[]byte{}, 1, []byte{1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, 1, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x12}, 1, []byte{18, 20, 2, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 1, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}}, + &deflateTest{[]byte{}, 9, []byte{1, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11}, 9, []byte{18, 4, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x12}, 9, []byte{18, 20, 2, 4, 0, 0, 255, 255}}, + &deflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}}, +} + +var deflateInflateTests = []*deflateInflateTest{ + &deflateInflateTest{[]byte{}}, + &deflateInflateTest{[]byte{0x11}}, + &deflateInflateTest{[]byte{0x11, 0x12}}, + &deflateInflateTest{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}}, + &deflateInflateTest{[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}}, + &deflateInflateTest{largeDataChunk()}, +} + +var reverseBitsTests = []*reverseBitsTest{ + &reverseBitsTest{1, 1, 1}, + &reverseBitsTest{1, 2, 2}, + &reverseBitsTest{1, 3, 4}, + &reverseBitsTest{1, 4, 8}, + &reverseBitsTest{1, 5, 16}, + &reverseBitsTest{17, 5, 17}, + &reverseBitsTest{257, 9, 257}, + &reverseBitsTest{29, 5, 23}, +} + +func largeDataChunk() []byte { + result := make([]byte, 100000) + for i := range result { + result[i] = byte(i * i & 0xFF) + } + return result +} + +func TestDeflate(t *testing.T) { + for _, h := range deflateTests { + var buf bytes.Buffer + w := NewWriter(&buf, h.level) + w.Write(h.in) + w.Close() + if !bytes.Equal(buf.Bytes(), h.out) { + t.Errorf("Deflate(%d, %x) = %x, want %x", h.level, h.in, buf.Bytes(), h.out) + } + } +} + +type syncBuffer struct { + buf bytes.Buffer + mu sync.RWMutex + closed bool + ready chan bool +} + +func newSyncBuffer() *syncBuffer { + return &syncBuffer{ready: make(chan bool, 1)} +} + +func (b *syncBuffer) Read(p []byte) (n int, err os.Error) { + for { + b.mu.RLock() + n, err = b.buf.Read(p) + b.mu.RUnlock() + if n > 0 || b.closed { + return + } + <-b.ready + } + panic("unreachable") +} + +func (b *syncBuffer) signal() { + select { + case b.ready <- true: + default: + } +} + +func (b *syncBuffer) Write(p []byte) (n int, err os.Error) { + n, err = b.buf.Write(p) + b.signal() + return +} + +func (b *syncBuffer) WriteMode() { + b.mu.Lock() +} + +func (b *syncBuffer) ReadMode() { + b.mu.Unlock() + b.signal() +} + +func (b *syncBuffer) Close() os.Error { + b.closed = true + b.signal() + return nil +} + +func testSync(t *testing.T, level int, input []byte, name string) { + if len(input) == 0 { + return + } + + t.Logf("--testSync %d, %d, %s", level, len(input), name) + buf := newSyncBuffer() + buf1 := new(bytes.Buffer) + buf.WriteMode() + w := NewWriter(io.MultiWriter(buf, buf1), level) + r := NewReader(buf) + + // Write half the input and read back. + for i := 0; i < 2; i++ { + var lo, hi int + if i == 0 { + lo, hi = 0, (len(input)+1)/2 + } else { + lo, hi = (len(input)+1)/2, len(input) + } + t.Logf("#%d: write %d-%d", i, lo, hi) + if _, err := w.Write(input[lo:hi]); err != nil { + t.Errorf("testSync: write: %v", err) + return + } + if i == 0 { + if err := w.Flush(); err != nil { + t.Errorf("testSync: flush: %v", err) + return + } + } else { + if err := w.Close(); err != nil { + t.Errorf("testSync: close: %v", err) + } + } + buf.ReadMode() + out := make([]byte, hi-lo+1) + m, err := io.ReadAtLeast(r, out, hi-lo) + t.Logf("#%d: read %d", i, m) + if m != hi-lo || err != nil { + t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len()) + return + } + if !bytes.Equal(input[lo:hi], out[:hi-lo]) { + t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) + return + } + // This test originally checked that after reading + // the first half of the input, there was nothing left + // in the read buffer (buf.buf.Len() != 0) but that is + // not necessarily the case: the write Flush may emit + // some extra framing bits that are not necessary + // to process to obtain the first half of the uncompressed + // data. The test ran correctly most of the time, because + // the background goroutine had usually read even + // those extra bits by now, but it's not a useful thing to + // check. + buf.WriteMode() + } + buf.ReadMode() + out := make([]byte, 10) + if n, err := r.Read(out); n > 0 || err != os.EOF { + t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n]) + } + if buf.buf.Len() != 0 { + t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name) + } + r.Close() + + // stream should work for ordinary reader too + r = NewReader(buf1) + out, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("testSync: read: %s", err) + return + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name) + } +} + +func testToFromWithLevel(t *testing.T, level int, input []byte, name string) os.Error { + buffer := bytes.NewBuffer(nil) + w := NewWriter(buffer, level) + w.Write(input) + w.Close() + r := NewReader(buffer) + out, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("read: %s", err) + return err + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) + } + + testSync(t, level, input, name) + return nil +} + +func testToFrom(t *testing.T, input []byte, name string) { + for i := 0; i < 10; i++ { + testToFromWithLevel(t, i, input, name) + } +} + +func TestDeflateInflate(t *testing.T) { + for i, h := range deflateInflateTests { + testToFrom(t, h.in, fmt.Sprintf("#%d", i)) + } +} + +func TestReverseBits(t *testing.T) { + for _, h := range reverseBitsTests { + if v := reverseBits(h.in, h.bitCount); v != h.out { + t.Errorf("reverseBits(%v,%v) = %v, want %v", + h.in, h.bitCount, v, h.out) + } + } +} + +func TestDeflateInflateString(t *testing.T) { + gold, err := ioutil.ReadFile("../testdata/e.txt") + if err != nil { + t.Error(err) + } + testToFromWithLevel(t, 1, gold, "2.718281828...") +} + +func TestReaderDict(t *testing.T) { + const ( + dict = "hello world" + text = "hello again world" + ) + var b bytes.Buffer + w := NewWriter(&b, 5) + w.Write([]byte(dict)) + w.Flush() + b.Reset() + w.Write([]byte(text)) + w.Close() + + r := NewReaderDict(&b, []byte(dict)) + data, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello again world" { + t.Fatalf("read returned %q want %q", string(data), text) + } +} + +func TestWriterDict(t *testing.T) { + const ( + dict = "hello world" + text = "hello again world" + ) + var b bytes.Buffer + w := NewWriter(&b, 5) + w.Write([]byte(dict)) + w.Flush() + b.Reset() + w.Write([]byte(text)) + w.Close() + + var b1 bytes.Buffer + w = NewWriterDict(&b1, 5, []byte(dict)) + w.Write([]byte(text)) + w.Close() + + if !bytes.Equal(b1.Bytes(), b.Bytes()) { + t.Fatalf("writer wrote %q want %q", b1.Bytes(), b.Bytes()) + } +} diff --git a/src/pkg/compress/flate/flate_test.go b/src/pkg/compress/flate/flate_test.go new file mode 100644 index 000000000..bfd3b8381 --- /dev/null +++ b/src/pkg/compress/flate/flate_test.go @@ -0,0 +1,139 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test tests some internals of the flate package. +// The tests in package compress/gzip serve as the +// end-to-end test of the decompressor. + +package flate + +import ( + "bytes" + "reflect" + "testing" +) + +// The Huffman code lengths used by the fixed-format Huffman blocks. +var fixedHuffmanBits = [...]int{ + // 0-143 length 8 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + + // 144-255 length 9 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + + // 256-279 length 7 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + + // 280-287 length 8 + 8, 8, 8, 8, 8, 8, 8, 8, +} + +type InitDecoderTest struct { + in []int + out huffmanDecoder + ok bool +} + +var initDecoderTests = []*InitDecoderTest{ + // Example from Connell 1973, + &InitDecoderTest{ + []int{3, 5, 2, 4, 3, 5, 5, 4, 4, 3, 4, 5}, + huffmanDecoder{ + 2, 5, + [maxCodeLen + 1]int{2: 0, 4, 13, 31}, + [maxCodeLen + 1]int{2: 0, 1, 6, 20}, + // Paper used different code assignment: + // 2, 9, 4, 0, 10, 8, 3, 7, 1, 5, 11, 6 + // Reordered here so that codes of same length + // are assigned to increasing numbers. + []int{2, 0, 4, 9, 3, 7, 8, 10, 1, 5, 6, 11}, + }, + true, + }, + + // Example from RFC 1951 section 3.2.2 + &InitDecoderTest{ + []int{2, 1, 3, 3}, + huffmanDecoder{ + 1, 3, + [maxCodeLen + 1]int{1: 0, 2, 7}, + [maxCodeLen + 1]int{1: 0, 1, 4}, + []int{1, 0, 2, 3}, + }, + true, + }, + + // Second example from RFC 1951 section 3.2.2 + &InitDecoderTest{ + []int{3, 3, 3, 3, 3, 2, 4, 4}, + huffmanDecoder{ + 2, 4, + [maxCodeLen + 1]int{2: 0, 6, 15}, + [maxCodeLen + 1]int{2: 0, 1, 8}, + []int{5, 0, 1, 2, 3, 4, 6, 7}, + }, + true, + }, + + // Static Huffman codes (RFC 1951 section 3.2.6) + &InitDecoderTest{ + fixedHuffmanBits[0:], + fixedHuffmanDecoder, + true, + }, + + // Illegal input. + &InitDecoderTest{ + []int{}, + huffmanDecoder{}, + false, + }, + + // Illegal input. + &InitDecoderTest{ + []int{0, 0, 0, 0, 0, 0, 0}, + huffmanDecoder{}, + false, + }, +} + +func TestInitDecoder(t *testing.T) { + for i, tt := range initDecoderTests { + var h huffmanDecoder + if h.init(tt.in) != tt.ok { + t.Errorf("test %d: init = %v", i, !tt.ok) + continue + } + if !reflect.DeepEqual(&h, &tt.out) { + t.Errorf("test %d:\nhave %v\nwant %v", i, h, tt.out) + } + } +} + +func TestUncompressedSource(t *testing.T) { + decoder := NewReader(bytes.NewBuffer([]byte{0x01, 0x01, 0x00, 0xfe, 0xff, 0x11})) + output := make([]byte, 1) + n, error := decoder.Read(output) + if n != 1 || error != nil { + t.Fatalf("decoder.Read() = %d, %v, want 1, nil", n, error) + } + if output[0] != 0x11 { + t.Errorf("output[0] = %x, want 0x11", output[0]) + } +} diff --git a/src/pkg/compress/flate/huffman_bit_writer.go b/src/pkg/compress/flate/huffman_bit_writer.go new file mode 100644 index 000000000..3981df5cb --- /dev/null +++ b/src/pkg/compress/flate/huffman_bit_writer.go @@ -0,0 +1,494 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" + "os" + "strconv" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 +) + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = []int8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = []uint32{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// offset code word extra bits. +var offsetExtraBits = []int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + /* extended window */ + 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, +} + +var offsetBase = []uint32{ + /* normal deflate */ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, + + /* extended window */ + 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000, + 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000, + 0x100000, 0x180000, 0x200000, 0x300000, +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + w io.Writer + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. + bits uint32 + nbits uint32 + bytes [64]byte + nbytes int + literalFreq []int32 + offsetFreq []int32 + codegen []uint8 + codegenFreq []int32 + literalEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err os.Error +} + +type WrongValueError struct { + name string + from int32 + to int32 + value int32 +} + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + w: w, + literalFreq: make([]int32, maxLit), + offsetFreq: make([]int32, offsetCodeCount), + codegen: make([]uint8, maxLit+offsetCodeCount+1), + codegenFreq: make([]int32, codegenCodeCount), + literalEncoding: newHuffmanEncoder(maxLit), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + } +} + +func (err WrongValueError) String() string { + return "huffmanBitWriter: " + err.name + " should belong to [" + strconv.Itoa64(int64(err.from)) + ";" + + strconv.Itoa64(int64(err.to)) + "] but actual value is " + strconv.Itoa64(int64(err.value)) +} + +func (w *huffmanBitWriter) flushBits() { + if w.err != nil { + w.nbits = 0 + return + } + bits := w.bits + w.bits >>= 16 + w.nbits -= 16 + n := w.nbytes + w.bytes[n] = byte(bits) + w.bytes[n+1] = byte(bits >> 8) + if n += 2; n >= len(w.bytes) { + _, w.err = w.w.Write(w.bytes[0:]) + n = 0 + } + w.nbytes = n +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + n := w.nbytes + if w.nbits > 8 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if w.nbits > 0 { + w.bytes[n] = byte(w.bits) + w.nbits = 0 + n++ + } + w.bits = 0 + _, w.err = w.w.Write(w.bytes[0:n]) + w.nbytes = 0 +} + +func (w *huffmanBitWriter) writeBits(b, nb int32) { + w.bits |= uint32(b) << w.nbits + if w.nbits += uint32(nb); w.nbits >= 16 { + w.flushBits() + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + if w.nbits == 8 { + w.bytes[n] = byte(w.bits) + w.nbits = 0 + n++ + } + if w.nbits != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + if n != 0 { + _, w.err = w.w.Write(w.bytes[0:n]) + if w.err != nil { + return + } + } + w.nbytes = 0 + _, w.err = w.w.Write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int) { + fillInt32s(w.codegenFreq, 0) + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + copyUint8s(codegen[0:numLiterals], w.literalEncoding.codeBits) + copyUint8s(codegen[numLiterals:numLiterals+numOffsets], w.offsetEncoding.codeBits) + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := min(count, 6) + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := min(count, 138) + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +func (w *huffmanBitWriter) writeCode(code *huffmanEncoder, literal uint32) { + if w.err != nil { + return + } + w.writeBits(int32(code.code[literal]), int32(code.codeBits[literal])) +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens The number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + w.writeBits(int32(numOffsets-1), 5) + w.writeBits(int32(numCodegens-4), 4) + + for i := 0; i < numCodegens; i++ { + value := w.codegenEncoding.codeBits[codegenOrder[i]] + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord int = int(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + // The low byte contains the actual code to generate. + w.writeCode(w.codegenEncoding, uint32(codeWord)) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + break + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + break + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + break + } + } +} + +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + fillInt32s(w.literalFreq, 0) + fillInt32s(w.offsetFreq, 0) + + n := len(tokens) + tokens = tokens[0 : n+1] + tokens[n] = endBlockMarker + + for _, t := range tokens { + switch t.typ() { + case literalType: + w.literalFreq[t.literal()]++ + case matchType: + length := t.length() + offset := t.offset() + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + } + } + + // get the number of literals + numLiterals := len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets := len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + + storedBytes := 0 + if input != nil { + storedBytes = len(input) + } + var extraBits int64 + var storedSize int64 = math.MaxInt64 + if storedBytes <= maxStoreBlockSize && input != nil { + storedSize = int64((storedBytes + 5) * 8) + // We only bother calculating the costs of the extra bits required by + // the length of offset fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { + // First eight length codes have extra size = 0. + extraBits += int64(w.literalFreq[lengthCode]) * int64(lengthExtraBits[lengthCode-lengthCodesStart]) + } + for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { + // First four offset codes have extra size = 0. + extraBits += int64(w.offsetFreq[offsetCode]) * int64(offsetExtraBits[offsetCode]) + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var size = int64(3) + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + + // Dynamic Huffman? + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets) + w.codegenEncoding.generate(w.codegenFreq, 7) + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + dynamicHeader := int64(3+5+5+4+(3*numCodegens)) + + w.codegenEncoding.bitLength(w.codegenFreq) + + int64(extraBits) + + int64(w.codegenFreq[16]*2) + + int64(w.codegenFreq[17]*3) + + int64(w.codegenFreq[18]*7) + dynamicSize := dynamicHeader + + w.literalEncoding.bitLength(w.literalFreq) + + w.offsetEncoding.bitLength(w.offsetFreq) + + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storedSize < size { + w.writeStoredHeader(storedBytes, eof) + w.writeBytes(input[0:storedBytes]) + return + } + + // Huffman. + if literalEncoding == fixedLiteralEncoding { + w.writeFixedHeader(eof) + } else { + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + } + for _, t := range tokens { + switch t.typ() { + case literalType: + w.writeCode(literalEncoding, t.literal()) + break + case matchType: + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(literalEncoding, lengthCode+lengthCodesStart) + extraLengthBits := int32(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(offsetEncoding, offsetCode) + extraOffsetBits := int32(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + break + default: + panic("unknown token type: " + string(t)) + } + } +} diff --git a/src/pkg/compress/flate/huffman_code.go b/src/pkg/compress/flate/huffman_code.go new file mode 100644 index 000000000..7ed603a4f --- /dev/null +++ b/src/pkg/compress/flate/huffman_code.go @@ -0,0 +1,378 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "math" + "sort" +) + +type huffmanEncoder struct { + codeBits []uint8 + code []uint16 +} + +type literalNode struct { + literal uint16 + freq int32 +} + +type chain struct { + // The sum of the leaves in this tree + freq int32 + + // The number of literals to the left of this item at this level + leafCount int32 + + // The right child of this chain in the previous level. + up *chain +} + +type levelInfo struct { + // Our level. for better printing + level int32 + + // The most recent chain generated for this level + lastChain *chain + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 + + // The levelInfo for level+1 + up *levelInfo + + // The levelInfo for level-1 + down *levelInfo +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + return &huffmanEncoder{make([]uint8, size), make([]uint16, size)} +} + +// Generates a HuffmanCode corresponding to the fixed literal table +func generateFixedLiteralEncoding() *huffmanEncoder { + h := newHuffmanEncoder(maxLit) + codeBits := h.codeBits + code := h.code + var ch uint16 + for ch = 0; ch < maxLit; ch++ { + var bits uint16 + var size uint8 + switch { + case ch < 144: + // size 8, 000110000 .. 10111111 + bits = ch + 48 + size = 8 + break + case ch < 256: + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144 + size = 9 + break + case ch < 280: + // size 7, 0000000 .. 0010111 + bits = ch - 256 + size = 7 + break + default: + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280 + size = 8 + } + codeBits[ch] = size + code[ch] = reverseBits(bits, size) + } + return h +} + +func generateFixedOffsetEncoding() *huffmanEncoder { + h := newHuffmanEncoder(30) + codeBits := h.codeBits + code := h.code + for ch := uint16(0); ch < 30; ch++ { + codeBits[ch] = 5 + code[ch] = reverseBits(ch, 5) + } + return h +} + +var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() +var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() + +func (h *huffmanEncoder) bitLength(freq []int32) int64 { + var total int64 + for i, f := range freq { + if f != 0 { + total += int64(f) * int64(h.codeBits[i]) + } + } + return total +} + +// Generate elements in the chain using an iterative algorithm. +func (h *huffmanEncoder) generateChains(top *levelInfo, list []literalNode) { + n := len(list) + list = list[0 : n+1] + list[n] = maxNode() + + l := top + for { + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To m sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.lastChain = nil + l.needed = 0 + l = l.up + l.nextPairFreq = math.MaxInt32 + continue + } + + prevFreq := l.lastChain.freq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := l.lastChain.leafCount + 1 + l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} + l.down.needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + up := l.up + if up == nil { + // All done! + return + } + up.nextPairFreq = prevFreq + l.lastChain.freq + l = up + } else { + // If we stole from below, move down temporarily to replenish it. + for l.down.needed > 0 { + l = l.down + } + } + } +} + +// Return the number of literals assigned to each bit size in the Huffman encoding +// +// This method is only called when list.length >= 3 +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list An array of the literals with non-zero frequencies +// and their associated frequencies. The array is in order of increasing +// frequency, and has as its last element a special element with frequency +// MaxInt32 +// maxBits The maximum number of bits that should be used to encode any literal. +// return An integer array in which array[i] indicates the number of literals +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + maxBits = minInt32(maxBits, n-1) + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + top := &levelInfo{needed: 0} + chain2 := &chain{list[1].freq, 2, new(chain)} + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + top = &levelInfo{ + level: level, + lastChain: chain2, + nextCharFreq: list[2].freq, + nextPairFreq: list[0].freq + list[1].freq, + down: top, + } + top.down.up = top + if level == 1 { + top.nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + top.needed = 2*n - 4 + + l := top + for { + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To m sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.lastChain = nil + l.needed = 0 + l = l.up + l.nextPairFreq = math.MaxInt32 + continue + } + + prevFreq := l.lastChain.freq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := l.lastChain.leafCount + 1 + l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} + l.down.needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + up := l.up + if up == nil { + // All done! + break + } + up.nextPairFreq = prevFreq + l.lastChain.freq + l = up + } else { + // If we stole from below, move down temporarily to replenish it. + for l.down.needed > 0 { + l = l.down + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if top.lastChain.leafCount != n { + panic("top.lastChain.leafCount != n") + } + + bitCount := make([]int32, maxBits+1) + bits := 1 + for chain := top.lastChain; chain.up != nil; chain = chain.up { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = chain.leafCount - chain.up.leafCount + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + sortByLiteral(chunk) + for _, node := range chunk { + h.codeBits[node.literal] = uint8(n) + h.code[node.literal] = reverseBits(code, uint8(n)) + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { + list := make([]literalNode, len(freq)+1) + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + h.codeBits[i] = 0 + } + } + // If freq[] is shorter than codeBits[], fill rest of codeBits[] with zeros + h.codeBits = h.codeBits[0:len(freq)] + list = list[0:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codeBits[node.literal] = 1 + h.code[node.literal] = uint16(i) + } + return + } + sortByFreq(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +type literalNodeSorter struct { + a []literalNode + less func(i, j int) bool +} + +func (s literalNodeSorter) Len() int { return len(s.a) } + +func (s literalNodeSorter) Less(i, j int) bool { + return s.less(i, j) +} + +func (s literalNodeSorter) Swap(i, j int) { s.a[i], s.a[j] = s.a[j], s.a[i] } + +func sortByFreq(a []literalNode) { + s := &literalNodeSorter{a, func(i, j int) bool { + if a[i].freq == a[j].freq { + return a[i].literal < a[j].literal + } + return a[i].freq < a[j].freq + }} + sort.Sort(s) +} + +func sortByLiteral(a []literalNode) { + s := &literalNodeSorter{a, func(i, j int) bool { return a[i].literal < a[j].literal }} + sort.Sort(s) +} diff --git a/src/pkg/compress/flate/inflate.go b/src/pkg/compress/flate/inflate.go new file mode 100644 index 000000000..3845f1204 --- /dev/null +++ b/src/pkg/compress/flate/inflate.go @@ -0,0 +1,708 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package flate implements the DEFLATE compressed data format, described in +// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file +// formats. +package flate + +import ( + "bufio" + "io" + "os" + "strconv" +) + +const ( + maxCodeLen = 16 // max length of Huffman code + maxHist = 32768 // max history required + maxLit = 286 + maxDist = 32 + numCodes = 19 // number of codes in Huffman meta-code +) + +// A CorruptInputError reports the presence of corrupt input at a given offset. +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "flate: corrupt input before offset " + strconv.Itoa64(int64(e)) +} + +// An InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) String() string { return "flate: internal error: " + string(e) } + +// A ReadError reports an error encountered while reading input. +type ReadError struct { + Offset int64 // byte offset where error occurred + Error os.Error // error returned by underlying Read +} + +func (e *ReadError) String() string { + return "flate: read error at offset " + strconv.Itoa64(e.Offset) + ": " + e.Error.String() +} + +// A WriteError reports an error encountered while writing output. +type WriteError struct { + Offset int64 // byte offset where error occurred + Error os.Error // error returned by underlying Write +} + +func (e *WriteError) String() string { + return "flate: write error at offset " + strconv.Itoa64(e.Offset) + ": " + e.Error.String() +} + +// Huffman decoder is based on +// J. Brian Connell, ``A Huffman-Shannon-Fano Code,'' +// Proceedings of the IEEE, 61(7) (July 1973), pp 1046-1047. +type huffmanDecoder struct { + // min, max code length + min, max int + + // limit[i] = largest code word of length i + // Given code v of length n, + // need more bits if v > limit[n]. + limit [maxCodeLen + 1]int + + // base[i] = smallest code word of length i - seq number + base [maxCodeLen + 1]int + + // codes[seq number] = output code. + // Given code v of length n, value is + // codes[v - base[n]]. + codes []int +} + +// Initialize Huffman decoding tables from array of code lengths. +func (h *huffmanDecoder) init(bits []int) bool { + // Count number of codes of each length, + // compute min and max length. + var count [maxCodeLen + 1]int + var min, max int + for _, n := range bits { + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if n > max { + max = n + } + count[n]++ + } + if max == 0 { + return false + } + + h.min = min + h.max = max + + // For each code range, compute + // nextcode (first code of that length), + // limit (last code of that length), and + // base (offset from first code to sequence number). + code := 0 + seq := 0 + var nextcode [maxCodeLen]int + for i := min; i <= max; i++ { + n := count[i] + nextcode[i] = code + h.base[i] = code - seq + code += n + seq += n + h.limit[i] = code - 1 + code <<= 1 + } + + // Make array mapping sequence numbers to codes. + if len(h.codes) < len(bits) { + h.codes = make([]int, len(bits)) + } + for i, n := range bits { + if n == 0 { + continue + } + code := nextcode[n] + nextcode[n]++ + seq := code - h.base[n] + h.codes[seq] = i + } + return true +} + +// Hard-coded Huffman tables for DEFLATE algorithm. +// See RFC 1951, section 3.2.6. +var fixedHuffmanDecoder = huffmanDecoder{ + 7, 9, + [maxCodeLen + 1]int{7: 23, 199, 511}, + [maxCodeLen + 1]int{7: 0, 24, 224}, + []int{ + // length 7: 256-279 + 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, + + // length 8: 0-143 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, + 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, + + // length 8: 280-287 + 280, 281, 282, 283, 284, 285, 286, 287, + + // length 9: 144-255 + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + }, +} + +// The actual read interface needed by NewReader. +// If the passed in io.Reader does not also have ReadByte, +// the NewReader will introduce its own buffering. +type Reader interface { + io.Reader + ReadByte() (c byte, err os.Error) +} + +// Decompress state. +type decompressor struct { + // Input source. + r Reader + roffset int64 + woffset int64 + + // Input bits, in top of b. + b uint32 + nb uint + + // Huffman decoders for literal/length, distance. + h1, h2 huffmanDecoder + + // Length arrays used to define Huffman codes. + bits [maxLit + maxDist]int + codebits [numCodes]int + + // Output history, buffer. + hist [maxHist]byte + hp int // current output position in buffer + hw int // have written hist[0:hw] already + hfull bool // buffer has filled at least once + + // Temporary buffer (avoids repeated allocation). + buf [4]byte + + // Next step in the decompression, + // and decompression state. + step func(*decompressor) + final bool + err os.Error + toRead []byte + hl, hd *huffmanDecoder + copyLen int + copyDist int +} + +func (f *decompressor) nextBlock() { + if f.final { + if f.hw != f.hp { + f.flush((*decompressor).nextBlock) + return + } + f.err = os.EOF + return + } + for f.nb < 1+2 { + if f.err = f.moreBits(); f.err != nil { + return + } + } + f.final = f.b&1 == 1 + f.b >>= 1 + typ := f.b & 3 + f.b >>= 2 + f.nb -= 1 + 2 + switch typ { + case 0: + f.dataBlock() + case 1: + // compressed, fixed Huffman tables + f.hl = &fixedHuffmanDecoder + f.hd = nil + f.huffmanBlock() + case 2: + // compressed, dynamic Huffman tables + if f.err = f.readHuffman(); f.err != nil { + break + } + f.hl = &f.h1 + f.hd = &f.h2 + f.huffmanBlock() + default: + // 3 is reserved. + f.err = CorruptInputError(f.roffset) + } +} + +func (f *decompressor) Read(b []byte) (int, os.Error) { + for { + if len(f.toRead) > 0 { + n := copy(b, f.toRead) + f.toRead = f.toRead[n:] + return n, nil + } + if f.err != nil { + return 0, f.err + } + f.step(f) + } + panic("unreachable") +} + +func (f *decompressor) Close() os.Error { + if f.err == os.EOF { + return nil + } + return f.err +} + +// RFC 1951 section 3.2.7. +// Compression with dynamic Huffman codes + +var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +func (f *decompressor) readHuffman() os.Error { + // HLIT[5], HDIST[5], HCLEN[4]. + for f.nb < 5+5+4 { + if err := f.moreBits(); err != nil { + return err + } + } + nlit := int(f.b&0x1F) + 257 + f.b >>= 5 + ndist := int(f.b&0x1F) + 1 + f.b >>= 5 + nclen := int(f.b&0xF) + 4 + f.b >>= 4 + f.nb -= 5 + 5 + 4 + + // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. + for i := 0; i < nclen; i++ { + for f.nb < 3 { + if err := f.moreBits(); err != nil { + return err + } + } + f.codebits[codeOrder[i]] = int(f.b & 0x7) + f.b >>= 3 + f.nb -= 3 + } + for i := nclen; i < len(codeOrder); i++ { + f.codebits[codeOrder[i]] = 0 + } + if !f.h1.init(f.codebits[0:]) { + return CorruptInputError(f.roffset) + } + + // HLIT + 257 code lengths, HDIST + 1 code lengths, + // using the code length Huffman code. + for i, n := 0, nlit+ndist; i < n; { + x, err := f.huffSym(&f.h1) + if err != nil { + return err + } + if x < 16 { + // Actual length. + f.bits[i] = x + i++ + continue + } + // Repeat previous length or zero. + var rep int + var nb uint + var b int + switch x { + default: + return InternalError("unexpected length code") + case 16: + rep = 3 + nb = 2 + if i == 0 { + return CorruptInputError(f.roffset) + } + b = f.bits[i-1] + case 17: + rep = 3 + nb = 3 + b = 0 + case 18: + rep = 11 + nb = 7 + b = 0 + } + for f.nb < nb { + if err := f.moreBits(); err != nil { + return err + } + } + rep += int(f.b & uint32(1<>= nb + f.nb -= nb + if i+rep > n { + return CorruptInputError(f.roffset) + } + for j := 0; j < rep; j++ { + f.bits[i] = b + i++ + } + } + + if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { + return CorruptInputError(f.roffset) + } + + return nil +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanBlock() { + for { + v, err := f.huffSym(f.hl) + if err != nil { + f.err = err + return + } + var n uint // number of bits extra + var length int + switch { + case v < 256: + f.hist[f.hp] = byte(v) + f.hp++ + if f.hp == len(f.hist) { + // After the flush, continue this loop. + f.flush((*decompressor).huffmanBlock) + return + } + continue + case v == 256: + // Done with huffman block; read next block. + f.step = (*decompressor).nextBlock + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + n = 0 + case v < 269: + length = v*2 - (265*2 - 11) + n = 1 + case v < 273: + length = v*4 - (269*4 - 19) + n = 2 + case v < 277: + length = v*8 - (273*8 - 35) + n = 3 + case v < 281: + length = v*16 - (277*16 - 67) + n = 4 + case v < 285: + length = v*32 - (281*32 - 131) + n = 5 + default: + length = 258 + n = 0 + } + if n > 0 { + for f.nb < n { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + length += int(f.b & uint32(1<>= n + f.nb -= n + } + + var dist int + if f.hd == nil { + for f.nb < 5 { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + dist = int(reverseByte[(f.b&0x1F)<<3]) + f.b >>= 5 + f.nb -= 5 + } else { + if dist, err = f.huffSym(f.hd); err != nil { + f.err = err + return + } + } + + switch { + case dist < 4: + dist++ + case dist >= 30: + f.err = CorruptInputError(f.roffset) + return + default: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << nb + for f.nb < nb { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + extra |= int(f.b & uint32(1<>= nb + f.nb -= nb + dist = 1<<(nb+1) + 1 + extra + } + + // Copy history[-dist:-dist+length] into output. + if dist > len(f.hist) { + f.err = InternalError("bad history distance") + return + } + + // No check on length; encoding can be prescient. + if !f.hfull && dist > f.hp { + f.err = CorruptInputError(f.roffset) + return + } + + p := f.hp - dist + if p < 0 { + p += len(f.hist) + } + for i := 0; i < length; i++ { + f.hist[f.hp] = f.hist[p] + f.hp++ + p++ + if f.hp == len(f.hist) { + // After flush continue copying out of history. + f.copyLen = length - (i + 1) + f.copyDist = dist + f.flush((*decompressor).copyHuff) + return + } + if p == len(f.hist) { + p = 0 + } + } + } + panic("unreached") +} + +func (f *decompressor) copyHuff() { + length := f.copyLen + dist := f.copyDist + p := f.hp - dist + if p < 0 { + p += len(f.hist) + } + for i := 0; i < length; i++ { + f.hist[f.hp] = f.hist[p] + f.hp++ + p++ + if f.hp == len(f.hist) { + f.copyLen = length - (i + 1) + f.flush((*decompressor).copyHuff) + return + } + if p == len(f.hist) { + p = 0 + } + } + + // Continue processing Huffman block. + f.huffmanBlock() +} + +// Copy a single uncompressed data block from input to output. +func (f *decompressor) dataBlock() { + // Uncompressed. + // Discard current half-byte. + f.nb = 0 + f.b = 0 + + // Length then ones-complement of length. + nr, err := io.ReadFull(f.r, f.buf[0:4]) + f.roffset += int64(nr) + if err != nil { + f.err = &ReadError{f.roffset, err} + return + } + n := int(f.buf[0]) | int(f.buf[1])<<8 + nn := int(f.buf[2]) | int(f.buf[3])<<8 + if uint16(nn) != uint16(^n) { + f.err = CorruptInputError(f.roffset) + return + } + + if n == 0 { + // 0-length block means sync + f.flush((*decompressor).nextBlock) + return + } + + f.copyLen = n + f.copyData() +} + +func (f *decompressor) copyData() { + // Read f.dataLen bytes into history, + // pausing for reads as history fills. + n := f.copyLen + for n > 0 { + m := len(f.hist) - f.hp + if m > n { + m = n + } + m, err := io.ReadFull(f.r, f.hist[f.hp:f.hp+m]) + f.roffset += int64(m) + if err != nil { + f.err = &ReadError{f.roffset, err} + return + } + n -= m + f.hp += m + if f.hp == len(f.hist) { + f.copyLen = n + f.flush((*decompressor).copyData) + return + } + } + f.step = (*decompressor).nextBlock +} + +func (f *decompressor) setDict(dict []byte) { + if len(dict) > len(f.hist) { + // Will only remember the tail. + dict = dict[len(dict)-len(f.hist):] + } + + f.hp = copy(f.hist[:], dict) + if f.hp == len(f.hist) { + f.hp = 0 + f.hfull = true + } + f.hw = f.hp +} + +func (f *decompressor) moreBits() os.Error { + c, err := f.r.ReadByte() + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 + return nil +} + +// Read the next Huffman-encoded symbol from f according to h. +func (f *decompressor) huffSym(h *huffmanDecoder) (int, os.Error) { + for n := uint(h.min); n <= uint(h.max); n++ { + lim := h.limit[n] + if lim == -1 { + continue + } + for f.nb < n { + if err := f.moreBits(); err != nil { + return 0, err + } + } + v := int(f.b & uint32(1<>8]) | int(reverseByte[v&0xFF])<<8 // reverse bits + if v <= lim { + f.b >>= n + f.nb -= n + return h.codes[v-h.base[n]], nil + } + } + return 0, CorruptInputError(f.roffset) +} + +// Flush any buffered output to the underlying writer. +func (f *decompressor) flush(step func(*decompressor)) { + f.toRead = f.hist[f.hw:f.hp] + f.woffset += int64(f.hp - f.hw) + f.hw = f.hp + if f.hp == len(f.hist) { + f.hp = 0 + f.hw = 0 + f.hfull = true + } + f.step = step +} + +func makeReader(r io.Reader) Reader { + if rr, ok := r.(Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. It is the caller's +// responsibility to call Close on the ReadCloser when +// finished reading. +func NewReader(r io.Reader) io.ReadCloser { + var f decompressor + f.r = makeReader(r) + f.step = (*decompressor).nextBlock + return &f +} + +// NewReaderDict is like NewReader but initializes the reader +// with a preset dictionary. The returned Reader behaves as if +// the uncompressed data stream started with the given dictionary, +// which has already been read. NewReaderDict is typically used +// to read data compressed by NewWriterDict. +func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { + var f decompressor + f.setDict(dict) + f.r = makeReader(r) + f.step = (*decompressor).nextBlock + return &f +} diff --git a/src/pkg/compress/flate/reverse_bits.go b/src/pkg/compress/flate/reverse_bits.go new file mode 100644 index 000000000..c1a02720d --- /dev/null +++ b/src/pkg/compress/flate/reverse_bits.go @@ -0,0 +1,48 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +var reverseByte = [256]byte{ + 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, + 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, + 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, + 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, + 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, + 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, + 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, + 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, + 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, + 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, + 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, + 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, + 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, + 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, + 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, + 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, + 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, + 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, + 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, + 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, + 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, + 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, + 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, + 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, + 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, + 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, + 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, + 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, + 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, + 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, + 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, + 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +} + +func reverseUint16(v uint16) uint16 { + return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 +} + +func reverseBits(number uint16, bitLength byte) uint16 { + return reverseUint16(number << uint8(16-bitLength)) +} diff --git a/src/pkg/compress/flate/token.go b/src/pkg/compress/flate/token.go new file mode 100644 index 000000000..38aea5fa6 --- /dev/null +++ b/src/pkg/compress/flate/token.go @@ -0,0 +1,103 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +const ( + // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused + // 8 bits: xlength = length - MIN_MATCH_LENGTH + // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal + lengthShift = 22 + offsetMask = 1< pair into a match token. +func matchToken(xlength uint32, xoffset uint32) token { + return token(matchType + xlength<> lengthShift) } + +func lengthCode(len uint32) uint32 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset +func offsetCode(off uint32) uint32 { + const n = uint32(len(offsetCodes)) + switch { + case off < n: + return offsetCodes[off] + case off>>7 < n: + return offsetCodes[off>>7] + 14 + default: + return offsetCodes[off>>14] + 28 + } + panic("unreachable") +} diff --git a/src/pkg/compress/flate/util.go b/src/pkg/compress/flate/util.go new file mode 100644 index 000000000..aca5c78b2 --- /dev/null +++ b/src/pkg/compress/flate/util.go @@ -0,0 +1,72 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +func min(left int, right int) int { + if left < right { + return left + } + return right +} + +func minInt32(left int32, right int32) int32 { + if left < right { + return left + } + return right +} + +func max(left int, right int) int { + if left > right { + return left + } + return right +} + +func fillInts(a []int, value int) { + for i := range a { + a[i] = value + } +} + +func fillInt32s(a []int32, value int32) { + for i := range a { + a[i] = value + } +} + +func fillBytes(a []byte, value byte) { + for i := range a { + a[i] = value + } +} + +func fillInt8s(a []int8, value int8) { + for i := range a { + a[i] = value + } +} + +func fillUint8s(a []uint8, value uint8) { + for i := range a { + a[i] = value + } +} + +func copyInt8s(dst []int8, src []int8) int { + cnt := min(len(dst), len(src)) + for i := 0; i < cnt; i++ { + dst[i] = src[i] + } + return cnt +} + +func copyUint8s(dst []uint8, src []uint8) int { + cnt := min(len(dst), len(src)) + for i := 0; i < cnt; i++ { + dst[i] = src[i] + } + return cnt +} diff --git a/src/pkg/compress/gzip/Makefile b/src/pkg/compress/gzip/Makefile new file mode 100644 index 000000000..b671fc72c --- /dev/null +++ b/src/pkg/compress/gzip/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/gzip +GOFILES=\ + gunzip.go\ + gzip.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/gzip/gunzip.go b/src/pkg/compress/gzip/gunzip.go new file mode 100644 index 000000000..6ac9293d7 --- /dev/null +++ b/src/pkg/compress/gzip/gunzip.go @@ -0,0 +1,230 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gzip implements reading and writing of gzip format compressed files, +// as specified in RFC 1952. +package gzip + +import ( + "bufio" + "compress/flate" + "hash" + "hash/crc32" + "io" + "os" +) + +// BUG(nigeltao): Comments and Names don't properly map UTF-8 character codes outside of +// the 0x00-0x7f range to ISO 8859-1 (Latin-1). + +const ( + gzipID1 = 0x1f + gzipID2 = 0x8b + gzipDeflate = 8 + flagText = 1 << 0 + flagHdrCrc = 1 << 1 + flagExtra = 1 << 2 + flagName = 1 << 3 + flagComment = 1 << 4 +) + +func makeReader(r io.Reader) flate.Reader { + if rr, ok := r.(flate.Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +var HeaderError = os.NewError("invalid gzip header") +var ChecksumError = os.NewError("gzip checksum error") + +// The gzip file stores a header giving metadata about the compressed file. +// That header is exposed as the fields of the Compressor and Decompressor structs. +type Header struct { + Comment string // comment + Extra []byte // "extra data" + Mtime uint32 // modification time (seconds since January 1, 1970) + Name string // file name + OS byte // operating system type +} + +// An Decompressor is an io.Reader that can be read to retrieve +// uncompressed data from a gzip-format compressed file. +// +// In general, a gzip file can be a concatenation of gzip files, +// each with its own header. Reads from the Decompressor +// return the concatenation of the uncompressed data of each. +// Only the first header is recorded in the Decompressor fields. +// +// Gzip files store a length and checksum of the uncompressed data. +// The Decompressor will return a ChecksumError when Read +// reaches the end of the uncompressed data if it does not +// have the expected length or checksum. Clients should treat data +// returned by Read as tentative until they receive the successful +// (zero length, nil error) Read marking the end of the data. +type Decompressor struct { + Header + r flate.Reader + decompressor io.ReadCloser + digest hash.Hash32 + size uint32 + flg byte + buf [512]byte + err os.Error +} + +// NewReader creates a new Decompressor reading the given reader. +// The implementation buffers input and may read more data than necessary from r. +// It is the caller's responsibility to call Close on the Decompressor when done. +func NewReader(r io.Reader) (*Decompressor, os.Error) { + z := new(Decompressor) + z.r = makeReader(r) + z.digest = crc32.NewIEEE() + if err := z.readHeader(true); err != nil { + z.err = err + return nil, err + } + return z, nil +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func get4(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func (z *Decompressor) readString() (string, os.Error) { + var err os.Error + for i := 0; ; i++ { + if i >= len(z.buf) { + return "", HeaderError + } + z.buf[i], err = z.r.ReadByte() + if err != nil { + return "", err + } + if z.buf[i] == 0 { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + // TODO(nigeltao): Convert from ISO 8859-1 (Latin-1) to UTF-8. + return string(z.buf[0:i]), nil + } + } + panic("not reached") +} + +func (z *Decompressor) read2() (uint32, os.Error) { + _, err := io.ReadFull(z.r, z.buf[0:2]) + if err != nil { + return 0, err + } + return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil +} + +func (z *Decompressor) readHeader(save bool) os.Error { + _, err := io.ReadFull(z.r, z.buf[0:10]) + if err != nil { + return err + } + if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { + return HeaderError + } + z.flg = z.buf[3] + if save { + z.Mtime = get4(z.buf[4:8]) + // z.buf[8] is xfl, ignored + z.OS = z.buf[9] + } + z.digest.Reset() + z.digest.Write(z.buf[0:10]) + + if z.flg&flagExtra != 0 { + n, err := z.read2() + if err != nil { + return err + } + data := make([]byte, n) + if _, err = io.ReadFull(z.r, data); err != nil { + return err + } + if save { + z.Extra = data + } + } + + var s string + if z.flg&flagName != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Name = s + } + } + + if z.flg&flagComment != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Comment = s + } + } + + if z.flg&flagHdrCrc != 0 { + n, err := z.read2() + if err != nil { + return err + } + sum := z.digest.Sum32() & 0xFFFF + if n != sum { + return HeaderError + } + } + + z.digest.Reset() + z.decompressor = flate.NewReader(z.r) + return nil +} + +func (z *Decompressor) Read(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + + n, err = z.decompressor.Read(p) + z.digest.Write(p[0:n]) + z.size += uint32(n) + if n != 0 || err != os.EOF { + z.err = err + return + } + + // Finished file; check checksum + size. + if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { + z.err = err + return 0, err + } + crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8]) + sum := z.digest.Sum32() + if sum != crc32 || isize != z.size { + z.err = ChecksumError + return 0, z.err + } + + // File is ok; is there another? + if err = z.readHeader(false); err != nil { + z.err = err + return + } + + // Yes. Reset and read from it. + z.digest.Reset() + z.size = 0 + return z.Read(p) +} + +// Calling Close does not close the wrapped io.Reader originally passed to NewReader. +func (z *Decompressor) Close() os.Error { return z.decompressor.Close() } diff --git a/src/pkg/compress/gzip/gunzip_test.go b/src/pkg/compress/gzip/gunzip_test.go new file mode 100644 index 000000000..1c08c7374 --- /dev/null +++ b/src/pkg/compress/gzip/gunzip_test.go @@ -0,0 +1,305 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "bytes" + "io" + "os" + "testing" +) + +type gunzipTest struct { + name string + desc string + raw string + gzip []byte + err os.Error +} + +var gunzipTests = []gunzipTest{ + { // has 1 empty fixed-huffman block + "empty.txt", + "empty.txt", + "", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xf7, 0x5e, 0x14, 0x4a, + 0x00, 0x03, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + nil, + }, + { // has 1 non-empty fixed huffman block + "hello.txt", + "hello.txt", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + }, + nil, + }, + { // concatenation + "hello.txt", + "hello.txt x2", + "hello world\n" + + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, + }, + nil, + }, + { // has a fixed huffman block with some length-distance pairs + "shesells.txt", + "shesells.txt", + "she sells seashells by the seashore\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x72, 0x66, 0x8b, 0x4a, + 0x00, 0x03, 0x73, 0x68, 0x65, 0x73, 0x65, 0x6c, + 0x6c, 0x73, 0x2e, 0x74, 0x78, 0x74, 0x00, 0x2b, + 0xce, 0x48, 0x55, 0x28, 0x4e, 0xcd, 0xc9, 0x29, + 0x06, 0x92, 0x89, 0xc5, 0x19, 0x60, 0x56, 0x52, + 0xa5, 0x42, 0x09, 0x58, 0x18, 0x28, 0x90, 0x5f, + 0x94, 0xca, 0x05, 0x00, 0x76, 0xb0, 0x3b, 0xeb, + 0x24, 0x00, 0x00, 0x00, + }, + nil, + }, + { // has dynamic huffman blocks + "gettysburg", + "gettysburg", + " Four score and seven years ago our fathers brought forth on\n" + + "this continent, a new nation, conceived in Liberty, and dedicated\n" + + "to the proposition that all men are created equal.\n" + + " Now we are engaged in a great Civil War, testing whether that\n" + + "nation, or any nation so conceived and so dedicated, can long\n" + + "endure.\n" + + " We are met on a great battle-field of that war.\n" + + " We have come to dedicate a portion of that field, as a final\n" + + "resting place for those who here gave their lives that that\n" + + "nation might live. It is altogether fitting and proper that\n" + + "we should do this.\n" + + " But, in a larger sense, we can not dedicate — we can not\n" + + "consecrate — we can not hallow — this ground.\n" + + " The brave men, living and dead, who struggled here, have\n" + + "consecrated it, far above our poor power to add or detract.\n" + + "The world will little note, nor long remember what we say here,\n" + + "but it can never forget what they did here.\n" + + " It is for us the living, rather, to be dedicated here to the\n" + + "unfinished work which they who fought here have thus far so\n" + + "nobly advanced. It is rather for us to be here dedicated to\n" + + "the great task remaining before us — that from these honored\n" + + "dead we take increased devotion to that cause for which they\n" + + "gave the last full measure of devotion —\n" + + " that we here highly resolve that these dead shall not have\n" + + "died in vain — that this nation, under God, shall have a new\n" + + "birth of freedom — and that government of the people, by the\n" + + "people, for the people, shall not perish from this earth.\n" + + "\n" + + "Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xd1, 0x12, 0x2b, 0x4a, + 0x00, 0x03, 0x67, 0x65, 0x74, 0x74, 0x79, 0x73, + 0x62, 0x75, 0x72, 0x67, 0x00, 0x65, 0x54, 0xcd, + 0x6e, 0xd4, 0x30, 0x10, 0xbe, 0xfb, 0x29, 0xe6, + 0x01, 0x42, 0xa5, 0x0a, 0x09, 0xc1, 0x11, 0x90, + 0x40, 0x48, 0xa8, 0xe2, 0x80, 0xd4, 0xf3, 0x24, + 0x9e, 0x24, 0x56, 0xbd, 0x9e, 0xc5, 0x76, 0x76, + 0x95, 0x1b, 0x0f, 0xc1, 0x13, 0xf2, 0x24, 0x7c, + 0x63, 0x77, 0x9b, 0x4a, 0x5c, 0xaa, 0x6e, 0x6c, + 0xcf, 0x7c, 0x7f, 0x33, 0x44, 0x5f, 0x74, 0xcb, + 0x54, 0x26, 0xcd, 0x42, 0x9c, 0x3c, 0x15, 0xb9, + 0x48, 0xa2, 0x5d, 0x38, 0x17, 0xe2, 0x45, 0xc9, + 0x4e, 0x67, 0xae, 0xab, 0xe0, 0xf7, 0x98, 0x75, + 0x5b, 0xd6, 0x4a, 0xb3, 0xe6, 0xba, 0x92, 0x26, + 0x57, 0xd7, 0x50, 0x68, 0xd2, 0x54, 0x43, 0x92, + 0x54, 0x07, 0x62, 0x4a, 0x72, 0xa5, 0xc4, 0x35, + 0x68, 0x1a, 0xec, 0x60, 0x92, 0x70, 0x11, 0x4f, + 0x21, 0xd1, 0xf7, 0x30, 0x4a, 0xae, 0xfb, 0xd0, + 0x9a, 0x78, 0xf1, 0x61, 0xe2, 0x2a, 0xde, 0x55, + 0x25, 0xd4, 0xa6, 0x73, 0xd6, 0xb3, 0x96, 0x60, + 0xef, 0xf0, 0x9b, 0x2b, 0x71, 0x8c, 0x74, 0x02, + 0x10, 0x06, 0xac, 0x29, 0x8b, 0xdd, 0x25, 0xf9, + 0xb5, 0x71, 0xbc, 0x73, 0x44, 0x0f, 0x7a, 0xa5, + 0xab, 0xb4, 0x33, 0x49, 0x0b, 0x2f, 0xbd, 0x03, + 0xd3, 0x62, 0x17, 0xe9, 0x73, 0xb8, 0x84, 0x48, + 0x8f, 0x9c, 0x07, 0xaa, 0x52, 0x00, 0x6d, 0xa1, + 0xeb, 0x2a, 0xc6, 0xa0, 0x95, 0x76, 0x37, 0x78, + 0x9a, 0x81, 0x65, 0x7f, 0x46, 0x4b, 0x45, 0x5f, + 0xe1, 0x6d, 0x42, 0xe8, 0x01, 0x13, 0x5c, 0x38, + 0x51, 0xd4, 0xb4, 0x38, 0x49, 0x7e, 0xcb, 0x62, + 0x28, 0x1e, 0x3b, 0x82, 0x93, 0x54, 0x48, 0xf1, + 0xd2, 0x7d, 0xe4, 0x5a, 0xa3, 0xbc, 0x99, 0x83, + 0x44, 0x4f, 0x3a, 0x77, 0x36, 0x57, 0xce, 0xcf, + 0x2f, 0x56, 0xbe, 0x80, 0x90, 0x9e, 0x84, 0xea, + 0x51, 0x1f, 0x8f, 0xcf, 0x90, 0xd4, 0x60, 0xdc, + 0x5e, 0xb4, 0xf7, 0x10, 0x0b, 0x26, 0xe0, 0xff, + 0xc4, 0xd1, 0xe5, 0x67, 0x2e, 0xe7, 0xc8, 0x93, + 0x98, 0x05, 0xb8, 0xa8, 0x45, 0xc0, 0x4d, 0x09, + 0xdc, 0x84, 0x16, 0x2b, 0x0d, 0x9a, 0x21, 0x53, + 0x04, 0x8b, 0xd2, 0x0b, 0xbd, 0xa2, 0x4c, 0xa7, + 0x60, 0xee, 0xd9, 0xe1, 0x1d, 0xd1, 0xb7, 0x4a, + 0x30, 0x8f, 0x63, 0xd5, 0xa5, 0x8b, 0x33, 0x87, + 0xda, 0x1a, 0x18, 0x79, 0xf3, 0xe3, 0xa6, 0x17, + 0x94, 0x2e, 0xab, 0x6e, 0xa0, 0xe3, 0xcd, 0xac, + 0x50, 0x8c, 0xca, 0xa7, 0x0d, 0x76, 0x37, 0xd1, + 0x23, 0xe7, 0x05, 0x57, 0x8b, 0xa4, 0x22, 0x83, + 0xd9, 0x62, 0x52, 0x25, 0xad, 0x07, 0xbb, 0xbf, + 0xbf, 0xff, 0xbc, 0xfa, 0xee, 0x20, 0x73, 0x91, + 0x29, 0xff, 0x7f, 0x02, 0x71, 0x62, 0x84, 0xb5, + 0xf6, 0xb5, 0x25, 0x6b, 0x41, 0xde, 0x92, 0xb7, + 0x76, 0x3f, 0x91, 0x91, 0x31, 0x1b, 0x41, 0x84, + 0x62, 0x30, 0x0a, 0x37, 0xa4, 0x5e, 0x18, 0x3a, + 0x99, 0x08, 0xa5, 0xe6, 0x6d, 0x59, 0x22, 0xec, + 0x33, 0x39, 0x86, 0x26, 0xf5, 0xab, 0x66, 0xc8, + 0x08, 0x20, 0xcf, 0x0c, 0xd7, 0x47, 0x45, 0x21, + 0x0b, 0xf6, 0x59, 0xd5, 0xfe, 0x5c, 0x8d, 0xaa, + 0x12, 0x7b, 0x6f, 0xa1, 0xf0, 0x52, 0x33, 0x4f, + 0xf5, 0xce, 0x59, 0xd3, 0xab, 0x66, 0x10, 0xbf, + 0x06, 0xc4, 0x31, 0x06, 0x73, 0xd6, 0x80, 0xa2, + 0x78, 0xc2, 0x45, 0xcb, 0x03, 0x65, 0x39, 0xc9, + 0x09, 0xd1, 0x06, 0x04, 0x33, 0x1a, 0x5a, 0xf1, + 0xde, 0x01, 0xb8, 0x71, 0x83, 0xc4, 0xb5, 0xb3, + 0xc3, 0x54, 0x65, 0x33, 0x0d, 0x5a, 0xf7, 0x9b, + 0x90, 0x7c, 0x27, 0x1f, 0x3a, 0x58, 0xa3, 0xd8, + 0xfd, 0x30, 0x5f, 0xb7, 0xd2, 0x66, 0xa2, 0x93, + 0x1c, 0x28, 0xb7, 0xe9, 0x1b, 0x0c, 0xe1, 0x28, + 0x47, 0x26, 0xbb, 0xe9, 0x7d, 0x7e, 0xdc, 0x96, + 0x10, 0x92, 0x50, 0x56, 0x7c, 0x06, 0xe2, 0x27, + 0xb4, 0x08, 0xd3, 0xda, 0x7b, 0x98, 0x34, 0x73, + 0x9f, 0xdb, 0xf6, 0x62, 0xed, 0x31, 0x41, 0x13, + 0xd3, 0xa2, 0xa8, 0x4b, 0x3a, 0xc6, 0x1d, 0xe4, + 0x2f, 0x8c, 0xf8, 0xfb, 0x97, 0x64, 0xf4, 0xb6, + 0x2f, 0x80, 0x5a, 0xf3, 0x56, 0xe0, 0x40, 0x50, + 0xd5, 0x19, 0xd0, 0x1e, 0xfc, 0xca, 0xe5, 0xc9, + 0xd4, 0x60, 0x00, 0x81, 0x2e, 0xa3, 0xcc, 0xb6, + 0x52, 0xf0, 0xb4, 0xdb, 0x69, 0x99, 0xce, 0x7a, + 0x32, 0x4c, 0x08, 0xed, 0xaa, 0x10, 0x10, 0xe3, + 0x6f, 0xee, 0x99, 0x68, 0x95, 0x9f, 0x04, 0x71, + 0xb2, 0x49, 0x2f, 0x62, 0xa6, 0x5e, 0xb4, 0xef, + 0x02, 0xed, 0x4f, 0x27, 0xde, 0x4a, 0x0f, 0xfd, + 0xc1, 0xcc, 0xdd, 0x02, 0x8f, 0x08, 0x16, 0x54, + 0xdf, 0xda, 0xca, 0xe0, 0x82, 0xf1, 0xb4, 0x31, + 0x7a, 0xa9, 0x81, 0xfe, 0x90, 0xb7, 0x3e, 0xdb, + 0xd3, 0x35, 0xc0, 0x20, 0x80, 0x33, 0x46, 0x4a, + 0x63, 0xab, 0xd1, 0x0d, 0x29, 0xd2, 0xe2, 0x84, + 0xb8, 0xdb, 0xfa, 0xe9, 0x89, 0x44, 0x86, 0x7c, + 0xe8, 0x0b, 0xe6, 0x02, 0x6a, 0x07, 0x9b, 0x96, + 0xd0, 0xdb, 0x2e, 0x41, 0x4c, 0xa1, 0xd5, 0x57, + 0x45, 0x14, 0xfb, 0xe3, 0xa6, 0x72, 0x5b, 0x87, + 0x6e, 0x0c, 0x6d, 0x5b, 0xce, 0xe0, 0x2f, 0xe2, + 0x21, 0x81, 0x95, 0xb0, 0xe8, 0xb6, 0x32, 0x0b, + 0xb2, 0x98, 0x13, 0x52, 0x5d, 0xfb, 0xec, 0x63, + 0x17, 0x8a, 0x9e, 0x23, 0x22, 0x36, 0xee, 0xcd, + 0xda, 0xdb, 0xcf, 0x3e, 0xf1, 0xc7, 0xf1, 0x01, + 0x12, 0x93, 0x0a, 0xeb, 0x6f, 0xf2, 0x02, 0x15, + 0x96, 0x77, 0x5d, 0xef, 0x9c, 0xfb, 0x88, 0x91, + 0x59, 0xf9, 0x84, 0xdd, 0x9b, 0x26, 0x8d, 0x80, + 0xf9, 0x80, 0x66, 0x2d, 0xac, 0xf7, 0x1f, 0x06, + 0xba, 0x7f, 0xff, 0xee, 0xed, 0x40, 0x5f, 0xa5, + 0xd6, 0xbd, 0x8c, 0x5b, 0x46, 0xd2, 0x7e, 0x48, + 0x4a, 0x65, 0x8f, 0x08, 0x42, 0x60, 0xf7, 0x0f, + 0xb9, 0x16, 0x0b, 0x0c, 0x1a, 0x06, 0x00, 0x00, + }, + nil, + }, + { // has 1 non-empty fixed huffman block then garbage + "hello.txt", + "hello.txt + garbage", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, 'g', 'a', 'r', 'b', 'a', 'g', 'e', '!', '!', '!', + }, + HeaderError, + }, + { // has 1 non-empty fixed huffman block not enough header + "hello.txt", + "hello.txt + garbage", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, + 0x00, 0x00, gzipID1, + }, + io.ErrUnexpectedEOF, + }, + { // has 1 non-empty fixed huffman block but corrupt checksum + "hello.txt", + "hello.txt + corrupt checksum", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0c, 0x00, + 0x00, 0x00, + }, + ChecksumError, + }, + { // has 1 non-empty fixed huffman block but corrupt size + "hello.txt", + "hello.txt + corrupt size", + "hello world\n", + []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, + 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, + 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, + 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, + 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0xff, 0x00, + 0x00, 0x00, + }, + ChecksumError, + }, +} + +func TestDecompressor(t *testing.T) { + b := new(bytes.Buffer) + for _, tt := range gunzipTests { + in := bytes.NewBuffer(tt.gzip) + gzip, err := NewReader(in) + if err != nil { + t.Errorf("%s: NewReader: %s", tt.name, err) + continue + } + defer gzip.Close() + if tt.name != gzip.Name { + t.Errorf("%s: got name %s", tt.name, gzip.Name) + } + b.Reset() + n, err := io.Copy(b, gzip) + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.name, err, tt.err) + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) + } + } +} diff --git a/src/pkg/compress/gzip/gzip.go b/src/pkg/compress/gzip/gzip.go new file mode 100644 index 000000000..8860d10af --- /dev/null +++ b/src/pkg/compress/gzip/gzip.go @@ -0,0 +1,187 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "compress/flate" + "hash" + "hash/crc32" + "io" + "os" +) + +// These constants are copied from the flate package, so that code that imports +// "compress/gzip" does not also have to import "compress/flate". +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression +) + +// A Compressor is an io.WriteCloser that satisfies writes by compressing data written +// to its wrapped io.Writer. +type Compressor struct { + Header + w io.Writer + level int + compressor io.WriteCloser + digest hash.Hash32 + size uint32 + closed bool + buf [10]byte + err os.Error +} + +// NewWriter calls NewWriterLevel with the default compression level. +func NewWriter(w io.Writer) (*Compressor, os.Error) { + return NewWriterLevel(w, DefaultCompression) +} + +// NewWriterLevel creates a new Compressor writing to the given writer. +// Writes may be buffered and not flushed until Close. +// Callers that wish to set the fields in Compressor.Header must +// do so before the first call to Write or Close. +// It is the caller's responsibility to call Close on the WriteCloser when done. +// level is the compression level, which can be DefaultCompression, NoCompression, +// or any integer value between BestSpeed and BestCompression (inclusive). +func NewWriterLevel(w io.Writer, level int) (*Compressor, os.Error) { + z := new(Compressor) + z.OS = 255 // unknown + z.w = w + z.level = level + z.digest = crc32.NewIEEE() + return z, nil +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func put2(p []byte, v uint16) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) +} + +func put4(p []byte, v uint32) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) + p[2] = uint8(v >> 16) + p[3] = uint8(v >> 24) +} + +// writeBytes writes a length-prefixed byte slice to z.w. +func (z *Compressor) writeBytes(b []byte) os.Error { + if len(b) > 0xffff { + return os.NewError("gzip.Write: Extra data is too large") + } + put2(z.buf[0:2], uint16(len(b))) + _, err := z.w.Write(z.buf[0:2]) + if err != nil { + return err + } + _, err = z.w.Write(b) + return err +} + +// writeString writes a string (in ISO 8859-1 (Latin-1) format) to z.w. +func (z *Compressor) writeString(s string) os.Error { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + // TODO(nigeltao): Convert from UTF-8 to ISO 8859-1 (Latin-1). + for _, v := range s { + if v == 0 || v > 0x7f { + return os.NewError("gzip.Write: non-ASCII header string") + } + } + _, err := io.WriteString(z.w, s) + if err != nil { + return err + } + // GZIP strings are NUL-terminated. + z.buf[0] = 0 + _, err = z.w.Write(z.buf[0:1]) + return err +} + +func (z *Compressor) Write(p []byte) (int, os.Error) { + if z.err != nil { + return 0, z.err + } + var n int + // Write the GZIP header lazily. + if z.compressor == nil { + z.buf[0] = gzipID1 + z.buf[1] = gzipID2 + z.buf[2] = gzipDeflate + z.buf[3] = 0 + if z.Extra != nil { + z.buf[3] |= 0x04 + } + if z.Name != "" { + z.buf[3] |= 0x08 + } + if z.Comment != "" { + z.buf[3] |= 0x10 + } + put4(z.buf[4:8], z.Mtime) + if z.level == BestCompression { + z.buf[8] = 2 + } else if z.level == BestSpeed { + z.buf[8] = 4 + } else { + z.buf[8] = 0 + } + z.buf[9] = z.OS + n, z.err = z.w.Write(z.buf[0:10]) + if z.err != nil { + return n, z.err + } + if z.Extra != nil { + z.err = z.writeBytes(z.Extra) + if z.err != nil { + return n, z.err + } + } + if z.Name != "" { + z.err = z.writeString(z.Name) + if z.err != nil { + return n, z.err + } + } + if z.Comment != "" { + z.err = z.writeString(z.Comment) + if z.err != nil { + return n, z.err + } + } + z.compressor = flate.NewWriter(z.w, z.level) + } + z.size += uint32(len(p)) + z.digest.Write(p) + n, z.err = z.compressor.Write(p) + return n, z.err +} + +// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. +func (z *Compressor) Close() os.Error { + if z.err != nil { + return z.err + } + if z.closed { + return nil + } + z.closed = true + if z.compressor == nil { + z.Write(nil) + if z.err != nil { + return z.err + } + } + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + put4(z.buf[0:4], z.digest.Sum32()) + put4(z.buf[4:8], z.size) + _, z.err = z.w.Write(z.buf[0:8]) + return z.err +} diff --git a/src/pkg/compress/gzip/gzip_test.go b/src/pkg/compress/gzip/gzip_test.go new file mode 100644 index 000000000..121e627e6 --- /dev/null +++ b/src/pkg/compress/gzip/gzip_test.go @@ -0,0 +1,84 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gzip + +import ( + "io" + "io/ioutil" + "testing" +) + +// pipe creates two ends of a pipe that gzip and gunzip, and runs dfunc at the +// writer end and cfunc at the reader end. +func pipe(t *testing.T, dfunc func(*Compressor), cfunc func(*Decompressor)) { + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer pipew.Close() + compressor, err := NewWriter(pipew) + if err != nil { + t.Fatalf("%v", err) + } + defer compressor.Close() + dfunc(compressor) + }() + decompressor, err := NewReader(piper) + if err != nil { + t.Fatalf("%v", err) + } + defer decompressor.Close() + cfunc(decompressor) +} + +// Tests that an empty payload still forms a valid GZIP stream. +func TestEmpty(t *testing.T) { + pipe(t, + func(compressor *Compressor) {}, + func(decompressor *Decompressor) { + b, err := ioutil.ReadAll(decompressor) + if err != nil { + t.Fatalf("%v", err) + } + if len(b) != 0 { + t.Fatalf("did not read an empty slice") + } + }) +} + +// Tests that gzipping and then gunzipping is the identity function. +func TestWriter(t *testing.T) { + pipe(t, + func(compressor *Compressor) { + compressor.Comment = "comment" + compressor.Extra = []byte("extra") + compressor.Mtime = 1e8 + compressor.Name = "name" + _, err := compressor.Write([]byte("payload")) + if err != nil { + t.Fatalf("%v", err) + } + }, + func(decompressor *Decompressor) { + b, err := ioutil.ReadAll(decompressor) + if err != nil { + t.Fatalf("%v", err) + } + if string(b) != "payload" { + t.Fatalf("payload is %q, want %q", string(b), "payload") + } + if decompressor.Comment != "comment" { + t.Fatalf("comment is %q, want %q", decompressor.Comment, "comment") + } + if string(decompressor.Extra) != "extra" { + t.Fatalf("extra is %q, want %q", decompressor.Extra, "extra") + } + if decompressor.Mtime != 1e8 { + t.Fatalf("mtime is %d, want %d", decompressor.Mtime, uint32(1e8)) + } + if decompressor.Name != "name" { + t.Fatalf("name is %q, want %q", decompressor.Name, "name") + } + }) +} diff --git a/src/pkg/compress/lzw/Makefile b/src/pkg/compress/lzw/Makefile new file mode 100644 index 000000000..28f5e6abc --- /dev/null +++ b/src/pkg/compress/lzw/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/lzw +GOFILES=\ + reader.go\ + writer.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/lzw/reader.go b/src/pkg/compress/lzw/reader.go new file mode 100644 index 000000000..21231c8e5 --- /dev/null +++ b/src/pkg/compress/lzw/reader.go @@ -0,0 +1,253 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lzw implements the Lempel-Ziv-Welch compressed data format, +// described in T. A. Welch, ``A Technique for High-Performance Data +// Compression'', Computer, 17(6) (June 1984), pp 8-19. +// +// In particular, it implements LZW as used by the GIF, TIFF and PDF file +// formats, which means variable-width codes up to 12 bits and the first +// two non-literal codes are a clear code and an EOF code. +package lzw + +// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF, +// modulo LSB/MSB packing order. + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// Order specifies the bit ordering in an LZW data stream. +type Order int + +const ( + // LSB means Least Significant Bits first, as used in the GIF file format. + LSB Order = iota + // MSB means Most Significant Bits first, as used in the TIFF and PDF + // file formats. + MSB +) + +const ( + maxWidth = 12 + decoderInvalidCode = 0xffff + flushBuffer = 1 << maxWidth +) + +// decoder is the state from which the readXxx method converts a byte +// stream into a code stream. +type decoder struct { + r io.ByteReader + bits uint32 + nBits uint + width uint + read func(*decoder) (uint16, os.Error) // readLSB or readMSB + litWidth int // width in bits of literal codes + err os.Error + + // The first 1<= 1<>= d.width + d.nBits -= d.width + return code, nil +} + +// readMSB returns the next code for "Most Significant Bits first" data. +func (d *decoder) readMSB() (uint16, os.Error) { + for d.nBits < d.width { + x, err := d.r.ReadByte() + if err != nil { + return 0, err + } + d.bits |= uint32(x) << (24 - d.nBits) + d.nBits += 8 + } + code := uint16(d.bits >> (32 - d.width)) + d.bits <<= d.width + d.nBits -= d.width + return code, nil +} + +func (d *decoder) Read(b []byte) (int, os.Error) { + for { + if len(d.toRead) > 0 { + n := copy(b, d.toRead) + d.toRead = d.toRead[n:] + return n, nil + } + if d.err != nil { + return 0, d.err + } + d.decode() + } + panic("unreachable") +} + +// decode decompresses bytes from r and leaves them in d.toRead. +// read specifies how to decode bytes into codes. +// litWidth is the width in bits of literal codes. +func (d *decoder) decode() { + // Loop over the code stream, converting codes into decompressed bytes. + for { + code, err := d.read(d) + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + d.err = err + return + } + switch { + case code < d.clear: + // We have a literal code. + d.output[d.o] = uint8(code) + d.o++ + if d.last != decoderInvalidCode { + // Save what the hi code expands to. + d.suffix[d.hi] = uint8(code) + d.prefix[d.hi] = d.last + } + case code == d.clear: + d.width = 1 + uint(d.litWidth) + d.hi = d.eof + d.overflow = 1 << d.width + d.last = decoderInvalidCode + continue + case code == d.eof: + d.flush() + d.err = os.EOF + return + case code <= d.hi: + c, i := code, len(d.output)-1 + if code == d.hi { + // code == hi is a special case which expands to the last expansion + // followed by the head of the last expansion. To find the head, we walk + // the prefix chain until we find a literal code. + c = d.last + for c >= d.clear { + c = d.prefix[c] + } + d.output[i] = uint8(c) + i-- + c = d.last + } + // Copy the suffix chain into output and then write that to w. + for c >= d.clear { + d.output[i] = d.suffix[c] + i-- + c = d.prefix[c] + } + d.output[i] = uint8(c) + d.o += copy(d.output[d.o:], d.output[i:]) + if d.last != decoderInvalidCode { + // Save what the hi code expands to. + d.suffix[d.hi] = uint8(c) + d.prefix[d.hi] = d.last + } + default: + d.err = os.NewError("lzw: invalid code") + return + } + d.last, d.hi = code, d.hi+1 + if d.hi >= d.overflow { + if d.width == maxWidth { + d.last = decoderInvalidCode + } else { + d.width++ + d.overflow <<= 1 + } + } + if d.o >= flushBuffer { + d.flush() + return + } + } + panic("unreachable") +} + +func (d *decoder) flush() { + d.toRead = d.output[:d.o] + d.o = 0 +} + +func (d *decoder) Close() os.Error { + d.err = os.EINVAL // in case any Reads come along + return nil +} + +// NewReader creates a new io.ReadCloser that satisfies reads by decompressing +// the data read from r. +// It is the caller's responsibility to call Close on the ReadCloser when +// finished reading. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser { + d := new(decoder) + switch order { + case LSB: + d.read = (*decoder).readLSB + case MSB: + d.read = (*decoder).readMSB + default: + d.err = os.NewError("lzw: unknown order") + return d + } + if litWidth < 2 || 8 < litWidth { + d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth) + return d + } + if br, ok := r.(io.ByteReader); ok { + d.r = br + } else { + d.r = bufio.NewReader(r) + } + d.litWidth = litWidth + d.width = 1 + uint(litWidth) + d.clear = uint16(1) << uint(litWidth) + d.eof, d.hi = d.clear+1, d.clear+1 + d.overflow = uint16(1) << d.width + d.last = decoderInvalidCode + + return d +} diff --git a/src/pkg/compress/lzw/reader_test.go b/src/pkg/compress/lzw/reader_test.go new file mode 100644 index 000000000..f8042b0d1 --- /dev/null +++ b/src/pkg/compress/lzw/reader_test.go @@ -0,0 +1,145 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "runtime" + "strconv" + "strings" + "testing" +) + +type lzwTest struct { + desc string + raw string + compressed string + err os.Error +} + +var lzwTests = []lzwTest{ + { + "empty;LSB;8", + "", + "\x01\x01", + nil, + }, + { + "empty;MSB;8", + "", + "\x80\x80", + nil, + }, + { + "tobe;LSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04\x12\x34\xb8\xb0\xe0\xc1\x84\x01\x01", + nil, + }, + { + "tobe;MSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;MSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x2a\x13\xc8\x44\x52\x79\x48\x9c\x4f\x2a\x40\xa0\x90\x68\x5c\x16\x0f\x09\x80\x80", + nil, + }, + { + "tobe-truncated;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04", + io.ErrUnexpectedEOF, + }, + // This example comes from http://en.wikipedia.org/wiki/Graphics_Interchange_Format. + { + "gif;LSB;8", + "\x28\xff\xff\xff\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x51\xfc\x1b\x28\x70\xa0\xc1\x83\x01\x01", + nil, + }, + // This example comes from http://compgroups.net/comp.lang.ruby/Decompressing-LZW-compression-from-PDF-file + { + "pdf;MSB;8", + "-----A---B", + "\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01", + nil, + }, +} + +func TestReader(t *testing.T) { + b := bytes.NewBuffer(nil) + for _, tt := range lzwTests { + d := strings.Split(tt.desc, ";") + var order Order + switch d[1] { + case "LSB": + order = LSB + case "MSB": + order = MSB + default: + t.Errorf("%s: bad order %q", tt.desc, d[1]) + } + litWidth, _ := strconv.Atoi(d[2]) + rc := NewReader(strings.NewReader(tt.compressed), order, litWidth) + defer rc.Close() + b.Reset() + n, err := io.Copy(b, rc) + if err != nil { + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err) + } + continue + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw) + } + } +} + +func benchmarkDecoder(b *testing.B, n int) { + b.StopTimer() + b.SetBytes(int64(n)) + buf0, _ := ioutil.ReadFile("../testdata/e.txt") + buf0 = buf0[:10000] + compressed := bytes.NewBuffer(nil) + w := NewWriter(compressed, LSB, 8) + for i := 0; i < n; i += len(buf0) { + io.Copy(w, bytes.NewBuffer(buf0)) + } + w.Close() + buf1 := compressed.Bytes() + buf0, compressed, w = nil, nil, nil + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + io.Copy(ioutil.Discard, NewReader(bytes.NewBuffer(buf1), LSB, 8)) + } +} + +func BenchmarkDecoder1e4(b *testing.B) { + benchmarkDecoder(b, 1e4) +} + +func BenchmarkDecoder1e5(b *testing.B) { + benchmarkDecoder(b, 1e5) +} + +func BenchmarkDecoder1e6(b *testing.B) { + benchmarkDecoder(b, 1e6) +} diff --git a/src/pkg/compress/lzw/writer.go b/src/pkg/compress/lzw/writer.go new file mode 100644 index 000000000..87143b7aa --- /dev/null +++ b/src/pkg/compress/lzw/writer.go @@ -0,0 +1,259 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// A writer is a buffered, flushable writer. +type writer interface { + WriteByte(byte) os.Error + Flush() os.Error +} + +// An errWriteCloser is an io.WriteCloser that always returns a given error. +type errWriteCloser struct { + err os.Error +} + +func (e *errWriteCloser) Write([]byte) (int, os.Error) { + return 0, e.err +} + +func (e *errWriteCloser) Close() os.Error { + return e.err +} + +const ( + // A code is a 12 bit value, stored as a uint32 when encoding to avoid + // type conversions when shifting bits. + maxCode = 1<<12 - 1 + invalidCode = 1<<32 - 1 + // There are 1<<12 possible codes, which is an upper bound on the number of + // valid hash table entries at any given point in time. tableSize is 4x that. + tableSize = 4 * 1 << 12 + tableMask = tableSize - 1 + // A hash table entry is a uint32. Zero is an invalid entry since the + // lower 12 bits of a valid entry must be a non-literal code. + invalidEntry = 0 +) + +// encoder is LZW compressor. +type encoder struct { + // w is the writer that compressed bytes are written to. + w writer + // write, bits, nBits and width are the state for converting a code stream + // into a byte stream. + write func(*encoder, uint32) os.Error + bits uint32 + nBits uint + width uint + // litWidth is the width in bits of literal codes. + litWidth uint + // hi is the code implied by the next code emission. + // overflow is the code at which hi overflows the code width. + hi, overflow uint32 + // savedCode is the accumulated code at the end of the most recent Write + // call. It is equal to invalidCode if there was no such call. + savedCode uint32 + // err is the first error encountered during writing. Closing the encoder + // will make any future Write calls return os.EINVAL. + err os.Error + // table is the hash table from 20-bit keys to 12-bit values. Each table + // entry contains key<<12|val and collisions resolve by linear probing. + // The keys consist of a 12-bit code prefix and an 8-bit byte suffix. + // The values are a 12-bit code. + table [tableSize]uint32 +} + +// writeLSB writes the code c for "Least Significant Bits first" data. +func (e *encoder) writeLSB(c uint32) os.Error { + e.bits |= c << e.nBits + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + e.bits >>= 8 + e.nBits -= 8 + } + return nil +} + +// writeMSB writes the code c for "Most Significant Bits first" data. +func (e *encoder) writeMSB(c uint32) os.Error { + e.bits |= c << (32 - e.width - e.nBits) + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits >> 24)); err != nil { + return err + } + e.bits <<= 8 + e.nBits -= 8 + } + return nil +} + +// errOutOfCodes is an internal error that means that the encoder has run out +// of unused codes and a clear code needs to be sent next. +var errOutOfCodes = os.NewError("lzw: out of codes") + +// incHi increments e.hi and checks for both overflow and running out of +// unused codes. In the latter case, incHi sends a clear code, resets the +// encoder state and returns errOutOfCodes. +func (e *encoder) incHi() os.Error { + e.hi++ + if e.hi == e.overflow { + e.width++ + e.overflow <<= 1 + } + if e.hi == maxCode { + clear := uint32(1) << e.litWidth + if err := e.write(e, clear); err != nil { + return err + } + e.width = uint(e.litWidth) + 1 + e.hi = clear + 1 + e.overflow = clear << 1 + for i := range e.table { + e.table[i] = invalidEntry + } + return errOutOfCodes + } + return nil +} + +// Write writes a compressed representation of p to e's underlying writer. +func (e *encoder) Write(p []byte) (int, os.Error) { + if e.err != nil { + return 0, e.err + } + if len(p) == 0 { + return 0, nil + } + litMask := uint32(1<>12 ^ key) & tableMask + for h, t := hash, e.table[hash]; t != invalidEntry; { + if key == t>>12 { + code = t & maxCode + continue loop + } + h = (h + 1) & tableMask + t = e.table[h] + } + // Otherwise, write the current code, and literal becomes the start of + // the next emitted code. + if e.err = e.write(e, code); e.err != nil { + return 0, e.err + } + code = literal + // Increment e.hi, the next implied code. If we run out of codes, reset + // the encoder state (including clearing the hash table) and continue. + if err := e.incHi(); err != nil { + if err == errOutOfCodes { + continue + } + e.err = err + return 0, e.err + } + // Otherwise, insert key -> e.hi into the map that e.table represents. + for { + if e.table[hash] == invalidEntry { + e.table[hash] = (key << 12) | e.hi + break + } + hash = (hash + 1) & tableMask + } + } + e.savedCode = code + return len(p), nil +} + +// Close closes the encoder, flushing any pending output. It does not close or +// flush e's underlying writer. +func (e *encoder) Close() os.Error { + if e.err != nil { + if e.err == os.EINVAL { + return nil + } + return e.err + } + // Make any future calls to Write return os.EINVAL. + e.err = os.EINVAL + // Write the savedCode if valid. + if e.savedCode != invalidCode { + if err := e.write(e, e.savedCode); err != nil { + return err + } + if err := e.incHi(); err != nil && err != errOutOfCodes { + return err + } + } + // Write the eof code. + eof := uint32(1)< 0 { + if e.write == (*encoder).writeMSB { + e.bits >>= 24 + } + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + } + return e.w.Flush() +} + +// NewWriter creates a new io.WriteCloser that satisfies writes by compressing +// the data and writing it to w. +// It is the caller's responsibility to call Close on the WriteCloser when +// finished writing. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewWriter(w io.Writer, order Order, litWidth int) io.WriteCloser { + var write func(*encoder, uint32) os.Error + switch order { + case LSB: + write = (*encoder).writeLSB + case MSB: + write = (*encoder).writeMSB + default: + return &errWriteCloser{os.NewError("lzw: unknown order")} + } + if litWidth < 2 || 8 < litWidth { + return &errWriteCloser{fmt.Errorf("lzw: litWidth %d out of range", litWidth)} + } + bw, ok := w.(writer) + if !ok { + bw = bufio.NewWriter(w) + } + lw := uint(litWidth) + return &encoder{ + w: bw, + write: write, + width: 1 + lw, + litWidth: lw, + hi: 1<> 24) + z.scratch[1] = uint8(checksum >> 16) + z.scratch[2] = uint8(checksum >> 8) + z.scratch[3] = uint8(checksum >> 0) + _, err = w.Write(z.scratch[0:4]) + if err != nil { + return nil, err + } + } + z.w = w + z.compressor = flate.NewWriterDict(w, level, dict) + z.digest = adler32.New() + return z, nil +} + +func (z *Writer) Write(p []byte) (n int, err os.Error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + n, err = z.compressor.Write(p) + if err != nil { + z.err = err + return + } + z.digest.Write(p) + return +} + +// Flush flushes the underlying compressor. +func (z *Writer) Flush() os.Error { + if z.err != nil { + return z.err + } + z.err = z.compressor.Flush() + return z.err +} + +// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. +func (z *Writer) Close() os.Error { + if z.err != nil { + return z.err + } + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + checksum := z.digest.Sum32() + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + z.scratch[0] = uint8(checksum >> 24) + z.scratch[1] = uint8(checksum >> 16) + z.scratch[2] = uint8(checksum >> 8) + z.scratch[3] = uint8(checksum >> 0) + _, z.err = z.w.Write(z.scratch[0:4]) + return z.err +} diff --git a/src/pkg/compress/zlib/writer_test.go b/src/pkg/compress/zlib/writer_test.go new file mode 100644 index 000000000..32f05ab68 --- /dev/null +++ b/src/pkg/compress/zlib/writer_test.go @@ -0,0 +1,144 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "testing" +) + +var filenames = []string{ + "../testdata/e.txt", + "../testdata/pi.txt", +} + +var data = []string{ + "test a reasonable sized string that can be compressed", +} + +// Tests that compressing and then decompressing the given file at the given compression level and dictionary +// yields equivalent bytes to the original file. +func testFileLevelDict(t *testing.T, fn string, level int, d string) { + // Read the file, as golden output. + golden, err := os.Open(fn) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + defer golden.Close() + b0, err0 := ioutil.ReadAll(golden) + if err0 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0) + return + } + testLevelDict(t, fn, b0, level, d) +} + +func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) { + // Make dictionary, if given. + var dict []byte + if d != "" { + dict = []byte(d) + } + + // Push data through a pipe that compresses at the write end, and decompresses at the read end. + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer pipew.Close() + zlibw, err := NewWriterDict(pipew, level, dict) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + defer zlibw.Close() + _, err = zlibw.Write(b0) + if err == os.EPIPE { + // Fail, but do not report the error, as some other (presumably reported) error broke the pipe. + return + } + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + }() + zlibr, err := NewReaderDict(piper, dict) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + return + } + defer zlibr.Close() + + // Compare the decompressed data. + b1, err1 := ioutil.ReadAll(zlibr) + if err1 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1) + return + } + if len(b0) != len(b1) { + t.Errorf("%s (level=%d, dict=%q): length mismatch %d versus %d", fn, level, d, len(b0), len(b1)) + return + } + for i := 0; i < len(b0); i++ { + if b0[i] != b1[i] { + t.Errorf("%s (level=%d, dict=%q): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, d, i, b0[i], b1[i]) + return + } + } +} + +func TestWriter(t *testing.T) { + for i, s := range data { + b := []byte(s) + tag := fmt.Sprintf("#%d", i) + testLevelDict(t, tag, b, DefaultCompression, "") + testLevelDict(t, tag, b, NoCompression, "") + for level := BestSpeed; level <= BestCompression; level++ { + testLevelDict(t, tag, b, level, "") + } + } +} + +func TestWriterBig(t *testing.T) { + for _, fn := range filenames { + testFileLevelDict(t, fn, DefaultCompression, "") + testFileLevelDict(t, fn, NoCompression, "") + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDict(t, fn, level, "") + } + } +} + +func TestWriterDict(t *testing.T) { + const dictionary = "0123456789." + for _, fn := range filenames { + testFileLevelDict(t, fn, DefaultCompression, dictionary) + testFileLevelDict(t, fn, NoCompression, dictionary) + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDict(t, fn, level, dictionary) + } + } +} + +func TestWriterDictIsUsed(t *testing.T) { + var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") + buf := bytes.NewBuffer(nil) + compressor, err := NewWriterDict(buf, BestCompression, input) + if err != nil { + t.Errorf("error in NewWriterDict: %s", err) + return + } + compressor.Write(input) + compressor.Close() + const expectedMaxSize = 25 + output := buf.Bytes() + if len(output) > expectedMaxSize { + t.Errorf("result too large (got %d, want <= %d bytes). Is the dictionary being used?", len(output), expectedMaxSize) + } +} diff --git a/src/pkg/container/heap/Makefile b/src/pkg/container/heap/Makefile new file mode 100644 index 000000000..4291d1122 --- /dev/null +++ b/src/pkg/container/heap/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/heap +GOFILES=\ + heap.go\ + +include ../../../Make.pkg diff --git a/src/pkg/container/heap/heap.go b/src/pkg/container/heap/heap.go new file mode 100644 index 000000000..2dfe5b43c --- /dev/null +++ b/src/pkg/container/heap/heap.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package heap provides heap operations for any type that implements +// heap.Interface. +// +package heap + +import "sort" + +// Any type that implements heap.Interface may be used as a +// min-heap with the following invariants (established after +// Init has been called): +// +// !h.Less(j, i) for 0 <= i < h.Len() and j = 2*i+1 or 2*i+2 and j < h.Len() +// +type Interface interface { + sort.Interface + Push(x interface{}) + Pop() interface{} +} + +// A heap must be initialized before any of the heap operations +// can be used. Init is idempotent with respect to the heap invariants +// and may be called whenever the heap invariants may have been invalidated. +// Its complexity is O(n) where n = h.Len(). +// +func Init(h Interface) { + // heapify + n := h.Len() + for i := n/2 - 1; i >= 0; i-- { + down(h, i, n) + } +} + +// Push pushes the element x onto the heap. The complexity is +// O(log(n)) where n = h.Len(). +// +func Push(h Interface, x interface{}) { + h.Push(x) + up(h, h.Len()-1) +} + +// Pop removes the minimum element (according to Less) from the heap +// and returns it. The complexity is O(log(n)) where n = h.Len(). +// Same as Remove(h, 0). +// +func Pop(h Interface) interface{} { + n := h.Len() - 1 + h.Swap(0, n) + down(h, 0, n) + return h.Pop() +} + +// Remove removes the element at index i from the heap. +// The complexity is O(log(n)) where n = h.Len(). +// +func Remove(h Interface, i int) interface{} { + n := h.Len() - 1 + if n != i { + h.Swap(i, n) + down(h, i, n) + up(h, i) + } + return h.Pop() +} + +func up(h Interface, j int) { + for { + i := (j - 1) / 2 // parent + if i == j || h.Less(i, j) { + break + } + h.Swap(i, j) + j = i + } +} + +func down(h Interface, i, n int) { + for { + j1 := 2*i + 1 + if j1 >= n { + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && !h.Less(j1, j2) { + j = j2 // = 2*i + 2 // right child + } + if h.Less(i, j) { + break + } + h.Swap(i, j) + i = j + } +} diff --git a/src/pkg/container/heap/heap_test.go b/src/pkg/container/heap/heap_test.go new file mode 100644 index 000000000..c5c1f76e1 --- /dev/null +++ b/src/pkg/container/heap/heap_test.go @@ -0,0 +1,158 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package heap_test + +import ( + "testing" + "container/vector" + . "container/heap" +) + +type myHeap struct { + // A vector.Vector implements sort.Interface except for Less, + // and it implements Push and Pop as required for heap.Interface. + vector.Vector +} + +func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) } + +func (h *myHeap) verify(t *testing.T, i int) { + n := h.Len() + j1 := 2*i + 1 + j2 := 2*i + 2 + if j1 < n { + if h.Less(j1, i) { + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1)) + return + } + h.verify(t, j1) + } + if j2 < n { + if h.Less(j2, i) { + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2)) + return + } + h.verify(t, j2) + } +} + +func TestInit0(t *testing.T) { + h := new(myHeap) + for i := 20; i > 0; i-- { + h.Push(0) // all elements are the same + } + Init(h) + h.verify(t, 0) + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + h.verify(t, 0) + if x != 0 { + t.Errorf("%d.th pop got %d; want %d", i, x, 0) + } + } +} + +func TestInit1(t *testing.T) { + h := new(myHeap) + for i := 20; i > 0; i-- { + h.Push(i) // all elements are different + } + Init(h) + h.verify(t, 0) + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + h.verify(t, 0) + if x != i { + t.Errorf("%d.th pop got %d; want %d", i, x, i) + } + } +} + +func Test(t *testing.T) { + h := new(myHeap) + h.verify(t, 0) + + for i := 20; i > 10; i-- { + h.Push(i) + } + Init(h) + h.verify(t, 0) + + for i := 10; i > 0; i-- { + Push(h, i) + h.verify(t, 0) + } + + for i := 1; h.Len() > 0; i++ { + x := Pop(h).(int) + if i < 20 { + Push(h, 20+i) + } + h.verify(t, 0) + if x != i { + t.Errorf("%d.th pop got %d; want %d", i, x, i) + } + } +} + +func TestRemove0(t *testing.T) { + h := new(myHeap) + for i := 0; i < 10; i++ { + h.Push(i) + } + h.verify(t, 0) + + for h.Len() > 0 { + i := h.Len() - 1 + x := Remove(h, i).(int) + if x != i { + t.Errorf("Remove(%d) got %d; want %d", i, x, i) + } + h.verify(t, 0) + } +} + +func TestRemove1(t *testing.T) { + h := new(myHeap) + for i := 0; i < 10; i++ { + h.Push(i) + } + h.verify(t, 0) + + for i := 0; h.Len() > 0; i++ { + x := Remove(h, 0).(int) + if x != i { + t.Errorf("Remove(0) got %d; want %d", x, i) + } + h.verify(t, 0) + } +} + +func TestRemove2(t *testing.T) { + N := 10 + + h := new(myHeap) + for i := 0; i < N; i++ { + h.Push(i) + } + h.verify(t, 0) + + m := make(map[int]bool) + for h.Len() > 0 { + m[Remove(h, (h.Len()-1)/2).(int)] = true + h.verify(t, 0) + } + + if len(m) != N { + t.Errorf("len(m) = %d; want %d", len(m), N) + } + for i := 0; i < len(m); i++ { + if !m[i] { + t.Errorf("m[%d] doesn't exist", i) + } + } +} diff --git a/src/pkg/container/list/Makefile b/src/pkg/container/list/Makefile new file mode 100644 index 000000000..7fcd5f99a --- /dev/null +++ b/src/pkg/container/list/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/list +GOFILES=\ + list.go\ + +include ../../../Make.pkg diff --git a/src/pkg/container/list/list.go b/src/pkg/container/list/list.go new file mode 100644 index 000000000..a3fd4b39f --- /dev/null +++ b/src/pkg/container/list/list.go @@ -0,0 +1,211 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package list implements a doubly linked list. +// +// To iterate over a list (where l is a *List): +// for e := l.Front(); e != nil; e = e.Next() { +// // do something with e.Value +// } +// +package list + +// Element is an element in the linked list. +type Element struct { + // Next and previous pointers in the doubly-linked list of elements. + // The front of the list has prev = nil, and the back has next = nil. + next, prev *Element + + // The list to which this element belongs. + list *List + + // The contents of this list element. + Value interface{} +} + +// Next returns the next list element or nil. +func (e *Element) Next() *Element { return e.next } + +// Prev returns the previous list element or nil. +func (e *Element) Prev() *Element { return e.prev } + +// List represents a doubly linked list. +// The zero value for List is an empty list ready to use. +type List struct { + front, back *Element + len int +} + +// Init initializes or clears a List. +func (l *List) Init() *List { + l.front = nil + l.back = nil + l.len = 0 + return l +} + +// New returns an initialized list. +func New() *List { return new(List) } + +// Front returns the first element in the list. +func (l *List) Front() *Element { return l.front } + +// Back returns the last element in the list. +func (l *List) Back() *Element { return l.back } + +// Remove removes the element from the list +// and returns its Value. +func (l *List) Remove(e *Element) interface{} { + l.remove(e) + e.list = nil // do what remove does not + return e.Value +} + +// remove the element from the list, but do not clear the Element's list field. +// This is so that other List methods may use remove when relocating Elements +// without needing to restore the list field. +func (l *List) remove(e *Element) { + if e.list != l { + return + } + if e.prev == nil { + l.front = e.next + } else { + e.prev.next = e.next + } + if e.next == nil { + l.back = e.prev + } else { + e.next.prev = e.prev + } + + e.prev = nil + e.next = nil + l.len-- +} + +func (l *List) insertBefore(e *Element, mark *Element) { + if mark.prev == nil { + // new front of the list + l.front = e + } else { + mark.prev.next = e + } + e.prev = mark.prev + mark.prev = e + e.next = mark + l.len++ +} + +func (l *List) insertAfter(e *Element, mark *Element) { + if mark.next == nil { + // new back of the list + l.back = e + } else { + mark.next.prev = e + } + e.next = mark.next + mark.next = e + e.prev = mark + l.len++ +} + +func (l *List) insertFront(e *Element) { + if l.front == nil { + // empty list + l.front, l.back = e, e + e.prev, e.next = nil, nil + l.len = 1 + return + } + l.insertBefore(e, l.front) +} + +func (l *List) insertBack(e *Element) { + if l.back == nil { + // empty list + l.front, l.back = e, e + e.prev, e.next = nil, nil + l.len = 1 + return + } + l.insertAfter(e, l.back) +} + +// PushFront inserts the value at the front of the list and returns a new Element containing the value. +func (l *List) PushFront(value interface{}) *Element { + e := &Element{nil, nil, l, value} + l.insertFront(e) + return e +} + +// PushBack inserts the value at the back of the list and returns a new Element containing the value. +func (l *List) PushBack(value interface{}) *Element { + e := &Element{nil, nil, l, value} + l.insertBack(e) + return e +} + +// InsertBefore inserts the value immediately before mark and returns a new Element containing the value. +func (l *List) InsertBefore(value interface{}, mark *Element) *Element { + if mark.list != l { + return nil + } + e := &Element{nil, nil, l, value} + l.insertBefore(e, mark) + return e +} + +// InsertAfter inserts the value immediately after mark and returns a new Element containing the value. +func (l *List) InsertAfter(value interface{}, mark *Element) *Element { + if mark.list != l { + return nil + } + e := &Element{nil, nil, l, value} + l.insertAfter(e, mark) + return e +} + +// MoveToFront moves the element to the front of the list. +func (l *List) MoveToFront(e *Element) { + if e.list != l || l.front == e { + return + } + l.remove(e) + l.insertFront(e) +} + +// MoveToBack moves the element to the back of the list. +func (l *List) MoveToBack(e *Element) { + if e.list != l || l.back == e { + return + } + l.remove(e) + l.insertBack(e) +} + +// Len returns the number of elements in the list. +func (l *List) Len() int { return l.len } + +// PushBackList inserts each element of ol at the back of the list. +func (l *List) PushBackList(ol *List) { + last := ol.Back() + for e := ol.Front(); e != nil; e = e.Next() { + l.PushBack(e.Value) + if e == last { + break + } + } +} + +// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved. +func (l *List) PushFrontList(ol *List) { + first := ol.Front() + for e := ol.Back(); e != nil; e = e.Prev() { + l.PushFront(e.Value) + if e == first { + break + } + } +} diff --git a/src/pkg/container/list/list_test.go b/src/pkg/container/list/list_test.go new file mode 100644 index 000000000..1d44ff84e --- /dev/null +++ b/src/pkg/container/list/list_test.go @@ -0,0 +1,209 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package list + +import ( + "testing" +) + +func checkListPointers(t *testing.T, l *List, es []*Element) { + if len(es) == 0 { + if l.front != nil || l.back != nil { + t.Errorf("l.front/l.back = %v/%v should be nil/nil", l.front, l.back) + } + return + } + + if l.front != es[0] { + t.Errorf("l.front = %v, want %v", l.front, es[0]) + } + if last := es[len(es)-1]; l.back != last { + t.Errorf("l.back = %v, want %v", l.back, last) + } + + for i, e := range es { + var e_prev, e_next *Element = nil, nil + if i > 0 { + e_prev = es[i-1] + } + if i < len(es)-1 { + e_next = es[i+1] + } + if e.prev != e_prev { + t.Errorf("elt #%d (%v) has prev=%v, want %v", i, e, e.prev, e_prev) + } + if e.next != e_next { + t.Errorf("elt #%d (%v) has next=%v, want %v", i, e, e.next, e_next) + } + } +} + +func checkListLen(t *testing.T, l *List, n int) { + if an := l.Len(); an != n { + t.Errorf("l.Len() = %d, want %d", an, n) + } +} + +func TestList(t *testing.T) { + l := New() + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) + + // Single element list + e := l.PushFront("a") + checkListLen(t, l, 1) + checkListPointers(t, l, []*Element{e}) + l.MoveToFront(e) + checkListPointers(t, l, []*Element{e}) + l.MoveToBack(e) + checkListPointers(t, l, []*Element{e}) + checkListLen(t, l, 1) + l.Remove(e) + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) + + // Bigger list + e2 := l.PushFront(2) + e1 := l.PushFront(1) + e3 := l.PushBack(3) + e4 := l.PushBack("banana") + checkListPointers(t, l, []*Element{e1, e2, e3, e4}) + checkListLen(t, l, 4) + + l.Remove(e2) + checkListPointers(t, l, []*Element{e1, e3, e4}) + checkListLen(t, l, 3) + + l.MoveToFront(e3) // move from middle + checkListPointers(t, l, []*Element{e3, e1, e4}) + + l.MoveToFront(e1) + l.MoveToBack(e3) // move from middle + checkListPointers(t, l, []*Element{e1, e4, e3}) + + l.MoveToFront(e3) // move from back + checkListPointers(t, l, []*Element{e3, e1, e4}) + l.MoveToFront(e3) // should be no-op + checkListPointers(t, l, []*Element{e3, e1, e4}) + + l.MoveToBack(e3) // move from front + checkListPointers(t, l, []*Element{e1, e4, e3}) + l.MoveToBack(e3) // should be no-op + checkListPointers(t, l, []*Element{e1, e4, e3}) + + e2 = l.InsertBefore(2, e1) // insert before front + checkListPointers(t, l, []*Element{e2, e1, e4, e3}) + l.Remove(e2) + e2 = l.InsertBefore(2, e4) // insert before middle + checkListPointers(t, l, []*Element{e1, e2, e4, e3}) + l.Remove(e2) + e2 = l.InsertBefore(2, e3) // insert before back + checkListPointers(t, l, []*Element{e1, e4, e2, e3}) + l.Remove(e2) + + e2 = l.InsertAfter(2, e1) // insert after front + checkListPointers(t, l, []*Element{e1, e2, e4, e3}) + l.Remove(e2) + e2 = l.InsertAfter(2, e4) // insert after middle + checkListPointers(t, l, []*Element{e1, e4, e2, e3}) + l.Remove(e2) + e2 = l.InsertAfter(2, e3) // insert after back + checkListPointers(t, l, []*Element{e1, e4, e3, e2}) + l.Remove(e2) + + // Check standard iteration. + sum := 0 + for e := l.Front(); e != nil; e = e.Next() { + if i, ok := e.Value.(int); ok { + sum += i + } + } + if sum != 4 { + t.Errorf("sum over l.Iter() = %d, want 4", sum) + } + + // Clear all elements by iterating + var next *Element + for e := l.Front(); e != nil; e = next { + next = e.Next() + l.Remove(e) + } + checkListPointers(t, l, []*Element{}) + checkListLen(t, l, 0) +} + +func checkList(t *testing.T, l *List, es []interface{}) { + if l.Len() != len(es) { + t.Errorf("list has len=%v, want %v", l.Len(), len(es)) + return + } + i := 0 + for e := l.Front(); e != nil; e = e.Next() { + le := e.Value.(int) + if le != es[i] { + t.Errorf("elt #%d has value=%v, want %v", i, le, es[i]) + } + i++ + } +} + +func TestExtending(t *testing.T) { + l1 := New() + l2 := New() + + l1.PushBack(1) + l1.PushBack(2) + l1.PushBack(3) + + l2.PushBack(4) + l2.PushBack(5) + + l3 := New() + l3.PushBackList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushBackList(l2) + checkList(t, l3, []interface{}{1, 2, 3, 4, 5}) + + l3 = New() + l3.PushFrontList(l2) + checkList(t, l3, []interface{}{4, 5}) + l3.PushFrontList(l1) + checkList(t, l3, []interface{}{1, 2, 3, 4, 5}) + + checkList(t, l1, []interface{}{1, 2, 3}) + checkList(t, l2, []interface{}{4, 5}) + + l3 = New() + l3.PushBackList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushBackList(l3) + checkList(t, l3, []interface{}{1, 2, 3, 1, 2, 3}) + + l3 = New() + l3.PushFrontList(l1) + checkList(t, l3, []interface{}{1, 2, 3}) + l3.PushFrontList(l3) + checkList(t, l3, []interface{}{1, 2, 3, 1, 2, 3}) + + l3 = New() + l1.PushBackList(l3) + checkList(t, l1, []interface{}{1, 2, 3}) + l1.PushFrontList(l3) + checkList(t, l1, []interface{}{1, 2, 3}) +} + +func TestRemove(t *testing.T) { + l := New() + e1 := l.PushBack(1) + e2 := l.PushBack(2) + checkListPointers(t, l, []*Element{e1, e2}) + e := l.Front() + l.Remove(e) + checkListPointers(t, l, []*Element{e2}) + checkListLen(t, l, 1) + l.Remove(e) + checkListPointers(t, l, []*Element{e2}) + checkListLen(t, l, 1) +} diff --git a/src/pkg/container/ring/Makefile b/src/pkg/container/ring/Makefile new file mode 100644 index 000000000..fb0900774 --- /dev/null +++ b/src/pkg/container/ring/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/ring +GOFILES=\ + ring.go\ + +include ../../../Make.pkg diff --git a/src/pkg/container/ring/ring.go b/src/pkg/container/ring/ring.go new file mode 100644 index 000000000..1d96918d3 --- /dev/null +++ b/src/pkg/container/ring/ring.go @@ -0,0 +1,141 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ring implements operations on circular lists. +package ring + +// A Ring is an element of a circular list, or ring. +// Rings do not have a beginning or end; a pointer to any ring element +// serves as reference to the entire ring. Empty rings are represented +// as nil Ring pointers. The zero value for a Ring is a one-element +// ring with a nil Value. +// +type Ring struct { + next, prev *Ring + Value interface{} // for use by client; untouched by this library +} + +func (r *Ring) init() *Ring { + r.next = r + r.prev = r + return r +} + +// Next returns the next ring element. r must not be empty. +func (r *Ring) Next() *Ring { + if r.next == nil { + return r.init() + } + return r.next +} + +// Prev returns the previous ring element. r must not be empty. +func (r *Ring) Prev() *Ring { + if r.next == nil { + return r.init() + } + return r.prev +} + +// Move moves n % r.Len() elements backward (n < 0) or forward (n >= 0) +// in the ring and returns that ring element. r must not be empty. +// +func (r *Ring) Move(n int) *Ring { + if r.next == nil { + return r.init() + } + switch { + case n < 0: + for ; n < 0; n++ { + r = r.prev + } + case n > 0: + for ; n > 0; n-- { + r = r.next + } + } + return r +} + +// New creates a ring of n elements. +func New(n int) *Ring { + if n <= 0 { + return nil + } + r := new(Ring) + p := r + for i := 1; i < n; i++ { + p.next = &Ring{prev: p} + p = p.next + } + p.next = r + r.prev = p + return r +} + +// Link connects ring r with with ring s such that r.Next() +// becomes s and returns the original value for r.Next(). +// r must not be empty. +// +// If r and s point to the same ring, linking +// them removes the elements between r and s from the ring. +// The removed elements form a subring and the result is a +// reference to that subring (if no elements were removed, +// the result is still the original value for r.Next(), +// and not nil). +// +// If r and s point to different rings, linking +// them creates a single ring with the elements of s inserted +// after r. The result points to the element following the +// last element of s after insertion. +// +func (r *Ring) Link(s *Ring) *Ring { + n := r.Next() + if s != nil { + p := s.Prev() + // Note: Cannot use multiple assignment because + // evaluation order of LHS is not specified. + r.next = s + s.prev = r + n.prev = p + p.next = n + } + return n +} + +// Unlink removes n % r.Len() elements from the ring r, starting +// at r.Next(). If n % r.Len() == 0, r remains unchanged. +// The result is the removed subring. r must not be empty. +// +func (r *Ring) Unlink(n int) *Ring { + if n <= 0 { + return nil + } + return r.Link(r.Move(n + 1)) +} + +// Len computes the number of elements in ring r. +// It executes in time proportional to the number of elements. +// +func (r *Ring) Len() int { + n := 0 + if r != nil { + n = 1 + for p := r.Next(); p != r; p = p.next { + n++ + } + } + return n +} + +// Do calls function f on each element of the ring, in forward order. +// The behavior of Do is undefined if f changes *r. +func (r *Ring) Do(f func(interface{})) { + if r != nil { + f(r.Value) + for p := r.Next(); p != r; p = p.next { + f(p.Value) + } + } +} diff --git a/src/pkg/container/ring/ring_test.go b/src/pkg/container/ring/ring_test.go new file mode 100644 index 000000000..099d92b25 --- /dev/null +++ b/src/pkg/container/ring/ring_test.go @@ -0,0 +1,220 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ring + +import ( + "fmt" + "testing" +) + +// For debugging - keep around. +func dump(r *Ring) { + if r == nil { + fmt.Println("empty") + return + } + i, n := 0, r.Len() + for p := r; i < n; p = p.next { + fmt.Printf("%4d: %p = {<- %p | %p ->}\n", i, p, p.prev, p.next) + i++ + } + fmt.Println() +} + +func verify(t *testing.T, r *Ring, N int, sum int) { + // Len + n := r.Len() + if n != N { + t.Errorf("r.Len() == %d; expected %d", n, N) + } + + // iteration + n = 0 + s := 0 + r.Do(func(p interface{}) { + n++ + if p != nil { + s += p.(int) + } + }) + if n != N { + t.Errorf("number of forward iterations == %d; expected %d", n, N) + } + if sum >= 0 && s != sum { + t.Errorf("forward ring sum = %d; expected %d", s, sum) + } + + if r == nil { + return + } + + // connections + if r.next != nil { + var p *Ring // previous element + for q := r; p == nil || q != r; q = q.next { + if p != nil && p != q.prev { + t.Errorf("prev = %p, expected q.prev = %p\n", p, q.prev) + } + p = q + } + if p != r.prev { + t.Errorf("prev = %p, expected r.prev = %p\n", p, r.prev) + } + } + + // Next, Prev + if r.Next() != r.next { + t.Errorf("r.Next() != r.next") + } + if r.Prev() != r.prev { + t.Errorf("r.Prev() != r.prev") + } + + // Move + if r.Move(0) != r { + t.Errorf("r.Move(0) != r") + } + if r.Move(N) != r { + t.Errorf("r.Move(%d) != r", N) + } + if r.Move(-N) != r { + t.Errorf("r.Move(%d) != r", -N) + } + for i := 0; i < 10; i++ { + ni := N + i + mi := ni % N + if r.Move(ni) != r.Move(mi) { + t.Errorf("r.Move(%d) != r.Move(%d)", ni, mi) + } + if r.Move(-ni) != r.Move(-mi) { + t.Errorf("r.Move(%d) != r.Move(%d)", -ni, -mi) + } + } +} + +func TestCornerCases(t *testing.T) { + var ( + r0 *Ring + r1 Ring + ) + // Basics + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Insert + r1.Link(r0) + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Insert + r1.Link(r0) + verify(t, r0, 0, 0) + verify(t, &r1, 1, 0) + // Unlink + r1.Unlink(0) + verify(t, &r1, 1, 0) +} + +func makeN(n int) *Ring { + r := New(n) + for i := 1; i <= n; i++ { + r.Value = i + r = r.Next() + } + return r +} + +func sumN(n int) int { return (n*n + n) / 2 } + +func TestNew(t *testing.T) { + for i := 0; i < 10; i++ { + r := New(i) + verify(t, r, i, -1) + } + for i := 0; i < 10; i++ { + r := makeN(i) + verify(t, r, i, sumN(i)) + } +} + +func TestLink1(t *testing.T) { + r1a := makeN(1) + var r1b Ring + r2a := r1a.Link(&r1b) + verify(t, r2a, 2, 1) + if r2a != r1a { + t.Errorf("a) 2-element link failed") + } + + r2b := r2a.Link(r2a.Next()) + verify(t, r2b, 2, 1) + if r2b != r2a.Next() { + t.Errorf("b) 2-element link failed") + } + + r1c := r2b.Link(r2b) + verify(t, r1c, 1, 1) + verify(t, r2b, 1, 0) +} + +func TestLink2(t *testing.T) { + var r0 *Ring + r1a := &Ring{Value: 42} + r1b := &Ring{Value: 77} + r10 := makeN(10) + + r1a.Link(r0) + verify(t, r1a, 1, 42) + + r1a.Link(r1b) + verify(t, r1a, 2, 42+77) + + r10.Link(r0) + verify(t, r10, 10, sumN(10)) + + r10.Link(r1a) + verify(t, r10, 12, sumN(10)+42+77) +} + +func TestLink3(t *testing.T) { + var r Ring + n := 1 + for i := 1; i < 100; i++ { + n += i + verify(t, r.Link(New(i)), n, -1) + } +} + +func TestUnlink(t *testing.T) { + r10 := makeN(10) + s10 := r10.Move(6) + + sum10 := sumN(10) + + verify(t, r10, 10, sum10) + verify(t, s10, 10, sum10) + + r0 := r10.Unlink(0) + verify(t, r0, 0, 0) + + r1 := r10.Unlink(1) + verify(t, r1, 1, 2) + verify(t, r10, 9, sum10-2) + + r9 := r10.Unlink(9) + verify(t, r9, 9, sum10-2) + verify(t, r10, 9, sum10-2) +} + +func TestLinkUnlink(t *testing.T) { + for i := 1; i < 4; i++ { + ri := New(i) + for j := 0; j < i; j++ { + rj := ri.Unlink(j) + verify(t, rj, j, -1) + verify(t, ri, i-j, -1) + ri.Link(rj) + verify(t, ri, i, -1) + } + } +} diff --git a/src/pkg/container/vector/Makefile b/src/pkg/container/vector/Makefile new file mode 100644 index 000000000..f6b50156f --- /dev/null +++ b/src/pkg/container/vector/Makefile @@ -0,0 +1,69 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=container/vector +GOFILES=\ + defs.go\ + intvector.go\ + stringvector.go\ + vector.go\ + +include ../../../Make.pkg + +generate: vector.go vector_test.go + < vector.go cat\ + | gofmt -r='Vector -> IntVector'\ + | gofmt -r='interface{} -> int'\ + > intvector.go\ + + < vector.go cat\ + | gofmt -r='Vector -> StringVector'\ + | gofmt -r='interface{} -> string'\ + > stringvector.go\ + + < vector_test.go cat\ + | gofmt -r='Vector -> IntVector'\ + | gofmt -r='zero -> intzero'\ + | gofmt -r='elem2Value -> elem2IntValue'\ + | gofmt -r='intf2Value -> intf2IntValue'\ + | gofmt -r='int2Value -> int2IntValue'\ + | gofmt -r='TestZeroLen -> TestIntZeroLen'\ + | gofmt -r='TestResize -> TestIntResize'\ + | gofmt -r='TestResize2 -> TestIntResize2'\ + | gofmt -r='checkZero -> checkIntZero'\ + | gofmt -r='TestTrailingElements -> TestIntTrailingElements'\ + | gofmt -r='TestAccess -> TestIntAccess'\ + | gofmt -r='TestInsertDeleteClear -> TestIntInsertDeleteClear'\ + | gofmt -r='verify_slice -> verify_sliceInt'\ + | gofmt -r='verify_pattern -> verify_patternInt'\ + | gofmt -r='make_vector -> make_vectorInt'\ + | gofmt -r='TestInsertVector -> TestIntInsertVector'\ + | gofmt -r='TestDo -> TestIntDo'\ + | gofmt -r='TestVectorCopy -> TestIntVectorCopy'\ + | gofmt -r='interface{} -> int'\ + > intvector_test.go\ + + < vector_test.go cat\ + | gofmt -r='Vector -> StringVector'\ + | gofmt -r='zero -> strzero'\ + | gofmt -r='int2Value -> int2StrValue'\ + | gofmt -r='intf2Value -> intf2StrValue'\ + | gofmt -r='elem2Value -> elem2StrValue'\ + | gofmt -r='TestZeroLen -> TestStrZeroLen'\ + | gofmt -r='TestResize -> TestStrResize'\ + | gofmt -r='TestResize2 -> TestStrResize2'\ + | gofmt -r='checkZero -> checkStrZero'\ + | gofmt -r='TestTrailingElements -> TestStrTrailingElements'\ + | gofmt -r='TestAccess -> TestStrAccess'\ + | gofmt -r='TestInsertDeleteClear -> TestStrInsertDeleteClear'\ + | gofmt -r='verify_slice -> verify_sliceStr'\ + | gofmt -r='verify_pattern -> verify_patternStr'\ + | gofmt -r='make_vector -> make_vectorStr'\ + | gofmt -r='TestInsertVector -> TestStrInsertVector'\ + | gofmt -r='TestDo -> TestStrDo'\ + | gofmt -r='TestVectorCopy -> TestStrVectorCopy'\ + | gofmt -r='interface{} -> string'\ + > stringvector_test.go diff --git a/src/pkg/container/vector/defs.go b/src/pkg/container/vector/defs.go new file mode 100644 index 000000000..6d6b2ac81 --- /dev/null +++ b/src/pkg/container/vector/defs.go @@ -0,0 +1,43 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package vector implements containers for managing sequences of elements. +// Vectors grow and shrink dynamically as necessary. +package vector + +// Vector is a container for numbered sequences of elements of type interface{}. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for Vector is an empty vector ready to use. +type Vector []interface{} + +// IntVector is a container for numbered sequences of elements of type int. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for IntVector is an empty vector ready to use. +type IntVector []int + +// StringVector is a container for numbered sequences of elements of type string. +// A vector's length and capacity adjusts automatically as necessary. +// The zero value for StringVector is an empty vector ready to use. +type StringVector []string + +// Initial underlying array size +const initialSize = 8 + +// Partial sort.Interface support + +// LessInterface provides partial support of the sort.Interface. +type LessInterface interface { + Less(y interface{}) bool +} + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *Vector) Less(i, j int) bool { return (*p)[i].(LessInterface).Less((*p)[j]) } + +// sort.Interface support + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *IntVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } + +// Less returns a boolean denoting whether the i'th element is less than the j'th element. +func (p *StringVector) Less(i, j int) bool { return (*p)[i] < (*p)[j] } diff --git a/src/pkg/container/vector/intvector.go b/src/pkg/container/vector/intvector.go new file mode 100644 index 000000000..aa88cfeb3 --- /dev/null +++ b/src/pkg/container/vector/intvector.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + +func (p *IntVector) realloc(length, capacity int) (b []int) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(IntVector, length, capacity) + copy(b, *p) + *p = b + return +} + +// Insert n elements at position i. +func (p *IntVector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + +// Insert n elements at the end of a vector. +func (p *IntVector) Extend(n int) { p.Expand(len(*p), n) } + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *IntVector) Resize(length, capacity int) *IntVector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero int + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *IntVector) Len() int { return len(*p) } + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *IntVector) Cap() int { return cap(*p) } + +// At returns the i'th element of the vector. +func (p *IntVector) At(i int) int { return (*p)[i] } + +// Set sets the i'th element of the vector to value x. +func (p *IntVector) Set(i int, x int) { (*p)[i] = x } + +// Last returns the element in the vector of highest index. +func (p *IntVector) Last() int { return (*p)[len(*p)-1] } + +// Copy makes a copy of the vector and returns it. +func (p *IntVector) Copy() IntVector { + arr := make(IntVector, len(*p)) + copy(arr, *p) + return arr +} + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *IntVector) Insert(i int, x int) { + p.Expand(i, 1) + (*p)[i] = x +} + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *IntVector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero int + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *IntVector) InsertVector(i int, x *IntVector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + +// Cut deletes elements i through j-1, inclusive. +func (p *IntVector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero int + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *IntVector) Slice(i, j int) *IntVector { + var s IntVector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *IntVector) Push(x int) { p.Insert(len(*p), x) } + +// Pop deletes the last element of the vector. +func (p *IntVector) Pop() int { + a := *p + + i := len(a) - 1 + x := a[i] + var zero int + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + +// AppendVector appends the entire vector x to the end of this vector. +func (p *IntVector) AppendVector(x *IntVector) { p.InsertVector(len(*p), x) } + +// Swap exchanges the elements at indexes i and j. +func (p *IntVector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *IntVector) Do(f func(elem int)) { + for _, e := range *p { + f(e) + } +} diff --git a/src/pkg/container/vector/intvector_test.go b/src/pkg/container/vector/intvector_test.go new file mode 100644 index 000000000..b825af912 --- /dev/null +++ b/src/pkg/container/vector/intvector_test.go @@ -0,0 +1,331 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + +func TestIntZeroLen(t *testing.T) { + a := new(IntVector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b IntVector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + +func TestIntResize(t *testing.T) { + var a IntVector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + +func TestIntResize2(t *testing.T) { + var a IntVector + checkSize(t, &a, 0, 0) + a.Push(int2IntValue(1)) + a.Push(int2IntValue(2)) + a.Push(int2IntValue(3)) + a.Push(int2IntValue(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != intzero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, intzero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != intzero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, intzero, a[i]) + } + } +} + +func checkIntZero(t *testing.T, a *IntVector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == intzero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == intzero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != intzero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, intzero, a.At(i)) + } + if (*a)[i] != intzero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, intzero, (*a)[i]) + } + } +} + +func TestIntTrailingElements(t *testing.T) { + var a IntVector + for i := 0; i < 10; i++ { + a.Push(int2IntValue(i + 1)) + } + checkIntZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkIntZero(t, &a, 5) +} + +func TestIntAccess(t *testing.T) { + const n = 100 + var a IntVector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2IntValue(val(i))) + } + for i := 0; i < n; i++ { + if elem2IntValue(a.At(i)) != int2IntValue(val(i)) { + t.Error(i) + } + } + var b IntVector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2IntValue(val(i)) + } + for i := 0; i < n; i++ { + if elem2IntValue(b[i]) != int2IntValue(val(i)) { + t.Error(i) + } + } +} + +func TestIntInsertDeleteClear(t *testing.T) { + const n = 100 + var a IntVector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2IntValue(val(i))) + if elem2IntValue(a.Last()) != int2IntValue(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2IntValue(a.Last()) != int2IntValue(val(0)) { + t.Errorf("%T: C", a) + } + if elem2IntValue(a.At(0)) != int2IntValue(val(i)) { + t.Errorf("%T: D", a) + } + if elem2IntValue(a[0]) != int2IntValue(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2IntValue(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2IntValue(a.Last()) != int2IntValue(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2IntValue(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2IntValue(x)) + if elem2IntValue(a.Pop()) != int2IntValue(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + +func verify_sliceInt(t *testing.T, x *IntVector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2IntValue(x.At(k)) != int2IntValue(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2IntValue(s.At(k)) != int2IntValue(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2IntValue(x.At(k)), int2IntValue(elt)) + } + } +} + +func verify_patternInt(t *testing.T, x *IntVector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_sliceInt(t, x, 0, 0, a) + verify_sliceInt(t, x, 1, a, a+b) + verify_sliceInt(t, x, 0, a+b, n) +} + +func make_vectorInt(elt, len int) *IntVector { + x := new(IntVector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2IntValue(elt)) + } + return x +} + +func TestIntInsertVector(t *testing.T) { + // 1 + a := make_vectorInt(0, 0) + b := make_vectorInt(1, 10) + a.InsertVector(0, b) + verify_patternInt(t, a, 0, 10, 0) + // 2 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 0) + a.InsertVector(5, b) + verify_patternInt(t, a, 5, 0, 5) + // 3 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 3) + a.InsertVector(3, b) + verify_patternInt(t, a, 3, 3, 7) + // 4 + a = make_vectorInt(0, 10) + b = make_vectorInt(1, 1000) + a.InsertVector(8, b) + verify_patternInt(t, a, 8, 1000, 2) +} + +func TestIntDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(IntVector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2IntValue(salt*i)) + } + count := 0 + a.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(IntVector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2IntValue(salt * i) + } + count = 0 + b.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c IntVector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2IntValue(salt * i) + } + count = 0 + c.Do(func(e int) { + i := intf2IntValue(e) + if i != int2IntValue(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + +func TestIntVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src IntVector + for i := 0; i < Len; i++ { + src.Push(int2IntValue(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2IntValue(-1) + v := elem2IntValue(dest[i]) + if v != int2IntValue(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/src/pkg/container/vector/nogen_test.go b/src/pkg/container/vector/nogen_test.go new file mode 100644 index 000000000..7b6a25952 --- /dev/null +++ b/src/pkg/container/vector/nogen_test.go @@ -0,0 +1,67 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + +import ( + "fmt" + "sort" + "testing" +) + +var ( + zero interface{} + intzero int + strzero string +) + +func int2Value(x int) int { return x } +func int2IntValue(x int) int { return x } +func int2StrValue(x int) string { return string(x) } + +func elem2Value(x interface{}) int { return x.(int) } +func elem2IntValue(x int) int { return x } +func elem2StrValue(x string) string { return x } + +func intf2Value(x interface{}) int { return x.(int) } +func intf2IntValue(x interface{}) int { return x.(int) } +func intf2StrValue(x interface{}) string { return x.(string) } + +type VectorInterface interface { + Len() int + Cap() int +} + +func checkSize(t *testing.T, v VectorInterface, len, cap int) { + if v.Len() != len { + t.Errorf("%T expected len = %d; found %d", v, len, v.Len()) + } + if v.Cap() < cap { + t.Errorf("%T expected cap >= %d; found %d", v, cap, v.Cap()) + } +} + +func val(i int) int { return i*991 - 1234 } + +func TestSorting(t *testing.T) { + const n = 100 + + a := new(IntVector).Resize(n, 0) + for i := n - 1; i >= 0; i-- { + a.Set(i, n-1-i) + } + if sort.IsSorted(a) { + t.Error("int vector not sorted") + } + + b := new(StringVector).Resize(n, 0) + for i := n - 1; i >= 0; i-- { + b.Set(i, fmt.Sprint(n-1-i)) + } + if sort.IsSorted(b) { + t.Error("string vector not sorted") + } +} + +func tname(x interface{}) string { return fmt.Sprintf("%T: ", x) } diff --git a/src/pkg/container/vector/numbers_test.go b/src/pkg/container/vector/numbers_test.go new file mode 100644 index 000000000..abe01a8fb --- /dev/null +++ b/src/pkg/container/vector/numbers_test.go @@ -0,0 +1,123 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +const memTestN = 1000000 + +func s(n uint64) string { + str := fmt.Sprintf("%d", n) + lens := len(str) + a := make([]string, (lens+2)/3) + start := lens + for i := range a { + start -= 3 + if start < 0 { + start = 0 + } + a[len(a)-i-1] = str[start:lens] + lens -= 3 + } + return strings.Join(a, " ") +} + +func TestVectorNums(t *testing.T) { + if testing.Short() { + return + } + var v Vector + c := int(0) + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + +func TestIntVectorNums(t *testing.T) { + if testing.Short() { + return + } + var v IntVector + c := int(0) + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + +func TestStringVectorNums(t *testing.T) { + if testing.Short() { + return + } + var v StringVector + c := "" + runtime.GC() + m0 := runtime.MemStats + v.Resize(memTestN, memTestN) + for i := 0; i < memTestN; i++ { + v.Set(i, c) + } + runtime.GC() + m := runtime.MemStats + v.Resize(0, 0) + runtime.GC() + n := m.Alloc - m0.Alloc + t.Logf("%T.Push(%#v), n = %s: Alloc/n = %.2f\n", v, c, s(memTestN), float64(n)/memTestN) +} + +func BenchmarkVectorNums(b *testing.B) { + c := int(0) + var v Vector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} + +func BenchmarkIntVectorNums(b *testing.B) { + c := int(0) + var v IntVector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} + +func BenchmarkStringVectorNums(b *testing.B) { + c := "" + var v StringVector + b.StopTimer() + runtime.GC() + b.StartTimer() + for i := 0; i < b.N; i++ { + v.Push(c) + } +} diff --git a/src/pkg/container/vector/stringvector.go b/src/pkg/container/vector/stringvector.go new file mode 100644 index 000000000..dc81f06b7 --- /dev/null +++ b/src/pkg/container/vector/stringvector.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + +func (p *StringVector) realloc(length, capacity int) (b []string) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(StringVector, length, capacity) + copy(b, *p) + *p = b + return +} + +// Insert n elements at position i. +func (p *StringVector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + +// Insert n elements at the end of a vector. +func (p *StringVector) Extend(n int) { p.Expand(len(*p), n) } + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *StringVector) Resize(length, capacity int) *StringVector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero string + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *StringVector) Len() int { return len(*p) } + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *StringVector) Cap() int { return cap(*p) } + +// At returns the i'th element of the vector. +func (p *StringVector) At(i int) string { return (*p)[i] } + +// Set sets the i'th element of the vector to value x. +func (p *StringVector) Set(i int, x string) { (*p)[i] = x } + +// Last returns the element in the vector of highest index. +func (p *StringVector) Last() string { return (*p)[len(*p)-1] } + +// Copy makes a copy of the vector and returns it. +func (p *StringVector) Copy() StringVector { + arr := make(StringVector, len(*p)) + copy(arr, *p) + return arr +} + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *StringVector) Insert(i int, x string) { + p.Expand(i, 1) + (*p)[i] = x +} + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *StringVector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero string + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *StringVector) InsertVector(i int, x *StringVector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + +// Cut deletes elements i through j-1, inclusive. +func (p *StringVector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero string + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *StringVector) Slice(i, j int) *StringVector { + var s StringVector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *StringVector) Push(x string) { p.Insert(len(*p), x) } + +// Pop deletes the last element of the vector. +func (p *StringVector) Pop() string { + a := *p + + i := len(a) - 1 + x := a[i] + var zero string + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + +// AppendVector appends the entire vector x to the end of this vector. +func (p *StringVector) AppendVector(x *StringVector) { p.InsertVector(len(*p), x) } + +// Swap exchanges the elements at indexes i and j. +func (p *StringVector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *StringVector) Do(f func(elem string)) { + for _, e := range *p { + f(e) + } +} diff --git a/src/pkg/container/vector/stringvector_test.go b/src/pkg/container/vector/stringvector_test.go new file mode 100644 index 000000000..c75676f78 --- /dev/null +++ b/src/pkg/container/vector/stringvector_test.go @@ -0,0 +1,331 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + +func TestStrZeroLen(t *testing.T) { + a := new(StringVector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b StringVector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + +func TestStrResize(t *testing.T) { + var a StringVector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + +func TestStrResize2(t *testing.T) { + var a StringVector + checkSize(t, &a, 0, 0) + a.Push(int2StrValue(1)) + a.Push(int2StrValue(2)) + a.Push(int2StrValue(3)) + a.Push(int2StrValue(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != strzero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, strzero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != strzero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, strzero, a[i]) + } + } +} + +func checkStrZero(t *testing.T, a *StringVector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == strzero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == strzero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != strzero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, strzero, a.At(i)) + } + if (*a)[i] != strzero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, strzero, (*a)[i]) + } + } +} + +func TestStrTrailingElements(t *testing.T) { + var a StringVector + for i := 0; i < 10; i++ { + a.Push(int2StrValue(i + 1)) + } + checkStrZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkStrZero(t, &a, 5) +} + +func TestStrAccess(t *testing.T) { + const n = 100 + var a StringVector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2StrValue(val(i))) + } + for i := 0; i < n; i++ { + if elem2StrValue(a.At(i)) != int2StrValue(val(i)) { + t.Error(i) + } + } + var b StringVector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2StrValue(val(i)) + } + for i := 0; i < n; i++ { + if elem2StrValue(b[i]) != int2StrValue(val(i)) { + t.Error(i) + } + } +} + +func TestStrInsertDeleteClear(t *testing.T) { + const n = 100 + var a StringVector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2StrValue(val(i))) + if elem2StrValue(a.Last()) != int2StrValue(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2StrValue(a.Last()) != int2StrValue(val(0)) { + t.Errorf("%T: C", a) + } + if elem2StrValue(a.At(0)) != int2StrValue(val(i)) { + t.Errorf("%T: D", a) + } + if elem2StrValue(a[0]) != int2StrValue(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2StrValue(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2StrValue(a.Last()) != int2StrValue(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2StrValue(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2StrValue(x)) + if elem2StrValue(a.Pop()) != int2StrValue(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + +func verify_sliceStr(t *testing.T, x *StringVector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2StrValue(x.At(k)) != int2StrValue(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2StrValue(s.At(k)) != int2StrValue(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2StrValue(x.At(k)), int2StrValue(elt)) + } + } +} + +func verify_patternStr(t *testing.T, x *StringVector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_sliceStr(t, x, 0, 0, a) + verify_sliceStr(t, x, 1, a, a+b) + verify_sliceStr(t, x, 0, a+b, n) +} + +func make_vectorStr(elt, len int) *StringVector { + x := new(StringVector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2StrValue(elt)) + } + return x +} + +func TestStrInsertVector(t *testing.T) { + // 1 + a := make_vectorStr(0, 0) + b := make_vectorStr(1, 10) + a.InsertVector(0, b) + verify_patternStr(t, a, 0, 10, 0) + // 2 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 0) + a.InsertVector(5, b) + verify_patternStr(t, a, 5, 0, 5) + // 3 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 3) + a.InsertVector(3, b) + verify_patternStr(t, a, 3, 3, 7) + // 4 + a = make_vectorStr(0, 10) + b = make_vectorStr(1, 1000) + a.InsertVector(8, b) + verify_patternStr(t, a, 8, 1000, 2) +} + +func TestStrDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(StringVector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2StrValue(salt*i)) + } + count := 0 + a.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(StringVector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2StrValue(salt * i) + } + count = 0 + b.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c StringVector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2StrValue(salt * i) + } + count = 0 + c.Do(func(e string) { + i := intf2StrValue(e) + if i != int2StrValue(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + +func TestStrVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src StringVector + for i := 0; i < Len; i++ { + src.Push(int2StrValue(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2StrValue(-1) + v := elem2StrValue(dest[i]) + if v != int2StrValue(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/src/pkg/container/vector/vector.go b/src/pkg/container/vector/vector.go new file mode 100644 index 000000000..8470ec067 --- /dev/null +++ b/src/pkg/container/vector/vector.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector.go, it was generated +// automatically from vector.go - DO NOT EDIT in that case! + +package vector + +func (p *Vector) realloc(length, capacity int) (b []interface{}) { + if capacity < initialSize { + capacity = initialSize + } + if capacity < length { + capacity = length + } + b = make(Vector, length, capacity) + copy(b, *p) + *p = b + return +} + +// Insert n elements at position i. +func (p *Vector) Expand(i, n int) { + a := *p + + // make sure we have enough space + len0 := len(a) + len1 := len0 + n + if len1 <= cap(a) { + // enough space - just expand + a = a[0:len1] + } else { + // not enough space - double capacity + capb := cap(a) * 2 + if capb < len1 { + // still not enough - use required length + capb = len1 + } + // capb >= len1 + a = p.realloc(len1, capb) + } + + // make a hole + for j := len0 - 1; j >= i; j-- { + a[j+n] = a[j] + } + + *p = a +} + +// Insert n elements at the end of a vector. +func (p *Vector) Extend(n int) { p.Expand(len(*p), n) } + +// Resize changes the length and capacity of a vector. +// If the new length is shorter than the current length, Resize discards +// trailing elements. If the new length is longer than the current length, +// Resize adds the respective zero values for the additional elements. The capacity +// parameter is ignored unless the new length or capacity is longer than the current +// capacity. The resized vector's capacity may be larger than the requested capacity. +func (p *Vector) Resize(length, capacity int) *Vector { + a := *p + + if length > cap(a) || capacity > cap(a) { + // not enough space or larger capacity requested explicitly + a = p.realloc(length, capacity) + } else if length < len(a) { + // clear trailing elements + for i := range a[length:] { + var zero interface{} + a[length+i] = zero + } + } + + *p = a[0:length] + return p +} + +// Len returns the number of elements in the vector. +// Same as len(*p). +func (p *Vector) Len() int { return len(*p) } + +// Cap returns the capacity of the vector; that is, the +// maximum length the vector can grow without resizing. +// Same as cap(*p). +func (p *Vector) Cap() int { return cap(*p) } + +// At returns the i'th element of the vector. +func (p *Vector) At(i int) interface{} { return (*p)[i] } + +// Set sets the i'th element of the vector to value x. +func (p *Vector) Set(i int, x interface{}) { (*p)[i] = x } + +// Last returns the element in the vector of highest index. +func (p *Vector) Last() interface{} { return (*p)[len(*p)-1] } + +// Copy makes a copy of the vector and returns it. +func (p *Vector) Copy() Vector { + arr := make(Vector, len(*p)) + copy(arr, *p) + return arr +} + +// Insert inserts into the vector an element of value x before +// the current element at index i. +func (p *Vector) Insert(i int, x interface{}) { + p.Expand(i, 1) + (*p)[i] = x +} + +// Delete deletes the i'th element of the vector. The gap is closed so the old +// element at index i+1 has index i afterwards. +func (p *Vector) Delete(i int) { + a := *p + n := len(a) + + copy(a[i:n-1], a[i+1:n]) + var zero interface{} + a[n-1] = zero // support GC, zero out entry + *p = a[0 : n-1] +} + +// InsertVector inserts into the vector the contents of the vector +// x such that the 0th element of x appears at index i after insertion. +func (p *Vector) InsertVector(i int, x *Vector) { + b := *x + + p.Expand(i, len(b)) + copy((*p)[i:i+len(b)], b) +} + +// Cut deletes elements i through j-1, inclusive. +func (p *Vector) Cut(i, j int) { + a := *p + n := len(a) + m := n - (j - i) + + copy(a[i:m], a[j:n]) + for k := m; k < n; k++ { //TODO(bflm) don't zero out the elements unless it's a Vector. + var zero interface{} + a[k] = zero // support GC, zero out entries + } + + *p = a[0:m] +} + +// Slice returns a new sub-vector by slicing the old one to extract slice [i:j]. +// The elements are copied. The original vector is unchanged. +func (p *Vector) Slice(i, j int) *Vector { + var s Vector + s.realloc(j-i, 0) // will fail in Init() if j < i + copy(s, (*p)[i:j]) + return &s +} + +// Convenience wrappers + +// Push appends x to the end of the vector. +func (p *Vector) Push(x interface{}) { p.Insert(len(*p), x) } + +// Pop deletes the last element of the vector. +func (p *Vector) Pop() interface{} { + a := *p + + i := len(a) - 1 + x := a[i] + var zero interface{} + a[i] = zero // support GC, zero out entry + *p = a[0:i] + return x +} + +// AppendVector appends the entire vector x to the end of this vector. +func (p *Vector) AppendVector(x *Vector) { p.InsertVector(len(*p), x) } + +// Swap exchanges the elements at indexes i and j. +func (p *Vector) Swap(i, j int) { + a := *p + a[i], a[j] = a[j], a[i] +} + +// Do calls function f for each element of the vector, in order. +// The behavior of Do is undefined if f changes *p. +func (p *Vector) Do(f func(elem interface{})) { + for _, e := range *p { + f(e) + } +} diff --git a/src/pkg/container/vector/vector_test.go b/src/pkg/container/vector/vector_test.go new file mode 100644 index 000000000..a7f47b8c2 --- /dev/null +++ b/src/pkg/container/vector/vector_test.go @@ -0,0 +1,331 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CAUTION: If this file is not vector_test.go, it was generated +// automatically from vector_test.go - DO NOT EDIT in that case! + +package vector + +import "testing" + +func TestZeroLen(t *testing.T) { + a := new(Vector) + if a.Len() != 0 { + t.Errorf("%T: B1) expected 0, got %d", a, a.Len()) + } + if len(*a) != 0 { + t.Errorf("%T: B2) expected 0, got %d", a, len(*a)) + } + var b Vector + if b.Len() != 0 { + t.Errorf("%T: B3) expected 0, got %d", b, b.Len()) + } + if len(b) != 0 { + t.Errorf("%T: B4) expected 0, got %d", b, len(b)) + } +} + +func TestResize(t *testing.T) { + var a Vector + checkSize(t, &a, 0, 0) + checkSize(t, a.Resize(0, 5), 0, 5) + checkSize(t, a.Resize(1, 0), 1, 5) + checkSize(t, a.Resize(10, 0), 10, 10) + checkSize(t, a.Resize(5, 0), 5, 10) + checkSize(t, a.Resize(3, 8), 3, 10) + checkSize(t, a.Resize(0, 100), 0, 100) + checkSize(t, a.Resize(11, 100), 11, 100) +} + +func TestResize2(t *testing.T) { + var a Vector + checkSize(t, &a, 0, 0) + a.Push(int2Value(1)) + a.Push(int2Value(2)) + a.Push(int2Value(3)) + a.Push(int2Value(4)) + checkSize(t, &a, 4, 4) + checkSize(t, a.Resize(10, 0), 10, 10) + for i := 4; i < a.Len(); i++ { + if a.At(i) != zero { + t.Errorf("%T: expected a.At(%d) == %v; found %v!", a, i, zero, a.At(i)) + } + } + for i := 4; i < len(a); i++ { + if a[i] != zero { + t.Errorf("%T: expected a[%d] == %v; found %v", a, i, zero, a[i]) + } + } +} + +func checkZero(t *testing.T, a *Vector, i int) { + for j := 0; j < i; j++ { + if a.At(j) == zero { + t.Errorf("%T: 1 expected a.At(%d) == %d; found %v", a, j, j, a.At(j)) + } + if (*a)[j] == zero { + t.Errorf("%T: 2 expected (*a)[%d] == %d; found %v", a, j, j, (*a)[j]) + } + } + for ; i < a.Len(); i++ { + if a.At(i) != zero { + t.Errorf("%T: 3 expected a.At(%d) == %v; found %v", a, i, zero, a.At(i)) + } + if (*a)[i] != zero { + t.Errorf("%T: 4 expected (*a)[%d] == %v; found %v", a, i, zero, (*a)[i]) + } + } +} + +func TestTrailingElements(t *testing.T) { + var a Vector + for i := 0; i < 10; i++ { + a.Push(int2Value(i + 1)) + } + checkZero(t, &a, 10) + checkSize(t, &a, 10, 16) + checkSize(t, a.Resize(5, 0), 5, 16) + checkSize(t, a.Resize(10, 0), 10, 16) + checkZero(t, &a, 5) +} + +func TestAccess(t *testing.T) { + const n = 100 + var a Vector + a.Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2Value(val(i))) + } + for i := 0; i < n; i++ { + if elem2Value(a.At(i)) != int2Value(val(i)) { + t.Error(i) + } + } + var b Vector + b.Resize(n, 0) + for i := 0; i < n; i++ { + b[i] = int2Value(val(i)) + } + for i := 0; i < n; i++ { + if elem2Value(b[i]) != int2Value(val(i)) { + t.Error(i) + } + } +} + +func TestInsertDeleteClear(t *testing.T) { + const n = 100 + var a Vector + + for i := 0; i < n; i++ { + if a.Len() != i { + t.Errorf("%T: A) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: A) wrong len() %d (expected %d)", a, len(a), i) + } + a.Insert(0, int2Value(val(i))) + if elem2Value(a.Last()) != int2Value(val(0)) { + t.Errorf("%T: B", a) + } + } + for i := n - 1; i >= 0; i-- { + if elem2Value(a.Last()) != int2Value(val(0)) { + t.Errorf("%T: C", a) + } + if elem2Value(a.At(0)) != int2Value(val(i)) { + t.Errorf("%T: D", a) + } + if elem2Value(a[0]) != int2Value(val(i)) { + t.Errorf("%T: D2", a) + } + a.Delete(0) + if a.Len() != i { + t.Errorf("%T: E) wrong Len() %d (expected %d)", a, a.Len(), i) + } + if len(a) != i { + t.Errorf("%T: E) wrong len() %d (expected %d)", a, len(a), i) + } + } + + if a.Len() != 0 { + t.Errorf("%T: F) wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: F) wrong len() %d (expected 0)", a, len(a)) + } + for i := 0; i < n; i++ { + a.Push(int2Value(val(i))) + if a.Len() != i+1 { + t.Errorf("%T: G) wrong Len() %d (expected %d)", a, a.Len(), i+1) + } + if len(a) != i+1 { + t.Errorf("%T: G) wrong len() %d (expected %d)", a, len(a), i+1) + } + if elem2Value(a.Last()) != int2Value(val(i)) { + t.Errorf("%T: H", a) + } + } + a.Resize(0, 0) + if a.Len() != 0 { + t.Errorf("%T: I wrong Len() %d (expected 0)", a, a.Len()) + } + if len(a) != 0 { + t.Errorf("%T: I wrong len() %d (expected 0)", a, len(a)) + } + + const m = 5 + for j := 0; j < m; j++ { + a.Push(int2Value(j)) + for i := 0; i < n; i++ { + x := val(i) + a.Push(int2Value(x)) + if elem2Value(a.Pop()) != int2Value(x) { + t.Errorf("%T: J", a) + } + if a.Len() != j+1 { + t.Errorf("%T: K) wrong Len() %d (expected %d)", a, a.Len(), j+1) + } + if len(a) != j+1 { + t.Errorf("%T: K) wrong len() %d (expected %d)", a, len(a), j+1) + } + } + } + if a.Len() != m { + t.Errorf("%T: L) wrong Len() %d (expected %d)", a, a.Len(), m) + } + if len(a) != m { + t.Errorf("%T: L) wrong len() %d (expected %d)", a, len(a), m) + } +} + +func verify_slice(t *testing.T, x *Vector, elt, i, j int) { + for k := i; k < j; k++ { + if elem2Value(x.At(k)) != int2Value(elt) { + t.Errorf("%T: M) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) + } + } + + s := x.Slice(i, j) + for k, n := 0, j-i; k < n; k++ { + if elem2Value(s.At(k)) != int2Value(elt) { + t.Errorf("%T: N) wrong [%d] element %v (expected %v)", x, k, elem2Value(x.At(k)), int2Value(elt)) + } + } +} + +func verify_pattern(t *testing.T, x *Vector, a, b, c int) { + n := a + b + c + if x.Len() != n { + t.Errorf("%T: O) wrong Len() %d (expected %d)", x, x.Len(), n) + } + if len(*x) != n { + t.Errorf("%T: O) wrong len() %d (expected %d)", x, len(*x), n) + } + verify_slice(t, x, 0, 0, a) + verify_slice(t, x, 1, a, a+b) + verify_slice(t, x, 0, a+b, n) +} + +func make_vector(elt, len int) *Vector { + x := new(Vector).Resize(len, 0) + for i := 0; i < len; i++ { + x.Set(i, int2Value(elt)) + } + return x +} + +func TestInsertVector(t *testing.T) { + // 1 + a := make_vector(0, 0) + b := make_vector(1, 10) + a.InsertVector(0, b) + verify_pattern(t, a, 0, 10, 0) + // 2 + a = make_vector(0, 10) + b = make_vector(1, 0) + a.InsertVector(5, b) + verify_pattern(t, a, 5, 0, 5) + // 3 + a = make_vector(0, 10) + b = make_vector(1, 3) + a.InsertVector(3, b) + verify_pattern(t, a, 3, 3, 7) + // 4 + a = make_vector(0, 10) + b = make_vector(1, 1000) + a.InsertVector(8, b) + verify_pattern(t, a, 8, 1000, 2) +} + +func TestDo(t *testing.T) { + const n = 25 + const salt = 17 + a := new(Vector).Resize(n, 0) + for i := 0; i < n; i++ { + a.Set(i, int2Value(salt*i)) + } + count := 0 + a.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(a), "value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(a), "should visit", n, "values; did visit", count) + } + + b := new(Vector).Resize(n, 0) + for i := 0; i < n; i++ { + (*b)[i] = int2Value(salt * i) + } + count = 0 + b.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(b), "b) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(b), "b) should visit", n, "values; did visit", count) + } + + var c Vector + c.Resize(n, 0) + for i := 0; i < n; i++ { + c[i] = int2Value(salt * i) + } + count = 0 + c.Do(func(e interface{}) { + i := intf2Value(e) + if i != int2Value(count*salt) { + t.Error(tname(c), "c) value at", count, "should be", count*salt, "not", i) + } + count++ + }) + if count != n { + t.Error(tname(c), "c) should visit", n, "values; did visit", count) + } + +} + +func TestVectorCopy(t *testing.T) { + // verify Copy() returns a copy, not simply a slice of the original vector + const Len = 10 + var src Vector + for i := 0; i < Len; i++ { + src.Push(int2Value(i * i)) + } + dest := src.Copy() + for i := 0; i < Len; i++ { + src[i] = int2Value(-1) + v := elem2Value(dest[i]) + if v != int2Value(i*i) { + t.Error(tname(src), "expected", i*i, "got", v) + } + } +} diff --git a/src/pkg/crypto/Makefile b/src/pkg/crypto/Makefile new file mode 100644 index 000000000..738a52062 --- /dev/null +++ b/src/pkg/crypto/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=crypto +GOFILES=\ + crypto.go\ + +include ../../Make.pkg diff --git a/src/pkg/crypto/aes/Makefile b/src/pkg/crypto/aes/Makefile new file mode 100644 index 000000000..9dc846ee3 --- /dev/null +++ b/src/pkg/crypto/aes/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/aes +GOFILES=\ + block.go\ + cipher.go\ + const.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/aes/aes_test.go b/src/pkg/crypto/aes/aes_test.go new file mode 100644 index 000000000..2136d447d --- /dev/null +++ b/src/pkg/crypto/aes/aes_test.go @@ -0,0 +1,350 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package aes + +import ( + "testing" +) + +// See const.go for overview of math here. + +// Test that powx is initialized correctly. +// (Can adapt this code to generate it too.) +func TestPowx(t *testing.T) { + p := 1 + for i := 0; i < len(powx); i++ { + if powx[i] != byte(p) { + t.Errorf("powx[%d] = %#x, want %#x", i, powx[i], p) + } + p <<= 1 + if p&0x100 != 0 { + p ^= poly + } + } +} + +// Multiply b and c as GF(2) polynomials modulo poly +func mul(b, c uint32) uint32 { + i := b + j := c + s := uint32(0) + for k := uint32(1); k < 0x100 && j != 0; k <<= 1 { + // Invariant: k == 1<>8 + } + } +} + +// Test that decryption tables are correct. +// (Can adapt this code to generate them too.) +func TestTd(t *testing.T) { + for i := 0; i < 256; i++ { + s := uint32(sbox1[i]) + s9 := mul(s, 0x9) + sb := mul(s, 0xb) + sd := mul(s, 0xd) + se := mul(s, 0xe) + w := se<<24 | s9<<16 | sd<<8 | sb + for j := 0; j < 4; j++ { + if x := td[j][i]; x != w { + t.Fatalf("td[%d][%d] = %#x, want %#x", j, i, x, w) + } + w = w<<24 | w>>8 + } + } +} + +// Test vectors are from FIPS 197: +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf + +// Appendix A of FIPS 197: Key expansion examples +type KeyTest struct { + key []byte + enc []uint32 + dec []uint32 // decryption expansion; not in FIPS 197, computed from C implementation. +} + +var keyTests = []KeyTest{ + { + // A.1. Expansion of a 128-bit Cipher Key + []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c}, + []uint32{ + 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x09cf4f3c, + 0xa0fafe17, 0x88542cb1, 0x23a33939, 0x2a6c7605, + 0xf2c295f2, 0x7a96b943, 0x5935807a, 0x7359f67f, + 0x3d80477d, 0x4716fe3e, 0x1e237e44, 0x6d7a883b, + 0xef44a541, 0xa8525b7f, 0xb671253b, 0xdb0bad00, + 0xd4d1c6f8, 0x7c839d87, 0xcaf2b8bc, 0x11f915bc, + 0x6d88a37a, 0x110b3efd, 0xdbf98641, 0xca0093fd, + 0x4e54f70e, 0x5f5fc9f3, 0x84a64fb2, 0x4ea6dc4f, + 0xead27321, 0xb58dbad2, 0x312bf560, 0x7f8d292f, + 0xac7766f3, 0x19fadc21, 0x28d12941, 0x575c006e, + 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + }, + []uint32{ + 0xd014f9a8, 0xc9ee2589, 0xe13f0cc8, 0xb6630ca6, + 0xc7b5a63, 0x1319eafe, 0xb0398890, 0x664cfbb4, + 0xdf7d925a, 0x1f62b09d, 0xa320626e, 0xd6757324, + 0x12c07647, 0xc01f22c7, 0xbc42d2f3, 0x7555114a, + 0x6efcd876, 0xd2df5480, 0x7c5df034, 0xc917c3b9, + 0x6ea30afc, 0xbc238cf6, 0xae82a4b4, 0xb54a338d, + 0x90884413, 0xd280860a, 0x12a12842, 0x1bc89739, + 0x7c1f13f7, 0x4208c219, 0xc021ae48, 0x969bf7b, + 0xcc7505eb, 0x3e17d1ee, 0x82296c51, 0xc9481133, + 0x2b3708a7, 0xf262d405, 0xbc3ebdbf, 0x4b617d62, + 0x2b7e1516, 0x28aed2a6, 0xabf71588, 0x9cf4f3c, + }, + }, + { + // A.2. Expansion of a 192-bit Cipher Key + []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, + }, + []uint32{ + 0x8e73b0f7, 0xda0e6452, 0xc810f32b, 0x809079e5, + 0x62f8ead2, 0x522c6b7b, 0xfe0c91f7, 0x2402f5a5, + 0xec12068e, 0x6c827f6b, 0x0e7a95b9, 0x5c56fec2, + 0x4db7b4bd, 0x69b54118, 0x85a74796, 0xe92538fd, + 0xe75fad44, 0xbb095386, 0x485af057, 0x21efb14f, + 0xa448f6d9, 0x4d6dce24, 0xaa326360, 0x113b30e6, + 0xa25e7ed5, 0x83b1cf9a, 0x27f93943, 0x6a94f767, + 0xc0a69407, 0xd19da4e1, 0xec1786eb, 0x6fa64971, + 0x485f7032, 0x22cb8755, 0xe26d1352, 0x33f0b7b3, + 0x40beeb28, 0x2f18a259, 0x6747d26b, 0x458c553e, + 0xa7e1466c, 0x9411f1df, 0x821f750a, 0xad07d753, + 0xca400538, 0x8fcc5006, 0x282d166a, 0xbc3ce7b5, + 0xe98ba06f, 0x448c773c, 0x8ecc7204, 0x01002202, + }, + nil, + }, + { + // A.3. Expansion of a 256-bit Cipher Key + []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, + }, + []uint32{ + 0x603deb10, 0x15ca71be, 0x2b73aef0, 0x857d7781, + 0x1f352c07, 0x3b6108d7, 0x2d9810a3, 0x0914dff4, + 0x9ba35411, 0x8e6925af, 0xa51a8b5f, 0x2067fcde, + 0xa8b09c1a, 0x93d194cd, 0xbe49846e, 0xb75d5b9a, + 0xd59aecb8, 0x5bf3c917, 0xfee94248, 0xde8ebe96, + 0xb5a9328a, 0x2678a647, 0x98312229, 0x2f6c79b3, + 0x812c81ad, 0xdadf48ba, 0x24360af2, 0xfab8b464, + 0x98c5bfc9, 0xbebd198e, 0x268c3ba7, 0x09e04214, + 0x68007bac, 0xb2df3316, 0x96e939e4, 0x6c518d80, + 0xc814e204, 0x76a9fb8a, 0x5025c02d, 0x59c58239, + 0xde136967, 0x6ccc5a71, 0xfa256395, 0x9674ee15, + 0x5886ca5d, 0x2e2f31d7, 0x7e0af1fa, 0x27cf73c3, + 0x749c47ab, 0x18501dda, 0xe2757e4f, 0x7401905a, + 0xcafaaae3, 0xe4d59b34, 0x9adf6ace, 0xbd10190d, + 0xfe4890d1, 0xe6188d0b, 0x046df344, 0x706c631e, + }, + nil, + }, +} + +// Test key expansion against FIPS 197 examples. +func TestExpandKey(t *testing.T) { +L: + for i, tt := range keyTests { + enc := make([]uint32, len(tt.enc)) + var dec []uint32 + if tt.dec != nil { + dec = make([]uint32, len(tt.dec)) + } + expandKey(tt.key, enc, dec) + for j, v := range enc { + if v != tt.enc[j] { + t.Errorf("key %d: enc[%d] = %#x, want %#x", i, j, v, tt.enc[j]) + continue L + } + } + if dec != nil { + for j, v := range dec { + if v != tt.dec[j] { + t.Errorf("key %d: dec[%d] = %#x, want %#x", i, j, v, tt.dec[j]) + continue L + } + } + } + } +} + +// Appendix B, C of FIPS 197: Cipher examples, Example vectors. +type CryptTest struct { + key []byte + in []byte + out []byte +} + +var encryptTests = []CryptTest{ + { + // Appendix B. + []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c}, + []byte{0x32, 0x43, 0xf6, 0xa8, 0x88, 0x5a, 0x30, 0x8d, 0x31, 0x31, 0x98, 0xa2, 0xe0, 0x37, 0x07, 0x34}, + []byte{0x39, 0x25, 0x84, 0x1d, 0x02, 0xdc, 0x09, 0xfb, 0xdc, 0x11, 0x85, 0x97, 0x19, 0x6a, 0x0b, 0x32}, + }, + { + // Appendix C.1. AES-128 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30, 0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a}, + }, + { + // Appendix C.2. AES-192 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + }, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0xdd, 0xa9, 0x7c, 0xa4, 0x86, 0x4c, 0xdf, 0xe0, 0x6e, 0xaf, 0x70, 0xa0, 0xec, 0x0d, 0x71, 0x91}, + }, + { + // Appendix C.3. AES-256 + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }, + []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + []byte{0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, 0x60, 0x89}, + }, +} + +// Test encryptBlock against FIPS 197 examples. +func TestEncryptBlock(t *testing.T) { + for i, tt := range encryptTests { + n := len(tt.key) + 28 + enc := make([]uint32, n) + dec := make([]uint32, n) + expandKey(tt.key, enc, dec) + out := make([]byte, len(tt.in)) + encryptBlock(enc, out, tt.in) + for j, v := range out { + if v != tt.out[j] { + t.Errorf("encryptBlock %d: out[%d] = %#x, want %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +// Test decryptBlock against FIPS 197 examples. +func TestDecryptBlock(t *testing.T) { + for i, tt := range encryptTests { + n := len(tt.key) + 28 + enc := make([]uint32, n) + dec := make([]uint32, n) + expandKey(tt.key, enc, dec) + plain := make([]byte, len(tt.in)) + decryptBlock(dec, plain, tt.out) + for j, v := range plain { + if v != tt.in[j] { + t.Errorf("decryptBlock %d: plain[%d] = %#x, want %#x", i, j, v, tt.in[j]) + break + } + } + } +} + +// Test Cipher Encrypt method against FIPS 197 examples. +func TestCipherEncrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + for j, v := range out { + if v != tt.out[j] { + t.Errorf("Cipher.Encrypt %d: out[%d] = %#x, want %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +// Test Cipher Decrypt against FIPS 197 examples. +func TestCipherDecrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + plain := make([]byte, len(tt.in)) + c.Decrypt(plain, tt.out) + for j, v := range plain { + if v != tt.in[j] { + t.Errorf("decryptBlock %d: plain[%d] = %#x, want %#x", i, j, v, tt.in[j]) + break + } + } + } +} diff --git a/src/pkg/crypto/aes/block.go b/src/pkg/crypto/aes/block.go new file mode 100644 index 000000000..130cd011c --- /dev/null +++ b/src/pkg/crypto/aes/block.go @@ -0,0 +1,176 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This Go implementation is derived in part from the reference +// ANSI C implementation, which carries the following notice: +// +// rijndael-alg-fst.c +// +// @version 3.0 (December 2000) +// +// Optimised ANSI C code for the Rijndael cipher (now AES) +// +// @author Vincent Rijmen +// @author Antoon Bosselaers +// @author Paulo Barreto +// +// This code is hereby placed in the public domain. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS +// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// See FIPS 197 for specification, and see Daemen and Rijmen's Rijndael submission +// for implementation details. +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +// http://csrc.nist.gov/archive/aes/rijndael/Rijndael-ammended.pdf + +package aes + +// Encrypt one block from src into dst, using the expanded key xk. +func encryptBlock(xk []uint32, dst, src []byte) { + var s0, s1, s2, s3, t0, t1, t2, t3 uint32 + + s0 = uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + s1 = uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + s2 = uint32(src[8])<<24 | uint32(src[9])<<16 | uint32(src[10])<<8 | uint32(src[11]) + s3 = uint32(src[12])<<24 | uint32(src[13])<<16 | uint32(src[14])<<8 | uint32(src[15]) + + // First round just XORs input with key. + s0 ^= xk[0] + s1 ^= xk[1] + s2 ^= xk[2] + s3 ^= xk[3] + + // Middle rounds shuffle using tables. + // Number of rounds is set by length of expanded key. + nr := len(xk)/4 - 2 // - 2: one above, one more below + k := 4 + for r := 0; r < nr; r++ { + t0 = xk[k+0] ^ te[0][s0>>24] ^ te[1][s1>>16&0xff] ^ te[2][s2>>8&0xff] ^ te[3][s3&0xff] + t1 = xk[k+1] ^ te[0][s1>>24] ^ te[1][s2>>16&0xff] ^ te[2][s3>>8&0xff] ^ te[3][s0&0xff] + t2 = xk[k+2] ^ te[0][s2>>24] ^ te[1][s3>>16&0xff] ^ te[2][s0>>8&0xff] ^ te[3][s1&0xff] + t3 = xk[k+3] ^ te[0][s3>>24] ^ te[1][s0>>16&0xff] ^ te[2][s1>>8&0xff] ^ te[3][s2&0xff] + k += 4 + s0, s1, s2, s3 = t0, t1, t2, t3 + } + + // Last round uses s-box directly and XORs to produce output. + s0 = uint32(sbox0[t0>>24])<<24 | uint32(sbox0[t1>>16&0xff])<<16 | uint32(sbox0[t2>>8&0xff])<<8 | uint32(sbox0[t3&0xff]) + s1 = uint32(sbox0[t1>>24])<<24 | uint32(sbox0[t2>>16&0xff])<<16 | uint32(sbox0[t3>>8&0xff])<<8 | uint32(sbox0[t0&0xff]) + s2 = uint32(sbox0[t2>>24])<<24 | uint32(sbox0[t3>>16&0xff])<<16 | uint32(sbox0[t0>>8&0xff])<<8 | uint32(sbox0[t1&0xff]) + s3 = uint32(sbox0[t3>>24])<<24 | uint32(sbox0[t0>>16&0xff])<<16 | uint32(sbox0[t1>>8&0xff])<<8 | uint32(sbox0[t2&0xff]) + + s0 ^= xk[k+0] + s1 ^= xk[k+1] + s2 ^= xk[k+2] + s3 ^= xk[k+3] + + dst[0], dst[1], dst[2], dst[3] = byte(s0>>24), byte(s0>>16), byte(s0>>8), byte(s0) + dst[4], dst[5], dst[6], dst[7] = byte(s1>>24), byte(s1>>16), byte(s1>>8), byte(s1) + dst[8], dst[9], dst[10], dst[11] = byte(s2>>24), byte(s2>>16), byte(s2>>8), byte(s2) + dst[12], dst[13], dst[14], dst[15] = byte(s3>>24), byte(s3>>16), byte(s3>>8), byte(s3) +} + +// Decrypt one block from src into dst, using the expanded key xk. +func decryptBlock(xk []uint32, dst, src []byte) { + var s0, s1, s2, s3, t0, t1, t2, t3 uint32 + + s0 = uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + s1 = uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + s2 = uint32(src[8])<<24 | uint32(src[9])<<16 | uint32(src[10])<<8 | uint32(src[11]) + s3 = uint32(src[12])<<24 | uint32(src[13])<<16 | uint32(src[14])<<8 | uint32(src[15]) + + // First round just XORs input with key. + s0 ^= xk[0] + s1 ^= xk[1] + s2 ^= xk[2] + s3 ^= xk[3] + + // Middle rounds shuffle using tables. + // Number of rounds is set by length of expanded key. + nr := len(xk)/4 - 2 // - 2: one above, one more below + k := 4 + for r := 0; r < nr; r++ { + t0 = xk[k+0] ^ td[0][s0>>24] ^ td[1][s3>>16&0xff] ^ td[2][s2>>8&0xff] ^ td[3][s1&0xff] + t1 = xk[k+1] ^ td[0][s1>>24] ^ td[1][s0>>16&0xff] ^ td[2][s3>>8&0xff] ^ td[3][s2&0xff] + t2 = xk[k+2] ^ td[0][s2>>24] ^ td[1][s1>>16&0xff] ^ td[2][s0>>8&0xff] ^ td[3][s3&0xff] + t3 = xk[k+3] ^ td[0][s3>>24] ^ td[1][s2>>16&0xff] ^ td[2][s1>>8&0xff] ^ td[3][s0&0xff] + k += 4 + s0, s1, s2, s3 = t0, t1, t2, t3 + } + + // Last round uses s-box directly and XORs to produce output. + s0 = uint32(sbox1[t0>>24])<<24 | uint32(sbox1[t3>>16&0xff])<<16 | uint32(sbox1[t2>>8&0xff])<<8 | uint32(sbox1[t1&0xff]) + s1 = uint32(sbox1[t1>>24])<<24 | uint32(sbox1[t0>>16&0xff])<<16 | uint32(sbox1[t3>>8&0xff])<<8 | uint32(sbox1[t2&0xff]) + s2 = uint32(sbox1[t2>>24])<<24 | uint32(sbox1[t1>>16&0xff])<<16 | uint32(sbox1[t0>>8&0xff])<<8 | uint32(sbox1[t3&0xff]) + s3 = uint32(sbox1[t3>>24])<<24 | uint32(sbox1[t2>>16&0xff])<<16 | uint32(sbox1[t1>>8&0xff])<<8 | uint32(sbox1[t0&0xff]) + + s0 ^= xk[k+0] + s1 ^= xk[k+1] + s2 ^= xk[k+2] + s3 ^= xk[k+3] + + dst[0], dst[1], dst[2], dst[3] = byte(s0>>24), byte(s0>>16), byte(s0>>8), byte(s0) + dst[4], dst[5], dst[6], dst[7] = byte(s1>>24), byte(s1>>16), byte(s1>>8), byte(s1) + dst[8], dst[9], dst[10], dst[11] = byte(s2>>24), byte(s2>>16), byte(s2>>8), byte(s2) + dst[12], dst[13], dst[14], dst[15] = byte(s3>>24), byte(s3>>16), byte(s3>>8), byte(s3) +} + +// Apply sbox0 to each byte in w. +func subw(w uint32) uint32 { + return uint32(sbox0[w>>24])<<24 | + uint32(sbox0[w>>16&0xff])<<16 | + uint32(sbox0[w>>8&0xff])<<8 | + uint32(sbox0[w&0xff]) +} + +// Rotate +func rotw(w uint32) uint32 { return w<<8 | w>>24 } + +// Key expansion algorithm. See FIPS-197, Figure 11. +// Their rcon[i] is our powx[i-1] << 24. +func expandKey(key []byte, enc, dec []uint32) { + // Encryption key setup. + var i int + nk := len(key) / 4 + for i = 0; i < nk; i++ { + enc[i] = uint32(key[4*i])<<24 | uint32(key[4*i+1])<<16 | uint32(key[4*i+2])<<8 | uint32(key[4*i+3]) + } + for ; i < len(enc); i++ { + t := enc[i-1] + if i%nk == 0 { + t = subw(rotw(t)) ^ (uint32(powx[i/nk-1]) << 24) + } else if nk > 6 && i%nk == 4 { + t = subw(t) + } + enc[i] = enc[i-nk] ^ t + } + + // Derive decryption key from encryption key. + // Reverse the 4-word round key sets from enc to produce dec. + // All sets but the first and last get the MixColumn transform applied. + if dec == nil { + return + } + n := len(enc) + for i := 0; i < n; i += 4 { + ei := n - i - 4 + for j := 0; j < 4; j++ { + x := enc[ei+j] + if i > 0 && i+4 < n { + x = td[0][sbox0[x>>24]] ^ td[1][sbox0[x>>16&0xff]] ^ td[2][sbox0[x>>8&0xff]] ^ td[3][sbox0[x&0xff]] + } + dec[i+j] = x + } + } +} diff --git a/src/pkg/crypto/aes/cipher.go b/src/pkg/crypto/aes/cipher.go new file mode 100644 index 000000000..73223531e --- /dev/null +++ b/src/pkg/crypto/aes/cipher.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package aes + +import ( + "os" + "strconv" +) + +// The AES block size in bytes. +const BlockSize = 16 + +// A Cipher is an instance of AES encryption using a particular key. +type Cipher struct { + enc []uint32 + dec []uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/aes: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. +// The key argument should be the AES key, +// either 16, 24, or 32 bytes to select +// AES-128, AES-192, or AES-256. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + switch k { + default: + return nil, KeySizeError(k) + case 16, 24, 32: + break + } + + n := k + 28 + c := &Cipher{make([]uint32, n), make([]uint32, n)} + expandKey(key, c.enc, c.dec) + return c, nil +} + +// BlockSize returns the AES block size, 16 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/cipher". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 16-byte buffer src using the key k +// and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c.enc, dst, src) } + +// Decrypt decrypts the 16-byte buffer src using the key k +// and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c.dec, dst, src) } + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.enc); i++ { + c.enc[i] = 0 + } + for i := 0; i < len(c.dec); i++ { + c.dec[i] = 0 + } +} diff --git a/src/pkg/crypto/aes/const.go b/src/pkg/crypto/aes/const.go new file mode 100644 index 000000000..25acd0d17 --- /dev/null +++ b/src/pkg/crypto/aes/const.go @@ -0,0 +1,362 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package aes implements AES encryption (formerly Rijndael), as defined in +// U.S. Federal Information Processing Standards Publication 197. +package aes + +// This file contains AES constants - 8720 bytes of initialized data. + +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf + +// AES is based on the mathematical behavior of binary polynomials +// (polynomials over GF(2)) modulo the irreducible polynomial x⁸ + x⁴ + x² + x + 1. +// Addition of these binary polynomials corresponds to binary xor. +// Reducing mod poly corresponds to binary xor with poly every +// time a 0x100 bit appears. +const poly = 1<<8 | 1<<4 | 1<<3 | 1<<1 | 1<<0 // x⁸ + x⁴ + x² + x + 1 + +// Powers of x mod poly in GF(2). +var powx = [16]byte{ + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + 0x1b, + 0x36, + 0x6c, + 0xd8, + 0xab, + 0x4d, + 0x9a, + 0x2f, +} + +// FIPS-197 Figure 7. S-box substitution values in hexadecimal format. +var sbox0 = [256]byte{ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, +} + +// FIPS-197 Figure 14. Inverse S-box substitution values in hexadecimal format. +var sbox1 = [256]byte{ + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, +} + +// Lookup tables for encryption. +// These can be recomputed by adapting the tests in aes_test.go. + +var te = [4][256]uint32{ + { + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, + 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, + 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, + 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, + 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, + 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, + 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, + 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, + 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, + 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, + 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d, + 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, + 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, + 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, + 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b, + 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, + 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, + 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, + 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, + 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, + 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, + 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, + }, + { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5, + 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, + 0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0, + 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, + 0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a, + 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, + 0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484, + 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, + 0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585, + 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, + 0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2, + 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, + 0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888, + 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, + 0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979, + 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, + 0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6, + 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, + 0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e, + 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, + 0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868, + 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, + }, + { + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, 0x6fb1de6f, 0xc55491c5, + 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, + 0xca458fca, 0x829d1f82, 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, 0x7296e472, 0xc05b9bc0, + 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, + 0x345c6834, 0xa5f451a5, 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, 0x050f0a05, 0x9ab52f9a, + 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, + 0x091b1209, 0x839e1d83, 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, 0x2f715e2f, 0x84971384, + 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, + 0x6abed46a, 0xcb468dcb, 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, 0x33556633, 0x85941185, + 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, + 0x51f3a251, 0xa3fe5da3, 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, 0xf30efdf3, 0xd26dbfd2, + 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, + 0xc45793c4, 0xa7f255a7, 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, 0x90ab3b90, 0x88830b88, + 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, + 0xe03bdbe0, 0x32566432, 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, 0xe437d3e4, 0x798bf279, + 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, + 0x6cb4d86c, 0x56faac56, 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, 0xb4c773b4, 0xc65197c6, + 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, + 0x7090e070, 0x3e427c3e, 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, 0x1d273a1d, 0x9eb9279e, + 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, + 0x9bb62d9b, 0x1e223c1e, 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, 0x42c68442, 0x68b8d068, + 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, + }, + { + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, 0x6f6fb1de, 0xc5c55491, + 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, + 0xcaca458f, 0x82829d1f, 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, 0x727296e4, 0xc0c05b9b, + 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, + 0x34345c68, 0xa5a5f451, 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, 0x05050f0a, 0x9a9ab52f, + 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, + 0x09091b12, 0x83839e1d, 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, 0x2f2f715e, 0x84849713, + 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, + 0x6a6abed4, 0xcbcb468d, 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, 0x33335566, 0x85859411, + 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, + 0x5151f3a2, 0xa3a3fe5d, 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, 0xf3f30efd, 0xd2d26dbf, + 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, + 0xc4c45793, 0xa7a7f255, 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, 0x9090ab3b, 0x8888830b, + 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, + 0xe0e03bdb, 0x32325664, 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, 0xe4e437d3, 0x79798bf2, + 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, + 0x6c6cb4d8, 0x5656faac, 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, 0xb4b4c773, 0xc6c65197, + 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, + 0x707090e0, 0x3e3e427c, 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, 0x1d1d273a, 0x9e9eb927, + 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, + 0x9b9bb62d, 0x1e1e223c, 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, 0x4242c684, 0x6868b8d0, + 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, + }, +} + +// Lookup tables for decryption. +// These can be recomputed by adapting the tests in aes_test.go. + +var td = [4][256]uint32{ + { + 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, + 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, + 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, + 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, + 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, + 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94, + 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, + 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c, + 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, + 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, + 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, + 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, + 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, + 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a, + 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, + 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8, + 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, + 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, + 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, + 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, + 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, + 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5, + 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, + 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6, + 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, + 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, + 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, + 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, + 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, + 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86, + 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, + 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742, + }, + { + 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e, 0xcb3bab6b, 0xf11f9d45, 0xabacfa58, 0x934be303, + 0x552030fa, 0xf6ad766d, 0x9188cc76, 0x25f5024c, 0xfc4fe5d7, 0xd7c52acb, 0x80263544, 0x8fb562a3, + 0x49deb15a, 0x6725ba1b, 0x9845ea0e, 0xe15dfec0, 0x02c32f75, 0x12814cf0, 0xa38d4697, 0xc66bd3f9, + 0xe7038f5f, 0x9515929c, 0xebbf6d7a, 0xda955259, 0x2dd4be83, 0xd3587421, 0x2949e069, 0x448ec9c8, + 0x6a75c289, 0x78f48e79, 0x6b99583e, 0xdd27b971, 0xb6bee14f, 0x17f088ad, 0x66c920ac, 0xb47dce3a, + 0x1863df4a, 0x82e51a31, 0x60975133, 0x4562537f, 0xe0b16477, 0x84bb6bae, 0x1cfe81a0, 0x94f9082b, + 0x58704868, 0x198f45fd, 0x8794de6c, 0xb7527bf8, 0x23ab73d3, 0xe2724b02, 0x57e31f8f, 0x2a6655ab, + 0x07b2eb28, 0x032fb5c2, 0x9a86c57b, 0xa5d33708, 0xf2302887, 0xb223bfa5, 0xba02036a, 0x5ced1682, + 0x2b8acf1c, 0x92a779b4, 0xf0f307f2, 0xa14e69e2, 0xcd65daf4, 0xd50605be, 0x1fd13462, 0x8ac4a6fe, + 0x9d342e53, 0xa0a2f355, 0x32058ae1, 0x75a4f6eb, 0x390b83ec, 0xaa4060ef, 0x065e719f, 0x51bd6e10, + 0xf93e218a, 0x3d96dd06, 0xaedd3e05, 0x464de6bd, 0xb591548d, 0x0571c45d, 0x6f0406d4, 0xff605015, + 0x241998fb, 0x97d6bde9, 0xcc894043, 0x7767d99e, 0xbdb0e842, 0x8807898b, 0x38e7195b, 0xdb79c8ee, + 0x47a17c0a, 0xe97c420f, 0xc9f8841e, 0x00000000, 0x83098086, 0x48322bed, 0xac1e1170, 0x4e6c5a72, + 0xfbfd0eff, 0x560f8538, 0x1e3daed5, 0x27362d39, 0x640a0fd9, 0x21685ca6, 0xd19b5b54, 0x3a24362e, + 0xb10c0a67, 0x0f9357e7, 0xd2b4ee96, 0x9e1b9b91, 0x4f80c0c5, 0xa261dc20, 0x695a774b, 0x161c121a, + 0x0ae293ba, 0xe5c0a02a, 0x433c22e0, 0x1d121b17, 0x0b0e090d, 0xadf28bc7, 0xb92db6a8, 0xc8141ea9, + 0x8557f119, 0x4caf7507, 0xbbee99dd, 0xfda37f60, 0x9ff70126, 0xbc5c72f5, 0xc544663b, 0x345bfb7e, + 0x768b4329, 0xdccb23c6, 0x68b6edfc, 0x63b8e4f1, 0xcad731dc, 0x10426385, 0x40139722, 0x2084c611, + 0x7d854a24, 0xf8d2bb3d, 0x11aef932, 0x6dc729a1, 0x4b1d9e2f, 0xf3dcb230, 0xec0d8652, 0xd077c1e3, + 0x6c2bb316, 0x99a970b9, 0xfa119448, 0x2247e964, 0xc4a8fc8c, 0x1aa0f03f, 0xd8567d2c, 0xef223390, + 0xc787494e, 0xc1d938d1, 0xfe8ccaa2, 0x3698d40b, 0xcfa6f581, 0x28a57ade, 0x26dab78e, 0xa43fadbf, + 0xe42c3a9d, 0x0d507892, 0x9b6a5fcc, 0x62547e46, 0xc2f68d13, 0xe890d8b8, 0x5e2e39f7, 0xf582c3af, + 0xbe9f5d80, 0x7c69d093, 0xa96fd52d, 0xb3cf2512, 0x3bc8ac99, 0xa710187d, 0x6ee89c63, 0x7bdb3bbb, + 0x09cd2678, 0xf46e5918, 0x01ec9ab7, 0xa8834f9a, 0x65e6956e, 0x7eaaffe6, 0x0821bccf, 0xe6ef15e8, + 0xd9bae79b, 0xce4a6f36, 0xd4ea9f09, 0xd629b07c, 0xaf31a4b2, 0x312a3f23, 0x30c6a594, 0xc035a266, + 0x37744ebc, 0xa6fc82ca, 0xb0e090d0, 0x1533a7d8, 0x4af10498, 0xf741ecda, 0x0e7fcd50, 0x2f1791f6, + 0x8d764dd6, 0x4d43efb0, 0x54ccaa4d, 0xdfe49604, 0xe39ed1b5, 0x1b4c6a88, 0xb8c12c1f, 0x7f466551, + 0x049d5eea, 0x5d018c35, 0x73fa8774, 0x2efb0b41, 0x5ab3671d, 0x5292dbd2, 0x33e91056, 0x136dd647, + 0x8c9ad761, 0x7a37a10c, 0x8e59f814, 0x89eb133c, 0xeecea927, 0x35b761c9, 0xede11ce5, 0x3c7a47b1, + 0x599cd2df, 0x3f55f273, 0x791814ce, 0xbf73c737, 0xea53f7cd, 0x5b5ffdaa, 0x14df3d6f, 0x867844db, + 0x81caaff3, 0x3eb968c4, 0x2c382434, 0x5fc2a340, 0x72161dc3, 0x0cbce225, 0x8b283c49, 0x41ff0d95, + 0x7139a801, 0xde080cb3, 0x9cd8b4e4, 0x906456c1, 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857, + }, + { + 0xa75051f4, 0x65537e41, 0xa4c31a17, 0x5e963a27, 0x6bcb3bab, 0x45f11f9d, 0x58abacfa, 0x03934be3, + 0xfa552030, 0x6df6ad76, 0x769188cc, 0x4c25f502, 0xd7fc4fe5, 0xcbd7c52a, 0x44802635, 0xa38fb562, + 0x5a49deb1, 0x1b6725ba, 0x0e9845ea, 0xc0e15dfe, 0x7502c32f, 0xf012814c, 0x97a38d46, 0xf9c66bd3, + 0x5fe7038f, 0x9c951592, 0x7aebbf6d, 0x59da9552, 0x832dd4be, 0x21d35874, 0x692949e0, 0xc8448ec9, + 0x896a75c2, 0x7978f48e, 0x3e6b9958, 0x71dd27b9, 0x4fb6bee1, 0xad17f088, 0xac66c920, 0x3ab47dce, + 0x4a1863df, 0x3182e51a, 0x33609751, 0x7f456253, 0x77e0b164, 0xae84bb6b, 0xa01cfe81, 0x2b94f908, + 0x68587048, 0xfd198f45, 0x6c8794de, 0xf8b7527b, 0xd323ab73, 0x02e2724b, 0x8f57e31f, 0xab2a6655, + 0x2807b2eb, 0xc2032fb5, 0x7b9a86c5, 0x08a5d337, 0x87f23028, 0xa5b223bf, 0x6aba0203, 0x825ced16, + 0x1c2b8acf, 0xb492a779, 0xf2f0f307, 0xe2a14e69, 0xf4cd65da, 0xbed50605, 0x621fd134, 0xfe8ac4a6, + 0x539d342e, 0x55a0a2f3, 0xe132058a, 0xeb75a4f6, 0xec390b83, 0xefaa4060, 0x9f065e71, 0x1051bd6e, + 0x8af93e21, 0x063d96dd, 0x05aedd3e, 0xbd464de6, 0x8db59154, 0x5d0571c4, 0xd46f0406, 0x15ff6050, + 0xfb241998, 0xe997d6bd, 0x43cc8940, 0x9e7767d9, 0x42bdb0e8, 0x8b880789, 0x5b38e719, 0xeedb79c8, + 0x0a47a17c, 0x0fe97c42, 0x1ec9f884, 0x00000000, 0x86830980, 0xed48322b, 0x70ac1e11, 0x724e6c5a, + 0xfffbfd0e, 0x38560f85, 0xd51e3dae, 0x3927362d, 0xd9640a0f, 0xa621685c, 0x54d19b5b, 0x2e3a2436, + 0x67b10c0a, 0xe70f9357, 0x96d2b4ee, 0x919e1b9b, 0xc54f80c0, 0x20a261dc, 0x4b695a77, 0x1a161c12, + 0xba0ae293, 0x2ae5c0a0, 0xe0433c22, 0x171d121b, 0x0d0b0e09, 0xc7adf28b, 0xa8b92db6, 0xa9c8141e, + 0x198557f1, 0x074caf75, 0xddbbee99, 0x60fda37f, 0x269ff701, 0xf5bc5c72, 0x3bc54466, 0x7e345bfb, + 0x29768b43, 0xc6dccb23, 0xfc68b6ed, 0xf163b8e4, 0xdccad731, 0x85104263, 0x22401397, 0x112084c6, + 0x247d854a, 0x3df8d2bb, 0x3211aef9, 0xa16dc729, 0x2f4b1d9e, 0x30f3dcb2, 0x52ec0d86, 0xe3d077c1, + 0x166c2bb3, 0xb999a970, 0x48fa1194, 0x642247e9, 0x8cc4a8fc, 0x3f1aa0f0, 0x2cd8567d, 0x90ef2233, + 0x4ec78749, 0xd1c1d938, 0xa2fe8cca, 0x0b3698d4, 0x81cfa6f5, 0xde28a57a, 0x8e26dab7, 0xbfa43fad, + 0x9de42c3a, 0x920d5078, 0xcc9b6a5f, 0x4662547e, 0x13c2f68d, 0xb8e890d8, 0xf75e2e39, 0xaff582c3, + 0x80be9f5d, 0x937c69d0, 0x2da96fd5, 0x12b3cf25, 0x993bc8ac, 0x7da71018, 0x636ee89c, 0xbb7bdb3b, + 0x7809cd26, 0x18f46e59, 0xb701ec9a, 0x9aa8834f, 0x6e65e695, 0xe67eaaff, 0xcf0821bc, 0xe8e6ef15, + 0x9bd9bae7, 0x36ce4a6f, 0x09d4ea9f, 0x7cd629b0, 0xb2af31a4, 0x23312a3f, 0x9430c6a5, 0x66c035a2, + 0xbc37744e, 0xcaa6fc82, 0xd0b0e090, 0xd81533a7, 0x984af104, 0xdaf741ec, 0x500e7fcd, 0xf62f1791, + 0xd68d764d, 0xb04d43ef, 0x4d54ccaa, 0x04dfe496, 0xb5e39ed1, 0x881b4c6a, 0x1fb8c12c, 0x517f4665, + 0xea049d5e, 0x355d018c, 0x7473fa87, 0x412efb0b, 0x1d5ab367, 0xd25292db, 0x5633e910, 0x47136dd6, + 0x618c9ad7, 0x0c7a37a1, 0x148e59f8, 0x3c89eb13, 0x27eecea9, 0xc935b761, 0xe5ede11c, 0xb13c7a47, + 0xdf599cd2, 0x733f55f2, 0xce791814, 0x37bf73c7, 0xcdea53f7, 0xaa5b5ffd, 0x6f14df3d, 0xdb867844, + 0xf381caaf, 0xc43eb968, 0x342c3824, 0x405fc2a3, 0xc372161d, 0x250cbce2, 0x498b283c, 0x9541ff0d, + 0x017139a8, 0xb3de080c, 0xe49cd8b4, 0xc1906456, 0x84617bcb, 0xb670d532, 0x5c74486c, 0x5742d0b8, + }, + { + 0xf4a75051, 0x4165537e, 0x17a4c31a, 0x275e963a, 0xab6bcb3b, 0x9d45f11f, 0xfa58abac, 0xe303934b, + 0x30fa5520, 0x766df6ad, 0xcc769188, 0x024c25f5, 0xe5d7fc4f, 0x2acbd7c5, 0x35448026, 0x62a38fb5, + 0xb15a49de, 0xba1b6725, 0xea0e9845, 0xfec0e15d, 0x2f7502c3, 0x4cf01281, 0x4697a38d, 0xd3f9c66b, + 0x8f5fe703, 0x929c9515, 0x6d7aebbf, 0x5259da95, 0xbe832dd4, 0x7421d358, 0xe0692949, 0xc9c8448e, + 0xc2896a75, 0x8e7978f4, 0x583e6b99, 0xb971dd27, 0xe14fb6be, 0x88ad17f0, 0x20ac66c9, 0xce3ab47d, + 0xdf4a1863, 0x1a3182e5, 0x51336097, 0x537f4562, 0x6477e0b1, 0x6bae84bb, 0x81a01cfe, 0x082b94f9, + 0x48685870, 0x45fd198f, 0xde6c8794, 0x7bf8b752, 0x73d323ab, 0x4b02e272, 0x1f8f57e3, 0x55ab2a66, + 0xeb2807b2, 0xb5c2032f, 0xc57b9a86, 0x3708a5d3, 0x2887f230, 0xbfa5b223, 0x036aba02, 0x16825ced, + 0xcf1c2b8a, 0x79b492a7, 0x07f2f0f3, 0x69e2a14e, 0xdaf4cd65, 0x05bed506, 0x34621fd1, 0xa6fe8ac4, + 0x2e539d34, 0xf355a0a2, 0x8ae13205, 0xf6eb75a4, 0x83ec390b, 0x60efaa40, 0x719f065e, 0x6e1051bd, + 0x218af93e, 0xdd063d96, 0x3e05aedd, 0xe6bd464d, 0x548db591, 0xc45d0571, 0x06d46f04, 0x5015ff60, + 0x98fb2419, 0xbde997d6, 0x4043cc89, 0xd99e7767, 0xe842bdb0, 0x898b8807, 0x195b38e7, 0xc8eedb79, + 0x7c0a47a1, 0x420fe97c, 0x841ec9f8, 0x00000000, 0x80868309, 0x2bed4832, 0x1170ac1e, 0x5a724e6c, + 0x0efffbfd, 0x8538560f, 0xaed51e3d, 0x2d392736, 0x0fd9640a, 0x5ca62168, 0x5b54d19b, 0x362e3a24, + 0x0a67b10c, 0x57e70f93, 0xee96d2b4, 0x9b919e1b, 0xc0c54f80, 0xdc20a261, 0x774b695a, 0x121a161c, + 0x93ba0ae2, 0xa02ae5c0, 0x22e0433c, 0x1b171d12, 0x090d0b0e, 0x8bc7adf2, 0xb6a8b92d, 0x1ea9c814, + 0xf1198557, 0x75074caf, 0x99ddbbee, 0x7f60fda3, 0x01269ff7, 0x72f5bc5c, 0x663bc544, 0xfb7e345b, + 0x4329768b, 0x23c6dccb, 0xedfc68b6, 0xe4f163b8, 0x31dccad7, 0x63851042, 0x97224013, 0xc6112084, + 0x4a247d85, 0xbb3df8d2, 0xf93211ae, 0x29a16dc7, 0x9e2f4b1d, 0xb230f3dc, 0x8652ec0d, 0xc1e3d077, + 0xb3166c2b, 0x70b999a9, 0x9448fa11, 0xe9642247, 0xfc8cc4a8, 0xf03f1aa0, 0x7d2cd856, 0x3390ef22, + 0x494ec787, 0x38d1c1d9, 0xcaa2fe8c, 0xd40b3698, 0xf581cfa6, 0x7ade28a5, 0xb78e26da, 0xadbfa43f, + 0x3a9de42c, 0x78920d50, 0x5fcc9b6a, 0x7e466254, 0x8d13c2f6, 0xd8b8e890, 0x39f75e2e, 0xc3aff582, + 0x5d80be9f, 0xd0937c69, 0xd52da96f, 0x2512b3cf, 0xac993bc8, 0x187da710, 0x9c636ee8, 0x3bbb7bdb, + 0x267809cd, 0x5918f46e, 0x9ab701ec, 0x4f9aa883, 0x956e65e6, 0xffe67eaa, 0xbccf0821, 0x15e8e6ef, + 0xe79bd9ba, 0x6f36ce4a, 0x9f09d4ea, 0xb07cd629, 0xa4b2af31, 0x3f23312a, 0xa59430c6, 0xa266c035, + 0x4ebc3774, 0x82caa6fc, 0x90d0b0e0, 0xa7d81533, 0x04984af1, 0xecdaf741, 0xcd500e7f, 0x91f62f17, + 0x4dd68d76, 0xefb04d43, 0xaa4d54cc, 0x9604dfe4, 0xd1b5e39e, 0x6a881b4c, 0x2c1fb8c1, 0x65517f46, + 0x5eea049d, 0x8c355d01, 0x877473fa, 0x0b412efb, 0x671d5ab3, 0xdbd25292, 0x105633e9, 0xd647136d, + 0xd7618c9a, 0xa10c7a37, 0xf8148e59, 0x133c89eb, 0xa927eece, 0x61c935b7, 0x1ce5ede1, 0x47b13c7a, + 0xd2df599c, 0xf2733f55, 0x14ce7918, 0xc737bf73, 0xf7cdea53, 0xfdaa5b5f, 0x3d6f14df, 0x44db8678, + 0xaff381ca, 0x68c43eb9, 0x24342c38, 0xa3405fc2, 0x1dc37216, 0xe2250cbc, 0x3c498b28, 0x0d9541ff, + 0xa8017139, 0x0cb3de08, 0xb4e49cd8, 0x56c19064, 0xcb84617b, 0x32b670d5, 0x6c5c7448, 0xb85742d0, + }, +} diff --git a/src/pkg/crypto/blowfish/Makefile b/src/pkg/crypto/blowfish/Makefile new file mode 100644 index 000000000..f370ab28b --- /dev/null +++ b/src/pkg/crypto/blowfish/Makefile @@ -0,0 +1,13 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/blowfish +GOFILES=\ + block.go\ + cipher.go\ + const.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/blowfish/block.go b/src/pkg/crypto/blowfish/block.go new file mode 100644 index 000000000..7fbe7eefb --- /dev/null +++ b/src/pkg/crypto/blowfish/block.go @@ -0,0 +1,101 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package blowfish + +func expandKey(key []byte, c *Cipher) { + copy(c.p[0:], p[0:]) + copy(c.s0[0:], s0[0:]) + copy(c.s1[0:], s1[0:]) + copy(c.s2[0:], s2[0:]) + copy(c.s3[0:], s3[0:]) + + j := 0 + for i := 0; i < 18; i++ { + var d uint32 + for k := 0; k < 4; k++ { + d = d<<8 | uint32(key[j])&0x000000FF + j++ + if j >= len(key) { + j = 0 + } + } + c.p[i] ^= d + } + + var l, r uint32 + for i := 0; i < 18; i += 2 { + l, r = encryptBlock(l, r, c) + c.p[i], c.p[i+1] = l, r + } + + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s0[i], c.s0[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s1[i], c.s1[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s2[i], c.s2[i+1] = l, r + } + for i := 0; i < 256; i += 2 { + l, r = encryptBlock(l, r, c) + c.s3[i], c.s3[i+1] = l, r + } +} + +func encryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { + xl, xr := l, r + xl ^= c.p[0] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[1] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[2] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[3] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[4] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[5] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[6] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[7] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[8] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[9] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[10] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[11] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[12] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[13] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[14] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[15] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[16] + xr ^= c.p[17] + return xr, xl +} + +func decryptBlock(l, r uint32, c *Cipher) (uint32, uint32) { + xl, xr := l, r + xl ^= c.p[17] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[16] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[15] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[14] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[13] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[12] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[11] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[10] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[9] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[8] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[7] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[6] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[5] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[4] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[3] + xr ^= ((c.s0[byte(xl>>24)] + c.s1[byte(xl>>16)]) ^ c.s2[byte(xl>>8)]) + c.s3[byte(xl)] ^ c.p[2] + xl ^= ((c.s0[byte(xr>>24)] + c.s1[byte(xr>>16)]) ^ c.s2[byte(xr>>8)]) + c.s3[byte(xr)] ^ c.p[1] + xr ^= c.p[0] + return xr, xl +} + +func zero(x []uint32) { + for i := range x { + x[i] = 0 + } +} diff --git a/src/pkg/crypto/blowfish/blowfish_test.go b/src/pkg/crypto/blowfish/blowfish_test.go new file mode 100644 index 000000000..3a7ab6c2a --- /dev/null +++ b/src/pkg/crypto/blowfish/blowfish_test.go @@ -0,0 +1,192 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package blowfish + +import ( + "testing" +) + +type CryptTest struct { + key []byte + in []byte + out []byte +} + +// Test vector values are from http://www.schneier.com/code/vectors.txt. +var encryptTests = []CryptTest{ + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78}}, + { + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x51, 0x86, 0x6F, 0xD5, 0xB8, 0x5E, 0xCB, 0x8A}}, + { + []byte{0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + []byte{0x7D, 0x85, 0x6F, 0x9A, 0x61, 0x30, 0x63, 0xF2}}, + { + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x24, 0x66, 0xDD, 0x87, 0x8B, 0x96, 0x3C, 0x9D}}, + + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x61, 0xF9, 0xC3, 0x80, 0x22, 0x81, 0xB0, 0x96}}, + { + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x7D, 0x0C, 0xC6, 0x30, 0xAF, 0xDA, 0x1E, 0xC7}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4E, 0xF9, 0x97, 0x45, 0x61, 0x98, 0xDD, 0x78}}, + { + []byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x0A, 0xCE, 0xAB, 0x0F, 0xC6, 0xA0, 0xA2, 0x8D}}, + { + []byte{0x7C, 0xA1, 0x10, 0x45, 0x4A, 0x1A, 0x6E, 0x57}, + []byte{0x01, 0xA1, 0xD6, 0xD0, 0x39, 0x77, 0x67, 0x42}, + []byte{0x59, 0xC6, 0x82, 0x45, 0xEB, 0x05, 0x28, 0x2B}}, + { + []byte{0x01, 0x31, 0xD9, 0x61, 0x9D, 0xC1, 0x37, 0x6E}, + []byte{0x5C, 0xD5, 0x4C, 0xA8, 0x3D, 0xEF, 0x57, 0xDA}, + []byte{0xB1, 0xB8, 0xCC, 0x0B, 0x25, 0x0F, 0x09, 0xA0}}, + { + []byte{0x07, 0xA1, 0x13, 0x3E, 0x4A, 0x0B, 0x26, 0x86}, + []byte{0x02, 0x48, 0xD4, 0x38, 0x06, 0xF6, 0x71, 0x72}, + []byte{0x17, 0x30, 0xE5, 0x77, 0x8B, 0xEA, 0x1D, 0xA4}}, + { + []byte{0x38, 0x49, 0x67, 0x4C, 0x26, 0x02, 0x31, 0x9E}, + []byte{0x51, 0x45, 0x4B, 0x58, 0x2D, 0xDF, 0x44, 0x0A}, + []byte{0xA2, 0x5E, 0x78, 0x56, 0xCF, 0x26, 0x51, 0xEB}}, + { + []byte{0x04, 0xB9, 0x15, 0xBA, 0x43, 0xFE, 0xB5, 0xB6}, + []byte{0x42, 0xFD, 0x44, 0x30, 0x59, 0x57, 0x7F, 0xA2}, + []byte{0x35, 0x38, 0x82, 0xB1, 0x09, 0xCE, 0x8F, 0x1A}}, + { + []byte{0x01, 0x13, 0xB9, 0x70, 0xFD, 0x34, 0xF2, 0xCE}, + []byte{0x05, 0x9B, 0x5E, 0x08, 0x51, 0xCF, 0x14, 0x3A}, + []byte{0x48, 0xF4, 0xD0, 0x88, 0x4C, 0x37, 0x99, 0x18}}, + { + []byte{0x01, 0x70, 0xF1, 0x75, 0x46, 0x8F, 0xB5, 0xE6}, + []byte{0x07, 0x56, 0xD8, 0xE0, 0x77, 0x47, 0x61, 0xD2}, + []byte{0x43, 0x21, 0x93, 0xB7, 0x89, 0x51, 0xFC, 0x98}}, + { + []byte{0x43, 0x29, 0x7F, 0xAD, 0x38, 0xE3, 0x73, 0xFE}, + []byte{0x76, 0x25, 0x14, 0xB8, 0x29, 0xBF, 0x48, 0x6A}, + []byte{0x13, 0xF0, 0x41, 0x54, 0xD6, 0x9D, 0x1A, 0xE5}}, + { + []byte{0x07, 0xA7, 0x13, 0x70, 0x45, 0xDA, 0x2A, 0x16}, + []byte{0x3B, 0xDD, 0x11, 0x90, 0x49, 0x37, 0x28, 0x02}, + []byte{0x2E, 0xED, 0xDA, 0x93, 0xFF, 0xD3, 0x9C, 0x79}}, + { + []byte{0x04, 0x68, 0x91, 0x04, 0xC2, 0xFD, 0x3B, 0x2F}, + []byte{0x26, 0x95, 0x5F, 0x68, 0x35, 0xAF, 0x60, 0x9A}, + []byte{0xD8, 0x87, 0xE0, 0x39, 0x3C, 0x2D, 0xA6, 0xE3}}, + { + []byte{0x37, 0xD0, 0x6B, 0xB5, 0x16, 0xCB, 0x75, 0x46}, + []byte{0x16, 0x4D, 0x5E, 0x40, 0x4F, 0x27, 0x52, 0x32}, + []byte{0x5F, 0x99, 0xD0, 0x4F, 0x5B, 0x16, 0x39, 0x69}}, + { + []byte{0x1F, 0x08, 0x26, 0x0D, 0x1A, 0xC2, 0x46, 0x5E}, + []byte{0x6B, 0x05, 0x6E, 0x18, 0x75, 0x9F, 0x5C, 0xCA}, + []byte{0x4A, 0x05, 0x7A, 0x3B, 0x24, 0xD3, 0x97, 0x7B}}, + { + []byte{0x58, 0x40, 0x23, 0x64, 0x1A, 0xBA, 0x61, 0x76}, + []byte{0x00, 0x4B, 0xD6, 0xEF, 0x09, 0x17, 0x60, 0x62}, + []byte{0x45, 0x20, 0x31, 0xC1, 0xE4, 0xFA, 0xDA, 0x8E}}, + { + []byte{0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xB0, 0x07}, + []byte{0x48, 0x0D, 0x39, 0x00, 0x6E, 0xE7, 0x62, 0xF2}, + []byte{0x75, 0x55, 0xAE, 0x39, 0xF5, 0x9B, 0x87, 0xBD}}, + { + []byte{0x49, 0x79, 0x3E, 0xBC, 0x79, 0xB3, 0x25, 0x8F}, + []byte{0x43, 0x75, 0x40, 0xC8, 0x69, 0x8F, 0x3C, 0xFA}, + []byte{0x53, 0xC5, 0x5F, 0x9C, 0xB4, 0x9F, 0xC0, 0x19}}, + { + []byte{0x4F, 0xB0, 0x5E, 0x15, 0x15, 0xAB, 0x73, 0xA7}, + []byte{0x07, 0x2D, 0x43, 0xA0, 0x77, 0x07, 0x52, 0x92}, + []byte{0x7A, 0x8E, 0x7B, 0xFA, 0x93, 0x7E, 0x89, 0xA3}}, + { + []byte{0x49, 0xE9, 0x5D, 0x6D, 0x4C, 0xA2, 0x29, 0xBF}, + []byte{0x02, 0xFE, 0x55, 0x77, 0x81, 0x17, 0xF1, 0x2A}, + []byte{0xCF, 0x9C, 0x5D, 0x7A, 0x49, 0x86, 0xAD, 0xB5}}, + { + []byte{0x01, 0x83, 0x10, 0xDC, 0x40, 0x9B, 0x26, 0xD6}, + []byte{0x1D, 0x9D, 0x5C, 0x50, 0x18, 0xF7, 0x28, 0xC2}, + []byte{0xD1, 0xAB, 0xB2, 0x90, 0x65, 0x8B, 0xC7, 0x78}}, + { + []byte{0x1C, 0x58, 0x7F, 0x1C, 0x13, 0x92, 0x4F, 0xEF}, + []byte{0x30, 0x55, 0x32, 0x28, 0x6D, 0x6F, 0x29, 0x5A}, + []byte{0x55, 0xCB, 0x37, 0x74, 0xD1, 0x3E, 0xF2, 0x01}}, + { + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xFA, 0x34, 0xEC, 0x48, 0x47, 0xB2, 0x68, 0xB2}}, + { + []byte{0x1F, 0x1F, 0x1F, 0x1F, 0x0E, 0x0E, 0x0E, 0x0E}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xA7, 0x90, 0x79, 0x51, 0x08, 0xEA, 0x3C, 0xAE}}, + { + []byte{0xE0, 0xFE, 0xE0, 0xFE, 0xF1, 0xFE, 0xF1, 0xFE}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0xC3, 0x9E, 0x07, 0x2D, 0x9F, 0xAC, 0x63, 0x1D}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x01, 0x49, 0x33, 0xE0, 0xCD, 0xAF, 0xF6, 0xE4}}, + { + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xF2, 0x1E, 0x9A, 0x77, 0xB7, 0x1C, 0x49, 0xBC}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x24, 0x59, 0x46, 0x88, 0x57, 0x54, 0x36, 0x9A}}, + { + []byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + []byte{0x6B, 0x5C, 0x5A, 0x9C, 0x5D, 0x9E, 0x0A, 0x5A}}, +} + +func TestCipherEncrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + ct := make([]byte, len(tt.out)) + c.Encrypt(ct, tt.in) + for j, v := range ct { + if v != tt.out[j] { + t.Errorf("Cipher.Encrypt, test vector #%d: cipher-text[%d] = %#x, expected %#x", i, j, v, tt.out[j]) + break + } + } + } +} + +func TestCipherDecrypt(t *testing.T) { + for i, tt := range encryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(tt.key), err) + continue + } + pt := make([]byte, len(tt.in)) + c.Decrypt(pt, tt.out) + for j, v := range pt { + if v != tt.in[j] { + t.Errorf("Cipher.Decrypt, test vector #%d: plain-text[%d] = %#x, expected %#x", i, j, v, tt.in[j]) + break + } + } + } +} diff --git a/src/pkg/crypto/blowfish/cipher.go b/src/pkg/crypto/blowfish/cipher.go new file mode 100644 index 000000000..6c37dfe94 --- /dev/null +++ b/src/pkg/crypto/blowfish/cipher.go @@ -0,0 +1,79 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package blowfish implements Bruce Schneier's Blowfish encryption algorithm. +package blowfish + +// The code is a port of Bruce Schneier's C implementation. +// See http://www.schneier.com/blowfish.html. + +import ( + "os" + "strconv" +) + +// The Blowfish block size in bytes. +const BlockSize = 8 + +// A Cipher is an instance of Blowfish encryption using a particular key. +type Cipher struct { + p [18]uint32 + s0, s1, s2, s3 [256]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/blowfish: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a Cipher. +// The key argument should be the Blowfish key, 4 to 56 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + if k < 4 || k > 56 { + return nil, KeySizeError(k) + } + var result Cipher + expandKey(key, &result) + return &result, nil +} + +// BlockSize returns the Blowfish block size, 8 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/cipher". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 8-byte buffer src using the key k +// and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + l, r = encryptBlock(l, r, c) + dst[0], dst[1], dst[2], dst[3] = byte(l>>24), byte(l>>16), byte(l>>8), byte(l) + dst[4], dst[5], dst[6], dst[7] = byte(r>>24), byte(r>>16), byte(r>>8), byte(r) +} + +// Decrypt decrypts the 8-byte buffer src using the key k +// and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + l, r = decryptBlock(l, r, c) + dst[0], dst[1], dst[2], dst[3] = byte(l>>24), byte(l>>16), byte(l>>8), byte(l) + dst[4], dst[5], dst[6], dst[7] = byte(r>>24), byte(r>>16), byte(r>>8), byte(r) +} + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + zero(c.p[0:]) + zero(c.s0[0:]) + zero(c.s1[0:]) + zero(c.s2[0:]) + zero(c.s3[0:]) +} diff --git a/src/pkg/crypto/blowfish/const.go b/src/pkg/crypto/blowfish/const.go new file mode 100644 index 000000000..8c5ee4cb0 --- /dev/null +++ b/src/pkg/crypto/blowfish/const.go @@ -0,0 +1,199 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The startup permutation array and substitution boxes. +// They are the hexadecimal digits of PI; see: +// http://www.schneier.com/code/constants.txt. + +package blowfish + +var s0 = [256]uint32{ + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, + 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, + 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, + 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, + 0xb4cc5c34, 0x1141e8ce, 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, + 0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, + 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, + 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176, + 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, 0x4ed3aa62, 0x363f7706, + 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, + 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, + 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, + 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, + 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, + 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, + 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, + 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, + 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, 0x80957705, + 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, + 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9, + 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, + 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, +} + +var s1 = [256]uint32{ + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, + 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, + 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, + 0x3c971814, 0x6b6a70a1, 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, + 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, + 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, + 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, 0x2e6b7124, + 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, + 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, + 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, + 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, + 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, + 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, + 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, + 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, + 0x11ed935f, 0x16681281, 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, + 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, + 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, + 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a, 0x0c55f5ea, + 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, + 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, + 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, +} + +var s2 = [256]uint32{ + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, + 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, + 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, + 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, + 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, + 0x6841e7f7, 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, + 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, + 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, + 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, + 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, + 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, + 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, + 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, + 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, + 0x60787bf8, 0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, + 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, + 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, + 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, 0x573906fe, + 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, + 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, + 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, + 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0, +} + +var s3 = [256]uint32{ + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, + 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, + 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, + 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, + 0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, + 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, + 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba, + 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, 0x7533d928, 0xb155fdf5, + 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, + 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, + 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, + 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, + 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, + 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, + 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, + 0x0f91fc71, 0x9b941525, 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, + 0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, + 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, + 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a, + 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, 0x53113ec0, 0x1640e3d3, + 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, + 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6, +} + +var p = [18]uint32{ + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, + 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, + 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b, +} diff --git a/src/pkg/crypto/cast5/Makefile b/src/pkg/crypto/cast5/Makefile new file mode 100644 index 000000000..346fadd94 --- /dev/null +++ b/src/pkg/crypto/cast5/Makefile @@ -0,0 +1,11 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/cast5 +GOFILES=\ + cast5.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/cast5/cast5.go b/src/pkg/crypto/cast5/cast5.go new file mode 100644 index 000000000..e9d4a24e2 --- /dev/null +++ b/src/pkg/crypto/cast5/cast5.go @@ -0,0 +1,536 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cast5 implements CAST5, as defined in RFC 2144. CAST5 is a common +// OpenPGP cipher. +package cast5 + +import ( + "os" +) + +const BlockSize = 8 +const KeySize = 16 + +type Cipher struct { + masking [16]uint32 + rotate [16]uint8 +} + +func NewCipher(key []byte) (c *Cipher, err os.Error) { + if len(key) != KeySize { + return nil, os.NewError("CAST5: keys must be 16 bytes") + } + + c = new(Cipher) + c.keySchedule(key) + return +} + +func (c *Cipher) BlockSize() int { + return BlockSize +} + +// Reset zeros the key material in memory. +func (c *Cipher) Reset() { + for i := 0; i < 16; i++ { + c.masking[i] = 0 + c.rotate[i] = 0 + } +} + +func (c *Cipher) Encrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +func (c *Cipher) Decrypt(dst, src []byte) { + l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + + l, r = r, l^f1(r, c.masking[15], c.rotate[15]) + l, r = r, l^f3(r, c.masking[14], c.rotate[14]) + l, r = r, l^f2(r, c.masking[13], c.rotate[13]) + l, r = r, l^f1(r, c.masking[12], c.rotate[12]) + + l, r = r, l^f3(r, c.masking[11], c.rotate[11]) + l, r = r, l^f2(r, c.masking[10], c.rotate[10]) + l, r = r, l^f1(r, c.masking[9], c.rotate[9]) + l, r = r, l^f3(r, c.masking[8], c.rotate[8]) + + l, r = r, l^f2(r, c.masking[7], c.rotate[7]) + l, r = r, l^f1(r, c.masking[6], c.rotate[6]) + l, r = r, l^f3(r, c.masking[5], c.rotate[5]) + l, r = r, l^f2(r, c.masking[4], c.rotate[4]) + + l, r = r, l^f1(r, c.masking[3], c.rotate[3]) + l, r = r, l^f3(r, c.masking[2], c.rotate[2]) + l, r = r, l^f2(r, c.masking[1], c.rotate[1]) + l, r = r, l^f1(r, c.masking[0], c.rotate[0]) + + dst[0] = uint8(r >> 24) + dst[1] = uint8(r >> 16) + dst[2] = uint8(r >> 8) + dst[3] = uint8(r) + dst[4] = uint8(l >> 24) + dst[5] = uint8(l >> 16) + dst[6] = uint8(l >> 8) + dst[7] = uint8(l) +} + +type keyScheduleA [4][7]uint8 +type keyScheduleB [4][5]uint8 + +// keyScheduleRound contains the magic values for a round of the key schedule. +// The keyScheduleA deals with the lines like: +// z0z1z2z3 = x0x1x2x3 ^ S5[xD] ^ S6[xF] ^ S7[xC] ^ S8[xE] ^ S7[x8] +// Conceptually, both x and z are in the same array, x first. The first +// element describes which word of this array gets written to and the +// second, which word gets read. So, for the line above, it's "4, 0", because +// it's writing to the first word of z, which, being after x, is word 4, and +// reading from the first word of x: word 0. +// +// Next are the indexes into the S-boxes. Now the array is treated as bytes. So +// "xD" is 0xd. The first byte of z is written as "16 + 0", just to be clear +// that it's z that we're indexing. +// +// keyScheduleB deals with lines like: +// K1 = S5[z8] ^ S6[z9] ^ S7[z7] ^ S8[z6] ^ S5[z2] +// "K1" is ignored because key words are always written in order. So the five +// elements are the S-box indexes. They use the same form as in keyScheduleA, +// above. + +type keyScheduleRound struct{} +type keySchedule []keyScheduleRound + +var schedule = []struct { + a keyScheduleA + b keyScheduleB +}{ + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 0x8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 8, 16 + 9, 16 + 7, 16 + 6, 16 + 2}, + {16 + 0xa, 16 + 0xb, 16 + 5, 16 + 4, 16 + 6}, + {16 + 0xc, 16 + 0xd, 16 + 3, 16 + 2, 16 + 9}, + {16 + 0xe, 16 + 0xf, 16 + 1, 16 + 0, 16 + 0xc}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {3, 2, 0xc, 0xd, 8}, + {1, 0, 0xe, 0xf, 0xd}, + {7, 6, 8, 9, 3}, + {5, 4, 0xa, 0xb, 7}, + }, + }, + { + keyScheduleA{ + {4, 0, 0xd, 0xf, 0xc, 0xe, 8}, + {5, 2, 16 + 0, 16 + 2, 16 + 1, 16 + 3, 0xa}, + {6, 3, 16 + 7, 16 + 6, 16 + 5, 16 + 4, 9}, + {7, 1, 16 + 0xa, 16 + 9, 16 + 0xb, 16 + 8, 0xb}, + }, + keyScheduleB{ + {16 + 3, 16 + 2, 16 + 0xc, 16 + 0xd, 16 + 9}, + {16 + 1, 16 + 0, 16 + 0xe, 16 + 0xf, 16 + 0xc}, + {16 + 7, 16 + 6, 16 + 8, 16 + 9, 16 + 2}, + {16 + 5, 16 + 4, 16 + 0xa, 16 + 0xb, 16 + 6}, + }, + }, + { + keyScheduleA{ + {0, 6, 16 + 5, 16 + 7, 16 + 4, 16 + 6, 16 + 0}, + {1, 4, 0, 2, 1, 3, 16 + 2}, + {2, 5, 7, 6, 5, 4, 16 + 1}, + {3, 7, 0xa, 9, 0xb, 8, 16 + 3}, + }, + keyScheduleB{ + {8, 9, 7, 6, 3}, + {0xa, 0xb, 5, 4, 7}, + {0xc, 0xd, 3, 2, 8}, + {0xe, 0xf, 1, 0, 0xd}, + }, + }, +} + +func (c *Cipher) keySchedule(in []byte) { + var t [8]uint32 + var k [32]uint32 + + for i := 0; i < 4; i++ { + j := i * 4 + t[i] = uint32(in[j])<<24 | uint32(in[j+1])<<16 | uint32(in[j+2])<<8 | uint32(in[j+3]) + } + + x := []byte{6, 7, 4, 5} + ki := 0 + + for half := 0; half < 2; half++ { + for _, round := range schedule { + for j := 0; j < 4; j++ { + var a [7]uint8 + copy(a[:], round.a[j][:]) + w := t[a[1]] + w ^= sBox[4][(t[a[2]>>2]>>(24-8*(a[2]&3)))&0xff] + w ^= sBox[5][(t[a[3]>>2]>>(24-8*(a[3]&3)))&0xff] + w ^= sBox[6][(t[a[4]>>2]>>(24-8*(a[4]&3)))&0xff] + w ^= sBox[7][(t[a[5]>>2]>>(24-8*(a[5]&3)))&0xff] + w ^= sBox[x[j]][(t[a[6]>>2]>>(24-8*(a[6]&3)))&0xff] + t[a[0]] = w + } + + for j := 0; j < 4; j++ { + var b [5]uint8 + copy(b[:], round.b[j][:]) + w := sBox[4][(t[b[0]>>2]>>(24-8*(b[0]&3)))&0xff] + w ^= sBox[5][(t[b[1]>>2]>>(24-8*(b[1]&3)))&0xff] + w ^= sBox[6][(t[b[2]>>2]>>(24-8*(b[2]&3)))&0xff] + w ^= sBox[7][(t[b[3]>>2]>>(24-8*(b[3]&3)))&0xff] + w ^= sBox[4+j][(t[b[4]>>2]>>(24-8*(b[4]&3)))&0xff] + k[ki] = w + ki++ + } + } + } + + for i := 0; i < 16; i++ { + c.masking[i] = k[i] + c.rotate[i] = uint8(k[16+i] & 0x1f) + } +} + +// These are the three 'f' functions. See RFC 2144, section 2.2. +func f1(d, m uint32, r uint8) uint32 { + t := m + d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] ^ sBox[1][(I>>16)&0xff]) - sBox[2][(I>>8)&0xff]) + sBox[3][I&0xff] +} + +func f2(d, m uint32, r uint8) uint32 { + t := m ^ d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] - sBox[1][(I>>16)&0xff]) + sBox[2][(I>>8)&0xff]) ^ sBox[3][I&0xff] +} + +func f3(d, m uint32, r uint8) uint32 { + t := m - d + I := (t << r) | (t >> (32 - r)) + return ((sBox[0][I>>24] + sBox[1][(I>>16)&0xff]) ^ sBox[2][(I>>8)&0xff]) - sBox[3][I&0xff] +} + +var sBox = [8][256]uint32{ + { + 0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f, 0x9c004dd3, 0x6003e540, 0xcf9fc949, + 0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0, 0x15c361d2, 0xc2e7661d, 0x22d4ff8e, + 0x28683b6f, 0xc07fd059, 0xff2379c8, 0x775f50e2, 0x43c340d3, 0xdf2f8656, 0x887ca41a, 0xa2d2bd2d, + 0xa1c9e0d6, 0x346c4819, 0x61b76d87, 0x22540f2f, 0x2abe32e1, 0xaa54166b, 0x22568e3a, 0xa2d341d0, + 0x66db40c8, 0xa784392f, 0x004dff2f, 0x2db9d2de, 0x97943fac, 0x4a97c1d8, 0x527644b7, 0xb5f437a7, + 0xb82cbaef, 0xd751d159, 0x6ff7f0ed, 0x5a097a1f, 0x827b68d0, 0x90ecf52e, 0x22b0c054, 0xbc8e5935, + 0x4b6d2f7f, 0x50bb64a2, 0xd2664910, 0xbee5812d, 0xb7332290, 0xe93b159f, 0xb48ee411, 0x4bff345d, + 0xfd45c240, 0xad31973f, 0xc4f6d02e, 0x55fc8165, 0xd5b1caad, 0xa1ac2dae, 0xa2d4b76d, 0xc19b0c50, + 0x882240f2, 0x0c6e4f38, 0xa4e4bfd7, 0x4f5ba272, 0x564c1d2f, 0xc59c5319, 0xb949e354, 0xb04669fe, + 0xb1b6ab8a, 0xc71358dd, 0x6385c545, 0x110f935d, 0x57538ad5, 0x6a390493, 0xe63d37e0, 0x2a54f6b3, + 0x3a787d5f, 0x6276a0b5, 0x19a6fcdf, 0x7a42206a, 0x29f9d4d5, 0xf61b1891, 0xbb72275e, 0xaa508167, + 0x38901091, 0xc6b505eb, 0x84c7cb8c, 0x2ad75a0f, 0x874a1427, 0xa2d1936b, 0x2ad286af, 0xaa56d291, + 0xd7894360, 0x425c750d, 0x93b39e26, 0x187184c9, 0x6c00b32d, 0x73e2bb14, 0xa0bebc3c, 0x54623779, + 0x64459eab, 0x3f328b82, 0x7718cf82, 0x59a2cea6, 0x04ee002e, 0x89fe78e6, 0x3fab0950, 0x325ff6c2, + 0x81383f05, 0x6963c5c8, 0x76cb5ad6, 0xd49974c9, 0xca180dcf, 0x380782d5, 0xc7fa5cf6, 0x8ac31511, + 0x35e79e13, 0x47da91d0, 0xf40f9086, 0xa7e2419e, 0x31366241, 0x051ef495, 0xaa573b04, 0x4a805d8d, + 0x548300d0, 0x00322a3c, 0xbf64cddf, 0xba57a68e, 0x75c6372b, 0x50afd341, 0xa7c13275, 0x915a0bf5, + 0x6b54bfab, 0x2b0b1426, 0xab4cc9d7, 0x449ccd82, 0xf7fbf265, 0xab85c5f3, 0x1b55db94, 0xaad4e324, + 0xcfa4bd3f, 0x2deaa3e2, 0x9e204d02, 0xc8bd25ac, 0xeadf55b3, 0xd5bd9e98, 0xe31231b2, 0x2ad5ad6c, + 0x954329de, 0xadbe4528, 0xd8710f69, 0xaa51c90f, 0xaa786bf6, 0x22513f1e, 0xaa51a79b, 0x2ad344cc, + 0x7b5a41f0, 0xd37cfbad, 0x1b069505, 0x41ece491, 0xb4c332e6, 0x032268d4, 0xc9600acc, 0xce387e6d, + 0xbf6bb16c, 0x6a70fb78, 0x0d03d9c9, 0xd4df39de, 0xe01063da, 0x4736f464, 0x5ad328d8, 0xb347cc96, + 0x75bb0fc3, 0x98511bfb, 0x4ffbcc35, 0xb58bcf6a, 0xe11f0abc, 0xbfc5fe4a, 0xa70aec10, 0xac39570a, + 0x3f04442f, 0x6188b153, 0xe0397a2e, 0x5727cb79, 0x9ceb418f, 0x1cacd68d, 0x2ad37c96, 0x0175cb9d, + 0xc69dff09, 0xc75b65f0, 0xd9db40d8, 0xec0e7779, 0x4744ead4, 0xb11c3274, 0xdd24cb9e, 0x7e1c54bd, + 0xf01144f9, 0xd2240eb1, 0x9675b3fd, 0xa3ac3755, 0xd47c27af, 0x51c85f4d, 0x56907596, 0xa5bb15e6, + 0x580304f0, 0xca042cf1, 0x011a37ea, 0x8dbfaadb, 0x35ba3e4a, 0x3526ffa0, 0xc37b4d09, 0xbc306ed9, + 0x98a52666, 0x5648f725, 0xff5e569d, 0x0ced63d0, 0x7c63b2cf, 0x700b45e1, 0xd5ea50f1, 0x85a92872, + 0xaf1fbda7, 0xd4234870, 0xa7870bf3, 0x2d3b4d79, 0x42e04198, 0x0cd0ede7, 0x26470db8, 0xf881814c, + 0x474d6ad7, 0x7c0c5e5c, 0xd1231959, 0x381b7298, 0xf5d2f4db, 0xab838653, 0x6e2f1e23, 0x83719c9e, + 0xbd91e046, 0x9a56456e, 0xdc39200c, 0x20c8c571, 0x962bda1c, 0xe1e696ff, 0xb141ab08, 0x7cca89b9, + 0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, 0x427b169c, 0x5ac9f049, 0xdd8f0f00, 0x5c8165bf, + }, + { + 0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a, 0xeec5207a, 0x55889c94, 0x72fc0651, + 0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef, 0x5f0c0794, 0x18dcdb7d, 0xa1d6eff3, + 0xa0b52f7b, 0x59e83605, 0xee15b094, 0xe9ffd909, 0xdc440086, 0xef944459, 0xba83ccb3, 0xe0c3cdfb, + 0xd1da4181, 0x3b092ab1, 0xf997f1c1, 0xa5e6cf7b, 0x01420ddb, 0xe4e7ef5b, 0x25a1ff41, 0xe180f806, + 0x1fc41080, 0x179bee7a, 0xd37ac6a9, 0xfe5830a4, 0x98de8b7f, 0x77e83f4e, 0x79929269, 0x24fa9f7b, + 0xe113c85b, 0xacc40083, 0xd7503525, 0xf7ea615f, 0x62143154, 0x0d554b63, 0x5d681121, 0xc866c359, + 0x3d63cf73, 0xcee234c0, 0xd4d87e87, 0x5c672b21, 0x071f6181, 0x39f7627f, 0x361e3084, 0xe4eb573b, + 0x602f64a4, 0xd63acd9c, 0x1bbc4635, 0x9e81032d, 0x2701f50c, 0x99847ab4, 0xa0e3df79, 0xba6cf38c, + 0x10843094, 0x2537a95e, 0xf46f6ffe, 0xa1ff3b1f, 0x208cfb6a, 0x8f458c74, 0xd9e0a227, 0x4ec73a34, + 0xfc884f69, 0x3e4de8df, 0xef0e0088, 0x3559648d, 0x8a45388c, 0x1d804366, 0x721d9bfd, 0xa58684bb, + 0xe8256333, 0x844e8212, 0x128d8098, 0xfed33fb4, 0xce280ae1, 0x27e19ba5, 0xd5a6c252, 0xe49754bd, + 0xc5d655dd, 0xeb667064, 0x77840b4d, 0xa1b6a801, 0x84db26a9, 0xe0b56714, 0x21f043b7, 0xe5d05860, + 0x54f03084, 0x066ff472, 0xa31aa153, 0xdadc4755, 0xb5625dbf, 0x68561be6, 0x83ca6b94, 0x2d6ed23b, + 0xeccf01db, 0xa6d3d0ba, 0xb6803d5c, 0xaf77a709, 0x33b4a34c, 0x397bc8d6, 0x5ee22b95, 0x5f0e5304, + 0x81ed6f61, 0x20e74364, 0xb45e1378, 0xde18639b, 0x881ca122, 0xb96726d1, 0x8049a7e8, 0x22b7da7b, + 0x5e552d25, 0x5272d237, 0x79d2951c, 0xc60d894c, 0x488cb402, 0x1ba4fe5b, 0xa4b09f6b, 0x1ca815cf, + 0xa20c3005, 0x8871df63, 0xb9de2fcb, 0x0cc6c9e9, 0x0beeff53, 0xe3214517, 0xb4542835, 0x9f63293c, + 0xee41e729, 0x6e1d2d7c, 0x50045286, 0x1e6685f3, 0xf33401c6, 0x30a22c95, 0x31a70850, 0x60930f13, + 0x73f98417, 0xa1269859, 0xec645c44, 0x52c877a9, 0xcdff33a6, 0xa02b1741, 0x7cbad9a2, 0x2180036f, + 0x50d99c08, 0xcb3f4861, 0xc26bd765, 0x64a3f6ab, 0x80342676, 0x25a75e7b, 0xe4e6d1fc, 0x20c710e6, + 0xcdf0b680, 0x17844d3b, 0x31eef84d, 0x7e0824e4, 0x2ccb49eb, 0x846a3bae, 0x8ff77888, 0xee5d60f6, + 0x7af75673, 0x2fdd5cdb, 0xa11631c1, 0x30f66f43, 0xb3faec54, 0x157fd7fa, 0xef8579cc, 0xd152de58, + 0xdb2ffd5e, 0x8f32ce19, 0x306af97a, 0x02f03ef8, 0x99319ad5, 0xc242fa0f, 0xa7e3ebb0, 0xc68e4906, + 0xb8da230c, 0x80823028, 0xdcdef3c8, 0xd35fb171, 0x088a1bc8, 0xbec0c560, 0x61a3c9e8, 0xbca8f54d, + 0xc72feffa, 0x22822e99, 0x82c570b4, 0xd8d94e89, 0x8b1c34bc, 0x301e16e6, 0x273be979, 0xb0ffeaa6, + 0x61d9b8c6, 0x00b24869, 0xb7ffce3f, 0x08dc283b, 0x43daf65a, 0xf7e19798, 0x7619b72f, 0x8f1c9ba4, + 0xdc8637a0, 0x16a7d3b1, 0x9fc393b7, 0xa7136eeb, 0xc6bcc63e, 0x1a513742, 0xef6828bc, 0x520365d6, + 0x2d6a77ab, 0x3527ed4b, 0x821fd216, 0x095c6e2e, 0xdb92f2fb, 0x5eea29cb, 0x145892f5, 0x91584f7f, + 0x5483697b, 0x2667a8cc, 0x85196048, 0x8c4bacea, 0x833860d4, 0x0d23e0f9, 0x6c387e8a, 0x0ae6d249, + 0xb284600c, 0xd835731d, 0xdcb1c647, 0xac4c56ea, 0x3ebd81b3, 0x230eabb0, 0x6438bc87, 0xf0b5b1fa, + 0x8f5ea2b3, 0xfc184642, 0x0a036b7a, 0x4fb089bd, 0x649da589, 0xa345415e, 0x5c038323, 0x3e5d3bb9, + 0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, 0x7160a539, 0x73bfbe70, 0x83877605, 0x4523ecf1, + }, + { + 0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff, 0x369fe44b, 0x8c1fc644, 0xaececa90, + 0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806, 0xf0ad0548, 0xe13c8d83, 0x927010d5, + 0x11107d9f, 0x07647db9, 0xb2e3e4d4, 0x3d4f285e, 0xb9afa820, 0xfade82e0, 0xa067268b, 0x8272792e, + 0x553fb2c0, 0x489ae22b, 0xd4ef9794, 0x125e3fbc, 0x21fffcee, 0x825b1bfd, 0x9255c5ed, 0x1257a240, + 0x4e1a8302, 0xbae07fff, 0x528246e7, 0x8e57140e, 0x3373f7bf, 0x8c9f8188, 0xa6fc4ee8, 0xc982b5a5, + 0xa8c01db7, 0x579fc264, 0x67094f31, 0xf2bd3f5f, 0x40fff7c1, 0x1fb78dfc, 0x8e6bd2c1, 0x437be59b, + 0x99b03dbf, 0xb5dbc64b, 0x638dc0e6, 0x55819d99, 0xa197c81c, 0x4a012d6e, 0xc5884a28, 0xccc36f71, + 0xb843c213, 0x6c0743f1, 0x8309893c, 0x0feddd5f, 0x2f7fe850, 0xd7c07f7e, 0x02507fbf, 0x5afb9a04, + 0xa747d2d0, 0x1651192e, 0xaf70bf3e, 0x58c31380, 0x5f98302e, 0x727cc3c4, 0x0a0fb402, 0x0f7fef82, + 0x8c96fdad, 0x5d2c2aae, 0x8ee99a49, 0x50da88b8, 0x8427f4a0, 0x1eac5790, 0x796fb449, 0x8252dc15, + 0xefbd7d9b, 0xa672597d, 0xada840d8, 0x45f54504, 0xfa5d7403, 0xe83ec305, 0x4f91751a, 0x925669c2, + 0x23efe941, 0xa903f12e, 0x60270df2, 0x0276e4b6, 0x94fd6574, 0x927985b2, 0x8276dbcb, 0x02778176, + 0xf8af918d, 0x4e48f79e, 0x8f616ddf, 0xe29d840e, 0x842f7d83, 0x340ce5c8, 0x96bbb682, 0x93b4b148, + 0xef303cab, 0x984faf28, 0x779faf9b, 0x92dc560d, 0x224d1e20, 0x8437aa88, 0x7d29dc96, 0x2756d3dc, + 0x8b907cee, 0xb51fd240, 0xe7c07ce3, 0xe566b4a1, 0xc3e9615e, 0x3cf8209d, 0x6094d1e3, 0xcd9ca341, + 0x5c76460e, 0x00ea983b, 0xd4d67881, 0xfd47572c, 0xf76cedd9, 0xbda8229c, 0x127dadaa, 0x438a074e, + 0x1f97c090, 0x081bdb8a, 0x93a07ebe, 0xb938ca15, 0x97b03cff, 0x3dc2c0f8, 0x8d1ab2ec, 0x64380e51, + 0x68cc7bfb, 0xd90f2788, 0x12490181, 0x5de5ffd4, 0xdd7ef86a, 0x76a2e214, 0xb9a40368, 0x925d958f, + 0x4b39fffa, 0xba39aee9, 0xa4ffd30b, 0xfaf7933b, 0x6d498623, 0x193cbcfa, 0x27627545, 0x825cf47a, + 0x61bd8ba0, 0xd11e42d1, 0xcead04f4, 0x127ea392, 0x10428db7, 0x8272a972, 0x9270c4a8, 0x127de50b, + 0x285ba1c8, 0x3c62f44f, 0x35c0eaa5, 0xe805d231, 0x428929fb, 0xb4fcdf82, 0x4fb66a53, 0x0e7dc15b, + 0x1f081fab, 0x108618ae, 0xfcfd086d, 0xf9ff2889, 0x694bcc11, 0x236a5cae, 0x12deca4d, 0x2c3f8cc5, + 0xd2d02dfe, 0xf8ef5896, 0xe4cf52da, 0x95155b67, 0x494a488c, 0xb9b6a80c, 0x5c8f82bc, 0x89d36b45, + 0x3a609437, 0xec00c9a9, 0x44715253, 0x0a874b49, 0xd773bc40, 0x7c34671c, 0x02717ef6, 0x4feb5536, + 0xa2d02fff, 0xd2bf60c4, 0xd43f03c0, 0x50b4ef6d, 0x07478cd1, 0x006e1888, 0xa2e53f55, 0xb9e6d4bc, + 0xa2048016, 0x97573833, 0xd7207d67, 0xde0f8f3d, 0x72f87b33, 0xabcc4f33, 0x7688c55d, 0x7b00a6b0, + 0x947b0001, 0x570075d2, 0xf9bb88f8, 0x8942019e, 0x4264a5ff, 0x856302e0, 0x72dbd92b, 0xee971b69, + 0x6ea22fde, 0x5f08ae2b, 0xaf7a616d, 0xe5c98767, 0xcf1febd2, 0x61efc8c2, 0xf1ac2571, 0xcc8239c2, + 0x67214cb8, 0xb1e583d1, 0xb7dc3e62, 0x7f10bdce, 0xf90a5c38, 0x0ff0443d, 0x606e6dc6, 0x60543a49, + 0x5727c148, 0x2be98a1d, 0x8ab41738, 0x20e1be24, 0xaf96da0f, 0x68458425, 0x99833be5, 0x600d457d, + 0x282f9350, 0x8334b362, 0xd91d1120, 0x2b6d8da0, 0x642b1e31, 0x9c305a00, 0x52bce688, 0x1b03588a, + 0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, 0xdfef4636, 0xa133c501, 0xe9d3531c, 0xee353783, + }, + { + 0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb, 0x64ad8c57, 0x85510443, 0xfa020ed1, + 0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43, 0x6497b7b1, 0xf3641f63, 0x241e4adf, + 0x28147f5f, 0x4fa2b8cd, 0xc9430040, 0x0cc32220, 0xfdd30b30, 0xc0a5374f, 0x1d2d00d9, 0x24147b15, + 0xee4d111a, 0x0fca5167, 0x71ff904c, 0x2d195ffe, 0x1a05645f, 0x0c13fefe, 0x081b08ca, 0x05170121, + 0x80530100, 0xe83e5efe, 0xac9af4f8, 0x7fe72701, 0xd2b8ee5f, 0x06df4261, 0xbb9e9b8a, 0x7293ea25, + 0xce84ffdf, 0xf5718801, 0x3dd64b04, 0xa26f263b, 0x7ed48400, 0x547eebe6, 0x446d4ca0, 0x6cf3d6f5, + 0x2649abdf, 0xaea0c7f5, 0x36338cc1, 0x503f7e93, 0xd3772061, 0x11b638e1, 0x72500e03, 0xf80eb2bb, + 0xabe0502e, 0xec8d77de, 0x57971e81, 0xe14f6746, 0xc9335400, 0x6920318f, 0x081dbb99, 0xffc304a5, + 0x4d351805, 0x7f3d5ce3, 0xa6c866c6, 0x5d5bcca9, 0xdaec6fea, 0x9f926f91, 0x9f46222f, 0x3991467d, + 0xa5bf6d8e, 0x1143c44f, 0x43958302, 0xd0214eeb, 0x022083b8, 0x3fb6180c, 0x18f8931e, 0x281658e6, + 0x26486e3e, 0x8bd78a70, 0x7477e4c1, 0xb506e07c, 0xf32d0a25, 0x79098b02, 0xe4eabb81, 0x28123b23, + 0x69dead38, 0x1574ca16, 0xdf871b62, 0x211c40b7, 0xa51a9ef9, 0x0014377b, 0x041e8ac8, 0x09114003, + 0xbd59e4d2, 0xe3d156d5, 0x4fe876d5, 0x2f91a340, 0x557be8de, 0x00eae4a7, 0x0ce5c2ec, 0x4db4bba6, + 0xe756bdff, 0xdd3369ac, 0xec17b035, 0x06572327, 0x99afc8b0, 0x56c8c391, 0x6b65811c, 0x5e146119, + 0x6e85cb75, 0xbe07c002, 0xc2325577, 0x893ff4ec, 0x5bbfc92d, 0xd0ec3b25, 0xb7801ab7, 0x8d6d3b24, + 0x20c763ef, 0xc366a5fc, 0x9c382880, 0x0ace3205, 0xaac9548a, 0xeca1d7c7, 0x041afa32, 0x1d16625a, + 0x6701902c, 0x9b757a54, 0x31d477f7, 0x9126b031, 0x36cc6fdb, 0xc70b8b46, 0xd9e66a48, 0x56e55a79, + 0x026a4ceb, 0x52437eff, 0x2f8f76b4, 0x0df980a5, 0x8674cde3, 0xedda04eb, 0x17a9be04, 0x2c18f4df, + 0xb7747f9d, 0xab2af7b4, 0xefc34d20, 0x2e096b7c, 0x1741a254, 0xe5b6a035, 0x213d42f6, 0x2c1c7c26, + 0x61c2f50f, 0x6552daf9, 0xd2c231f8, 0x25130f69, 0xd8167fa2, 0x0418f2c8, 0x001a96a6, 0x0d1526ab, + 0x63315c21, 0x5e0a72ec, 0x49bafefd, 0x187908d9, 0x8d0dbd86, 0x311170a7, 0x3e9b640c, 0xcc3e10d7, + 0xd5cad3b6, 0x0caec388, 0xf73001e1, 0x6c728aff, 0x71eae2a1, 0x1f9af36e, 0xcfcbd12f, 0xc1de8417, + 0xac07be6b, 0xcb44a1d8, 0x8b9b0f56, 0x013988c3, 0xb1c52fca, 0xb4be31cd, 0xd8782806, 0x12a3a4e2, + 0x6f7de532, 0x58fd7eb6, 0xd01ee900, 0x24adffc2, 0xf4990fc5, 0x9711aac5, 0x001d7b95, 0x82e5e7d2, + 0x109873f6, 0x00613096, 0xc32d9521, 0xada121ff, 0x29908415, 0x7fbb977f, 0xaf9eb3db, 0x29c9ed2a, + 0x5ce2a465, 0xa730f32c, 0xd0aa3fe8, 0x8a5cc091, 0xd49e2ce7, 0x0ce454a9, 0xd60acd86, 0x015f1919, + 0x77079103, 0xdea03af6, 0x78a8565e, 0xdee356df, 0x21f05cbe, 0x8b75e387, 0xb3c50651, 0xb8a5c3ef, + 0xd8eeb6d2, 0xe523be77, 0xc2154529, 0x2f69efdf, 0xafe67afb, 0xf470c4b2, 0xf3e0eb5b, 0xd6cc9876, + 0x39e4460c, 0x1fda8538, 0x1987832f, 0xca007367, 0xa99144f8, 0x296b299e, 0x492fc295, 0x9266beab, + 0xb5676e69, 0x9bd3ddda, 0xdf7e052f, 0xdb25701c, 0x1b5e51ee, 0xf65324e6, 0x6afce36c, 0x0316cc04, + 0x8644213e, 0xb7dc59d0, 0x7965291f, 0xccd6fd43, 0x41823979, 0x932bcdf6, 0xb657c34d, 0x4edfd282, + 0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, 0x13ecf0b0, 0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2, + }, + { + 0x7ec90c04, 0x2c6e74b9, 0x9b0e66df, 0xa6337911, 0xb86a7fff, 0x1dd358f5, 0x44dd9d44, 0x1731167f, + 0x08fbf1fa, 0xe7f511cc, 0xd2051b00, 0x735aba00, 0x2ab722d8, 0x386381cb, 0xacf6243a, 0x69befd7a, + 0xe6a2e77f, 0xf0c720cd, 0xc4494816, 0xccf5c180, 0x38851640, 0x15b0a848, 0xe68b18cb, 0x4caadeff, + 0x5f480a01, 0x0412b2aa, 0x259814fc, 0x41d0efe2, 0x4e40b48d, 0x248eb6fb, 0x8dba1cfe, 0x41a99b02, + 0x1a550a04, 0xba8f65cb, 0x7251f4e7, 0x95a51725, 0xc106ecd7, 0x97a5980a, 0xc539b9aa, 0x4d79fe6a, + 0xf2f3f763, 0x68af8040, 0xed0c9e56, 0x11b4958b, 0xe1eb5a88, 0x8709e6b0, 0xd7e07156, 0x4e29fea7, + 0x6366e52d, 0x02d1c000, 0xc4ac8e05, 0x9377f571, 0x0c05372a, 0x578535f2, 0x2261be02, 0xd642a0c9, + 0xdf13a280, 0x74b55bd2, 0x682199c0, 0xd421e5ec, 0x53fb3ce8, 0xc8adedb3, 0x28a87fc9, 0x3d959981, + 0x5c1ff900, 0xfe38d399, 0x0c4eff0b, 0x062407ea, 0xaa2f4fb1, 0x4fb96976, 0x90c79505, 0xb0a8a774, + 0xef55a1ff, 0xe59ca2c2, 0xa6b62d27, 0xe66a4263, 0xdf65001f, 0x0ec50966, 0xdfdd55bc, 0x29de0655, + 0x911e739a, 0x17af8975, 0x32c7911c, 0x89f89468, 0x0d01e980, 0x524755f4, 0x03b63cc9, 0x0cc844b2, + 0xbcf3f0aa, 0x87ac36e9, 0xe53a7426, 0x01b3d82b, 0x1a9e7449, 0x64ee2d7e, 0xcddbb1da, 0x01c94910, + 0xb868bf80, 0x0d26f3fd, 0x9342ede7, 0x04a5c284, 0x636737b6, 0x50f5b616, 0xf24766e3, 0x8eca36c1, + 0x136e05db, 0xfef18391, 0xfb887a37, 0xd6e7f7d4, 0xc7fb7dc9, 0x3063fcdf, 0xb6f589de, 0xec2941da, + 0x26e46695, 0xb7566419, 0xf654efc5, 0xd08d58b7, 0x48925401, 0xc1bacb7f, 0xe5ff550f, 0xb6083049, + 0x5bb5d0e8, 0x87d72e5a, 0xab6a6ee1, 0x223a66ce, 0xc62bf3cd, 0x9e0885f9, 0x68cb3e47, 0x086c010f, + 0xa21de820, 0xd18b69de, 0xf3f65777, 0xfa02c3f6, 0x407edac3, 0xcbb3d550, 0x1793084d, 0xb0d70eba, + 0x0ab378d5, 0xd951fb0c, 0xded7da56, 0x4124bbe4, 0x94ca0b56, 0x0f5755d1, 0xe0e1e56e, 0x6184b5be, + 0x580a249f, 0x94f74bc0, 0xe327888e, 0x9f7b5561, 0xc3dc0280, 0x05687715, 0x646c6bd7, 0x44904db3, + 0x66b4f0a3, 0xc0f1648a, 0x697ed5af, 0x49e92ff6, 0x309e374f, 0x2cb6356a, 0x85808573, 0x4991f840, + 0x76f0ae02, 0x083be84d, 0x28421c9a, 0x44489406, 0x736e4cb8, 0xc1092910, 0x8bc95fc6, 0x7d869cf4, + 0x134f616f, 0x2e77118d, 0xb31b2be1, 0xaa90b472, 0x3ca5d717, 0x7d161bba, 0x9cad9010, 0xaf462ba2, + 0x9fe459d2, 0x45d34559, 0xd9f2da13, 0xdbc65487, 0xf3e4f94e, 0x176d486f, 0x097c13ea, 0x631da5c7, + 0x445f7382, 0x175683f4, 0xcdc66a97, 0x70be0288, 0xb3cdcf72, 0x6e5dd2f3, 0x20936079, 0x459b80a5, + 0xbe60e2db, 0xa9c23101, 0xeba5315c, 0x224e42f2, 0x1c5c1572, 0xf6721b2c, 0x1ad2fff3, 0x8c25404e, + 0x324ed72f, 0x4067b7fd, 0x0523138e, 0x5ca3bc78, 0xdc0fd66e, 0x75922283, 0x784d6b17, 0x58ebb16e, + 0x44094f85, 0x3f481d87, 0xfcfeae7b, 0x77b5ff76, 0x8c2302bf, 0xaaf47556, 0x5f46b02a, 0x2b092801, + 0x3d38f5f7, 0x0ca81f36, 0x52af4a8a, 0x66d5e7c0, 0xdf3b0874, 0x95055110, 0x1b5ad7a8, 0xf61ed5ad, + 0x6cf6e479, 0x20758184, 0xd0cefa65, 0x88f7be58, 0x4a046826, 0x0ff6f8f3, 0xa09c7f70, 0x5346aba0, + 0x5ce96c28, 0xe176eda3, 0x6bac307f, 0x376829d2, 0x85360fa9, 0x17e3fe2a, 0x24b79767, 0xf5a96b20, + 0xd6cd2595, 0x68ff1ebf, 0x7555442c, 0xf19f06be, 0xf9e0659a, 0xeeb9491d, 0x34010718, 0xbb30cab8, + 0xe822fe15, 0x88570983, 0x750e6249, 0xda627e55, 0x5e76ffa8, 0xb1534546, 0x6d47de08, 0xefe9e7d4, + }, + { + 0xf6fa8f9d, 0x2cac6ce1, 0x4ca34867, 0xe2337f7c, 0x95db08e7, 0x016843b4, 0xeced5cbc, 0x325553ac, + 0xbf9f0960, 0xdfa1e2ed, 0x83f0579d, 0x63ed86b9, 0x1ab6a6b8, 0xde5ebe39, 0xf38ff732, 0x8989b138, + 0x33f14961, 0xc01937bd, 0xf506c6da, 0xe4625e7e, 0xa308ea99, 0x4e23e33c, 0x79cbd7cc, 0x48a14367, + 0xa3149619, 0xfec94bd5, 0xa114174a, 0xeaa01866, 0xa084db2d, 0x09a8486f, 0xa888614a, 0x2900af98, + 0x01665991, 0xe1992863, 0xc8f30c60, 0x2e78ef3c, 0xd0d51932, 0xcf0fec14, 0xf7ca07d2, 0xd0a82072, + 0xfd41197e, 0x9305a6b0, 0xe86be3da, 0x74bed3cd, 0x372da53c, 0x4c7f4448, 0xdab5d440, 0x6dba0ec3, + 0x083919a7, 0x9fbaeed9, 0x49dbcfb0, 0x4e670c53, 0x5c3d9c01, 0x64bdb941, 0x2c0e636a, 0xba7dd9cd, + 0xea6f7388, 0xe70bc762, 0x35f29adb, 0x5c4cdd8d, 0xf0d48d8c, 0xb88153e2, 0x08a19866, 0x1ae2eac8, + 0x284caf89, 0xaa928223, 0x9334be53, 0x3b3a21bf, 0x16434be3, 0x9aea3906, 0xefe8c36e, 0xf890cdd9, + 0x80226dae, 0xc340a4a3, 0xdf7e9c09, 0xa694a807, 0x5b7c5ecc, 0x221db3a6, 0x9a69a02f, 0x68818a54, + 0xceb2296f, 0x53c0843a, 0xfe893655, 0x25bfe68a, 0xb4628abc, 0xcf222ebf, 0x25ac6f48, 0xa9a99387, + 0x53bddb65, 0xe76ffbe7, 0xe967fd78, 0x0ba93563, 0x8e342bc1, 0xe8a11be9, 0x4980740d, 0xc8087dfc, + 0x8de4bf99, 0xa11101a0, 0x7fd37975, 0xda5a26c0, 0xe81f994f, 0x9528cd89, 0xfd339fed, 0xb87834bf, + 0x5f04456d, 0x22258698, 0xc9c4c83b, 0x2dc156be, 0x4f628daa, 0x57f55ec5, 0xe2220abe, 0xd2916ebf, + 0x4ec75b95, 0x24f2c3c0, 0x42d15d99, 0xcd0d7fa0, 0x7b6e27ff, 0xa8dc8af0, 0x7345c106, 0xf41e232f, + 0x35162386, 0xe6ea8926, 0x3333b094, 0x157ec6f2, 0x372b74af, 0x692573e4, 0xe9a9d848, 0xf3160289, + 0x3a62ef1d, 0xa787e238, 0xf3a5f676, 0x74364853, 0x20951063, 0x4576698d, 0xb6fad407, 0x592af950, + 0x36f73523, 0x4cfb6e87, 0x7da4cec0, 0x6c152daa, 0xcb0396a8, 0xc50dfe5d, 0xfcd707ab, 0x0921c42f, + 0x89dff0bb, 0x5fe2be78, 0x448f4f33, 0x754613c9, 0x2b05d08d, 0x48b9d585, 0xdc049441, 0xc8098f9b, + 0x7dede786, 0xc39a3373, 0x42410005, 0x6a091751, 0x0ef3c8a6, 0x890072d6, 0x28207682, 0xa9a9f7be, + 0xbf32679d, 0xd45b5b75, 0xb353fd00, 0xcbb0e358, 0x830f220a, 0x1f8fb214, 0xd372cf08, 0xcc3c4a13, + 0x8cf63166, 0x061c87be, 0x88c98f88, 0x6062e397, 0x47cf8e7a, 0xb6c85283, 0x3cc2acfb, 0x3fc06976, + 0x4e8f0252, 0x64d8314d, 0xda3870e3, 0x1e665459, 0xc10908f0, 0x513021a5, 0x6c5b68b7, 0x822f8aa0, + 0x3007cd3e, 0x74719eef, 0xdc872681, 0x073340d4, 0x7e432fd9, 0x0c5ec241, 0x8809286c, 0xf592d891, + 0x08a930f6, 0x957ef305, 0xb7fbffbd, 0xc266e96f, 0x6fe4ac98, 0xb173ecc0, 0xbc60b42a, 0x953498da, + 0xfba1ae12, 0x2d4bd736, 0x0f25faab, 0xa4f3fceb, 0xe2969123, 0x257f0c3d, 0x9348af49, 0x361400bc, + 0xe8816f4a, 0x3814f200, 0xa3f94043, 0x9c7a54c2, 0xbc704f57, 0xda41e7f9, 0xc25ad33a, 0x54f4a084, + 0xb17f5505, 0x59357cbe, 0xedbd15c8, 0x7f97c5ab, 0xba5ac7b5, 0xb6f6deaf, 0x3a479c3a, 0x5302da25, + 0x653d7e6a, 0x54268d49, 0x51a477ea, 0x5017d55b, 0xd7d25d88, 0x44136c76, 0x0404a8c8, 0xb8e5a121, + 0xb81a928a, 0x60ed5869, 0x97c55b96, 0xeaec991b, 0x29935913, 0x01fdb7f1, 0x088e8dfa, 0x9ab6f6f5, + 0x3b4cbf9f, 0x4a5de3ab, 0xe6051d35, 0xa0e1d855, 0xd36b4cf1, 0xf544edeb, 0xb0e93524, 0xbebb8fbd, + 0xa2d762cf, 0x49c92f54, 0x38b5f331, 0x7128a454, 0x48392905, 0xa65b1db8, 0x851c97bd, 0xd675cf2f, + }, + { + 0x85e04019, 0x332bf567, 0x662dbfff, 0xcfc65693, 0x2a8d7f6f, 0xab9bc912, 0xde6008a1, 0x2028da1f, + 0x0227bce7, 0x4d642916, 0x18fac300, 0x50f18b82, 0x2cb2cb11, 0xb232e75c, 0x4b3695f2, 0xb28707de, + 0xa05fbcf6, 0xcd4181e9, 0xe150210c, 0xe24ef1bd, 0xb168c381, 0xfde4e789, 0x5c79b0d8, 0x1e8bfd43, + 0x4d495001, 0x38be4341, 0x913cee1d, 0x92a79c3f, 0x089766be, 0xbaeeadf4, 0x1286becf, 0xb6eacb19, + 0x2660c200, 0x7565bde4, 0x64241f7a, 0x8248dca9, 0xc3b3ad66, 0x28136086, 0x0bd8dfa8, 0x356d1cf2, + 0x107789be, 0xb3b2e9ce, 0x0502aa8f, 0x0bc0351e, 0x166bf52a, 0xeb12ff82, 0xe3486911, 0xd34d7516, + 0x4e7b3aff, 0x5f43671b, 0x9cf6e037, 0x4981ac83, 0x334266ce, 0x8c9341b7, 0xd0d854c0, 0xcb3a6c88, + 0x47bc2829, 0x4725ba37, 0xa66ad22b, 0x7ad61f1e, 0x0c5cbafa, 0x4437f107, 0xb6e79962, 0x42d2d816, + 0x0a961288, 0xe1a5c06e, 0x13749e67, 0x72fc081a, 0xb1d139f7, 0xf9583745, 0xcf19df58, 0xbec3f756, + 0xc06eba30, 0x07211b24, 0x45c28829, 0xc95e317f, 0xbc8ec511, 0x38bc46e9, 0xc6e6fa14, 0xbae8584a, + 0xad4ebc46, 0x468f508b, 0x7829435f, 0xf124183b, 0x821dba9f, 0xaff60ff4, 0xea2c4e6d, 0x16e39264, + 0x92544a8b, 0x009b4fc3, 0xaba68ced, 0x9ac96f78, 0x06a5b79a, 0xb2856e6e, 0x1aec3ca9, 0xbe838688, + 0x0e0804e9, 0x55f1be56, 0xe7e5363b, 0xb3a1f25d, 0xf7debb85, 0x61fe033c, 0x16746233, 0x3c034c28, + 0xda6d0c74, 0x79aac56c, 0x3ce4e1ad, 0x51f0c802, 0x98f8f35a, 0x1626a49f, 0xeed82b29, 0x1d382fe3, + 0x0c4fb99a, 0xbb325778, 0x3ec6d97b, 0x6e77a6a9, 0xcb658b5c, 0xd45230c7, 0x2bd1408b, 0x60c03eb7, + 0xb9068d78, 0xa33754f4, 0xf430c87d, 0xc8a71302, 0xb96d8c32, 0xebd4e7be, 0xbe8b9d2d, 0x7979fb06, + 0xe7225308, 0x8b75cf77, 0x11ef8da4, 0xe083c858, 0x8d6b786f, 0x5a6317a6, 0xfa5cf7a0, 0x5dda0033, + 0xf28ebfb0, 0xf5b9c310, 0xa0eac280, 0x08b9767a, 0xa3d9d2b0, 0x79d34217, 0x021a718d, 0x9ac6336a, + 0x2711fd60, 0x438050e3, 0x069908a8, 0x3d7fedc4, 0x826d2bef, 0x4eeb8476, 0x488dcf25, 0x36c9d566, + 0x28e74e41, 0xc2610aca, 0x3d49a9cf, 0xbae3b9df, 0xb65f8de6, 0x92aeaf64, 0x3ac7d5e6, 0x9ea80509, + 0xf22b017d, 0xa4173f70, 0xdd1e16c3, 0x15e0d7f9, 0x50b1b887, 0x2b9f4fd5, 0x625aba82, 0x6a017962, + 0x2ec01b9c, 0x15488aa9, 0xd716e740, 0x40055a2c, 0x93d29a22, 0xe32dbf9a, 0x058745b9, 0x3453dc1e, + 0xd699296e, 0x496cff6f, 0x1c9f4986, 0xdfe2ed07, 0xb87242d1, 0x19de7eae, 0x053e561a, 0x15ad6f8c, + 0x66626c1c, 0x7154c24c, 0xea082b2a, 0x93eb2939, 0x17dcb0f0, 0x58d4f2ae, 0x9ea294fb, 0x52cf564c, + 0x9883fe66, 0x2ec40581, 0x763953c3, 0x01d6692e, 0xd3a0c108, 0xa1e7160e, 0xe4f2dfa6, 0x693ed285, + 0x74904698, 0x4c2b0edd, 0x4f757656, 0x5d393378, 0xa132234f, 0x3d321c5d, 0xc3f5e194, 0x4b269301, + 0xc79f022f, 0x3c997e7e, 0x5e4f9504, 0x3ffafbbd, 0x76f7ad0e, 0x296693f4, 0x3d1fce6f, 0xc61e45be, + 0xd3b5ab34, 0xf72bf9b7, 0x1b0434c0, 0x4e72b567, 0x5592a33d, 0xb5229301, 0xcfd2a87f, 0x60aeb767, + 0x1814386b, 0x30bcc33d, 0x38a0c07d, 0xfd1606f2, 0xc363519b, 0x589dd390, 0x5479f8e6, 0x1cb8d647, + 0x97fd61a9, 0xea7759f4, 0x2d57539d, 0x569a58cf, 0xe84e63ad, 0x462e1b78, 0x6580f87e, 0xf3817914, + 0x91da55f4, 0x40a230f3, 0xd1988f35, 0xb6e318d2, 0x3ffa50bc, 0x3d40f021, 0xc3c0bdae, 0x4958c24c, + 0x518f36b2, 0x84b1d370, 0x0fedce83, 0x878ddada, 0xf2a279c7, 0x94e01be8, 0x90716f4b, 0x954b8aa3, + }, + { + 0xe216300d, 0xbbddfffc, 0xa7ebdabd, 0x35648095, 0x7789f8b7, 0xe6c1121b, 0x0e241600, 0x052ce8b5, + 0x11a9cfb0, 0xe5952f11, 0xece7990a, 0x9386d174, 0x2a42931c, 0x76e38111, 0xb12def3a, 0x37ddddfc, + 0xde9adeb1, 0x0a0cc32c, 0xbe197029, 0x84a00940, 0xbb243a0f, 0xb4d137cf, 0xb44e79f0, 0x049eedfd, + 0x0b15a15d, 0x480d3168, 0x8bbbde5a, 0x669ded42, 0xc7ece831, 0x3f8f95e7, 0x72df191b, 0x7580330d, + 0x94074251, 0x5c7dcdfa, 0xabbe6d63, 0xaa402164, 0xb301d40a, 0x02e7d1ca, 0x53571dae, 0x7a3182a2, + 0x12a8ddec, 0xfdaa335d, 0x176f43e8, 0x71fb46d4, 0x38129022, 0xce949ad4, 0xb84769ad, 0x965bd862, + 0x82f3d055, 0x66fb9767, 0x15b80b4e, 0x1d5b47a0, 0x4cfde06f, 0xc28ec4b8, 0x57e8726e, 0x647a78fc, + 0x99865d44, 0x608bd593, 0x6c200e03, 0x39dc5ff6, 0x5d0b00a3, 0xae63aff2, 0x7e8bd632, 0x70108c0c, + 0xbbd35049, 0x2998df04, 0x980cf42a, 0x9b6df491, 0x9e7edd53, 0x06918548, 0x58cb7e07, 0x3b74ef2e, + 0x522fffb1, 0xd24708cc, 0x1c7e27cd, 0xa4eb215b, 0x3cf1d2e2, 0x19b47a38, 0x424f7618, 0x35856039, + 0x9d17dee7, 0x27eb35e6, 0xc9aff67b, 0x36baf5b8, 0x09c467cd, 0xc18910b1, 0xe11dbf7b, 0x06cd1af8, + 0x7170c608, 0x2d5e3354, 0xd4de495a, 0x64c6d006, 0xbcc0c62c, 0x3dd00db3, 0x708f8f34, 0x77d51b42, + 0x264f620f, 0x24b8d2bf, 0x15c1b79e, 0x46a52564, 0xf8d7e54e, 0x3e378160, 0x7895cda5, 0x859c15a5, + 0xe6459788, 0xc37bc75f, 0xdb07ba0c, 0x0676a3ab, 0x7f229b1e, 0x31842e7b, 0x24259fd7, 0xf8bef472, + 0x835ffcb8, 0x6df4c1f2, 0x96f5b195, 0xfd0af0fc, 0xb0fe134c, 0xe2506d3d, 0x4f9b12ea, 0xf215f225, + 0xa223736f, 0x9fb4c428, 0x25d04979, 0x34c713f8, 0xc4618187, 0xea7a6e98, 0x7cd16efc, 0x1436876c, + 0xf1544107, 0xbedeee14, 0x56e9af27, 0xa04aa441, 0x3cf7c899, 0x92ecbae6, 0xdd67016d, 0x151682eb, + 0xa842eedf, 0xfdba60b4, 0xf1907b75, 0x20e3030f, 0x24d8c29e, 0xe139673b, 0xefa63fb8, 0x71873054, + 0xb6f2cf3b, 0x9f326442, 0xcb15a4cc, 0xb01a4504, 0xf1e47d8d, 0x844a1be5, 0xbae7dfdc, 0x42cbda70, + 0xcd7dae0a, 0x57e85b7a, 0xd53f5af6, 0x20cf4d8c, 0xcea4d428, 0x79d130a4, 0x3486ebfb, 0x33d3cddc, + 0x77853b53, 0x37effcb5, 0xc5068778, 0xe580b3e6, 0x4e68b8f4, 0xc5c8b37e, 0x0d809ea2, 0x398feb7c, + 0x132a4f94, 0x43b7950e, 0x2fee7d1c, 0x223613bd, 0xdd06caa2, 0x37df932b, 0xc4248289, 0xacf3ebc3, + 0x5715f6b7, 0xef3478dd, 0xf267616f, 0xc148cbe4, 0x9052815e, 0x5e410fab, 0xb48a2465, 0x2eda7fa4, + 0xe87b40e4, 0xe98ea084, 0x5889e9e1, 0xefd390fc, 0xdd07d35b, 0xdb485694, 0x38d7e5b2, 0x57720101, + 0x730edebc, 0x5b643113, 0x94917e4f, 0x503c2fba, 0x646f1282, 0x7523d24a, 0xe0779695, 0xf9c17a8f, + 0x7a5b2121, 0xd187b896, 0x29263a4d, 0xba510cdf, 0x81f47c9f, 0xad1163ed, 0xea7b5965, 0x1a00726e, + 0x11403092, 0x00da6d77, 0x4a0cdd61, 0xad1f4603, 0x605bdfb0, 0x9eedc364, 0x22ebe6a8, 0xcee7d28a, + 0xa0e736a0, 0x5564a6b9, 0x10853209, 0xc7eb8f37, 0x2de705ca, 0x8951570f, 0xdf09822b, 0xbd691a6c, + 0xaa12e4f2, 0x87451c0f, 0xe0f6a27a, 0x3ada4819, 0x4cf1764f, 0x0d771c2b, 0x67cdb156, 0x350d8384, + 0x5938fa0f, 0x42399ef3, 0x36997b07, 0x0e84093d, 0x4aa93e61, 0x8360d87b, 0x1fa98b0c, 0x1149382c, + 0xe97625a5, 0x0614d1b7, 0x0e25244b, 0x0c768347, 0x589e8d82, 0x0d2059d1, 0xa466bb1e, 0xf8da0a82, + 0x04f19130, 0xba6e4ec0, 0x99265164, 0x1ee7230d, 0x50b2ad80, 0xeaee6801, 0x8db2a283, 0xea8bf59e, + }, +} diff --git a/src/pkg/crypto/cast5/cast5_test.go b/src/pkg/crypto/cast5/cast5_test.go new file mode 100644 index 000000000..5f7025ff2 --- /dev/null +++ b/src/pkg/crypto/cast5/cast5_test.go @@ -0,0 +1,104 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cast5 + +import ( + "bytes" + "encoding/hex" + "testing" +) + +// This test vector is taken from RFC 2144, App B.1. +// Since the other two test vectors are for reduced-round variants, we can't +// use them. +var basicTests = []struct { + key, plainText, cipherText string +}{ + { + "0123456712345678234567893456789a", + "0123456789abcdef", + "238b4fe5847e44b2", + }, +} + +func TestBasic(t *testing.T) { + for i, test := range basicTests { + key, _ := hex.DecodeString(test.key) + plainText, _ := hex.DecodeString(test.plainText) + expected, _ := hex.DecodeString(test.cipherText) + + c, err := NewCipher(key) + if err != nil { + t.Errorf("#%d: failed to create Cipher: %s", i, err) + continue + } + var cipherText [BlockSize]byte + c.Encrypt(cipherText[:], plainText) + if !bytes.Equal(cipherText[:], expected) { + t.Errorf("#%d: got:%x want:%x", i, cipherText, expected) + } + + var plainTextAgain [BlockSize]byte + c.Decrypt(plainTextAgain[:], cipherText[:]) + if !bytes.Equal(plainTextAgain[:], plainText) { + t.Errorf("#%d: got:%x want:%x", i, plainTextAgain, plainText) + } + } +} + +// TestFull performs the test specified in RFC 2144, App B.2. +// However, due to the length of time taken, it's disabled here and a more +// limited version is included, below. +func TestFull(t *testing.T) { + // This is too slow for normal testing + return + + a, b := iterate(1000000) + + const expectedA = "eea9d0a249fd3ba6b3436fb89d6dca92" + const expectedB = "b2c95eb00c31ad7180ac05b8e83d696e" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} + +func iterate(iterations int) ([]byte, []byte) { + const initValueHex = "0123456712345678234567893456789a" + + initValue, _ := hex.DecodeString(initValueHex) + + var a, b [16]byte + copy(a[:], initValue) + copy(b[:], initValue) + + for i := 0; i < iterations; i++ { + c, _ := NewCipher(b[:]) + c.Encrypt(a[:8], a[:8]) + c.Encrypt(a[8:], a[8:]) + c, _ = NewCipher(a[:]) + c.Encrypt(b[:8], b[:8]) + c.Encrypt(b[8:], b[8:]) + } + + return a[:], b[:] +} + +func TestLimited(t *testing.T) { + a, b := iterate(1000) + + const expectedA = "23f73b14b02a2ad7dfb9f2c35644798d" + const expectedB = "e5bf37eff14c456a40b21ce369370a9f" + + if hex.EncodeToString(a) != expectedA { + t.Errorf("a: got:%x want:%s", a, expectedA) + } + if hex.EncodeToString(b) != expectedB { + t.Errorf("b: got:%x want:%s", b, expectedB) + } +} diff --git a/src/pkg/crypto/cipher/Makefile b/src/pkg/crypto/cipher/Makefile new file mode 100644 index 000000000..8f61cf20b --- /dev/null +++ b/src/pkg/crypto/cipher/Makefile @@ -0,0 +1,17 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/cipher +GOFILES=\ + cbc.go\ + cfb.go\ + cipher.go\ + ctr.go\ + io.go\ + ocfb.go\ + ofb.go + +include ../../../Make.pkg diff --git a/src/pkg/crypto/cipher/cbc.go b/src/pkg/crypto/cipher/cbc.go new file mode 100644 index 000000000..4632f882a --- /dev/null +++ b/src/pkg/crypto/cipher/cbc.go @@ -0,0 +1,78 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cipher block chaining (CBC) mode. + +// CBC provides confidentiality by xoring (chaining) each plaintext block +// with the previous ciphertext block before applying the block cipher. + +// See NIST SP 800-38A, pp 10-11 + +package cipher + +type cbc struct { + b Block + blockSize int + iv []byte + tmp []byte +} + +func newCBC(b Block, iv []byte) *cbc { + return &cbc{ + b: b, + blockSize: b.BlockSize(), + iv: dup(iv), + tmp: make([]byte, b.BlockSize()), + } +} + +type cbcEncrypter cbc + +// NewCBCEncrypter returns a BlockMode which encrypts in cipher block chaining +// mode, using the given Block. The length of iv must be the same as the +// Block's block size. +func NewCBCEncrypter(b Block, iv []byte) BlockMode { + return (*cbcEncrypter)(newCBC(b, iv)) +} + +func (x *cbcEncrypter) BlockSize() int { return x.blockSize } + +func (x *cbcEncrypter) CryptBlocks(dst, src []byte) { + for len(src) > 0 { + for i := 0; i < x.blockSize; i++ { + x.iv[i] ^= src[i] + } + x.b.Encrypt(x.iv, x.iv) + for i := 0; i < x.blockSize; i++ { + dst[i] = x.iv[i] + } + src = src[x.blockSize:] + dst = dst[x.blockSize:] + } +} + +type cbcDecrypter cbc + +// NewCBCDecrypter returns a BlockMode which decrypts in cipher block chaining +// mode, using the given Block. The length of iv must be the same as the +// Block's block size as must match the iv used to encrypt the data. +func NewCBCDecrypter(b Block, iv []byte) BlockMode { + return (*cbcDecrypter)(newCBC(b, iv)) +} + +func (x *cbcDecrypter) BlockSize() int { return x.blockSize } + +func (x *cbcDecrypter) CryptBlocks(dst, src []byte) { + for len(src) > 0 { + x.b.Decrypt(x.tmp, src[:x.blockSize]) + for i := 0; i < x.blockSize; i++ { + x.tmp[i] ^= x.iv[i] + x.iv[i] = src[i] + dst[i] = x.tmp[i] + } + + src = src[x.blockSize:] + dst = dst[x.blockSize:] + } +} diff --git a/src/pkg/crypto/cipher/cbc_aes_test.go b/src/pkg/crypto/cipher/cbc_aes_test.go new file mode 100644 index 000000000..944ca1ba8 --- /dev/null +++ b/src/pkg/crypto/cipher/cbc_aes_test.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CBC AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 24-29. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +var cbcAESTests = []struct { + name string + key []byte + iv []byte + in []byte + out []byte +}{ + // NIST SP 800-38A pp 27-29 + { + "CBC-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7, + }, + }, + { + "CBC-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8, + 0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4, 0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a, + 0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0, 0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0, + 0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81, 0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd, + }, + }, + { + "CBC-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6, + 0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d, 0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d, + 0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf, 0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61, + 0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc, 0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b, + }, + }, +} + +func TestCBC_AES(t *testing.T) { + for _, tt := range cbcAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + encrypter := NewCBCEncrypter(c, tt.iv) + d := make([]byte, len(tt.in)) + encrypter.CryptBlocks(d, tt.in) + if !bytes.Equal(tt.out, d) { + t.Errorf("%s: CBCEncrypter\nhave %x\nwant %x", test, d, tt.out) + } + + decrypter := NewCBCDecrypter(c, tt.iv) + p := make([]byte, len(d)) + decrypter.CryptBlocks(p, d) + if !bytes.Equal(tt.in, p) { + t.Errorf("%s: CBCDecrypter\nhave %x\nwant %x", test, d, tt.in) + } + } +} diff --git a/src/pkg/crypto/cipher/cfb.go b/src/pkg/crypto/cipher/cfb.go new file mode 100644 index 000000000..d14165a86 --- /dev/null +++ b/src/pkg/crypto/cipher/cfb.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CFB (Cipher Feedback) Mode. + +package cipher + +type cfb struct { + b Block + out []byte + outUsed int + decrypt bool +} + +// NewCFBEncrypter returns a Stream which encrypts with cipher feedback mode, +// using the given Block. The iv must be the same length as the Block's block +// size. +func NewCFBEncrypter(block Block, iv []byte) Stream { + return newCFB(block, iv, false) +} + +// NewCFBDecrypter returns a Stream which decrypts with cipher feedback mode, +// using the given Block. The iv must be the same length as the Block's block +// size. +func NewCFBDecrypter(block Block, iv []byte) Stream { + return newCFB(block, iv, true) +} + +func newCFB(block Block, iv []byte, decrypt bool) Stream { + blockSize := block.BlockSize() + if len(iv) != blockSize { + return nil + } + + x := &cfb{ + b: block, + out: make([]byte, blockSize), + outUsed: 0, + decrypt: decrypt, + } + block.Encrypt(x.out, iv) + + return x +} + +func (x *cfb) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.out) { + x.b.Encrypt(x.out, x.out) + x.outUsed = 0 + } + + if x.decrypt { + t := src[i] + dst[i] = src[i] ^ x.out[x.outUsed] + x.out[x.outUsed] = t + } else { + x.out[x.outUsed] ^= src[i] + dst[i] = x.out[x.outUsed] + } + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/cfb_test.go b/src/pkg/crypto/cipher/cfb_test.go new file mode 100644 index 000000000..9547bfceb --- /dev/null +++ b/src/pkg/crypto/cipher/cfb_test.go @@ -0,0 +1,35 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +func TestCFB(t *testing.T) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext") + iv := make([]byte, block.BlockSize()) + rand.Reader.Read(iv) + cfb := NewCFBEncrypter(block, iv) + ciphertext := make([]byte, len(plaintext)) + cfb.XORKeyStream(ciphertext, plaintext) + + cfbdec := NewCFBDecrypter(block, iv) + plaintextCopy := make([]byte, len(plaintext)) + cfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x", plaintextCopy, plaintext) + } +} diff --git a/src/pkg/crypto/cipher/cipher.go b/src/pkg/crypto/cipher/cipher.go new file mode 100644 index 000000000..1ffaa8c2c --- /dev/null +++ b/src/pkg/crypto/cipher/cipher.go @@ -0,0 +1,63 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cipher implements standard block cipher modes that can be wrapped +// around low-level block cipher implementations. +// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html +// and NIST Special Publication 800-38A. +package cipher + +// A Block represents an implementation of block cipher +// using a given key. It provides the capability to encrypt +// or decrypt individual blocks. The mode implementations +// extend that capability to streams of blocks. +type Block interface { + // BlockSize returns the cipher's block size. + BlockSize() int + + // Encrypt encrypts the first block in src into dst. + // Dst and src may point at the same memory. + Encrypt(dst, src []byte) + + // Decrypt decrypts the first block in src into dst. + // Dst and src may point at the same memory. + Decrypt(dst, src []byte) +} + +// A Stream represents a stream cipher. +type Stream interface { + // XORKeyStream XORs each byte in the given slice with a byte from the + // cipher's key stream. Dst and src may point to the same memory. + XORKeyStream(dst, src []byte) +} + +// A BlockMode represents a block cipher running in a block-based mode (CBC, +// ECB etc). +type BlockMode interface { + // BlockSize returns the mode's block size. + BlockSize() int + + // CryptBlocks encrypts or decrypts a number of blocks. The length of + // src must be a multiple of the block size. Dst and src may point to + // the same memory. + CryptBlocks(dst, src []byte) +} + +// Utility routines + +func shift1(dst, src []byte) byte { + var b byte + for i := len(src) - 1; i >= 0; i-- { + bb := src[i] >> 7 + dst[i] = src[i]<<1 | b + b = bb + } + return b +} + +func dup(p []byte) []byte { + q := make([]byte, len(p)) + copy(q, p) + return q +} diff --git a/src/pkg/crypto/cipher/common_test.go b/src/pkg/crypto/cipher/common_test.go new file mode 100644 index 000000000..fb755757c --- /dev/null +++ b/src/pkg/crypto/cipher/common_test.go @@ -0,0 +1,28 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +// Common values for tests. + +var commonInput = []byte{ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, +} + +var commonKey128 = []byte{0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c} + +var commonKey192 = []byte{ + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b, +} + +var commonKey256 = []byte{ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, +} + +var commonIV = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} diff --git a/src/pkg/crypto/cipher/ctr.go b/src/pkg/crypto/cipher/ctr.go new file mode 100644 index 000000000..147b74fc2 --- /dev/null +++ b/src/pkg/crypto/cipher/ctr.go @@ -0,0 +1,55 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Counter (CTR) mode. + +// CTR converts a block cipher into a stream cipher by +// repeatedly encrypting an incrementing counter and +// xoring the resulting stream of data with the input. + +// See NIST SP 800-38A, pp 13-15 + +package cipher + +type ctr struct { + b Block + ctr []byte + out []byte + outUsed int +} + +// NewCTR returns a Stream which encrypts/decrypts using the given Block in +// counter mode. The length of iv must be the same as the Block's block size. +func NewCTR(block Block, iv []byte) Stream { + if len(iv) != block.BlockSize() { + panic("cipher.NewCTR: iv length must equal block size") + } + + return &ctr{ + b: block, + ctr: dup(iv), + out: make([]byte, len(iv)), + outUsed: len(iv), + } +} + +func (x *ctr) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.ctr) { + x.b.Encrypt(x.out, x.ctr) + x.outUsed = 0 + + // Increment counter + for i := len(x.ctr) - 1; i >= 0; i-- { + x.ctr[i]++ + if x.ctr[i] != 0 { + break + } + } + } + + dst[i] = src[i] ^ x.out[x.outUsed] + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/ctr_aes_test.go b/src/pkg/crypto/cipher/ctr_aes_test.go new file mode 100644 index 000000000..8dca9968c --- /dev/null +++ b/src/pkg/crypto/cipher/ctr_aes_test.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CTR AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 55-58. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +var commonCounter = []byte{0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff} + +var ctrAESTests = []struct { + name string + key []byte + iv []byte + in []byte + out []byte +}{ + // NIST SP 800-38A pp 55-58 + { + "CTR-AES128", + commonKey128, + commonCounter, + commonInput, + []byte{ + 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce, + 0x98, 0x06, 0xf6, 0x6b, 0x79, 0x70, 0xfd, 0xff, 0x86, 0x17, 0x18, 0x7b, 0xb9, 0xff, 0xfd, 0xff, + 0x5a, 0xe4, 0xdf, 0x3e, 0xdb, 0xd5, 0xd3, 0x5e, 0x5b, 0x4f, 0x09, 0x02, 0x0d, 0xb0, 0x3e, 0xab, + 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee, + }, + }, + { + "CTR-AES192", + commonKey192, + commonCounter, + commonInput, + []byte{ + 0x1a, 0xbc, 0x93, 0x24, 0x17, 0x52, 0x1c, 0xa2, 0x4f, 0x2b, 0x04, 0x59, 0xfe, 0x7e, 0x6e, 0x0b, + 0x09, 0x03, 0x39, 0xec, 0x0a, 0xa6, 0xfa, 0xef, 0xd5, 0xcc, 0xc2, 0xc6, 0xf4, 0xce, 0x8e, 0x94, + 0x1e, 0x36, 0xb2, 0x6b, 0xd1, 0xeb, 0xc6, 0x70, 0xd1, 0xbd, 0x1d, 0x66, 0x56, 0x20, 0xab, 0xf7, + 0x4f, 0x78, 0xa7, 0xf6, 0xd2, 0x98, 0x09, 0x58, 0x5a, 0x97, 0xda, 0xec, 0x58, 0xc6, 0xb0, 0x50, + }, + }, + { + "CTR-AES256", + commonKey256, + commonCounter, + commonInput, + []byte{ + 0x60, 0x1e, 0xc3, 0x13, 0x77, 0x57, 0x89, 0xa5, 0xb7, 0xa7, 0xf5, 0x04, 0xbb, 0xf3, 0xd2, 0x28, + 0xf4, 0x43, 0xe3, 0xca, 0x4d, 0x62, 0xb5, 0x9a, 0xca, 0x84, 0xe9, 0x90, 0xca, 0xca, 0xf5, 0xc5, + 0x2b, 0x09, 0x30, 0xda, 0xa2, 0x3d, 0xe9, 0x4c, 0xe8, 0x70, 0x17, 0xba, 0x2d, 0x84, 0x98, 0x8d, + 0xdf, 0xc9, 0xc5, 0x8d, 0xb6, 0x7a, 0xad, 0xa6, 0x13, 0xc2, 0xdd, 0x08, 0x45, 0x79, 0x41, 0xa6, + }, + }, +} + +func TestCTR_AES(t *testing.T) { + for _, tt := range ctrAESTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + in := tt.in[0 : len(tt.in)-j] + ctr := NewCTR(c, tt.iv) + encrypted := make([]byte, len(in)) + ctr.XORKeyStream(encrypted, in) + if out := tt.out[0:len(in)]; !bytes.Equal(out, encrypted) { + t.Errorf("%s/%d: CTR\ninpt %x\nhave %x\nwant %x", test, len(in), in, encrypted, out) + } + } + + for j := 0; j <= 7; j += 7 { + in := tt.out[0 : len(tt.out)-j] + ctr := NewCTR(c, tt.iv) + plain := make([]byte, len(in)) + ctr.XORKeyStream(plain, in) + if out := tt.in[0:len(in)]; !bytes.Equal(out, plain) { + t.Errorf("%s/%d: CTRReader\nhave %x\nwant %x", test, len(out), plain, out) + } + } + + if t.Failed() { + break + } + } +} diff --git a/src/pkg/crypto/cipher/io.go b/src/pkg/crypto/cipher/io.go new file mode 100644 index 000000000..97f40b8e7 --- /dev/null +++ b/src/pkg/crypto/cipher/io.go @@ -0,0 +1,57 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "os" + "io" +) + +// The Stream* objects are so simple that all their members are public. Users +// can create them themselves. + +// StreamReader wraps a Stream into an io.Reader. It simply calls XORKeyStream +// to process each slice of data which passes through. +type StreamReader struct { + S Stream + R io.Reader +} + +func (r StreamReader) Read(dst []byte) (n int, err os.Error) { + n, err = r.R.Read(dst) + r.S.XORKeyStream(dst[:n], dst[:n]) + return +} + +// StreamWriter wraps a Stream into an io.Writer. It simply calls XORKeyStream +// to process each slice of data which passes through. If any Write call +// returns short then the StreamWriter is out of sync and must be discarded. +type StreamWriter struct { + S Stream + W io.Writer + Err os.Error +} + +func (w StreamWriter) Write(src []byte) (n int, err os.Error) { + if w.Err != nil { + return 0, w.Err + } + c := make([]byte, len(src)) + w.S.XORKeyStream(c, src) + n, err = w.W.Write(c) + if n != len(src) { + if err == nil { // should never happen + err = io.ErrShortWrite + } + w.Err = err + } + return +} + +func (w StreamWriter) Close() os.Error { + // This saves us from either requiring a WriteCloser or having a + // StreamWriterCloser. + return w.W.(io.Closer).Close() +} diff --git a/src/pkg/crypto/cipher/ocfb.go b/src/pkg/crypto/cipher/ocfb.go new file mode 100644 index 000000000..031e74a9d --- /dev/null +++ b/src/pkg/crypto/cipher/ocfb.go @@ -0,0 +1,138 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OpenPGP CFB Mode. http://tools.ietf.org/html/rfc4880#section-13.9 + +package cipher + +type ocfbEncrypter struct { + b Block + fre []byte + outUsed int +} + +// An OCFBResyncOption determines if the "resynchronization step" of OCFB is +// performed. +type OCFBResyncOption bool + +const ( + OCFBResync OCFBResyncOption = true + OCFBNoResync OCFBResyncOption = false +) + +// NewOCFBEncrypter returns a Stream which encrypts data with OpenPGP's cipher +// feedback mode using the given Block, and an initial amount of ciphertext. +// randData must be random bytes and be the same length as the Block's block +// size. Resync determines if the "resynchronization step" from RFC 4880, 13.9 +// step 7 is performed. Different parts of OpenPGP vary on this point. +func NewOCFBEncrypter(block Block, randData []byte, resync OCFBResyncOption) (Stream, []byte) { + blockSize := block.BlockSize() + if len(randData) != blockSize { + return nil, nil + } + + x := &ocfbEncrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefix := make([]byte, blockSize+2) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefix[i] = randData[i] ^ x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefix[blockSize] = x.fre[0] ^ randData[blockSize-2] + prefix[blockSize+1] = x.fre[1] ^ randData[blockSize-1] + + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } + return x, prefix +} + +func (x *ocfbEncrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + x.fre[x.outUsed] ^= src[i] + dst[i] = x.fre[x.outUsed] + x.outUsed++ + } +} + +type ocfbDecrypter struct { + b Block + fre []byte + outUsed int +} + +// NewOCFBDecrypter returns a Stream which decrypts data with OpenPGP's cipher +// feedback mode using the given Block. Prefix must be the first blockSize + 2 +// bytes of the ciphertext, where blockSize is the Block's block size. If an +// incorrect key is detected then nil is returned. On successful exit, +// blockSize+2 bytes of decrypted data are written into prefix. Resync +// determines if the "resynchronization step" from RFC 4880, 13.9 step 7 is +// performed. Different parts of OpenPGP vary on this point. +func NewOCFBDecrypter(block Block, prefix []byte, resync OCFBResyncOption) Stream { + blockSize := block.BlockSize() + if len(prefix) != blockSize+2 { + return nil + } + + x := &ocfbDecrypter{ + b: block, + fre: make([]byte, blockSize), + outUsed: 0, + } + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + + block.Encrypt(x.fre, x.fre) + for i := 0; i < blockSize; i++ { + prefixCopy[i] ^= x.fre[i] + } + + block.Encrypt(x.fre, prefix[:blockSize]) + prefixCopy[blockSize] ^= x.fre[0] + prefixCopy[blockSize+1] ^= x.fre[1] + + if prefixCopy[blockSize-2] != prefixCopy[blockSize] || + prefixCopy[blockSize-1] != prefixCopy[blockSize+1] { + return nil + } + + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } + copy(prefix, prefixCopy) + return x +} + +func (x *ocfbDecrypter) XORKeyStream(dst, src []byte) { + for i := 0; i < len(src); i++ { + if x.outUsed == len(x.fre) { + x.b.Encrypt(x.fre, x.fre) + x.outUsed = 0 + } + + c := src[i] + dst[i] = x.fre[x.outUsed] ^ src[i] + x.fre[x.outUsed] = c + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/ocfb_test.go b/src/pkg/crypto/cipher/ocfb_test.go new file mode 100644 index 000000000..40938b589 --- /dev/null +++ b/src/pkg/crypto/cipher/ocfb_test.go @@ -0,0 +1,44 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cipher + +import ( + "bytes" + "crypto/aes" + "crypto/rand" + "testing" +) + +func testOCFB(t *testing.T, resync OCFBResyncOption) { + block, err := aes.NewCipher(commonKey128) + if err != nil { + t.Error(err) + return + } + + plaintext := []byte("this is the plaintext, which is long enough to span several blocks.") + randData := make([]byte, block.BlockSize()) + rand.Reader.Read(randData) + ocfb, prefix := NewOCFBEncrypter(block, randData, resync) + ciphertext := make([]byte, len(plaintext)) + ocfb.XORKeyStream(ciphertext, plaintext) + + ocfbdec := NewOCFBDecrypter(block, prefix, resync) + if ocfbdec == nil { + t.Errorf("NewOCFBDecrypter failed (resync: %t)", resync) + return + } + plaintextCopy := make([]byte, len(plaintext)) + ocfbdec.XORKeyStream(plaintextCopy, ciphertext) + + if !bytes.Equal(plaintextCopy, plaintext) { + t.Errorf("got: %x, want: %x (resync: %t)", plaintextCopy, plaintext, resync) + } +} + +func TestOCFB(t *testing.T) { + testOCFB(t, OCFBNoResync) + testOCFB(t, OCFBResync) +} diff --git a/src/pkg/crypto/cipher/ofb.go b/src/pkg/crypto/cipher/ofb.go new file mode 100644 index 000000000..85e5f02b0 --- /dev/null +++ b/src/pkg/crypto/cipher/ofb.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB (Output Feedback) Mode. + +package cipher + +type ofb struct { + b Block + out []byte + outUsed int +} + +// NewOFB returns a Stream that encrypts or decrypts using the block cipher b +// in output feedback mode. The initialization vector iv's length must be equal +// to b's block size. +func NewOFB(b Block, iv []byte) Stream { + blockSize := b.BlockSize() + if len(iv) != blockSize { + return nil + } + + x := &ofb{ + b: b, + out: make([]byte, blockSize), + outUsed: 0, + } + b.Encrypt(x.out, iv) + + return x +} + +func (x *ofb) XORKeyStream(dst, src []byte) { + for i, s := range src { + if x.outUsed == len(x.out) { + x.b.Encrypt(x.out, x.out) + x.outUsed = 0 + } + + dst[i] = s ^ x.out[x.outUsed] + x.outUsed++ + } +} diff --git a/src/pkg/crypto/cipher/ofb_test.go b/src/pkg/crypto/cipher/ofb_test.go new file mode 100644 index 000000000..9b4495c88 --- /dev/null +++ b/src/pkg/crypto/cipher/ofb_test.go @@ -0,0 +1,101 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// OFB AES test vectors. + +// See U.S. National Institute of Standards and Technology (NIST) +// Special Publication 800-38A, ``Recommendation for Block Cipher +// Modes of Operation,'' 2001 Edition, pp. 52-55. + +package cipher + +import ( + "bytes" + "crypto/aes" + "testing" +) + +type ofbTest struct { + name string + key []byte + iv []byte + in []byte + out []byte +} + +var ofbTests = []ofbTest{ + // NIST SP 800-38A pp 52-55 + { + "OFB-AES128", + commonKey128, + commonIV, + commonInput, + []byte{ + 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20, 0x33, 0x34, 0x49, 0xf8, 0xe8, 0x3c, 0xfb, 0x4a, + 0x77, 0x89, 0x50, 0x8d, 0x16, 0x91, 0x8f, 0x03, 0xf5, 0x3c, 0x52, 0xda, 0xc5, 0x4e, 0xd8, 0x25, + 0x97, 0x40, 0x05, 0x1e, 0x9c, 0x5f, 0xec, 0xf6, 0x43, 0x44, 0xf7, 0xa8, 0x22, 0x60, 0xed, 0xcc, + 0x30, 0x4c, 0x65, 0x28, 0xf6, 0x59, 0xc7, 0x78, 0x66, 0xa5, 0x10, 0xd9, 0xc1, 0xd6, 0xae, 0x5e, + }, + }, + { + "OFB-AES192", + commonKey192, + commonIV, + commonInput, + []byte{ + 0xcd, 0xc8, 0x0d, 0x6f, 0xdd, 0xf1, 0x8c, 0xab, 0x34, 0xc2, 0x59, 0x09, 0xc9, 0x9a, 0x41, 0x74, + 0xfc, 0xc2, 0x8b, 0x8d, 0x4c, 0x63, 0x83, 0x7c, 0x09, 0xe8, 0x17, 0x00, 0xc1, 0x10, 0x04, 0x01, + 0x8d, 0x9a, 0x9a, 0xea, 0xc0, 0xf6, 0x59, 0x6f, 0x55, 0x9c, 0x6d, 0x4d, 0xaf, 0x59, 0xa5, 0xf2, + 0x6d, 0x9f, 0x20, 0x08, 0x57, 0xca, 0x6c, 0x3e, 0x9c, 0xac, 0x52, 0x4b, 0xd9, 0xac, 0xc9, 0x2a, + }, + }, + { + "OFB-AES256", + commonKey256, + commonIV, + commonInput, + []byte{ + 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60, + 0x4f, 0xeb, 0xdc, 0x67, 0x40, 0xd2, 0x0b, 0x3a, 0xc8, 0x8f, 0x6a, 0xd8, 0x2a, 0x4f, 0xb0, 0x8d, + 0x71, 0xab, 0x47, 0xa0, 0x86, 0xe8, 0x6e, 0xed, 0xf3, 0x9d, 0x1c, 0x5b, 0xba, 0x97, 0xc4, 0x08, + 0x01, 0x26, 0x14, 0x1d, 0x67, 0xf3, 0x7b, 0xe8, 0x53, 0x8f, 0x5a, 0x8b, 0xe7, 0x40, 0xe4, 0x84, + }, + }, +} + +func TestOFB(t *testing.T) { + for _, tt := range ofbTests { + test := tt.name + + c, err := aes.NewCipher(tt.key) + if err != nil { + t.Errorf("%s: NewCipher(%d bytes) = %s", test, len(tt.key), err) + continue + } + + for j := 0; j <= 5; j += 5 { + plaintext := tt.in[0 : len(tt.in)-j] + ofb := NewOFB(c, tt.iv) + ciphertext := make([]byte, len(plaintext)) + ofb.XORKeyStream(ciphertext, plaintext) + if !bytes.Equal(ciphertext, tt.out[:len(plaintext)]) { + t.Errorf("%s/%d: encrypting\ninput % x\nhave % x\nwant % x", test, len(plaintext), plaintext, ciphertext, tt.out) + } + } + + for j := 0; j <= 5; j += 5 { + ciphertext := tt.out[0 : len(tt.in)-j] + ofb := NewOFB(c, tt.iv) + plaintext := make([]byte, len(ciphertext)) + ofb.XORKeyStream(plaintext, ciphertext) + if !bytes.Equal(plaintext, tt.in[:len(ciphertext)]) { + t.Errorf("%s/%d: decrypting\nhave % x\nwant % x", test, len(ciphertext), plaintext, tt.in) + } + } + + if t.Failed() { + break + } + } +} diff --git a/src/pkg/crypto/crypto.go b/src/pkg/crypto/crypto.go new file mode 100644 index 000000000..53672a4da --- /dev/null +++ b/src/pkg/crypto/crypto.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package crypto collects common cryptographic constants. +package crypto + +import ( + "hash" +) + +// Hash identifies a cryptographic hash function that is implemented in another +// package. +type Hash uint + +const ( + MD4 Hash = 1 + iota // in package crypto/md4 + MD5 // in package crypto/md5 + SHA1 // in package crypto/sha1 + SHA224 // in package crypto/sha256 + SHA256 // in package crypto/sha256 + SHA384 // in package crypto/sha512 + SHA512 // in package crypto/sha512 + MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA + RIPEMD160 // in package crypto/ripemd160 + maxHash +) + +var digestSizes = []uint8{ + MD4: 16, + MD5: 16, + SHA1: 20, + SHA224: 28, + SHA256: 32, + SHA384: 48, + SHA512: 64, + MD5SHA1: 36, + RIPEMD160: 20, +} + +// Size returns the length, in bytes, of a digest resulting from the given hash +// function. It doesn't require that the hash function in question be linked +// into the program. +func (h Hash) Size() int { + if h > 0 && h < maxHash { + return int(digestSizes[h]) + } + panic("crypto: Size of unknown hash function") +} + +var hashes = make([]func() hash.Hash, maxHash) + +// New returns a new hash.Hash calculating the given hash function. If the +// hash function is not linked into the binary, New returns nil. +func (h Hash) New() hash.Hash { + if h > 0 && h < maxHash { + f := hashes[h] + if f != nil { + return f() + } + } + return nil +} + +// RegisterHash registers a function that returns a new instance of the given +// hash function. This is intended to be called from the init function in +// packages that implement hash functions. +func RegisterHash(h Hash, f func() hash.Hash) { + if h >= maxHash { + panic("crypto: RegisterHash of unknown hash function") + } + hashes[h] = f +} diff --git a/src/pkg/crypto/des/Makefile b/src/pkg/crypto/des/Makefile new file mode 100644 index 000000000..94b0fc0fa --- /dev/null +++ b/src/pkg/crypto/des/Makefile @@ -0,0 +1,13 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/des +GOFILES=\ + block.go\ + cipher.go\ + const.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/des/block.go b/src/pkg/crypto/des/block.go new file mode 100644 index 000000000..e18eaedf5 --- /dev/null +++ b/src/pkg/crypto/des/block.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package des + +import ( + "encoding/binary" +) + +func cryptBlock(subkeys []uint64, dst, src []byte, decrypt bool) { + b := binary.BigEndian.Uint64(src) + b = permuteBlock(b, initialPermutation[:]) + left, right := uint32(b>>32), uint32(b) + + var subkey uint64 + for i := 0; i < 16; i++ { + if decrypt { + subkey = subkeys[15-i] + } else { + subkey = subkeys[i] + } + + left, right = right, left^feistel(right, subkey) + } + // switch left & right and perform final permutation + preOutput := (uint64(right) << 32) | uint64(left) + binary.BigEndian.PutUint64(dst, permuteBlock(preOutput, finalPermutation[:])) +} + +// Encrypt one block from src into dst, using the subkeys. +func encryptBlock(subkeys []uint64, dst, src []byte) { + cryptBlock(subkeys, dst, src, false) +} + +// Decrypt one block from src into dst, using the subkeys. +func decryptBlock(subkeys []uint64, dst, src []byte) { + cryptBlock(subkeys, dst, src, true) +} + +// DES Feistel function +func feistel(right uint32, key uint64) (result uint32) { + sBoxLocations := key ^ permuteBlock(uint64(right), expansionFunction[:]) + var sBoxResult uint32 + for i := uint8(0); i < 8; i++ { + sBoxLocation := uint8(sBoxLocations>>42) & 0x3f + sBoxLocations <<= 6 + // row determined by 1st and 6th bit + row := (sBoxLocation & 0x1) | ((sBoxLocation & 0x20) >> 4) + // column is middle four bits + column := (sBoxLocation >> 1) & 0xf + sBoxResult |= uint32(sBoxes[i][row][column]) << (4 * (7 - i)) + } + return uint32(permuteBlock(uint64(sBoxResult), permutationFunction[:])) +} + +// general purpose function to perform DES block permutations +func permuteBlock(src uint64, permutation []uint8) (block uint64) { + for position, n := range permutation { + bit := (src >> n) & 1 + block |= bit << uint((len(permutation)-1)-position) + } + return +} + +// creates 16 28-bit blocks rotated according +// to the rotation schedule +func ksRotate(in uint32) (out []uint32) { + out = make([]uint32, 16) + last := in + for i := 0; i < 16; i++ { + // 28-bit circular left shift + left := (last << (4 + ksRotations[i])) >> 4 + right := (last << 4) >> (32 - ksRotations[i]) + out[i] = left | right + last = out[i] + } + return +} + +// creates 16 56-bit subkeys from the original key +func (c *Cipher) generateSubkeys(keyBytes []byte) { + // apply PC1 permutation to key + key := binary.BigEndian.Uint64(keyBytes) + permutedKey := permuteBlock(key, permutedChoice1[:]) + + // rotate halves of permuted key according to the rotation schedule + leftRotations := ksRotate(uint32(permutedKey >> 28)) + rightRotations := ksRotate(uint32(permutedKey<<4) >> 4) + + // generate subkeys + for i := 0; i < 16; i++ { + // combine halves to form 56-bit input to PC2 + pc2Input := uint64(leftRotations[i])<<28 | uint64(rightRotations[i]) + // apply PC2 permutation to 7 byte input + c.subkeys[i] = permuteBlock(pc2Input, permutedChoice2[:]) + } +} diff --git a/src/pkg/crypto/des/cipher.go b/src/pkg/crypto/des/cipher.go new file mode 100644 index 000000000..d17a1a783 --- /dev/null +++ b/src/pkg/crypto/des/cipher.go @@ -0,0 +1,103 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package des + +import ( + "os" + "strconv" +) + +// The DES block size in bytes. +const BlockSize = 8 + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/des: invalid key size " + strconv.Itoa(int(k)) +} + +// Cipher is an instance of DES encryption. +type Cipher struct { + subkeys [16]uint64 +} + +// NewCipher creates and returns a new Cipher. +func NewCipher(key []byte) (*Cipher, os.Error) { + if len(key) != 8 { + return nil, KeySizeError(len(key)) + } + + c := new(Cipher) + c.generateSubkeys(key) + return c, nil +} + +// BlockSize returns the DES block size, 8 bytes. +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypts the 8-byte buffer src and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c.subkeys[:], dst, src) } + +// Decrypts the 8-byte buffer src and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c.subkeys[:], dst, src) } + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.subkeys); i++ { + c.subkeys[i] = 0 + } +} + +// A TripleDESCipher is an instance of TripleDES encryption. +type TripleDESCipher struct { + cipher1, cipher2, cipher3 Cipher +} + +// NewCipher creates and returns a new Cipher. +func NewTripleDESCipher(key []byte) (*TripleDESCipher, os.Error) { + if len(key) != 24 { + return nil, KeySizeError(len(key)) + } + + c := new(TripleDESCipher) + c.cipher1.generateSubkeys(key[:8]) + c.cipher2.generateSubkeys(key[8:16]) + c.cipher3.generateSubkeys(key[16:]) + return c, nil +} + +// BlockSize returns the TripleDES block size, 8 bytes. +// It is necessary to satisfy the Block interface in the +// package "crypto/cipher". +func (c *TripleDESCipher) BlockSize() int { return BlockSize } + +// Encrypts the 8-byte buffer src and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *TripleDESCipher) Encrypt(dst, src []byte) { + c.cipher1.Encrypt(dst, src) + c.cipher2.Decrypt(dst, dst) + c.cipher3.Encrypt(dst, dst) +} + +// Decrypts the 8-byte buffer src and stores the result in dst. +func (c *TripleDESCipher) Decrypt(dst, src []byte) { + c.cipher3.Decrypt(dst, src) + c.cipher2.Encrypt(dst, dst) + c.cipher1.Decrypt(dst, dst) +} + +// Reset zeros the key data, so that it will no longer +// appear in the process's memory. +func (c *TripleDESCipher) Reset() { + c.cipher1.Reset() + c.cipher2.Reset() + c.cipher3.Reset() +} diff --git a/src/pkg/crypto/des/const.go b/src/pkg/crypto/des/const.go new file mode 100644 index 000000000..2bd485ee8 --- /dev/null +++ b/src/pkg/crypto/des/const.go @@ -0,0 +1,139 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package des implements the Data Encryption Standard (DES) and the +// Triple Data Encryption Algorithm (TDEA) as defined +// in U.S. Federal Information Processing Standards Publication 46-3. +package des + +// Used to perform an initial permutation of a 64-bit input block. +var initialPermutation = [64]byte{ + 6, 14, 22, 30, 38, 46, 54, 62, + 4, 12, 20, 28, 36, 44, 52, 60, + 2, 10, 18, 26, 34, 42, 50, 58, + 0, 8, 16, 24, 32, 40, 48, 56, + 7, 15, 23, 31, 39, 47, 55, 63, + 5, 13, 21, 29, 37, 45, 53, 61, + 3, 11, 19, 27, 35, 43, 51, 59, + 1, 9, 17, 25, 33, 41, 49, 57, +} + +// Used to perform a final permutation of a 4-bit preoutput block. This is the +// inverse of initialPermutation +var finalPermutation = [64]byte{ + 24, 56, 16, 48, 8, 40, 0, 32, + 25, 57, 17, 49, 9, 41, 1, 33, + 26, 58, 18, 50, 10, 42, 2, 34, + 27, 59, 19, 51, 11, 43, 3, 35, + 28, 60, 20, 52, 12, 44, 4, 36, + 29, 61, 21, 53, 13, 45, 5, 37, + 30, 62, 22, 54, 14, 46, 6, 38, + 31, 63, 23, 55, 15, 47, 7, 39, +} + +// Used to expand an input block of 32 bits, producing an output block of 48 +// bits. +var expansionFunction = [48]byte{ + 0, 31, 30, 29, 28, 27, 28, 27, + 26, 25, 24, 23, 24, 23, 22, 21, + 20, 19, 20, 19, 18, 17, 16, 15, + 16, 15, 14, 13, 12, 11, 12, 11, + 10, 9, 8, 7, 8, 7, 6, 5, + 4, 3, 4, 3, 2, 1, 0, 31, +} + +// Yields a 32-bit output from a 32-bit input +var permutationFunction = [32]byte{ + 16, 25, 12, 11, 3, 20, 4, 15, + 31, 17, 9, 6, 27, 14, 1, 22, + 30, 24, 8, 18, 0, 5, 29, 23, + 13, 19, 2, 26, 10, 21, 28, 7, +} + +// Used in the key schedule to select 56 bits +// from a 64-bit input. +var permutedChoice1 = [56]byte{ + 7, 15, 23, 31, 39, 47, 55, 63, + 6, 14, 22, 30, 38, 46, 54, 62, + 5, 13, 21, 29, 37, 45, 53, 61, + 4, 12, 20, 28, 1, 9, 17, 25, + 33, 41, 49, 57, 2, 10, 18, 26, + 34, 42, 50, 58, 3, 11, 19, 27, + 35, 43, 51, 59, 36, 44, 52, 60, +} + +// Used in the key schedule to produce each subkey by selecting 48 bits from +// the 56-bit input +var permutedChoice2 = [48]byte{ + 42, 39, 45, 32, 55, 51, 53, 28, + 41, 50, 35, 46, 33, 37, 44, 52, + 30, 48, 40, 49, 29, 36, 43, 54, + 15, 4, 25, 19, 9, 1, 26, 16, + 5, 11, 23, 8, 12, 7, 17, 0, + 22, 3, 10, 14, 6, 20, 27, 24, +} + +// 8 S-boxes composed of 4 rows and 16 columns +// Used in the DES cipher function +var sBoxes = [8][4][16]uint8{ + // S-box 1 + { + {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7}, + {0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8}, + {4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0}, + {15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}, + }, + // S-box 2 + { + {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10}, + {3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5}, + {0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15}, + {13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}, + }, + // S-box 3 + { + {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8}, + {13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1}, + {13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7}, + {1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}, + }, + // S-box 4 + { + {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15}, + {13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9}, + {10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4}, + {3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}, + }, + // S-box 5 + { + {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9}, + {14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6}, + {4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14}, + {11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}, + }, + // S-box 6 + { + {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11}, + {10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8}, + {9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6}, + {4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}, + }, + // S-box 7 + { + {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1}, + {13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6}, + {1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2}, + {6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}, + }, + // S-box 8 + { + {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7}, + {1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2}, + {7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8}, + {2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}, + }, +} + +// Size of left rotation per round in each half of the key schedule +var ksRotations = [16]uint8{1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1} diff --git a/src/pkg/crypto/des/des_test.go b/src/pkg/crypto/des/des_test.go new file mode 100644 index 000000000..d1f3aa71a --- /dev/null +++ b/src/pkg/crypto/des/des_test.go @@ -0,0 +1,1497 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package des + +import ( + "bytes" + "testing" +) + +type CryptTest struct { + key []byte + in []byte + out []byte +} + +// some custom tests for DES +var encryptDESTests = []CryptTest{ + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x8c, 0xa6, 0x4d, 0xe9, 0xc1, 0xb1, 0x23, 0xa7}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x35, 0x55, 0x50, 0xb2, 0x15, 0x0e, 0x24, 0x51}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x61, 0x7b, 0x3a, 0x0c, 0xe8, 0xf0, 0x71, 0x00}}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x92, 0x31, 0xf2, 0x36, 0xff, 0x9a, 0xa9, 0x5c}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xca, 0xaa, 0xaf, 0x4d, 0xea, 0xf1, 0xdb, 0xae}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x73, 0x59, 0xb2, 0x16, 0x3e, 0x4e, 0xdc, 0x58}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x6d, 0xce, 0x0d, 0xc9, 0x00, 0x65, 0x56, 0xa3}}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x9e, 0x84, 0xc5, 0xf3, 0x17, 0x0f, 0x8e, 0xff}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xd5, 0xd4, 0x4f, 0xf7, 0x20, 0x68, 0x3d, 0x0d}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x59, 0x73, 0x23, 0x56, 0xf3, 0x6f, 0xde, 0x06}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x56, 0xcc, 0x09, 0xe7, 0xcf, 0xdc, 0x4c, 0xef}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x12, 0xc6, 0x26, 0xaf, 0x05, 0x8b, 0x43, 0x3b}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xa6, 0x8c, 0xdc, 0xa9, 0x0c, 0x90, 0x21, 0xf9}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x2a, 0x2b, 0xb0, 0x08, 0xdf, 0x97, 0xc2, 0xf2}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0xed, 0x39, 0xd9, 0x50, 0xfa, 0x74, 0xbc, 0xc4}}, + { + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, + []byte{0xa9, 0x33, 0xf6, 0x18, 0x30, 0x23, 0xb3, 0x10}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, + []byte{0x17, 0x66, 0x8d, 0xfc, 0x72, 0x92, 0x53, 0x2d}}, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0xb4, 0xfd, 0x23, 0x16, 0x47, 0xa5, 0xbe, 0xc0}}, + { + []byte{0x0e, 0x32, 0x92, 0x32, 0xea, 0x6d, 0x0d, 0x73}, + []byte{0x87, 0x87, 0x87, 0x87, 0x87, 0x87, 0x87, 0x87}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, + { + []byte{0x73, 0x65, 0x63, 0x52, 0x33, 0x74, 0x24, 0x3b}, // "secR3t$;" + []byte{0x61, 0x20, 0x74, 0x65, 0x73, 0x74, 0x31, 0x32}, // "a test12" + []byte{0x37, 0x0d, 0xee, 0x2c, 0x1f, 0xb4, 0xf7, 0xa5}}, + { + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x2a, 0x8d, 0x69, 0xde, 0x9d, 0x5f, 0xdf, 0xf9}}, + { + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}, // "12345678" + []byte{0x21, 0xc6, 0x0d, 0xa5, 0x34, 0x24, 0x8b, 0xce}}, + { + []byte{0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}, // "12345678" + []byte{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68}, // "abcdefgh" + []byte{0x94, 0xd4, 0x43, 0x6b, 0xc3, 0xb5, 0xb6, 0x93}}, + { + []byte{0x1f, 0x79, 0x90, 0x5f, 0x88, 0x01, 0xc8, 0x88}, // random + []byte{0xc7, 0x46, 0x18, 0x73, 0xaf, 0x48, 0x5f, 0xb3}, // random + []byte{0xb0, 0x93, 0x50, 0x88, 0xf9, 0x92, 0x44, 0x6a}}, + { + []byte{0xe6, 0xf4, 0xf2, 0xdb, 0x31, 0x42, 0x53, 0x01}, // random + []byte{0xff, 0x3d, 0x25, 0x50, 0x12, 0xe3, 0x4a, 0xc5}, // random + []byte{0x86, 0x08, 0xd3, 0xd1, 0x6c, 0x2f, 0xd2, 0x55}}, + { + []byte{0x69, 0xc1, 0x9d, 0xc1, 0x15, 0xc5, 0xfb, 0x2b}, // random + []byte{0x1a, 0x22, 0x5c, 0xaf, 0x1f, 0x1d, 0xa3, 0xf9}, // random + []byte{0x64, 0xba, 0x31, 0x67, 0x56, 0x91, 0x1e, 0xa7}}, + { + []byte{0x6e, 0x5e, 0xe2, 0x47, 0xc4, 0xbf, 0xf6, 0x51}, // random + []byte{0x11, 0xc9, 0x57, 0xff, 0x66, 0x89, 0x0e, 0xf0}, // random + []byte{0x94, 0xc5, 0x35, 0xb2, 0xc5, 0x8b, 0x39, 0x72}}, +} + +var weakKeyTests = []CryptTest{ + { + []byte{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + []byte{0x55, 0x74, 0xc0, 0xbd, 0x7c, 0xdf, 0xf7, 0x39}, // random + nil}, + { + []byte{0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe}, + []byte{0xe8, 0xe1, 0xa7, 0xc1, 0xde, 0x11, 0x89, 0xaa}, // random + nil}, + { + []byte{0xe0, 0xe0, 0xe0, 0xe0, 0xf1, 0xf1, 0xf1, 0xf1}, + []byte{0x50, 0x6a, 0x4b, 0x94, 0x3b, 0xed, 0x7d, 0xdc}, // random + nil}, + { + []byte{0x1f, 0x1f, 0x1f, 0x1f, 0x0e, 0x0e, 0x0e, 0x0e}, + []byte{0x88, 0x81, 0x56, 0x38, 0xec, 0x3b, 0x1c, 0x97}, // random + nil}, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x17, 0xa0, 0x83, 0x62, 0x32, 0xfe, 0x9a, 0x0b}, // random + nil}, + { + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xca, 0x8f, 0xca, 0x1f, 0x50, 0xc5, 0x7b, 0x49}, // random + nil}, + { + []byte{0xe1, 0xe1, 0xe1, 0xe1, 0xf0, 0xf0, 0xf0, 0xf0}, + []byte{0xb1, 0xea, 0xad, 0x7d, 0xe7, 0xc3, 0x7a, 0x43}, // random + nil}, + { + []byte{0x1e, 0x1e, 0x1e, 0x1e, 0x0f, 0x0f, 0x0f, 0x0f}, + []byte{0xae, 0x74, 0x7d, 0x6f, 0xef, 0x16, 0xbb, 0x81}, // random + nil}, +} + +var semiWeakKeyTests = []CryptTest{ + // key and out contain the semi-weak key pair + { + []byte{0x01, 0x1f, 0x01, 0x1f, 0x01, 0x0e, 0x01, 0x0e}, + []byte{0x12, 0xfa, 0x31, 0x16, 0xf9, 0xc5, 0x0a, 0xe4}, // random + []byte{0x1f, 0x01, 0x1f, 0x01, 0x0e, 0x01, 0x0e, 0x01}}, + { + []byte{0x01, 0xe0, 0x01, 0xe0, 0x01, 0xf1, 0x01, 0xf1}, + []byte{0xb0, 0x4c, 0x7a, 0xee, 0xd2, 0xe5, 0x4d, 0xb7}, // random + []byte{0xe0, 0x01, 0xe0, 0x01, 0xf1, 0x01, 0xf1, 0x01}}, + { + []byte{0x01, 0xfe, 0x01, 0xfe, 0x01, 0xfe, 0x01, 0xfe}, + []byte{0xa4, 0x81, 0xcd, 0xb1, 0x64, 0x6f, 0xd3, 0xbc}, // random + []byte{0xfe, 0x01, 0xfe, 0x01, 0xfe, 0x01, 0xfe, 0x01}}, + { + []byte{0x1f, 0xe0, 0x1f, 0xe0, 0x0e, 0xf1, 0x0e, 0xf1}, + []byte{0xee, 0x27, 0xdd, 0x88, 0x4c, 0x22, 0xcd, 0xce}, // random + []byte{0xe0, 0x1f, 0xe0, 0x1f, 0xf1, 0x0e, 0xf1, 0x0e}}, + { + []byte{0x1f, 0xfe, 0x1f, 0xfe, 0x0e, 0xfe, 0x0e, 0xfe}, + []byte{0x19, 0x3d, 0xcf, 0x97, 0x70, 0xfb, 0xab, 0xe1}, // random + []byte{0xfe, 0x1f, 0xfe, 0x1f, 0xfe, 0x0e, 0xfe, 0x0e}}, + { + []byte{0xe0, 0xfe, 0xe0, 0xfe, 0xf1, 0xfe, 0xf1, 0xfe}, + []byte{0x7c, 0x82, 0x69, 0xe4, 0x1e, 0x86, 0x99, 0xd7}, // random + []byte{0xfe, 0xe0, 0xfe, 0xe0, 0xfe, 0xf1, 0xfe, 0xf1}}, +} + +// some custom tests for TripleDES +var encryptTripleDESTests = []CryptTest{ + { + []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x92, 0x95, 0xb5, 0x9b, 0xb3, 0x84, 0x73, 0x6e}}, + { + []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xc1, 0x97, 0xf5, 0x58, 0x74, 0x8a, 0x20, 0xe7}}, + { + []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x3e, 0x68, 0x0a, 0xa7, 0x8b, 0x75, 0xdf, 0x18}}, + { + []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + []byte{0x6d, 0x6a, 0x4a, 0x64, 0x4c, 0x7b, 0x8c, 0x91}}, + { + []byte{ // "abcdefgh12345678ABCDEFGH" + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30}, // "00000000" + []byte{0xe4, 0x61, 0xb7, 0x59, 0x68, 0x8b, 0xff, 0x66}}, + { + []byte{ // "abcdefgh12345678ABCDEFGH" + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}, // "12345678" + []byte{0xdb, 0xd0, 0x92, 0xde, 0xf8, 0x34, 0xff, 0x58}}, + { + []byte{ // "abcdefgh12345678ABCDEFGH" + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0xf0, 0xc5, 0x82, 0x22, 0xd3, 0xe6, 0x12, 0xd2}, // random + []byte{0xba, 0xe4, 0x41, 0xb1, 0x3c, 0x37, 0x4d, 0xf4}}, + { + []byte{ // random + 0xd3, 0x7d, 0x45, 0xee, 0x22, 0xe9, 0xcf, 0x52, + 0xf4, 0x65, 0xa2, 0x4f, 0x70, 0xd1, 0x81, 0x8a, + 0x3d, 0xbe, 0x2f, 0x39, 0xc7, 0x71, 0xd2, 0xe9}, + []byte{0x49, 0x53, 0xc3, 0xe9, 0x78, 0xdf, 0x9f, 0xaf}, // random + []byte{0x53, 0x40, 0x51, 0x24, 0xd8, 0x3c, 0xf9, 0x88}}, + { + []byte{ // random + 0xcb, 0x10, 0x7d, 0xda, 0x7e, 0x96, 0x57, 0x0a, + 0xe8, 0xeb, 0xe8, 0x07, 0x8e, 0x87, 0xd3, 0x57, + 0xb2, 0x61, 0x12, 0xb8, 0x2a, 0x90, 0xb7, 0x2f}, + []byte{0xa3, 0xc2, 0x60, 0xb1, 0x0b, 0xb7, 0x28, 0x6e}, // random + []byte{0x56, 0x73, 0x7d, 0xfb, 0xb5, 0xa1, 0xc3, 0xde}}, +} + +// NIST Special Publication 800-20, Appendix A +// Key for use with Table A.1 tests +var tableA1Key = []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, +} + +// Table A.1 Resulting Ciphertext from the Variable Plaintext Known Answer Test +var tableA1Tests = []CryptTest{ + {nil, // 0 + []byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x95, 0xf8, 0xa5, 0xe5, 0xdd, 0x31, 0xd9, 0x00}}, + {nil, // 1 + []byte{0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xdd, 0x7f, 0x12, 0x1c, 0xa5, 0x01, 0x56, 0x19}}, + {nil, // 2 + []byte{0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x2e, 0x86, 0x53, 0x10, 0x4f, 0x38, 0x34, 0xea}}, + {nil, // 3 + []byte{0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x4b, 0xd3, 0x88, 0xff, 0x6c, 0xd8, 0x1d, 0x4f}}, + {nil, // 4 + []byte{0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x20, 0xb9, 0xe7, 0x67, 0xb2, 0xfb, 0x14, 0x56}}, + {nil, // 5 + []byte{0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x55, 0x57, 0x93, 0x80, 0xd7, 0x71, 0x38, 0xef}}, + {nil, // 6 + []byte{0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x6c, 0xc5, 0xde, 0xfa, 0xaf, 0x04, 0x51, 0x2f}}, + {nil, // 7 + []byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x0d, 0x9f, 0x27, 0x9b, 0xa5, 0xd8, 0x72, 0x60}}, + {nil, // 8 + []byte{0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xd9, 0x03, 0x1b, 0x02, 0x71, 0xbd, 0x5a, 0x0a}}, + {nil, // 9 + []byte{0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x42, 0x42, 0x50, 0xb3, 0x7c, 0x3d, 0xd9, 0x51}}, + {nil, // 10 + []byte{0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xb8, 0x06, 0x1b, 0x7e, 0xcd, 0x9a, 0x21, 0xe5}}, + {nil, // 11 + []byte{0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xf1, 0x5d, 0x0f, 0x28, 0x6b, 0x65, 0xbd, 0x28}}, + {nil, // 12 + []byte{0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xad, 0xd0, 0xcc, 0x8d, 0x6e, 0x5d, 0xeb, 0xa1}}, + {nil, // 13 + []byte{0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe6, 0xd5, 0xf8, 0x27, 0x52, 0xad, 0x63, 0xd1}}, + {nil, // 14 + []byte{0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xec, 0xbf, 0xe3, 0xbd, 0x3f, 0x59, 0x1a, 0x5e}}, + {nil, // 15 + []byte{0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xf3, 0x56, 0x83, 0x43, 0x79, 0xd1, 0x65, 0xcd}}, + {nil, // 16 + []byte{0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x2b, 0x9f, 0x98, 0x2f, 0x20, 0x03, 0x7f, 0xa9}}, + {nil, // 17 + []byte{0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x88, 0x9d, 0xe0, 0x68, 0xa1, 0x6f, 0x0b, 0xe6}}, + {nil, // 18 + []byte{0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe1, 0x9e, 0x27, 0x5d, 0x84, 0x6a, 0x12, 0x98}}, + {nil, // 19 + []byte{0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x32, 0x9a, 0x8e, 0xd5, 0x23, 0xd7, 0x1a, 0xec}}, + {nil, // 20 + []byte{0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe7, 0xfc, 0xe2, 0x25, 0x57, 0xd2, 0x3c, 0x97}}, + {nil, // 21 + []byte{0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x12, 0xa9, 0xf5, 0x81, 0x7f, 0xf2, 0xd6, 0x5d}}, + {nil, // 22 + []byte{0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xa4, 0x84, 0xc3, 0xad, 0x38, 0xdc, 0x9c, 0x19}}, + {nil, // 23 + []byte{0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xfb, 0xe0, 0x0a, 0x8a, 0x1e, 0xf8, 0xad, 0x72}}, + {nil, // 24 + []byte{0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}, + []byte{0x75, 0x0d, 0x07, 0x94, 0x07, 0x52, 0x13, 0x63}}, + {nil, // 25 + []byte{0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00}, + []byte{0x64, 0xfe, 0xed, 0x9c, 0x72, 0x4c, 0x2f, 0xaf}}, + {nil, // 26 + []byte{0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00}, + []byte{0xf0, 0x2b, 0x26, 0x3b, 0x32, 0x8e, 0x2b, 0x60}}, + {nil, // 27 + []byte{0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00}, + []byte{0x9d, 0x64, 0x55, 0x5a, 0x9a, 0x10, 0xb8, 0x52}}, + {nil, // 28 + []byte{0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00}, + []byte{0xd1, 0x06, 0xff, 0x0b, 0xed, 0x52, 0x55, 0xd7}}, + {nil, // 29 + []byte{0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe1, 0x65, 0x2c, 0x6b, 0x13, 0x8c, 0x64, 0xa5}}, + {nil, // 30 + []byte{0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00}, + []byte{0xe4, 0x28, 0x58, 0x11, 0x86, 0xec, 0x8f, 0x46}}, + {nil, // 31 + []byte{0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}, + []byte{0xae, 0xb5, 0xf5, 0xed, 0xe2, 0x2d, 0x1a, 0x36}}, + {nil, // 32 + []byte{0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}, + []byte{0xe9, 0x43, 0xd7, 0x56, 0x8a, 0xec, 0x0c, 0x5c}}, + {nil, // 33 + []byte{0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00}, + []byte{0xdf, 0x98, 0xc8, 0x27, 0x6f, 0x54, 0xb0, 0x4b}}, + {nil, // 34 + []byte{0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00}, + []byte{0xb1, 0x60, 0xe4, 0x68, 0x0f, 0x6c, 0x69, 0x6f}}, + {nil, // 35 + []byte{0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00}, + []byte{0xfa, 0x07, 0x52, 0xb0, 0x7d, 0x9c, 0x4a, 0xb8}}, + {nil, // 36 + []byte{0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00}, + []byte{0xca, 0x3a, 0x2b, 0x03, 0x6d, 0xbc, 0x85, 0x02}}, + {nil, // 37 + []byte{0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00}, + []byte{0x5e, 0x09, 0x05, 0x51, 0x7b, 0xb5, 0x9b, 0xcf}}, + {nil, // 38 + []byte{0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00}, + []byte{0x81, 0x4e, 0xeb, 0x3b, 0x91, 0xd9, 0x07, 0x26}}, + {nil, // 39 + []byte{0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00}, + []byte{0x4d, 0x49, 0xdb, 0x15, 0x32, 0x91, 0x9c, 0x9f}}, + {nil, // 40 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00}, + []byte{0x25, 0xeb, 0x5f, 0xc3, 0xf8, 0xcf, 0x06, 0x21}}, + {nil, // 41 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00}, + []byte{0xab, 0x6a, 0x20, 0xc0, 0x62, 0x0d, 0x1c, 0x6f}}, + {nil, // 42 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00}, + []byte{0x79, 0xe9, 0x0d, 0xbc, 0x98, 0xf9, 0x2c, 0xca}}, + {nil, // 43 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00}, + []byte{0x86, 0x6e, 0xce, 0xdd, 0x80, 0x72, 0xbb, 0x0e}}, + {nil, // 44 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00}, + []byte{0x8b, 0x54, 0x53, 0x6f, 0x2f, 0x3e, 0x64, 0xa8}}, + {nil, // 45 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00}, + []byte{0xea, 0x51, 0xd3, 0x97, 0x55, 0x95, 0xb8, 0x6b}}, + {nil, // 46 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00}, + []byte{0xca, 0xff, 0xc6, 0xac, 0x45, 0x42, 0xde, 0x31}}, + {nil, // 47 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00}, + []byte{0x8d, 0xd4, 0x5a, 0x2d, 0xdf, 0x90, 0x79, 0x6c}}, + {nil, // 48 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00}, + []byte{0x10, 0x29, 0xd5, 0x5e, 0x88, 0x0e, 0xc2, 0xd0}}, + {nil, // 49 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00}, + []byte{0x5d, 0x86, 0xcb, 0x23, 0x63, 0x9d, 0xbe, 0xa9}}, + {nil, // 50 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00}, + []byte{0x1d, 0x1c, 0xa8, 0x53, 0xae, 0x7c, 0x0c, 0x5f}}, + {nil, // 51 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00}, + []byte{0xce, 0x33, 0x23, 0x29, 0x24, 0x8f, 0x32, 0x28}}, + {nil, // 52 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00}, + []byte{0x84, 0x05, 0xd1, 0xab, 0xe2, 0x4f, 0xb9, 0x42}}, + {nil, // 53 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00}, + []byte{0xe6, 0x43, 0xd7, 0x80, 0x90, 0xca, 0x42, 0x07}}, + {nil, // 54 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00}, + []byte{0x48, 0x22, 0x1b, 0x99, 0x37, 0x74, 0x8a, 0x23}}, + {nil, // 55 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00}, + []byte{0xdd, 0x7c, 0x0b, 0xbd, 0x61, 0xfa, 0xfd, 0x54}}, + {nil, // 56 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, + []byte{0x2f, 0xbc, 0x29, 0x1a, 0x57, 0x0d, 0xb5, 0xc4}}, + {nil, // 57 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40}, + []byte{0xe0, 0x7c, 0x30, 0xd7, 0xe4, 0xe2, 0x6e, 0x12}}, + {nil, // 58 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20}, + []byte{0x09, 0x53, 0xe2, 0x25, 0x8e, 0x8e, 0x90, 0xa1}}, + {nil, // 59 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10}, + []byte{0x5b, 0x71, 0x1b, 0xc4, 0xce, 0xeb, 0xf2, 0xee}}, + {nil, // 60 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08}, + []byte{0xcc, 0x08, 0x3f, 0x1e, 0x6d, 0x9e, 0x85, 0xf6}}, + {nil, // 61 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04}, + []byte{0xd2, 0xfd, 0x88, 0x67, 0xd5, 0x0d, 0x2d, 0xfe}}, + {nil, // 62 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}, + []byte{0x06, 0xe7, 0xea, 0x22, 0xce, 0x92, 0x70, 0x8f}}, + {nil, // 63 + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + []byte{0x16, 0x6b, 0x40, 0xb4, 0x4a, 0xba, 0x4b, 0xd6}}, +} + +// Plaintext for use with Table A.2 tests +var tableA2Plaintext = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + +// Table A.2 Resulting Ciphertext from the Variable Key Known Answer Test +var tableA2Tests = []CryptTest{ + { // 0 + []byte{ + 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x95, 0xa8, 0xd7, 0x28, 0x13, 0xda, 0xa9, 0x4d}}, + { // 1 + []byte{ + 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x0e, 0xec, 0x14, 0x87, 0xdd, 0x8c, 0x26, 0xd5}}, + { // 2 + []byte{ + 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x7a, 0xd1, 0x6f, 0xfb, 0x79, 0xc4, 0x59, 0x26}}, + { // 3 + []byte{ + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xd3, 0x74, 0x62, 0x94, 0xca, 0x6a, 0x6c, 0xf3}}, + { // 4 + []byte{ + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x80, 0x9f, 0x5f, 0x87, 0x3c, 0x1f, 0xd7, 0x61}}, + { // 5 + []byte{ + 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xc0, 0x2f, 0xaf, 0xfe, 0xc9, 0x89, 0xd1, 0xfc}}, + { // 6 + []byte{ + 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x46, 0x15, 0xaa, 0x1d, 0x33, 0xe7, 0x2f, 0x10}}, + { // 7 + []byte{ + 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x20, 0x55, 0x12, 0x33, 0x50, 0xc0, 0x08, 0x58}}, + { // 8 + []byte{ + 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xdf, 0x3b, 0x99, 0xd6, 0x57, 0x73, 0x97, 0xc8}}, + { // 9 + []byte{ + 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x31, 0xfe, 0x17, 0x36, 0x9b, 0x52, 0x88, 0xc9}}, + { // 10 + []byte{ + 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xdf, 0xdd, 0x3c, 0xc6, 0x4d, 0xae, 0x16, 0x42}}, + { // 11 + []byte{ + 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x17, 0x8c, 0x83, 0xce, 0x2b, 0x39, 0x9d, 0x94}}, + { // 12 + []byte{ + 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x50, 0xf6, 0x36, 0x32, 0x4a, 0x9b, 0x7f, 0x80}}, + { // 13 + []byte{ + 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xa8, 0x46, 0x8e, 0xe3, 0xbc, 0x18, 0xf0, 0x6d}}, + { // 14 + []byte{ + 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xa2, 0xdc, 0x9e, 0x92, 0xfd, 0x3c, 0xde, 0x92}}, + { // 15 + []byte{ + 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xca, 0xc0, 0x9f, 0x79, 0x7d, 0x03, 0x12, 0x87}}, + { // 16 + []byte{ + 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x90, 0xba, 0x68, 0x0b, 0x22, 0xae, 0xb5, 0x25}}, + { // 17 + []byte{ + 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xce, 0x7a, 0x24, 0xf3, 0x50, 0xe2, 0x80, 0xb6}}, + { // 18 + []byte{ + 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x88, 0x2b, 0xff, 0x0a, 0xa0, 0x1a, 0x0b, 0x87}}, + { // 19 + []byte{ + 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x25, 0x61, 0x02, 0x88, 0x92, 0x45, 0x11, 0xc2}}, + { // 20 + []byte{ + 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xc7, 0x15, 0x16, 0xc2, 0x9c, 0x75, 0xd1, 0x70}}, + { // 21 + []byte{ + 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x51, 0x99, 0xc2, 0x9a, 0x52, 0xc9, 0xf0, 0x59}}, + { // 22 + []byte{ + 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xc2, 0x2f, 0x0a, 0x29, 0x4a, 0x71, 0xf2, 0x9f}}, + { // 23 + []byte{ + 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xee, 0x37, 0x14, 0x83, 0x71, 0x4c, 0x02, 0xea}}, + { // 24 + []byte{ + 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xa8, 0x1f, 0xbd, 0x44, 0x8f, 0x9e, 0x52, 0x2f}}, + { // 25 + []byte{ + 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x4f, 0x64, 0x4c, 0x92, 0xe1, 0x92, 0xdf, 0xed}}, + { // 26 + []byte{ + 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0x1a, 0xfa, 0x9a, 0x66, 0xa6, 0xdf, 0x92, 0xae}}, + { // 27 + []byte{ + 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01}, + nil, + []byte{0xb3, 0xc1, 0xcc, 0x71, 0x5c, 0xb8, 0x79, 0xd8}}, + { // 28 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, 0x01}, + nil, + []byte{0x19, 0xd0, 0x32, 0xe6, 0x4a, 0xb0, 0xbd, 0x8b}}, + { // 29 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, 0x01}, + nil, + []byte{0x3c, 0xfa, 0xa7, 0xa7, 0xdc, 0x87, 0x20, 0xdc}}, + { // 30 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, 0x01}, + nil, + []byte{0xb7, 0x26, 0x5f, 0x7f, 0x44, 0x7a, 0xc6, 0xf3}}, + { // 31 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, 0x01}, + nil, + []byte{0x9d, 0xb7, 0x3b, 0x3c, 0x0d, 0x16, 0x3f, 0x54}}, + { // 32 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, 0x01}, + nil, + []byte{0x81, 0x81, 0xb6, 0x5b, 0xab, 0xf4, 0xa9, 0x75}}, + { // 33 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, 0x01}, + nil, + []byte{0x93, 0xc9, 0xb6, 0x40, 0x42, 0xea, 0xa2, 0x40}}, + { // 34 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01}, + nil, + []byte{0x55, 0x70, 0x53, 0x08, 0x29, 0x70, 0x55, 0x92}}, + { // 35 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, 0x01}, + nil, + []byte{0x86, 0x38, 0x80, 0x9e, 0x87, 0x87, 0x87, 0xa0}}, + { // 36 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, 0x01}, + nil, + []byte{0x41, 0xb9, 0xa7, 0x9a, 0xf7, 0x9a, 0xc2, 0x08}}, + { // 37 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x01}, + nil, + []byte{0x7a, 0x9b, 0xe4, 0x2f, 0x20, 0x09, 0xa8, 0x92}}, + { // 38 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, 0x01}, + nil, + []byte{0x29, 0x03, 0x8d, 0x56, 0xba, 0x6d, 0x27, 0x45}}, + { // 39 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, 0x01}, + nil, + []byte{0x54, 0x95, 0xc6, 0xab, 0xf1, 0xe5, 0xdf, 0x51}}, + { // 40 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, 0x01}, + nil, + []byte{0xae, 0x13, 0xdb, 0xd5, 0x61, 0x48, 0x89, 0x33}}, + { // 41 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01}, + nil, + []byte{0x02, 0x4d, 0x1f, 0xfa, 0x89, 0x04, 0xe3, 0x89}}, + { // 42 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, 0x01}, + nil, + []byte{0xd1, 0x39, 0x97, 0x12, 0xf9, 0x9b, 0xf0, 0x2e}}, + { // 43 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, 0x01}, + nil, + []byte{0x14, 0xc1, 0xd7, 0xc1, 0xcf, 0xfe, 0xc7, 0x9e}}, + { // 44 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01}, + nil, + []byte{0x1d, 0xe5, 0x27, 0x9d, 0xae, 0x3b, 0xed, 0x6f}}, + { // 45 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x01}, + nil, + []byte{0xe9, 0x41, 0xa3, 0x3f, 0x85, 0x50, 0x13, 0x03}}, + { // 46 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, 0x01}, + nil, + []byte{0xda, 0x99, 0xdb, 0xbc, 0x9a, 0x03, 0xf3, 0x79}}, + { // 47 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01}, + nil, + []byte{0xb7, 0xfc, 0x92, 0xf9, 0x1d, 0x8e, 0x92, 0xe9}}, + { // 48 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01}, + nil, + []byte{0xae, 0x8e, 0x5c, 0xaa, 0x3c, 0xa0, 0x4e, 0x85}}, + { // 49 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x80}, + nil, + []byte{0x9c, 0xc6, 0x2d, 0xf4, 0x3b, 0x6e, 0xed, 0x74}}, + { // 50 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x40}, + nil, + []byte{0xd8, 0x63, 0xdb, 0xb5, 0xc5, 0x9a, 0x91, 0xa0}}, + { // 50 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20}, + nil, + []byte{0xa1, 0xab, 0x21, 0x90, 0x54, 0x5b, 0x91, 0xd7}}, + { // 52 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10}, + nil, + []byte{0x08, 0x75, 0x04, 0x1e, 0x64, 0xc5, 0x70, 0xf7}}, + { // 53 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x08}, + nil, + []byte{0x5a, 0x59, 0x45, 0x28, 0xbe, 0xbe, 0xf1, 0xcc}}, + { // 54 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04}, + nil, + []byte{0xfc, 0xdb, 0x32, 0x91, 0xde, 0x21, 0xf0, 0xc0}}, + { // 55 + []byte{ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02}, + nil, + []byte{0x86, 0x9e, 0xfd, 0x7f, 0x9f, 0x26, 0x5a, 0x09}}, +} + +// Plaintext for use with Table A.3 tests +var tableA3Plaintext = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + +// Table A.3 Values To Be Used for the Permutation Operation Known Answer Test +var tableA3Tests = []CryptTest{ + { // 0 + []byte{ + 0x10, 0x46, 0x91, 0x34, 0x89, 0x98, 0x01, 0x31, + 0x10, 0x46, 0x91, 0x34, 0x89, 0x98, 0x01, 0x31, + 0x10, 0x46, 0x91, 0x34, 0x89, 0x98, 0x01, 0x31, + }, + nil, + []byte{0x88, 0xd5, 0x5e, 0x54, 0xf5, 0x4c, 0x97, 0xb4}}, + { // 1 + []byte{ + 0x10, 0x07, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x07, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x07, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + }, + nil, + []byte{0x0c, 0x0c, 0xc0, 0x0c, 0x83, 0xea, 0x48, 0xfd}}, + { // 2 + []byte{ + 0x10, 0x07, 0x10, 0x34, 0xc8, 0x98, 0x01, 0x20, + 0x10, 0x07, 0x10, 0x34, 0xc8, 0x98, 0x01, 0x20, + 0x10, 0x07, 0x10, 0x34, 0xc8, 0x98, 0x01, 0x20, + }, + nil, + []byte{0x83, 0xbc, 0x8e, 0xf3, 0xa6, 0x57, 0x01, 0x83}}, + { // 3 + []byte{ + 0x10, 0x46, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x46, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + 0x10, 0x46, 0x10, 0x34, 0x89, 0x98, 0x80, 0x20, + }, + nil, + []byte{0xdf, 0x72, 0x5d, 0xca, 0xd9, 0x4e, 0xa2, 0xe9}}, + { // 4 + []byte{ + 0x10, 0x86, 0x91, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xe6, 0x52, 0xb5, 0x3b, 0x55, 0x0b, 0xe8, 0xb0}}, + { // 5 + []byte{ + 0x10, 0x86, 0x91, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x10, 0x86, 0x91, 0x15, 0x19, 0x58, 0x01, 0x01, + }, + nil, + []byte{0xaf, 0x52, 0x71, 0x20, 0xc4, 0x85, 0xcb, 0xb0}}, + { // 6 + []byte{ + 0x51, 0x07, 0xb0, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x51, 0x07, 0xb0, 0x15, 0x19, 0x58, 0x01, 0x01, + 0x51, 0x07, 0xb0, 0x15, 0x19, 0x58, 0x01, 0x01, + }, + nil, + []byte{0x0f, 0x04, 0xce, 0x39, 0x3d, 0xb9, 0x26, 0xd5}}, + { // 7 + []byte{ + 0x10, 0x07, 0xb0, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x07, 0xb0, 0x15, 0x19, 0x19, 0x01, 0x01, + 0x10, 0x07, 0xb0, 0x15, 0x19, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xc9, 0xf0, 0x0f, 0xfc, 0x74, 0x07, 0x90, 0x67}}, + { // 8 + []byte{ + 0x31, 0x07, 0x91, 0x54, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x54, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x54, 0x98, 0x08, 0x01, 0x01, + }, + nil, + []byte{0x7c, 0xfd, 0x82, 0xa5, 0x93, 0x25, 0x2b, 0x4e}}, + { // 9 + []byte{ + 0x31, 0x07, 0x91, 0x94, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x94, 0x98, 0x08, 0x01, 0x01, + 0x31, 0x07, 0x91, 0x94, 0x98, 0x08, 0x01, 0x01, + }, + nil, + []byte{0xcb, 0x49, 0xa2, 0xf9, 0xe9, 0x13, 0x63, 0xe3}}, + { // 10 + []byte{ + 0x10, 0x07, 0x91, 0x15, 0xb9, 0x08, 0x01, 0x40, + 0x10, 0x07, 0x91, 0x15, 0xb9, 0x08, 0x01, 0x40, + 0x10, 0x07, 0x91, 0x15, 0xb9, 0x08, 0x01, 0x40, + }, + nil, + []byte{0x00, 0xb5, 0x88, 0xbe, 0x70, 0xd2, 0x3f, 0x56}}, + { // 11 + []byte{ + 0x31, 0x07, 0x91, 0x15, 0x98, 0x08, 0x01, 0x40, + 0x31, 0x07, 0x91, 0x15, 0x98, 0x08, 0x01, 0x40, + 0x31, 0x07, 0x91, 0x15, 0x98, 0x08, 0x01, 0x40, + }, + nil, + []byte{0x40, 0x6a, 0x9a, 0x6a, 0xb4, 0x33, 0x99, 0xae}}, + { // 12 + []byte{ + 0x10, 0x07, 0xd0, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x10, 0x07, 0xd0, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x10, 0x07, 0xd0, 0x15, 0x89, 0x98, 0x01, 0x01, + }, + nil, + []byte{0x6c, 0xb7, 0x73, 0x61, 0x1d, 0xca, 0x9a, 0xda}}, + { // 13 + []byte{ + 0x91, 0x07, 0x91, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x91, 0x07, 0x91, 0x15, 0x89, 0x98, 0x01, 0x01, + 0x91, 0x07, 0x91, 0x15, 0x89, 0x98, 0x01, 0x01, + }, + nil, + []byte{0x67, 0xfd, 0x21, 0xc1, 0x7d, 0xbb, 0x5d, 0x70}}, + { // 14 + []byte{ + 0x91, 0x07, 0xd0, 0x15, 0x89, 0x19, 0x01, 0x01, + 0x91, 0x07, 0xd0, 0x15, 0x89, 0x19, 0x01, 0x01, + 0x91, 0x07, 0xd0, 0x15, 0x89, 0x19, 0x01, 0x01, + }, + nil, + []byte{0x95, 0x92, 0xcb, 0x41, 0x10, 0x43, 0x07, 0x87}}, + { // 15 + []byte{ + 0x10, 0x07, 0xd0, 0x15, 0x98, 0x98, 0x01, 0x20, + 0x10, 0x07, 0xd0, 0x15, 0x98, 0x98, 0x01, 0x20, + 0x10, 0x07, 0xd0, 0x15, 0x98, 0x98, 0x01, 0x20, + }, + nil, + []byte{0xa6, 0xb7, 0xff, 0x68, 0xa3, 0x18, 0xdd, 0xd3}}, + { // 16 + []byte{ + 0x10, 0x07, 0x94, 0x04, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x94, 0x04, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x94, 0x04, 0x98, 0x19, 0x01, 0x01, + }, + nil, + []byte{0x4d, 0x10, 0x21, 0x96, 0xc9, 0x14, 0xca, 0x16}}, + { // 17 + []byte{ + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x04, 0x01, + }, + nil, + []byte{0x2d, 0xfa, 0x9f, 0x45, 0x73, 0x59, 0x49, 0x65}}, + { // 18 + []byte{ + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x01, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x01, 0x01, + 0x01, 0x07, 0x91, 0x04, 0x91, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xb4, 0x66, 0x04, 0x81, 0x6c, 0x0e, 0x07, 0x74}}, + { // 19 + []byte{ + 0x01, 0x07, 0x94, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x94, 0x04, 0x91, 0x19, 0x04, 0x01, + 0x01, 0x07, 0x94, 0x04, 0x91, 0x19, 0x04, 0x01, + }, + nil, + []byte{0x6e, 0x7e, 0x62, 0x21, 0xa4, 0xf3, 0x4e, 0x87}}, + { // 20 + []byte{ + 0x19, 0x07, 0x92, 0x10, 0x98, 0x1a, 0x01, 0x01, + 0x19, 0x07, 0x92, 0x10, 0x98, 0x1a, 0x01, 0x01, + 0x19, 0x07, 0x92, 0x10, 0x98, 0x1a, 0x01, 0x01, + }, + nil, + []byte{0xaa, 0x85, 0xe7, 0x46, 0x43, 0x23, 0x31, 0x99}}, + { // 21 + []byte{ + 0x10, 0x07, 0x91, 0x19, 0x98, 0x19, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x19, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x19, 0x08, 0x01, + }, + nil, + []byte{0x2e, 0x5a, 0x19, 0xdb, 0x4d, 0x19, 0x62, 0xd6}}, + { // 22 + []byte{ + 0x10, 0x07, 0x91, 0x19, 0x98, 0x1a, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x1a, 0x08, 0x01, + 0x10, 0x07, 0x91, 0x19, 0x98, 0x1a, 0x08, 0x01, + }, + nil, + []byte{0x23, 0xa8, 0x66, 0xa8, 0x09, 0xd3, 0x08, 0x94}}, + { // 23 + []byte{ + 0x10, 0x07, 0x92, 0x10, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x92, 0x10, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x07, 0x92, 0x10, 0x98, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xd8, 0x12, 0xd9, 0x61, 0xf0, 0x17, 0xd3, 0x20}}, + { // 24 + []byte{ + 0x10, 0x07, 0x91, 0x15, 0x98, 0x19, 0x01, 0x0b, + 0x10, 0x07, 0x91, 0x15, 0x98, 0x19, 0x01, 0x0b, + 0x10, 0x07, 0x91, 0x15, 0x98, 0x19, 0x01, 0x0b, + }, + nil, + []byte{0x05, 0x56, 0x05, 0x81, 0x6e, 0x58, 0x60, 0x8f}}, + { // 25 + []byte{ + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x01, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x01, + }, + nil, + []byte{0xab, 0xd8, 0x8e, 0x8b, 0x1b, 0x77, 0x16, 0xf1}}, + { // 26 + []byte{ + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x02, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x02, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x02, + }, + nil, + []byte{0x53, 0x7a, 0xc9, 0x5b, 0xe6, 0x9d, 0xa1, 0xe1}}, + { // 27 + []byte{ + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x08, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x08, + 0x10, 0x04, 0x80, 0x15, 0x98, 0x19, 0x01, 0x08, + }, + nil, + []byte{0xae, 0xd0, 0xf6, 0xae, 0x3c, 0x25, 0xcd, 0xd8}}, + { // 28 + []byte{ + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x01, 0x04, + }, + nil, + []byte{0xb3, 0xe3, 0x5a, 0x5e, 0xe5, 0x3e, 0x7b, 0x8d}}, + { // 29 + []byte{ + 0x10, 0x02, 0x91, 0x15, 0x98, 0x19, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x19, 0x01, 0x04, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x19, 0x01, 0x04, + }, + nil, + []byte{0x61, 0xc7, 0x9c, 0x71, 0x92, 0x1a, 0x2e, 0xf8}}, + { // 30 + []byte{ + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x02, 0x01, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x02, 0x01, + 0x10, 0x02, 0x91, 0x15, 0x98, 0x10, 0x02, 0x01, + }, + nil, + []byte{0xe2, 0xf5, 0x72, 0x8f, 0x09, 0x95, 0x01, 0x3c}}, + { // 31 + []byte{ + 0x10, 0x02, 0x91, 0x16, 0x98, 0x10, 0x01, 0x01, + 0x10, 0x02, 0x91, 0x16, 0x98, 0x10, 0x01, 0x01, + 0x10, 0x02, 0x91, 0x16, 0x98, 0x10, 0x01, 0x01, + }, + nil, + []byte{0x1a, 0xea, 0xc3, 0x9a, 0x61, 0xf0, 0xa4, 0x64}}, +} + +// Table A.4 Values To Be Used for the Substitution Table Known Answer Test +var tableA4Tests = []CryptTest{ + { // 0 + []byte{ + 0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57, + 0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57, + 0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57}, + []byte{0x01, 0xa1, 0xd6, 0xd0, 0x39, 0x77, 0x67, 0x42}, + []byte{0x69, 0x0f, 0x5b, 0x0d, 0x9a, 0x26, 0x93, 0x9b}}, + { // 1 + []byte{ + 0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e, + 0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e, + 0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e}, + []byte{0x5c, 0xd5, 0x4c, 0xa8, 0x3d, 0xef, 0x57, 0xda}, + []byte{0x7a, 0x38, 0x9d, 0x10, 0x35, 0x4b, 0xd2, 0x71}}, + { // 2 + []byte{ + 0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86, + 0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86, + 0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86}, + []byte{0x02, 0x48, 0xd4, 0x38, 0x06, 0xf6, 0x71, 0x72}, + []byte{0x86, 0x8e, 0xbb, 0x51, 0xca, 0xb4, 0x59, 0x9a}}, + { // 3 + []byte{ + 0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e, + 0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e, + 0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e}, + []byte{0x51, 0x45, 0x4b, 0x58, 0x2d, 0xdf, 0x44, 0x0a}, + []byte{0x71, 0x78, 0x87, 0x6e, 0x01, 0xf1, 0x9b, 0x2a}}, + { // 4 + []byte{ + 0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6, + 0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6, + 0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6}, + []byte{0x42, 0xfd, 0x44, 0x30, 0x59, 0x57, 0x7f, 0xa2}, + []byte{0xaf, 0x37, 0xfb, 0x42, 0x1f, 0x8c, 0x40, 0x95}}, + { // 5 + []byte{ + 0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce, + 0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce, + 0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce}, + []byte{0x05, 0x9b, 0x5e, 0x08, 0x51, 0xcf, 0x14, 0x3a}, + []byte{0x86, 0xa5, 0x60, 0xf1, 0x0e, 0xc6, 0xd8, 0x5b}}, + { // 6 + []byte{ + 0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6, + 0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6, + 0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6}, + []byte{0x07, 0x56, 0xd8, 0xe0, 0x77, 0x47, 0x61, 0xd2}, + []byte{0x0c, 0xd3, 0xda, 0x02, 0x00, 0x21, 0xdc, 0x09}}, + { // 7 + []byte{ + 0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe, + 0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe, + 0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe}, + []byte{0x76, 0x25, 0x14, 0xb8, 0x29, 0xbf, 0x48, 0x6a}, + []byte{0xea, 0x67, 0x6b, 0x2c, 0xb7, 0xdb, 0x2b, 0x7a}}, + { // 8 + []byte{ + 0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16, + 0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16, + 0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16}, + []byte{0x3b, 0xdd, 0x11, 0x90, 0x49, 0x37, 0x28, 0x02}, + []byte{0xdf, 0xd6, 0x4a, 0x81, 0x5c, 0xaf, 0x1a, 0x0f}}, + { // 9 + []byte{ + 0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f, + 0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f, + 0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f}, + []byte{0x26, 0x95, 0x5f, 0x68, 0x35, 0xaf, 0x60, 0x9a}, + []byte{0x5c, 0x51, 0x3c, 0x9c, 0x48, 0x86, 0xc0, 0x88}}, + { // 10 + []byte{ + 0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46, + 0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46, + 0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46}, + []byte{0x16, 0x4d, 0x5e, 0x40, 0x4f, 0x27, 0x52, 0x32}, + []byte{0x0a, 0x2a, 0xee, 0xae, 0x3f, 0xf4, 0xab, 0x77}}, + { // 11 + []byte{ + 0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e, + 0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e, + 0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e}, + []byte{0x6b, 0x05, 0x6e, 0x18, 0x75, 0x9f, 0x5c, 0xca}, + []byte{0xef, 0x1b, 0xf0, 0x3e, 0x5d, 0xfa, 0x57, 0x5a}}, + { // 12 + []byte{ + 0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76, + 0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76, + 0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76}, + []byte{0x00, 0x4b, 0xd6, 0xef, 0x09, 0x17, 0x60, 0x62}, + []byte{0x88, 0xbf, 0x0d, 0xb6, 0xd7, 0x0d, 0xee, 0x56}}, + { // 13 + []byte{ + 0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07, + 0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07, + 0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07}, + []byte{0x48, 0x0d, 0x39, 0x00, 0x6e, 0xe7, 0x62, 0xf2}, + []byte{0xa1, 0xf9, 0x91, 0x55, 0x41, 0x02, 0x0b, 0x56}}, + { // 14 + []byte{ + 0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f, + 0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f, + 0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f}, + []byte{0x43, 0x75, 0x40, 0xc8, 0x69, 0x8f, 0x3c, 0xfa}, + []byte{0x6f, 0xbf, 0x1c, 0xaf, 0xcf, 0xfd, 0x05, 0x56}}, + { // 15 + []byte{ + 0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7, + 0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7, + 0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7}, + []byte{0x07, 0x2d, 0x43, 0xa0, 0x77, 0x07, 0x52, 0x92}, + []byte{0x2f, 0x22, 0xe4, 0x9b, 0xab, 0x7c, 0xa1, 0xac}}, + { // 16 + []byte{ + 0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf, + 0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf, + 0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf}, + []byte{0x02, 0xfe, 0x55, 0x77, 0x81, 0x17, 0xf1, 0x2a}, + []byte{0x5a, 0x6b, 0x61, 0x2c, 0xc2, 0x6c, 0xce, 0x4a}}, + { // 17 + []byte{ + 0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6, + 0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6, + 0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6}, + []byte{0x1d, 0x9d, 0x5c, 0x50, 0x18, 0xf7, 0x28, 0xc2}, + []byte{0x5f, 0x4c, 0x03, 0x8e, 0xd1, 0x2b, 0x2e, 0x41}}, + { // 18 + []byte{ + 0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef, + 0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef, + 0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef}, + []byte{0x30, 0x55, 0x32, 0x28, 0x6d, 0x6f, 0x29, 0x5a}, + []byte{0x63, 0xfa, 0xc0, 0xd0, 0x34, 0xd9, 0xf7, 0x93}}, +} + +// Use the known weak keys to test DES implementation +func TestWeakKeys(t *testing.T) { + for i, tt := range weakKeyTests { + var encrypt = func(in []byte) (out []byte) { + c, _ := NewCipher(tt.key) + out = make([]byte, len(in)) + encryptBlock(c.subkeys[:], out, in) + return + } + + // Encrypting twice with a DES weak + // key should reproduce the original input + result := encrypt(tt.in) + result = encrypt(result) + + if !bytes.Equal(result, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, result, tt.in) + } + } +} + +// Use the known semi-weak key pairs to test DES implementation +func TestSemiWeakKeyPairs(t *testing.T) { + for i, tt := range semiWeakKeyTests { + var encrypt = func(key, in []byte) (out []byte) { + c, _ := NewCipher(key) + out = make([]byte, len(in)) + encryptBlock(c.subkeys[:], out, in) + return + } + + // Encrypting with one member of the semi-weak pair + // and then encrypting the result with the other member + // should reproduce the original input. + result := encrypt(tt.key, tt.in) + result = encrypt(tt.out, result) + + if !bytes.Equal(result, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, result, tt.in) + } + } +} + +func TestDESEncryptBlock(t *testing.T) { + for i, tt := range encryptDESTests { + c, _ := NewCipher(tt.key) + out := make([]byte, len(tt.in)) + encryptBlock(c.subkeys[:], out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +func TestDESDecryptBlock(t *testing.T) { + for i, tt := range encryptDESTests { + c, _ := NewCipher(tt.key) + plain := make([]byte, len(tt.in)) + decryptBlock(c.subkeys[:], plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +func TestEncryptTripleDES(t *testing.T) { + for i, tt := range encryptTripleDESTests { + c, _ := NewTripleDESCipher(tt.key) + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +func TestDecryptTripleDES(t *testing.T) { + for i, tt := range encryptTripleDESTests { + c, _ := NewTripleDESCipher(tt.key) + + plain := make([]byte, len(tt.in)) + c.Decrypt(plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +// Defined in Pub 800-20 +func TestVariablePlaintextKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestVariableCiphertextKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + plain := make([]byte, len(tt.out)) + c.Decrypt(plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +// Defined in Pub 800-20 +// Encrypting the Table A.1 ciphertext with the +// 0x01... key produces the original plaintext +func TestInversePermutationKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + plain := make([]byte, len(tt.in)) + c.Encrypt(plain, tt.out) + + if !bytes.Equal(plain, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, plain, tt.in) + } + } +} + +// Defined in Pub 800-20 +// Decrypting the Table A.1 plaintext with the +// 0x01... key produces the corresponding ciphertext +func TestInitialPermutationKnownAnswer(t *testing.T) { + for i, tt := range tableA1Tests { + c, _ := NewTripleDESCipher(tableA1Key) + + out := make([]byte, len(tt.in)) + c.Decrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestVariableKeyKnownAnswerEncrypt(t *testing.T) { + for i, tt := range tableA2Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tableA2Plaintext)) + c.Encrypt(out, tableA2Plaintext) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestVariableKeyKnownAnswerDecrypt(t *testing.T) { + for i, tt := range tableA2Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.out)) + c.Decrypt(out, tt.out) + + if !bytes.Equal(out, tableA2Plaintext) { + t.Errorf("#%d: result: %x want: %x", i, out, tableA2Plaintext) + } + } +} + +// Defined in Pub 800-20 +func TestPermutationOperationKnownAnswerEncrypt(t *testing.T) { + for i, tt := range tableA3Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tableA3Plaintext)) + c.Encrypt(out, tableA3Plaintext) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestPermutationOperationKnownAnswerDecrypt(t *testing.T) { + for i, tt := range tableA3Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.out)) + c.Decrypt(out, tt.out) + + if !bytes.Equal(out, tableA3Plaintext) { + t.Errorf("#%d: result: %x want: %x", i, out, tableA3Plaintext) + } + } +} + +// Defined in Pub 800-20 +func TestSubstitutionTableKnownAnswerEncrypt(t *testing.T) { + for i, tt := range tableA4Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.in)) + c.Encrypt(out, tt.in) + + if !bytes.Equal(out, tt.out) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.out) + } + } +} + +// Defined in Pub 800-20 +func TestSubstitutionTableKnownAnswerDecrypt(t *testing.T) { + for i, tt := range tableA4Tests { + c, _ := NewTripleDESCipher(tt.key) + + out := make([]byte, len(tt.out)) + c.Decrypt(out, tt.out) + + if !bytes.Equal(out, tt.in) { + t.Errorf("#%d: result: %x want: %x", i, out, tt.in) + } + } +} diff --git a/src/pkg/crypto/dsa/Makefile b/src/pkg/crypto/dsa/Makefile new file mode 100644 index 000000000..fa89d4ab2 --- /dev/null +++ b/src/pkg/crypto/dsa/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/dsa +GOFILES=\ + dsa.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/dsa/dsa.go b/src/pkg/crypto/dsa/dsa.go new file mode 100644 index 000000000..a5f96fe94 --- /dev/null +++ b/src/pkg/crypto/dsa/dsa.go @@ -0,0 +1,276 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3 +package dsa + +import ( + "big" + "io" + "os" +) + +// Parameters represents the domain parameters for a key. These parameters can +// be shared across many keys. The bit length of Q must be a multiple of 8. +type Parameters struct { + P, Q, G *big.Int +} + +// PublicKey represents a DSA public key. +type PublicKey struct { + Parameters + Y *big.Int +} + +// PrivateKey represents a DSA private key. +type PrivateKey struct { + PublicKey + X *big.Int +} + +type invalidPublicKeyError int + +func (invalidPublicKeyError) String() string { + return "crypto/dsa: invalid public key" +} + +// InvalidPublicKeyError results when a public key is not usable by this code. +// FIPS is quite strict about the format of DSA keys, but other code may be +// less so. Thus, when using keys which may have been generated by other code, +// this error must be handled. +var InvalidPublicKeyError = invalidPublicKeyError(0) + +// ParameterSizes is a enumeration of the acceptable bit lengths of the primes +// in a set of DSA parameters. See FIPS 186-3, section 4.2. +type ParameterSizes int + +const ( + L1024N160 ParameterSizes = iota + L2048N224 + L2048N256 + L3072N256 +) + +// numMRTests is the number of Miller-Rabin primality tests that we perform. We +// pick the largest recommended number from table C.1 of FIPS 186-3. +const numMRTests = 64 + +// GenerateParameters puts a random, valid set of DSA parameters into params. +// This function takes many seconds, even on fast machines. +func GenerateParameters(params *Parameters, rand io.Reader, sizes ParameterSizes) (err os.Error) { + // This function doesn't follow FIPS 186-3 exactly in that it doesn't + // use a verification seed to generate the primes. The verification + // seed doesn't appear to be exported or used by other code and + // omitting it makes the code cleaner. + + var L, N int + switch sizes { + case L1024N160: + L = 1024 + N = 160 + case L2048N224: + L = 2048 + N = 224 + case L2048N256: + L = 2048 + N = 256 + case L3072N256: + L = 3072 + N = 256 + default: + return os.NewError("crypto/dsa: invalid ParameterSizes") + } + + qBytes := make([]byte, N/8) + pBytes := make([]byte, L/8) + + q := new(big.Int) + p := new(big.Int) + rem := new(big.Int) + one := new(big.Int) + one.SetInt64(1) + +GeneratePrimes: + for { + _, err = io.ReadFull(rand, qBytes) + if err != nil { + return + } + + qBytes[len(qBytes)-1] |= 1 + qBytes[0] |= 0x80 + q.SetBytes(qBytes) + + if !big.ProbablyPrime(q, numMRTests) { + continue + } + + for i := 0; i < 4*L; i++ { + _, err = io.ReadFull(rand, pBytes) + if err != nil { + return + } + + pBytes[len(pBytes)-1] |= 1 + pBytes[0] |= 0x80 + + p.SetBytes(pBytes) + rem.Mod(p, q) + rem.Sub(rem, one) + p.Sub(p, rem) + if p.BitLen() < L { + continue + } + + if !big.ProbablyPrime(p, numMRTests) { + continue + } + + params.P = p + params.Q = q + break GeneratePrimes + } + } + + h := new(big.Int) + h.SetInt64(2) + g := new(big.Int) + + pm1 := new(big.Int).Sub(p, one) + e := new(big.Int).Div(pm1, q) + + for { + g.Exp(h, e, p) + if g.Cmp(one) == 0 { + h.Add(h, one) + continue + } + + params.G = g + return + } + + panic("unreachable") +} + +// GenerateKey generates a public&private key pair. The Parameters of the +// PrivateKey must already be valid (see GenerateParameters). +func GenerateKey(priv *PrivateKey, rand io.Reader) os.Error { + if priv.P == nil || priv.Q == nil || priv.G == nil { + return os.NewError("crypto/dsa: parameters not set up before generating key") + } + + x := new(big.Int) + xBytes := make([]byte, priv.Q.BitLen()/8) + + for { + _, err := io.ReadFull(rand, xBytes) + if err != nil { + return err + } + x.SetBytes(xBytes) + if x.Sign() != 0 && x.Cmp(priv.Q) < 0 { + break + } + } + + priv.X = x + priv.Y = new(big.Int) + priv.Y.Exp(priv.G, x, priv.P) + return nil +} + +// Sign signs an arbitrary length hash (which should be the result of hashing a +// larger message) using the private key, priv. It returns the signature as a +// pair of integers. The security of the private key depends on the entropy of +// rand. +func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err os.Error) { + // FIPS 186-3, section 4.6 + + n := priv.Q.BitLen() + if n&7 != 0 { + err = InvalidPublicKeyError + return + } + n >>= 3 + + for { + k := new(big.Int) + buf := make([]byte, n) + for { + _, err = io.ReadFull(rand, buf) + if err != nil { + return + } + k.SetBytes(buf) + if k.Sign() > 0 && k.Cmp(priv.Q) < 0 { + break + } + } + + kInv := new(big.Int).ModInverse(k, priv.Q) + + r = new(big.Int).Exp(priv.G, k, priv.P) + r.Mod(r, priv.Q) + + if r.Sign() == 0 { + continue + } + + if n > len(hash) { + n = len(hash) + } + z := k.SetBytes(hash[:n]) + + s = new(big.Int).Mul(priv.X, r) + s.Add(s, z) + s.Mod(s, priv.Q) + s.Mul(s, kInv) + s.Mod(s, priv.Q) + + if s.Sign() != 0 { + break + } + } + + return +} + +// Verify verifies the signature in r, s of hash using the public key, pub. It +// returns true iff the signature is valid. +func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool { + // FIPS 186-3, section 4.7 + + if r.Sign() < 1 || r.Cmp(pub.Q) >= 0 { + return false + } + if s.Sign() < 1 || s.Cmp(pub.Q) >= 0 { + return false + } + + w := new(big.Int).ModInverse(s, pub.Q) + + n := pub.Q.BitLen() + if n&7 != 0 { + return false + } + n >>= 3 + + if n > len(hash) { + n = len(hash) + } + z := new(big.Int).SetBytes(hash[:n]) + + u1 := new(big.Int).Mul(z, w) + u1.Mod(u1, pub.Q) + u2 := w.Mul(r, w) + u2.Mod(u2, pub.Q) + v := u1.Exp(pub.G, u1, pub.P) + u2.Exp(pub.Y, u2, pub.P) + v.Mul(v, u2) + v.Mod(v, pub.P) + v.Mod(v, pub.Q) + + return v.Cmp(r) == 0 +} diff --git a/src/pkg/crypto/dsa/dsa_test.go b/src/pkg/crypto/dsa/dsa_test.go new file mode 100644 index 000000000..deec08dfd --- /dev/null +++ b/src/pkg/crypto/dsa/dsa_test.go @@ -0,0 +1,84 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dsa + +import ( + "big" + "crypto/rand" + "testing" +) + +func testSignAndVerify(t *testing.T, i int, priv *PrivateKey) { + hashed := []byte("testing") + r, s, err := Sign(rand.Reader, priv, hashed) + if err != nil { + t.Errorf("%d: error signing: %s", i, err) + return + } + + if !Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%d: Verify failed", i) + } +} + +func testParameterGeneration(t *testing.T, sizes ParameterSizes, L, N int) { + var priv PrivateKey + params := &priv.Parameters + + err := GenerateParameters(params, rand.Reader, sizes) + if err != nil { + t.Errorf("%d: %s", int(sizes), err) + return + } + + if params.P.BitLen() != L { + t.Errorf("%d: params.BitLen got:%d want:%d", int(sizes), params.P.BitLen(), L) + } + + if params.Q.BitLen() != N { + t.Errorf("%d: q.BitLen got:%d want:%d", int(sizes), params.Q.BitLen(), L) + } + + one := new(big.Int) + one.SetInt64(1) + pm1 := new(big.Int).Sub(params.P, one) + quo, rem := new(big.Int).DivMod(pm1, params.Q, new(big.Int)) + if rem.Sign() != 0 { + t.Errorf("%d: p-1 mod q != 0", int(sizes)) + } + x := new(big.Int).Exp(params.G, quo, params.P) + if x.Cmp(one) == 0 { + t.Errorf("%d: invalid generator", int(sizes)) + } + + err = GenerateKey(&priv, rand.Reader) + if err != nil { + t.Errorf("error generating key: %s", err) + return + } + + testSignAndVerify(t, int(sizes), &priv) +} + +func TestParameterGeneration(t *testing.T) { + // This test is too slow to run all the time. + return + + testParameterGeneration(t, L1024N160, 1024, 160) + testParameterGeneration(t, L2048N224, 2048, 224) + testParameterGeneration(t, L2048N256, 2048, 256) + testParameterGeneration(t, L3072N256, 3072, 256) +} + +func TestSignAndVerify(t *testing.T) { + var priv PrivateKey + priv.P, _ = new(big.Int).SetString("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF", 16) + priv.Q, _ = new(big.Int).SetString("E1D3391245933D68A0714ED34BBCB7A1F422B9C1", 16) + priv.G, _ = new(big.Int).SetString("634364FC25248933D01D1993ECABD0657CC0CB2CEED7ED2E3E8AECDFCDC4A25C3B15E9E3B163ACA2984B5539181F3EFF1A5E8903D71D5B95DA4F27202B77D2C44B430BB53741A8D59A8F86887525C9F2A6A5980A195EAA7F2FF910064301DEF89D3AA213E1FAC7768D89365318E370AF54A112EFBA9246D9158386BA1B4EEFDA", 16) + priv.Y, _ = new(big.Int).SetString("32969E5780CFE1C849A1C276D7AEB4F38A23B591739AA2FE197349AEEBD31366AEE5EB7E6C6DDB7C57D02432B30DB5AA66D9884299FAA72568944E4EEDC92EA3FBC6F39F53412FBCC563208F7C15B737AC8910DBC2D9C9B8C001E72FDC40EB694AB1F06A5A2DBD18D9E36C66F31F566742F11EC0A52E9F7B89355C02FB5D32D2", 16) + priv.X, _ = new(big.Int).SetString("5078D4D29795CBE76D3AACFE48C9AF0BCDBEE91A", 16) + + testSignAndVerify(t, 0, &priv) +} diff --git a/src/pkg/crypto/ecdsa/Makefile b/src/pkg/crypto/ecdsa/Makefile new file mode 100644 index 000000000..0af24c2f2 --- /dev/null +++ b/src/pkg/crypto/ecdsa/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/ecdsa +GOFILES=\ + ecdsa.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/ecdsa/ecdsa.go b/src/pkg/crypto/ecdsa/ecdsa.go new file mode 100644 index 000000000..7bce1bc96 --- /dev/null +++ b/src/pkg/crypto/ecdsa/ecdsa.go @@ -0,0 +1,149 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ecdsa implements the Elliptic Curve Digital Signature Algorithm, as +// defined in FIPS 186-3. +package ecdsa + +// References: +// [NSA]: Suite B implementor's guide to FIPS 186-3, +// http://www.nsa.gov/ia/_files/ecdsa.pdf +// [SECG]: SECG, SEC1 +// http://www.secg.org/download/aid-780/sec1-v2.pdf + +import ( + "big" + "crypto/elliptic" + "io" + "os" +) + +// PublicKey represents an ECDSA public key. +type PublicKey struct { + *elliptic.Curve + X, Y *big.Int +} + +// PrivateKey represents a ECDSA private key. +type PrivateKey struct { + PublicKey + D *big.Int +} + +var one = new(big.Int).SetInt64(1) + +// randFieldElement returns a random element of the field underlying the given +// curve using the procedure given in [NSA] A.2.1. +func randFieldElement(c *elliptic.Curve, rand io.Reader) (k *big.Int, err os.Error) { + b := make([]byte, c.BitSize/8+8) + _, err = io.ReadFull(rand, b) + if err != nil { + return + } + + k = new(big.Int).SetBytes(b) + n := new(big.Int).Sub(c.N, one) + k.Mod(k, n) + k.Add(k, one) + return +} + +// GenerateKey generates a public&private key pair. +func GenerateKey(c *elliptic.Curve, rand io.Reader) (priv *PrivateKey, err os.Error) { + k, err := randFieldElement(c, rand) + if err != nil { + return + } + + priv = new(PrivateKey) + priv.PublicKey.Curve = c + priv.D = k + priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes()) + return +} + +// hashToInt converts a hash value to an integer. There is some disagreement +// about how this is done. [NSA] suggests that this is done in the obvious +// manner, but [SECG] truncates the hash to the bit-length of the curve order +// first. We follow [SECG] because that's what OpenSSL does. +func hashToInt(hash []byte, c *elliptic.Curve) *big.Int { + orderBits := c.N.BitLen() + orderBytes := (orderBits + 7) / 8 + if len(hash) > orderBytes { + hash = hash[:orderBytes] + } + + ret := new(big.Int).SetBytes(hash) + excess := orderBytes*8 - orderBits + if excess > 0 { + ret.Rsh(ret, uint(excess)) + } + return ret +} + +// Sign signs an arbitrary length hash (which should be the result of hashing a +// larger message) using the private key, priv. It returns the signature as a +// pair of integers. The security of the private key depends on the entropy of +// rand. +func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err os.Error) { + // See [NSA] 3.4.1 + c := priv.PublicKey.Curve + + var k, kInv *big.Int + for { + for { + k, err = randFieldElement(c, rand) + if err != nil { + r = nil + return + } + + kInv = new(big.Int).ModInverse(k, c.N) + r, _ = priv.Curve.ScalarBaseMult(k.Bytes()) + r.Mod(r, priv.Curve.N) + if r.Sign() != 0 { + break + } + } + + e := hashToInt(hash, c) + s = new(big.Int).Mul(priv.D, r) + s.Add(s, e) + s.Mul(s, kInv) + s.Mod(s, priv.PublicKey.Curve.N) + if s.Sign() != 0 { + break + } + } + + return +} + +// Verify verifies the signature in r, s of hash using the public key, pub. It +// returns true iff the signature is valid. +func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool { + // See [NSA] 3.4.2 + c := pub.Curve + + if r.Sign() == 0 || s.Sign() == 0 { + return false + } + if r.Cmp(c.N) >= 0 || s.Cmp(c.N) >= 0 { + return false + } + e := hashToInt(hash, c) + w := new(big.Int).ModInverse(s, c.N) + + u1 := e.Mul(e, w) + u2 := w.Mul(r, w) + + x1, y1 := c.ScalarBaseMult(u1.Bytes()) + x2, y2 := c.ScalarMult(pub.X, pub.Y, u2.Bytes()) + if x1.Cmp(x2) == 0 { + return false + } + x, _ := c.Add(x1, y1, x2, y2) + x.Mod(x, c.N) + return x.Cmp(r) == 0 +} diff --git a/src/pkg/crypto/ecdsa/ecdsa_test.go b/src/pkg/crypto/ecdsa/ecdsa_test.go new file mode 100644 index 000000000..d6b403914 --- /dev/null +++ b/src/pkg/crypto/ecdsa/ecdsa_test.go @@ -0,0 +1,227 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ecdsa + +import ( + "big" + "crypto/elliptic" + "crypto/sha1" + "crypto/rand" + "encoding/hex" + "testing" +) + +func testKeyGeneration(t *testing.T, c *elliptic.Curve, tag string) { + priv, err := GenerateKey(c, rand.Reader) + if err != nil { + t.Errorf("%s: error: %s", tag, err) + return + } + if !c.IsOnCurve(priv.PublicKey.X, priv.PublicKey.Y) { + t.Errorf("%s: public key invalid: %s", tag, err) + } +} + +func TestKeyGeneration(t *testing.T) { + testKeyGeneration(t, elliptic.P224(), "p224") + if testing.Short() { + return + } + testKeyGeneration(t, elliptic.P256(), "p256") + testKeyGeneration(t, elliptic.P384(), "p384") + testKeyGeneration(t, elliptic.P521(), "p521") +} + +func testSignAndVerify(t *testing.T, c *elliptic.Curve, tag string) { + priv, _ := GenerateKey(c, rand.Reader) + + hashed := []byte("testing") + r, s, err := Sign(rand.Reader, priv, hashed) + if err != nil { + t.Errorf("%s: error signing: %s", tag, err) + return + } + + if !Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%s: Verify failed", tag) + } + + hashed[0] ^= 0xff + if Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%s: Verify always works!", tag) + } +} + +func TestSignAndVerify(t *testing.T) { + testSignAndVerify(t, elliptic.P224(), "p224") + if testing.Short() { + return + } + testSignAndVerify(t, elliptic.P256(), "p256") + testSignAndVerify(t, elliptic.P384(), "p384") + testSignAndVerify(t, elliptic.P521(), "p521") +} + +func fromHex(s string) *big.Int { + r, ok := new(big.Int).SetString(s, 16) + if !ok { + panic("bad hex") + } + return r +} + +// These test vectors were taken from +// http://csrc.nist.gov/groups/STM/cavp/documents/dss/ecdsatestvectors.zip +var testVectors = []struct { + msg string + Qx, Qy string + r, s string + ok bool +}{ + { + "09626b45493672e48f3d1226a3aff3201960e577d33a7f72c7eb055302db8fe8ed61685dd036b554942a5737cd1512cdf811ee0c00e6dd2f08c69f08643be396e85dafda664801e772cdb7396868ac47b172245b41986aa2648cb77fbbfa562581be06651355a0c4b090f9d17d8f0ab6cced4e0c9d386cf465a516630f0231bd", + "9504b5b82d97a264d8b3735e0568decabc4b6ca275bc53cbadfc1c40", + "03426f80e477603b10dee670939623e3da91a94267fc4e51726009ed", + "81d3ac609f9575d742028dd496450a58a60eea2dcf8b9842994916e1", + "96a8c5f382c992e8f30ccce9af120b067ec1d74678fa8445232f75a5", + false, + }, + { + "96b2b6536f6df29be8567a72528aceeaccbaa66c66c534f3868ca9778b02faadb182e4ed34662e73b9d52ecbe9dc8e875fc05033c493108b380689ebf47e5b062e6a0cdb3dd34ce5fe347d92768d72f7b9b377c20aea927043b509c078ed2467d7113405d2ddd458811e6faf41c403a2a239240180f1430a6f4330df5d77de37", + "851e3100368a22478a0029353045ae40d1d8202ef4d6533cfdddafd8", + "205302ac69457dd345e86465afa72ee8c74ca97e2b0b999aec1f10c2", + "4450c2d38b697e990721aa2dbb56578d32b4f5aeb3b9072baa955ee0", + "e26d4b589166f7b4ba4b1c8fce823fa47aad22f8c9c396b8c6526e12", + false, + }, + { + "86778dbb4a068a01047a8d245d632f636c11d2ad350740b36fad90428b454ad0f120cb558d12ea5c8a23db595d87543d06d1ef489263d01ee529871eb68737efdb8ff85bc7787b61514bed85b7e01d6be209e0a4eb0db5c8df58a5c5bf706d76cb2bdf7800208639e05b89517155d11688236e6a47ed37d8e5a2b1e0adea338e", + "ad5bda09d319a717c1721acd6688d17020b31b47eef1edea57ceeffc", + "c8ce98e181770a7c9418c73c63d01494b8b80a41098c5ea50692c984", + "de5558c257ab4134e52c19d8db3b224a1899cbd08cc508ce8721d5e9", + "745db7af5a477e5046705c0a5eff1f52cb94a79d481f0c5a5e108ecd", + true, + }, + { + "4bc6ef1958556686dab1e39c3700054a304cbd8f5928603dcd97fafd1f29e69394679b638f71c9344ce6a535d104803d22119f57b5f9477e253817a52afa9bfbc9811d6cc8c8be6b6566c6ef48b439bbb532abe30627548c598867f3861ba0b154dc1c3deca06eb28df8efd28258554b5179883a36fbb1eecf4f93ee19d41e3d", + "cc5eea2edf964018bdc0504a3793e4d2145142caa09a72ac5fb8d3e8", + "a48d78ae5d08aa725342773975a00d4219cf7a8029bb8cf3c17c374a", + "67b861344b4e416d4094472faf4272f6d54a497177fbc5f9ef292836", + "1d54f3fcdad795bf3b23408ecbac3e1321d1d66f2e4e3d05f41f7020", + false, + }, + { + "bb658732acbf3147729959eb7318a2058308b2739ec58907dd5b11cfa3ecf69a1752b7b7d806fe00ec402d18f96039f0b78dbb90a59c4414fb33f1f4e02e4089de4122cd93df5263a95be4d7084e2126493892816e6a5b4ed123cb705bf930c8f67af0fb4514d5769232a9b008a803af225160ce63f675bd4872c4c97b146e5e", + "6234c936e27bf141fc7534bfc0a7eedc657f91308203f1dcbd642855", + "27983d87ca785ef4892c3591ef4a944b1deb125dd58bd351034a6f84", + "e94e05b42d01d0b965ffdd6c3a97a36a771e8ea71003de76c4ecb13f", + "1dc6464ffeefbd7872a081a5926e9fc3e66d123f1784340ba17737e9", + false, + }, + { + "7c00be9123bfa2c4290be1d8bc2942c7f897d9a5b7917e3aabd97ef1aab890f148400a89abd554d19bec9d8ed911ce57b22fbcf6d30ca2115f13ce0a3f569a23bad39ee645f624c49c60dcfc11e7d2be24de9c905596d8f23624d63dc46591d1f740e46f982bfae453f107e80db23545782be23ce43708245896fc54e1ee5c43", + "9f3f037282aaf14d4772edffff331bbdda845c3f65780498cde334f1", + "8308ee5a16e3bcb721b6bc30000a0419bc1aaedd761be7f658334066", + "6381d7804a8808e3c17901e4d283b89449096a8fba993388fa11dc54", + "8e858f6b5b253686a86b757bad23658cda53115ac565abca4e3d9f57", + false, + }, + { + "cffc122a44840dc705bb37130069921be313d8bde0b66201aebc48add028ca131914ef2e705d6bedd19dc6cf9459bbb0f27cdfe3c50483808ffcdaffbeaa5f062e097180f07a40ef4ab6ed03fe07ed6bcfb8afeb42c97eafa2e8a8df469de07317c5e1494c41547478eff4d8c7d9f0f484ad90fedf6e1c35ee68fa73f1691601", + "a03b88a10d930002c7b17ca6af2fd3e88fa000edf787dc594f8d4fd4", + "e0cf7acd6ddc758e64847fe4df9915ebda2f67cdd5ec979aa57421f5", + "387b84dcf37dc343c7d2c5beb82f0bf8bd894b395a7b894565d296c1", + "4adc12ce7d20a89ce3925e10491c731b15ddb3f339610857a21b53b4", + false, + }, + { + "26e0e0cafd85b43d16255908ccfd1f061c680df75aba3081246b337495783052ba06c60f4a486c1591a4048bae11b4d7fec4f161d80bdc9a7b79d23e44433ed625eab280521a37f23dd3e1bdc5c6a6cfaa026f3c45cf703e76dab57add93fe844dd4cda67dc3bddd01f9152579e49df60969b10f09ce9372fdd806b0c7301866", + "9a8983c42f2b5a87c37a00458b5970320d247f0c8a88536440173f7d", + "15e489ec6355351361900299088cfe8359f04fe0cab78dde952be80c", + "929a21baa173d438ec9f28d6a585a2f9abcfc0a4300898668e476dc0", + "59a853f046da8318de77ff43f26fe95a92ee296fa3f7e56ce086c872", + true, + }, + { + "1078eac124f48ae4f807e946971d0de3db3748dd349b14cca5c942560fb25401b2252744f18ad5e455d2d97ed5ae745f55ff509c6c8e64606afe17809affa855c4c4cdcaf6b69ab4846aa5624ed0687541aee6f2224d929685736c6a23906d974d3c257abce1a3fb8db5951b89ecb0cda92b5207d93f6618fd0f893c32cf6a6e", + "d6e55820bb62c2be97650302d59d667a411956138306bd566e5c3c2b", + "631ab0d64eaf28a71b9cbd27a7a88682a2167cee6251c44e3810894f", + "65af72bc7721eb71c2298a0eb4eed3cec96a737cc49125706308b129", + "bd5a987c78e2d51598dbd9c34a9035b0069c580edefdacee17ad892a", + false, + }, + { + "919deb1fdd831c23481dfdb2475dcbe325b04c34f82561ced3d2df0b3d749b36e255c4928973769d46de8b95f162b53cd666cad9ae145e7fcfba97919f703d864efc11eac5f260a5d920d780c52899e5d76f8fe66936ff82130761231f536e6a3d59792f784902c469aa897aabf9a0678f93446610d56d5e0981e4c8a563556b", + "269b455b1024eb92d860a420f143ac1286b8cce43031562ae7664574", + "baeb6ca274a77c44a0247e5eb12ca72bdd9a698b3f3ae69c9f1aaa57", + "cb4ec2160f04613eb0dfe4608486091a25eb12aa4dec1afe91cfb008", + "40b01d8cd06589481574f958b98ca08ade9d2a8fe31024375c01bb40", + false, + }, + { + "6e012361250dacf6166d2dd1aa7be544c3206a9d43464b3fcd90f3f8cf48d08ec099b59ba6fe7d9bdcfaf244120aed1695d8be32d1b1cd6f143982ab945d635fb48a7c76831c0460851a3d62b7209c30cd9c2abdbe3d2a5282a9fcde1a6f418dd23c409bc351896b9b34d7d3a1a63bbaf3d677e612d4a80fa14829386a64b33f", + "6d2d695efc6b43b13c14111f2109608f1020e3e03b5e21cfdbc82fcd", + "26a4859296b7e360b69cf40be7bd97ceaffa3d07743c8489fc47ca1b", + "9a8cb5f2fdc288b7183c5b32d8e546fc2ed1ca4285eeae00c8b572ad", + "8c623f357b5d0057b10cdb1a1593dab57cda7bdec9cf868157a79b97", + true, + }, + { + "bf6bd7356a52b234fe24d25557200971fc803836f6fec3cade9642b13a8e7af10ab48b749de76aada9d8927f9b12f75a2c383ca7358e2566c4bb4f156fce1fd4e87ef8c8d2b6b1bdd351460feb22cdca0437ac10ca5e0abbbce9834483af20e4835386f8b1c96daaa41554ceee56730aac04f23a5c765812efa746051f396566", + "14250131b2599939cf2d6bc491be80ddfe7ad9de644387ee67de2d40", + "b5dc473b5d014cd504022043c475d3f93c319a8bdcb7262d9e741803", + "4f21642f2201278a95339a80f75cc91f8321fcb3c9462562f6cbf145", + "452a5f816ea1f75dee4fd514fa91a0d6a43622981966c59a1b371ff8", + false, + }, + { + "0eb7f4032f90f0bd3cf9473d6d9525d264d14c031a10acd31a053443ed5fe919d5ac35e0be77813071b4062f0b5fdf58ad5f637b76b0b305aec18f82441b6e607b44cdf6e0e3c7c57f24e6fd565e39430af4a6b1d979821ed0175fa03e3125506847654d7e1ae904ce1190ae38dc5919e257bdac2db142a6e7cd4da6c2e83770", + "d1f342b7790a1667370a1840255ac5bbbdc66f0bc00ae977d99260ac", + "76416cabae2de9a1000b4646338b774baabfa3db4673790771220cdb", + "bc85e3fc143d19a7271b2f9e1c04b86146073f3fab4dda1c3b1f35ca", + "9a5c70ede3c48d5f43307a0c2a4871934424a3303b815df4bb0f128e", + false, + }, + { + "5cc25348a05d85e56d4b03cec450128727bc537c66ec3a9fb613c151033b5e86878632249cba83adcefc6c1e35dcd31702929c3b57871cda5c18d1cf8f9650a25b917efaed56032e43b6fc398509f0d2997306d8f26675f3a8683b79ce17128e006aa0903b39eeb2f1001be65de0520115e6f919de902b32c38d691a69c58c92", + "7e49a7abf16a792e4c7bbc4d251820a2abd22d9f2fc252a7bf59c9a6", + "44236a8fb4791c228c26637c28ae59503a2f450d4cfb0dc42aa843b9", + "084461b4050285a1a85b2113be76a17878d849e6bc489f4d84f15cd8", + "079b5bddcc4d45de8dbdfd39f69817c7e5afa454a894d03ee1eaaac3", + false, + }, + { + "1951533ce33afb58935e39e363d8497a8dd0442018fd96dff167b3b23d7206a3ee182a3194765df4768a3284e23b8696c199b4686e670d60c9d782f08794a4bccc05cffffbd1a12acd9eb1cfa01f7ebe124da66ecff4599ea7720c3be4bb7285daa1a86ebf53b042bd23208d468c1b3aa87381f8e1ad63e2b4c2ba5efcf05845", + "31945d12ebaf4d81f02be2b1768ed80784bf35cf5e2ff53438c11493", + "a62bebffac987e3b9d3ec451eb64c462cdf7b4aa0b1bbb131ceaa0a4", + "bc3c32b19e42b710bca5c6aaa128564da3ddb2726b25f33603d2af3c", + "ed1a719cc0c507edc5239d76fe50e2306c145ad252bd481da04180c0", + false, + }, +} + +func TestVectors(t *testing.T) { + sha := sha1.New() + + for i, test := range testVectors { + pub := PublicKey{ + Curve: elliptic.P224(), + X: fromHex(test.Qx), + Y: fromHex(test.Qy), + } + msg, _ := hex.DecodeString(test.msg) + sha.Reset() + sha.Write(msg) + hashed := sha.Sum() + r := fromHex(test.r) + s := fromHex(test.s) + if Verify(&pub, hashed, r, s) != test.ok { + t.Errorf("%d: bad result", i) + } + if testing.Short() { + break + } + } +} diff --git a/src/pkg/crypto/elliptic/Makefile b/src/pkg/crypto/elliptic/Makefile new file mode 100644 index 000000000..4db5d7de5 --- /dev/null +++ b/src/pkg/crypto/elliptic/Makefile @@ -0,0 +1,11 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/elliptic +GOFILES=\ + elliptic.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/elliptic/elliptic.go b/src/pkg/crypto/elliptic/elliptic.go new file mode 100644 index 000000000..41835f1a9 --- /dev/null +++ b/src/pkg/crypto/elliptic/elliptic.go @@ -0,0 +1,381 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elliptic implements several standard elliptic curves over prime +// fields. +package elliptic + +// This package operates, internally, on Jacobian coordinates. For a given +// (x, y) position on the curve, the Jacobian coordinates are (x1, y1, z1) +// where x = x1/z1² and y = y1/z1³. The greatest speedups come when the whole +// calculation can be performed within the transform (as in ScalarMult and +// ScalarBaseMult). But even for Add and Double, it's faster to apply and +// reverse the transform than to operate in affine coordinates. + +import ( + "big" + "io" + "os" + "sync" +) + +// A Curve represents a short-form Weierstrass curve with a=-3. +// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw.html +type Curve struct { + P *big.Int // the order of the underlying field + N *big.Int // the order of the base point + B *big.Int // the constant of the curve equation + Gx, Gy *big.Int // (x,y) of the base point + BitSize int // the size of the underlying field +} + +// IsOnCurve returns true if the given (x,y) lies on the curve. +func (curve *Curve) IsOnCurve(x, y *big.Int) bool { + // y² = x³ - 3x + b + y2 := new(big.Int).Mul(y, y) + y2.Mod(y2, curve.P) + + x3 := new(big.Int).Mul(x, x) + x3.Mul(x3, x) + + threeX := new(big.Int).Lsh(x, 1) + threeX.Add(threeX, x) + + x3.Sub(x3, threeX) + x3.Add(x3, curve.B) + x3.Mod(x3, curve.P) + + return x3.Cmp(y2) == 0 +} + +// affineFromJacobian reverses the Jacobian transform. See the comment at the +// top of the file. +func (curve *Curve) affineFromJacobian(x, y, z *big.Int) (xOut, yOut *big.Int) { + zinv := new(big.Int).ModInverse(z, curve.P) + zinvsq := new(big.Int).Mul(zinv, zinv) + + xOut = new(big.Int).Mul(x, zinvsq) + xOut.Mod(xOut, curve.P) + zinvsq.Mul(zinvsq, zinv) + yOut = new(big.Int).Mul(y, zinvsq) + yOut.Mod(yOut, curve.P) + return +} + +// Add returns the sum of (x1,y1) and (x2,y2) +func (curve *Curve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) { + z := new(big.Int).SetInt64(1) + return curve.affineFromJacobian(curve.addJacobian(x1, y1, z, x2, y2, z)) +} + +// addJacobian takes two points in Jacobian coordinates, (x1, y1, z1) and +// (x2, y2, z2) and returns their sum, also in Jacobian form. +func (curve *Curve) addJacobian(x1, y1, z1, x2, y2, z2 *big.Int) (*big.Int, *big.Int, *big.Int) { + // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl + z1z1 := new(big.Int).Mul(z1, z1) + z1z1.Mod(z1z1, curve.P) + z2z2 := new(big.Int).Mul(z2, z2) + z2z2.Mod(z2z2, curve.P) + + u1 := new(big.Int).Mul(x1, z2z2) + u1.Mod(u1, curve.P) + u2 := new(big.Int).Mul(x2, z1z1) + u2.Mod(u2, curve.P) + h := new(big.Int).Sub(u2, u1) + if h.Sign() == -1 { + h.Add(h, curve.P) + } + i := new(big.Int).Lsh(h, 1) + i.Mul(i, i) + j := new(big.Int).Mul(h, i) + + s1 := new(big.Int).Mul(y1, z2) + s1.Mul(s1, z2z2) + s1.Mod(s1, curve.P) + s2 := new(big.Int).Mul(y2, z1) + s2.Mul(s2, z1z1) + s2.Mod(s2, curve.P) + r := new(big.Int).Sub(s2, s1) + if r.Sign() == -1 { + r.Add(r, curve.P) + } + r.Lsh(r, 1) + v := new(big.Int).Mul(u1, i) + + x3 := new(big.Int).Set(r) + x3.Mul(x3, x3) + x3.Sub(x3, j) + x3.Sub(x3, v) + x3.Sub(x3, v) + x3.Mod(x3, curve.P) + + y3 := new(big.Int).Set(r) + v.Sub(v, x3) + y3.Mul(y3, v) + s1.Mul(s1, j) + s1.Lsh(s1, 1) + y3.Sub(y3, s1) + y3.Mod(y3, curve.P) + + z3 := new(big.Int).Add(z1, z2) + z3.Mul(z3, z3) + z3.Sub(z3, z1z1) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Sub(z3, z2z2) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Mul(z3, h) + z3.Mod(z3, curve.P) + + return x3, y3, z3 +} + +// Double returns 2*(x,y) +func (curve *Curve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) { + z1 := new(big.Int).SetInt64(1) + return curve.affineFromJacobian(curve.doubleJacobian(x1, y1, z1)) +} + +// doubleJacobian takes a point in Jacobian coordinates, (x, y, z), and +// returns its double, also in Jacobian form. +func (curve *Curve) doubleJacobian(x, y, z *big.Int) (*big.Int, *big.Int, *big.Int) { + // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b + delta := new(big.Int).Mul(z, z) + delta.Mod(delta, curve.P) + gamma := new(big.Int).Mul(y, y) + gamma.Mod(gamma, curve.P) + alpha := new(big.Int).Sub(x, delta) + if alpha.Sign() == -1 { + alpha.Add(alpha, curve.P) + } + alpha2 := new(big.Int).Add(x, delta) + alpha.Mul(alpha, alpha2) + alpha2.Set(alpha) + alpha.Lsh(alpha, 1) + alpha.Add(alpha, alpha2) + + beta := alpha2.Mul(x, gamma) + + x3 := new(big.Int).Mul(alpha, alpha) + beta8 := new(big.Int).Lsh(beta, 3) + x3.Sub(x3, beta8) + for x3.Sign() == -1 { + x3.Add(x3, curve.P) + } + x3.Mod(x3, curve.P) + + z3 := new(big.Int).Add(y, z) + z3.Mul(z3, z3) + z3.Sub(z3, gamma) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Sub(z3, delta) + if z3.Sign() == -1 { + z3.Add(z3, curve.P) + } + z3.Mod(z3, curve.P) + + beta.Lsh(beta, 2) + beta.Sub(beta, x3) + if beta.Sign() == -1 { + beta.Add(beta, curve.P) + } + y3 := alpha.Mul(alpha, beta) + + gamma.Mul(gamma, gamma) + gamma.Lsh(gamma, 3) + gamma.Mod(gamma, curve.P) + + y3.Sub(y3, gamma) + if y3.Sign() == -1 { + y3.Add(y3, curve.P) + } + y3.Mod(y3, curve.P) + + return x3, y3, z3 +} + +// ScalarMult returns k*(Bx,By) where k is a number in big-endian form. +func (curve *Curve) ScalarMult(Bx, By *big.Int, k []byte) (*big.Int, *big.Int) { + // We have a slight problem in that the identity of the group (the + // point at infinity) cannot be represented in (x, y) form on a finite + // machine. Thus the standard add/double algorithm has to be tweaked + // slightly: our initial state is not the identity, but x, and we + // ignore the first true bit in |k|. If we don't find any true bits in + // |k|, then we return nil, nil, because we cannot return the identity + // element. + + Bz := new(big.Int).SetInt64(1) + x := Bx + y := By + z := Bz + + seenFirstTrue := false + for _, byte := range k { + for bitNum := 0; bitNum < 8; bitNum++ { + if seenFirstTrue { + x, y, z = curve.doubleJacobian(x, y, z) + } + if byte&0x80 == 0x80 { + if !seenFirstTrue { + seenFirstTrue = true + } else { + x, y, z = curve.addJacobian(Bx, By, Bz, x, y, z) + } + } + byte <<= 1 + } + } + + if !seenFirstTrue { + return nil, nil + } + + return curve.affineFromJacobian(x, y, z) +} + +// ScalarBaseMult returns k*G, where G is the base point of the group and k is +// an integer in big-endian form. +func (curve *Curve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) { + return curve.ScalarMult(curve.Gx, curve.Gy, k) +} + +var mask = []byte{0xff, 0x1, 0x3, 0x7, 0xf, 0x1f, 0x3f, 0x7f} + +// GenerateKey returns a public/private key pair. The private key is generated +// using the given reader, which must return random data. +func (curve *Curve) GenerateKey(rand io.Reader) (priv []byte, x, y *big.Int, err os.Error) { + byteLen := (curve.BitSize + 7) >> 3 + priv = make([]byte, byteLen) + + for x == nil { + _, err = io.ReadFull(rand, priv) + if err != nil { + return + } + // We have to mask off any excess bits in the case that the size of the + // underlying field is not a whole number of bytes. + priv[0] &= mask[curve.BitSize%8] + // This is because, in tests, rand will return all zeros and we don't + // want to get the point at infinity and loop forever. + priv[1] ^= 0x42 + x, y = curve.ScalarBaseMult(priv) + } + return +} + +// Marshal converts a point into the form specified in section 4.3.6 of ANSI +// X9.62. +func (curve *Curve) Marshal(x, y *big.Int) []byte { + byteLen := (curve.BitSize + 7) >> 3 + + ret := make([]byte, 1+2*byteLen) + ret[0] = 4 // uncompressed point + + xBytes := x.Bytes() + copy(ret[1+byteLen-len(xBytes):], xBytes) + yBytes := y.Bytes() + copy(ret[1+2*byteLen-len(yBytes):], yBytes) + return ret +} + +// Unmarshal converts a point, serialized by Marshal, into an x, y pair. On +// error, x = nil. +func (curve *Curve) Unmarshal(data []byte) (x, y *big.Int) { + byteLen := (curve.BitSize + 7) >> 3 + if len(data) != 1+2*byteLen { + return + } + if data[0] != 4 { // uncompressed form + return + } + x = new(big.Int).SetBytes(data[1 : 1+byteLen]) + y = new(big.Int).SetBytes(data[1+byteLen:]) + return +} + +var initonce sync.Once +var p224 *Curve +var p256 *Curve +var p384 *Curve +var p521 *Curve + +func initAll() { + initP224() + initP256() + initP384() + initP521() +} + +func initP224() { + // See FIPS 186-3, section D.2.2 + p224 = new(Curve) + p224.P, _ = new(big.Int).SetString("26959946667150639794667015087019630673557916260026308143510066298881", 10) + p224.N, _ = new(big.Int).SetString("26959946667150639794667015087019625940457807714424391721682722368061", 10) + p224.B, _ = new(big.Int).SetString("b4050a850c04b3abf54132565044b0b7d7bfd8ba270b39432355ffb4", 16) + p224.Gx, _ = new(big.Int).SetString("b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", 16) + p224.Gy, _ = new(big.Int).SetString("bd376388b5f723fb4c22dfe6cd4375a05a07476444d5819985007e34", 16) + p224.BitSize = 224 +} + +func initP256() { + // See FIPS 186-3, section D.2.3 + p256 = new(Curve) + p256.P, _ = new(big.Int).SetString("115792089210356248762697446949407573530086143415290314195533631308867097853951", 10) + p256.N, _ = new(big.Int).SetString("115792089210356248762697446949407573529996955224135760342422259061068512044369", 10) + p256.B, _ = new(big.Int).SetString("5ac635d8aa3a93e7b3ebbd55769886bc651d06b0cc53b0f63bce3c3e27d2604b", 16) + p256.Gx, _ = new(big.Int).SetString("6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296", 16) + p256.Gy, _ = new(big.Int).SetString("4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5", 16) + p256.BitSize = 256 +} + +func initP384() { + // See FIPS 186-3, section D.2.4 + p384 = new(Curve) + p384.P, _ = new(big.Int).SetString("39402006196394479212279040100143613805079739270465446667948293404245721771496870329047266088258938001861606973112319", 10) + p384.N, _ = new(big.Int).SetString("39402006196394479212279040100143613805079739270465446667946905279627659399113263569398956308152294913554433653942643", 10) + p384.B, _ = new(big.Int).SetString("b3312fa7e23ee7e4988e056be3f82d19181d9c6efe8141120314088f5013875ac656398d8a2ed19d2a85c8edd3ec2aef", 16) + p384.Gx, _ = new(big.Int).SetString("aa87ca22be8b05378eb1c71ef320ad746e1d3b628ba79b9859f741e082542a385502f25dbf55296c3a545e3872760ab7", 16) + p384.Gy, _ = new(big.Int).SetString("3617de4a96262c6f5d9e98bf9292dc29f8f41dbd289a147ce9da3113b5f0b8c00a60b1ce1d7e819d7a431d7c90ea0e5f", 16) + p384.BitSize = 384 +} + +func initP521() { + // See FIPS 186-3, section D.2.5 + p521 = new(Curve) + p521.P, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397656052122559640661454554977296311391480858037121987999716643812574028291115057151", 10) + p521.N, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397655394245057746333217197532963996371363321113864768612440380340372808892707005449", 10) + p521.B, _ = new(big.Int).SetString("051953eb9618e1c9a1f929a21a0b68540eea2da725b99b315f3b8b489918ef109e156193951ec7e937b1652c0bd3bb1bf073573df883d2c34f1ef451fd46b503f00", 16) + p521.Gx, _ = new(big.Int).SetString("c6858e06b70404e9cd9e3ecb662395b4429c648139053fb521f828af606b4d3dbaa14b5e77efe75928fe1dc127a2ffa8de3348b3c1856a429bf97e7e31c2e5bd66", 16) + p521.Gy, _ = new(big.Int).SetString("11839296a789a3bc0045c8a5fb42c7d1bd998f54449579b446817afbd17273e662c97ee72995ef42640c550b9013fad0761353c7086a272c24088be94769fd16650", 16) + p521.BitSize = 521 +} + +// P224 returns a Curve which implements P-224 (see FIPS 186-3, section D.2.2) +func P224() *Curve { + initonce.Do(initAll) + return p224 +} + +// P256 returns a Curve which implements P-256 (see FIPS 186-3, section D.2.3) +func P256() *Curve { + initonce.Do(initAll) + return p256 +} + +// P384 returns a Curve which implements P-384 (see FIPS 186-3, section D.2.4) +func P384() *Curve { + initonce.Do(initAll) + return p384 +} + +// P256 returns a Curve which implements P-521 (see FIPS 186-3, section D.2.5) +func P521() *Curve { + initonce.Do(initAll) + return p521 +} diff --git a/src/pkg/crypto/elliptic/elliptic_test.go b/src/pkg/crypto/elliptic/elliptic_test.go new file mode 100644 index 000000000..b7e7f035f --- /dev/null +++ b/src/pkg/crypto/elliptic/elliptic_test.go @@ -0,0 +1,334 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elliptic + +import ( + "big" + "crypto/rand" + "fmt" + "testing" +) + +func TestOnCurve(t *testing.T) { + p224 := P224() + if !p224.IsOnCurve(p224.Gx, p224.Gy) { + t.Errorf("FAIL") + } +} + +type baseMultTest struct { + k string + x, y string +} + +var p224BaseMultTests = []baseMultTest{ + { + "1", + "b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", + "bd376388b5f723fb4c22dfe6cd4375a05a07476444d5819985007e34", + }, + { + "2", + "706a46dc76dcb76798e60e6d89474788d16dc18032d268fd1a704fa6", + "1c2b76a7bc25e7702a704fa986892849fca629487acf3709d2e4e8bb", + }, + { + "3", + "df1b1d66a551d0d31eff822558b9d2cc75c2180279fe0d08fd896d04", + "a3f7f03cadd0be444c0aa56830130ddf77d317344e1af3591981a925", + }, + { + "4", + "ae99feebb5d26945b54892092a8aee02912930fa41cd114e40447301", + "482580a0ec5bc47e88bc8c378632cd196cb3fa058a7114eb03054c9", + }, + { + "5", + "31c49ae75bce7807cdff22055d94ee9021fedbb5ab51c57526f011aa", + "27e8bff1745635ec5ba0c9f1c2ede15414c6507d29ffe37e790a079b", + }, + { + "6", + "1f2483f82572251fca975fea40db821df8ad82a3c002ee6c57112408", + "89faf0ccb750d99b553c574fad7ecfb0438586eb3952af5b4b153c7e", + }, + { + "7", + "db2f6be630e246a5cf7d99b85194b123d487e2d466b94b24a03c3e28", + "f3a30085497f2f611ee2517b163ef8c53b715d18bb4e4808d02b963", + }, + { + "8", + "858e6f9cc6c12c31f5df124aa77767b05c8bc021bd683d2b55571550", + "46dcd3ea5c43898c5c5fc4fdac7db39c2f02ebee4e3541d1e78047a", + }, + { + "9", + "2fdcccfee720a77ef6cb3bfbb447f9383117e3daa4a07e36ed15f78d", + "371732e4f41bf4f7883035e6a79fcedc0e196eb07b48171697517463", + }, + { + "10", + "aea9e17a306517eb89152aa7096d2c381ec813c51aa880e7bee2c0fd", + "39bb30eab337e0a521b6cba1abe4b2b3a3e524c14a3fe3eb116b655f", + }, + { + "11", + "ef53b6294aca431f0f3c22dc82eb9050324f1d88d377e716448e507c", + "20b510004092e96636cfb7e32efded8265c266dfb754fa6d6491a6da", + }, + { + "12", + "6e31ee1dc137f81b056752e4deab1443a481033e9b4c93a3044f4f7a", + "207dddf0385bfdeab6e9acda8da06b3bbef224a93ab1e9e036109d13", + }, + { + "13", + "34e8e17a430e43289793c383fac9774247b40e9ebd3366981fcfaeca", + "252819f71c7fb7fbcb159be337d37d3336d7feb963724fdfb0ecb767", + }, + { + "14", + "a53640c83dc208603ded83e4ecf758f24c357d7cf48088b2ce01e9fa", + "d5814cd724199c4a5b974a43685fbf5b8bac69459c9469bc8f23ccaf", + }, + { + "15", + "baa4d8635511a7d288aebeedd12ce529ff102c91f97f867e21916bf9", + "979a5f4759f80f4fb4ec2e34f5566d595680a11735e7b61046127989", + }, + { + "16", + "b6ec4fe1777382404ef679997ba8d1cc5cd8e85349259f590c4c66d", + "3399d464345906b11b00e363ef429221f2ec720d2f665d7dead5b482", + }, + { + "17", + "b8357c3a6ceef288310e17b8bfeff9200846ca8c1942497c484403bc", + "ff149efa6606a6bd20ef7d1b06bd92f6904639dce5174db6cc554a26", + }, + { + "18", + "c9ff61b040874c0568479216824a15eab1a838a797d189746226e4cc", + "ea98d60e5ffc9b8fcf999fab1df7e7ef7084f20ddb61bb045a6ce002", + }, + { + "19", + "a1e81c04f30ce201c7c9ace785ed44cc33b455a022f2acdbc6cae83c", + "dcf1f6c3db09c70acc25391d492fe25b4a180babd6cea356c04719cd", + }, + { + "20", + "fcc7f2b45df1cd5a3c0c0731ca47a8af75cfb0347e8354eefe782455", + "d5d7110274cba7cdee90e1a8b0d394c376a5573db6be0bf2747f530", + }, + { + "112233445566778899", + "61f077c6f62ed802dad7c2f38f5c67f2cc453601e61bd076bb46179e", + "2272f9e9f5933e70388ee652513443b5e289dd135dcc0d0299b225e4", + }, + { + "112233445566778899112233445566778899", + "29895f0af496bfc62b6ef8d8a65c88c613949b03668aab4f0429e35", + "3ea6e53f9a841f2019ec24bde1a75677aa9b5902e61081c01064de93", + }, + { + "6950511619965839450988900688150712778015737983940691968051900319680", + "ab689930bcae4a4aa5f5cb085e823e8ae30fd365eb1da4aba9cf0379", + "3345a121bbd233548af0d210654eb40bab788a03666419be6fbd34e7", + }, + { + "13479972933410060327035789020509431695094902435494295338570602119423", + "bdb6a8817c1f89da1c2f3dd8e97feb4494f2ed302a4ce2bc7f5f4025", + "4c7020d57c00411889462d77a5438bb4e97d177700bf7243a07f1680", + }, + { + "13479971751745682581351455311314208093898607229429740618390390702079", + "d58b61aa41c32dd5eba462647dba75c5d67c83606c0af2bd928446a9", + "d24ba6a837be0460dd107ae77725696d211446c5609b4595976b16bd", + }, + { + "13479972931865328106486971546324465392952975980343228160962702868479", + "dc9fa77978a005510980e929a1485f63716df695d7a0c18bb518df03", + "ede2b016f2ddffc2a8c015b134928275ce09e5661b7ab14ce0d1d403", + }, + { + "11795773708834916026404142434151065506931607341523388140225443265536", + "499d8b2829cfb879c901f7d85d357045edab55028824d0f05ba279ba", + "bf929537b06e4015919639d94f57838fa33fc3d952598dcdbb44d638", + }, + { + "784254593043826236572847595991346435467177662189391577090", + "8246c999137186632c5f9eddf3b1b0e1764c5e8bd0e0d8a554b9cb77", + "e80ed8660bc1cb17ac7d845be40a7a022d3306f116ae9f81fea65947", + }, + { + "13479767645505654746623887797783387853576174193480695826442858012671", + "6670c20afcceaea672c97f75e2e9dd5c8460e54bb38538ebb4bd30eb", + "f280d8008d07a4caf54271f993527d46ff3ff46fd1190a3f1faa4f74", + }, + { + "205688069665150753842126177372015544874550518966168735589597183", + "eca934247425cfd949b795cb5ce1eff401550386e28d1a4c5a8eb", + "d4c01040dba19628931bc8855370317c722cbd9ca6156985f1c2e9ce", + }, + { + "13479966930919337728895168462090683249159702977113823384618282123295", + "ef353bf5c73cd551b96d596fbc9a67f16d61dd9fe56af19de1fba9cd", + "21771b9cdce3e8430c09b3838be70b48c21e15bc09ee1f2d7945b91f", + }, + { + "50210731791415612487756441341851895584393717453129007497216", + "4036052a3091eb481046ad3289c95d3ac905ca0023de2c03ecd451cf", + "d768165a38a2b96f812586a9d59d4136035d9c853a5bf2e1c86a4993", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368041", + "fcc7f2b45df1cd5a3c0c0731ca47a8af75cfb0347e8354eefe782455", + "f2a28eefd8b345832116f1e574f2c6b2c895aa8c24941f40d8b80ad1", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368042", + "a1e81c04f30ce201c7c9ace785ed44cc33b455a022f2acdbc6cae83c", + "230e093c24f638f533dac6e2b6d01da3b5e7f45429315ca93fb8e634", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368043", + "c9ff61b040874c0568479216824a15eab1a838a797d189746226e4cc", + "156729f1a003647030666054e208180f8f7b0df2249e44fba5931fff", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368044", + "b8357c3a6ceef288310e17b8bfeff9200846ca8c1942497c484403bc", + "eb610599f95942df1082e4f9426d086fb9c6231ae8b24933aab5db", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368045", + "b6ec4fe1777382404ef679997ba8d1cc5cd8e85349259f590c4c66d", + "cc662b9bcba6f94ee4ff1c9c10bd6ddd0d138df2d099a282152a4b7f", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368046", + "baa4d8635511a7d288aebeedd12ce529ff102c91f97f867e21916bf9", + "6865a0b8a607f0b04b13d1cb0aa992a5a97f5ee8ca1849efb9ed8678", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368047", + "a53640c83dc208603ded83e4ecf758f24c357d7cf48088b2ce01e9fa", + "2a7eb328dbe663b5a468b5bc97a040a3745396ba636b964370dc3352", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368048", + "34e8e17a430e43289793c383fac9774247b40e9ebd3366981fcfaeca", + "dad7e608e380480434ea641cc82c82cbc92801469c8db0204f13489a", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368049", + "6e31ee1dc137f81b056752e4deab1443a481033e9b4c93a3044f4f7a", + "df82220fc7a4021549165325725f94c3410ddb56c54e161fc9ef62ee", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368050", + "ef53b6294aca431f0f3c22dc82eb9050324f1d88d377e716448e507c", + "df4aefffbf6d1699c930481cd102127c9a3d992048ab05929b6e5927", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368051", + "aea9e17a306517eb89152aa7096d2c381ec813c51aa880e7bee2c0fd", + "c644cf154cc81f5ade49345e541b4d4b5c1adb3eb5c01c14ee949aa2", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368052", + "2fdcccfee720a77ef6cb3bfbb447f9383117e3daa4a07e36ed15f78d", + "c8e8cd1b0be40b0877cfca1958603122f1e6914f84b7e8e968ae8b9e", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368053", + "858e6f9cc6c12c31f5df124aa77767b05c8bc021bd683d2b55571550", + "fb9232c15a3bc7673a3a03b0253824c53d0fd1411b1cabe2e187fb87", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368054", + "db2f6be630e246a5cf7d99b85194b123d487e2d466b94b24a03c3e28", + "f0c5cff7ab680d09ee11dae84e9c1072ac48ea2e744b1b7f72fd469e", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368055", + "1f2483f82572251fca975fea40db821df8ad82a3c002ee6c57112408", + "76050f3348af2664aac3a8b05281304ebc7a7914c6ad50a4b4eac383", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368056", + "31c49ae75bce7807cdff22055d94ee9021fedbb5ab51c57526f011aa", + "d817400e8ba9ca13a45f360e3d121eaaeb39af82d6001c8186f5f866", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368057", + "ae99feebb5d26945b54892092a8aee02912930fa41cd114e40447301", + "fb7da7f5f13a43b81774373c879cd32d6934c05fa758eeb14fcfab38", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368058", + "df1b1d66a551d0d31eff822558b9d2cc75c2180279fe0d08fd896d04", + "5c080fc3522f41bbb3f55a97cfecf21f882ce8cbb1e50ca6e67e56dc", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368059", + "706a46dc76dcb76798e60e6d89474788d16dc18032d268fd1a704fa6", + "e3d4895843da188fd58fb0567976d7b50359d6b78530c8f62d1b1746", + }, + { + "26959946667150639794667015087019625940457807714424391721682722368060", + "b70e0cbd6bb4bf7f321390b94a03c1d356c21122343280d6115c1d21", + "42c89c774a08dc04b3dd201932bc8a5ea5f8b89bbb2a7e667aff81cd", + }, +} + +func TestBaseMult(t *testing.T) { + p224 := P224() + for i, e := range p224BaseMultTests { + k, ok := new(big.Int).SetString(e.k, 10) + if !ok { + t.Errorf("%d: bad value for k: %s", i, e.k) + } + x, y := p224.ScalarBaseMult(k.Bytes()) + if fmt.Sprintf("%x", x) != e.x || fmt.Sprintf("%x", y) != e.y { + t.Errorf("%d: bad output for k=%s: got (%x, %s), want (%s, %s)", i, e.k, x, y, e.x, e.y) + } + if testing.Short() && i > 5 { + break + } + } +} + +func BenchmarkBaseMult(b *testing.B) { + b.ResetTimer() + p224 := P224() + e := p224BaseMultTests[25] + k, _ := new(big.Int).SetString(e.k, 10) + b.StartTimer() + for i := 0; i < b.N; i++ { + p224.ScalarBaseMult(k.Bytes()) + } +} + +func TestMarshal(t *testing.T) { + p224 := P224() + _, x, y, err := p224.GenerateKey(rand.Reader) + if err != nil { + t.Error(err) + return + } + serialized := p224.Marshal(x, y) + xx, yy := p224.Unmarshal(serialized) + if xx == nil { + t.Error("failed to unmarshal") + return + } + if xx.Cmp(x) != 0 || yy.Cmp(y) != 0 { + t.Error("unmarshal returned different values") + return + } +} diff --git a/src/pkg/crypto/hmac/Makefile b/src/pkg/crypto/hmac/Makefile new file mode 100644 index 000000000..cc69abf60 --- /dev/null +++ b/src/pkg/crypto/hmac/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/hmac +GOFILES=\ + hmac.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/hmac/hmac.go b/src/pkg/crypto/hmac/hmac.go new file mode 100644 index 000000000..04ec86e9a --- /dev/null +++ b/src/pkg/crypto/hmac/hmac.go @@ -0,0 +1,100 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package hmac implements the Keyed-Hash Message Authentication Code (HMAC) as +// defined in U.S. Federal Information Processing Standards Publication 198. +// An HMAC is a cryptographic hash that uses a key to sign a message. +// The receiver verifies the hash by recomputing it using the same key. +package hmac + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "hash" + "os" +) + +// FIPS 198: +// http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + +// key is zero padded to 64 bytes +// ipad = 0x36 byte repeated to 64 bytes +// opad = 0x5c byte repeated to 64 bytes +// hmac = H([key ^ opad] H([key ^ ipad] text)) + +const ( + // NOTE(rsc): This constant is actually the + // underlying hash function's block size. + // HMAC is only conventionally used with + // MD5 and SHA1, and both use 64-byte blocks. + // The hash.Hash interface doesn't provide a + // way to find out the block size. + padSize = 64 +) + +type hmac struct { + size int + key, tmp []byte + outer, inner hash.Hash +} + +func (h *hmac) tmpPad(xor byte) { + for i, k := range h.key { + h.tmp[i] = xor ^ k + } + for i := len(h.key); i < padSize; i++ { + h.tmp[i] = xor + } +} + +func (h *hmac) Sum() []byte { + sum := h.inner.Sum() + h.tmpPad(0x5c) + for i, b := range sum { + h.tmp[padSize+i] = b + } + h.outer.Reset() + h.outer.Write(h.tmp) + return h.outer.Sum() +} + +func (h *hmac) Write(p []byte) (n int, err os.Error) { + return h.inner.Write(p) +} + +func (h *hmac) Size() int { return h.size } + +func (h *hmac) Reset() { + h.inner.Reset() + h.tmpPad(0x36) + h.inner.Write(h.tmp[0:padSize]) +} + +// New returns a new HMAC hash using the given hash generator and key. +func New(h func() hash.Hash, key []byte) hash.Hash { + hm := new(hmac) + hm.outer = h() + hm.inner = h() + hm.size = hm.inner.Size() + hm.tmp = make([]byte, padSize+hm.size) + if len(key) > padSize { + // If key is too big, hash it. + hm.outer.Write(key) + key = hm.outer.Sum() + } + hm.key = make([]byte, len(key)) + copy(hm.key, key) + hm.Reset() + return hm +} + +// NewMD5 returns a new HMAC-MD5 hash using the given key. +func NewMD5(key []byte) hash.Hash { return New(md5.New, key) } + +// NewSHA1 returns a new HMAC-SHA1 hash using the given key. +func NewSHA1(key []byte) hash.Hash { return New(sha1.New, key) } + +// NewSHA256 returns a new HMAC-SHA256 hash using the given key. +func NewSHA256(key []byte) hash.Hash { return New(sha256.New, key) } diff --git a/src/pkg/crypto/hmac/hmac_test.go b/src/pkg/crypto/hmac/hmac_test.go new file mode 100644 index 000000000..bcae63b8a --- /dev/null +++ b/src/pkg/crypto/hmac/hmac_test.go @@ -0,0 +1,205 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hmac + +import ( + "hash" + "fmt" + "testing" +) + +type hmacTest struct { + hash func([]byte) hash.Hash + key []byte + in []byte + out string +} + +var hmacTests = []hmacTest{ + // Tests from US FIPS 198 + // http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + { + NewSHA1, + []byte{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + }, + []byte("Sample #1"), + "4f4ca3d5d68ba7cc0a1208c9c61e9c5da0403c0a", + }, + { + NewSHA1, + []byte{ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, + }, + []byte("Sample #2"), + "0922d3405faa3d194f82a45830737d5cc6c75d24", + }, + { + NewSHA1, + []byte{ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, + }, + []byte("Sample #3"), + "bcf41eab8bb2d802f3d05caf7cb092ecf8d1a3aa", + }, + + // Test from Plan 9. + { + NewMD5, + []byte("Jefe"), + []byte("what do ya want for nothing?"), + "750c783e6ab0b503eaa86e310a5db738", + }, + + // Tests from RFC 4231 + { + NewSHA256, + []byte{ + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, + }, + []byte("Hi There"), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", + }, + { + NewSHA256, + []byte("Jefe"), + []byte("what do ya want for nothing?"), + "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, + }, + []byte{ + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, + }, + "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", + }, + { + NewSHA256, + []byte{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, + }, + []byte{ + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, + 0xcd, 0xcd, + }, + "82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, + }, + []byte("Test Using Larger Than Block-Size Key - Hash Key First"), + "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", + }, + { + NewSHA256, + []byte{ + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, + }, + []byte("This is a test using a larger than block-size key " + + "and a larger than block-size data. The key needs to " + + "be hashed before being used by the HMAC algorithm."), + "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", + }, +} + +func TestHMAC(t *testing.T) { + for i, tt := range hmacTests { + h := tt.hash(tt.key) + for j := 0; j < 2; j++ { + n, err := h.Write(tt.in) + if n != len(tt.in) || err != nil { + t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err) + continue + } + + // Repetitive Sum() calls should return the same value + for k := 0; k < 2; k++ { + sum := fmt.Sprintf("%x", h.Sum()) + if sum != tt.out { + t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out) + } + } + + // Second iteration: make sure reset works. + h.Reset() + } + } +} diff --git a/src/pkg/crypto/md4/Makefile b/src/pkg/crypto/md4/Makefile new file mode 100644 index 000000000..eef05ab70 --- /dev/null +++ b/src/pkg/crypto/md4/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/md4 +GOFILES=\ + md4.go\ + md4block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/md4/md4.go b/src/pkg/crypto/md4/md4.go new file mode 100644 index 000000000..848d9552d --- /dev/null +++ b/src/pkg/crypto/md4/md4.go @@ -0,0 +1,117 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package md4 implements the MD4 hash algorithm as defined in RFC 1320. +package md4 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.MD4, New) +} + +// The size of an MD4 checksum in bytes. +const Size = 16 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [4]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.s[0] = _Init0 + d.s[1] = _Init1 + d.s[2] = _Init2 + d.s[3] = _Init3 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the MD4 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0, so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 16) + j := 0 + for _, s := range d.s { + p[j+0] = byte(s >> 0) + p[j+1] = byte(s >> 8) + p[j+2] = byte(s >> 16) + p[j+3] = byte(s >> 24) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/md4/md4_test.go b/src/pkg/crypto/md4/md4_test.go new file mode 100644 index 000000000..721bd4cbc --- /dev/null +++ b/src/pkg/crypto/md4/md4_test.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package md4 + +import ( + "fmt" + "io" + "testing" +) + +type md4Test struct { + out string + in string +} + +var golden = []md4Test{ + {"31d6cfe0d16ae931b73c59d7e0c089c0", ""}, + {"bde52cb31de33e46245e05fbdbd6fb24", "a"}, + {"ec388dd78999dfc7cf4632465693b6bf", "ab"}, + {"a448017aaf21d8525fc10ae87aa6729d", "abc"}, + {"41decd8f579255c5200f86a4bb3ba740", "abcd"}, + {"9803f4a34e8eb14f96adba49064a0c41", "abcde"}, + {"804e7f1c2586e50b49ac65db5b645131", "abcdef"}, + {"752f4adfe53d1da0241b5bc216d098fc", "abcdefg"}, + {"ad9daf8d49d81988590a6f0e745d15dd", "abcdefgh"}, + {"1e4e28b05464316b56402b3815ed2dfd", "abcdefghi"}, + {"dc959c6f5d6f9e04e4380777cc964b3d", "abcdefghij"}, + {"1b5701e265778898ef7de5623bbe7cc0", "Discard medicine more than two years old."}, + {"d7f087e090fe7ad4a01cb59dacc9a572", "He who has a shady past knows that nice guys finish last."}, + {"a6f8fd6df617c72837592fc3570595c9", "I wouldn't marry him with a ten foot pole."}, + {"c92a84a9526da8abc240c05d6b1a1ce0", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"f6013160c4dcb00847069fee3bb09803", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"2c3bb64f50b9107ed57640fe94bec09f", "Nepal premier won't resign."}, + {"45b7d8a32c7806f2f7f897332774d6e4", "For every action there is an equal and opposite government program."}, + {"b5b4f9026b175c62d7654bdc3a1cd438", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"caf44e80f2c20ce19b5ba1cab766e7bd", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"191fae6707f496aa54a6bce9f2ecf74d", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"9ddc753e7a4ccee6081cd1b45b23a834", "size: a.out: bad magic"}, + {"8d050f55b1cadb9323474564be08a521", "The major problem is with sendmail. -Mark Horton"}, + {"ad6e2587f74c3e3cc19146f6127fa2e3", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"1d616d60a5fabe85589c3f1566ca7fca", "If the enemy is within range, then so are you."}, + {"aec3326a4f496a2ced65a1963f84577f", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"77b4fd762d6b9245e61c50bf6ebf118b", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"e8f48c726bae5e516f6ddb1a4fe62438", "C is as portable as Stonehedge!!"}, + {"a3a84366e7219e887423b01f9be7166e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"a6b7aa35157e984ef5d9b7f32e5fbb52", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"75661f0545955f8f9abeeb17845f3fd6", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("md4[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/md4/md4block.go b/src/pkg/crypto/md4/md4block.go new file mode 100644 index 000000000..3fed475f3 --- /dev/null +++ b/src/pkg/crypto/md4/md4block.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MD4 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package md4 + +var shift1 = []uint{3, 7, 11, 19} +var shift2 = []uint{3, 5, 9, 13} +var shift3 = []uint{3, 9, 11, 15} + +var xIndex2 = []uint{0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15} +var xIndex3 = []uint{0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15} + +func _Block(dig *digest, p []byte) int { + a := dig.s[0] + b := dig.s[1] + c := dig.s[2] + d := dig.s[3] + n := 0 + var X [16]uint32 + for len(p) >= _Chunk { + aa, bb, cc, dd := a, b, c, d + + j := 0 + for i := 0; i < 16; i++ { + X[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // If this needs to be made faster in the future, + // the usual trick is to unroll each of these + // loops by a factor of 4; that lets you replace + // the shift[] lookups with constants and, + // with suitable variable renaming in each + // unrolled body, delete the a, b, c, d = d, a, b, c + // (or you can let the optimizer do the renaming). + // + // The index variables are uint so that % by a power + // of two can be optimized easily by a compiler. + + // Round 1. + for i := uint(0); i < 16; i++ { + x := i + s := shift1[i%4] + f := ((c ^ d) & b) ^ d + a += f + X[x] + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + // Round 2. + for i := uint(0); i < 16; i++ { + x := xIndex2[i] + s := shift2[i%4] + g := (b & c) | (b & d) | (c & d) + a += g + X[x] + 0x5a827999 + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + // Round 3. + for i := uint(0); i < 16; i++ { + x := xIndex3[i] + s := shift3[i%4] + h := b ^ c ^ d + a += h + X[x] + 0x6ed9eba1 + a = a<>(32-s) + a, b, c, d = d, a, b, c + } + + a += aa + b += bb + c += cc + d += dd + + p = p[_Chunk:] + n += _Chunk + } + + dig.s[0] = a + dig.s[1] = b + dig.s[2] = c + dig.s[3] = d + return n +} diff --git a/src/pkg/crypto/md5/Makefile b/src/pkg/crypto/md5/Makefile new file mode 100644 index 000000000..5cde3e6d6 --- /dev/null +++ b/src/pkg/crypto/md5/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/md5 +GOFILES=\ + md5.go\ + md5block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/md5/md5.go b/src/pkg/crypto/md5/md5.go new file mode 100644 index 000000000..378faa6ec --- /dev/null +++ b/src/pkg/crypto/md5/md5.go @@ -0,0 +1,117 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package md5 implements the MD5 hash algorithm as defined in RFC 1321. +package md5 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.MD5, New) +} + +// The size of an MD5 checksum in bytes. +const Size = 16 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [4]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.s[0] = _Init0 + d.s[1] = _Init1 + d.s[2] = _Init2 + d.s[3] = _Init3 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the MD5 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 16) + j := 0 + for _, s := range d.s { + p[j+0] = byte(s >> 0) + p[j+1] = byte(s >> 8) + p[j+2] = byte(s >> 16) + p[j+3] = byte(s >> 24) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/md5/md5_test.go b/src/pkg/crypto/md5/md5_test.go new file mode 100644 index 000000000..857002b70 --- /dev/null +++ b/src/pkg/crypto/md5/md5_test.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package md5 + +import ( + "fmt" + "io" + "testing" +) + +type md5Test struct { + out string + in string +} + +var golden = []md5Test{ + {"d41d8cd98f00b204e9800998ecf8427e", ""}, + {"0cc175b9c0f1b6a831c399e269772661", "a"}, + {"187ef4436122d1cc2f40dc2b92f0eba0", "ab"}, + {"900150983cd24fb0d6963f7d28e17f72", "abc"}, + {"e2fc714c4727ee9395f324cd2e7f331f", "abcd"}, + {"ab56b4d92b40713acc5af89985d4b786", "abcde"}, + {"e80b5017098950fc58aad83c8c14978e", "abcdef"}, + {"7ac66c0f148de9519b8bd264312c4d64", "abcdefg"}, + {"e8dc4081b13434b45189a720b77b6818", "abcdefgh"}, + {"8aa99b1f439ff71293e95357bac6fd94", "abcdefghi"}, + {"a925576942e94b2ef57a066101b48876", "abcdefghij"}, + {"d747fc1719c7eacb84058196cfe56d57", "Discard medicine more than two years old."}, + {"bff2dcb37ef3a44ba43ab144768ca837", "He who has a shady past knows that nice guys finish last."}, + {"0441015ecb54a7342d017ed1bcfdbea5", "I wouldn't marry him with a ten foot pole."}, + {"9e3cac8e9e9757a60c3ea391130d3689", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"a0f04459b031f916a59a35cc482dc039", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"e7a48e0fe884faf31475d2a04b1362cc", "Nepal premier won't resign."}, + {"637d2fe925c07c113800509964fb0e06", "For every action there is an equal and opposite government program."}, + {"834a8d18d5c6562119cf4c7f5086cb71", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"de3a4d2fd6c73ec2db2abad23b444281", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"acf203f997e2cf74ea3aff86985aefaf", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"e1c1384cb4d2221dfdd7c795a4222c9a", "size: a.out: bad magic"}, + {"c90f3ddecc54f34228c063d7525bf644", "The major problem is with sendmail. -Mark Horton"}, + {"cdf7ab6c1fd49bd9933c43f3ea5af185", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"83bc85234942fc883c063cbd7f0ad5d0", "If the enemy is within range, then so are you."}, + {"277cbe255686b48dd7e8f389394d9299", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"fd3fb0a7ffb8af16603f3d3af98f8e1f", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"469b13a78ebf297ecda64d4723655154", "C is as portable as Stonehedge!!"}, + {"63eb3a2f466410104731c4b037600110", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"72c2ed7592debca1c90fc0100f931a2f", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"132f7619d33b523b1d9e5bd8e0928355", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("md5[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/md5/md5block.go b/src/pkg/crypto/md5/md5block.go new file mode 100644 index 000000000..a887e2e05 --- /dev/null +++ b/src/pkg/crypto/md5/md5block.go @@ -0,0 +1,172 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MD5 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package md5 + +// table[i] = int((1<<32) * abs(sin(i+1 radians))). +var table = []uint32{ + // round 1 + 0xd76aa478, + 0xe8c7b756, + 0x242070db, + 0xc1bdceee, + 0xf57c0faf, + 0x4787c62a, + 0xa8304613, + 0xfd469501, + 0x698098d8, + 0x8b44f7af, + 0xffff5bb1, + 0x895cd7be, + 0x6b901122, + 0xfd987193, + 0xa679438e, + 0x49b40821, + + // round 2 + 0xf61e2562, + 0xc040b340, + 0x265e5a51, + 0xe9b6c7aa, + 0xd62f105d, + 0x2441453, + 0xd8a1e681, + 0xe7d3fbc8, + 0x21e1cde6, + 0xc33707d6, + 0xf4d50d87, + 0x455a14ed, + 0xa9e3e905, + 0xfcefa3f8, + 0x676f02d9, + 0x8d2a4c8a, + + // round3 + 0xfffa3942, + 0x8771f681, + 0x6d9d6122, + 0xfde5380c, + 0xa4beea44, + 0x4bdecfa9, + 0xf6bb4b60, + 0xbebfbc70, + 0x289b7ec6, + 0xeaa127fa, + 0xd4ef3085, + 0x4881d05, + 0xd9d4d039, + 0xe6db99e5, + 0x1fa27cf8, + 0xc4ac5665, + + // round 4 + 0xf4292244, + 0x432aff97, + 0xab9423a7, + 0xfc93a039, + 0x655b59c3, + 0x8f0ccc92, + 0xffeff47d, + 0x85845dd1, + 0x6fa87e4f, + 0xfe2ce6e0, + 0xa3014314, + 0x4e0811a1, + 0xf7537e82, + 0xbd3af235, + 0x2ad7d2bb, + 0xeb86d391, +} + +var shift1 = []uint{7, 12, 17, 22} +var shift2 = []uint{5, 9, 14, 20} +var shift3 = []uint{4, 11, 16, 23} +var shift4 = []uint{6, 10, 15, 21} + +func _Block(dig *digest, p []byte) int { + a := dig.s[0] + b := dig.s[1] + c := dig.s[2] + d := dig.s[3] + n := 0 + var X [16]uint32 + for len(p) >= _Chunk { + aa, bb, cc, dd := a, b, c, d + + j := 0 + for i := 0; i < 16; i++ { + X[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // If this needs to be made faster in the future, + // the usual trick is to unroll each of these + // loops by a factor of 4; that lets you replace + // the shift[] lookups with constants and, + // with suitable variable renaming in each + // unrolled body, delete the a, b, c, d = d, a, b, c + // (or you can let the optimizer do the renaming). + // + // The index variables are uint so that % by a power + // of two can be optimized easily by a compiler. + + // Round 1. + for i := uint(0); i < 16; i++ { + x := i + s := shift1[i%4] + f := ((c ^ d) & b) ^ d + a += f + X[x] + table[i] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 2. + for i := uint(0); i < 16; i++ { + x := (1 + 5*i) % 16 + s := shift2[i%4] + g := ((b ^ c) & d) ^ c + a += g + X[x] + table[i+16] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 3. + for i := uint(0); i < 16; i++ { + x := (5 + 3*i) % 16 + s := shift3[i%4] + h := b ^ c ^ d + a += h + X[x] + table[i+32] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + // Round 4. + for i := uint(0); i < 16; i++ { + x := (7 * i) % 16 + s := shift4[i%4] + j := c ^ (b | ^d) + a += j + X[x] + table[i+48] + a = a<>(32-s) + b + a, b, c, d = d, a, b, c + } + + a += aa + b += bb + c += cc + d += dd + + p = p[_Chunk:] + n += _Chunk + } + + dig.s[0] = a + dig.s[1] = b + dig.s[2] = c + dig.s[3] = d + return n +} diff --git a/src/pkg/crypto/ocsp/Makefile b/src/pkg/crypto/ocsp/Makefile new file mode 100644 index 000000000..6e132ff9b --- /dev/null +++ b/src/pkg/crypto/ocsp/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/ocsp +GOFILES=\ + ocsp.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/ocsp/ocsp.go b/src/pkg/crypto/ocsp/ocsp.go new file mode 100644 index 000000000..7ea7a1e82 --- /dev/null +++ b/src/pkg/crypto/ocsp/ocsp.go @@ -0,0 +1,192 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ocsp parses OCSP responses as specified in RFC 2560. OCSP responses +// are signed messages attesting to the validity of a certificate for a small +// period of time. This is used to manage revocation for X.509 certificates. +package ocsp + +import ( + "asn1" + "crypto" + "crypto/rsa" + _ "crypto/sha1" + "crypto/x509" + "crypto/x509/pkix" + "os" + "time" +) + +var idPKIXOCSPBasic = asn1.ObjectIdentifier([]int{1, 3, 6, 1, 5, 5, 7, 48, 1, 1}) +var idSHA1WithRSA = asn1.ObjectIdentifier([]int{1, 2, 840, 113549, 1, 1, 5}) + +// These are internal structures that reflect the ASN.1 structure of an OCSP +// response. See RFC 2560, section 4.2. + +const ( + ocspSuccess = 0 + ocspMalformed = 1 + ocspInternalError = 2 + ocspTryLater = 3 + ocspSigRequired = 4 + ocspUnauthorized = 5 +) + +type certID struct { + HashAlgorithm pkix.AlgorithmIdentifier + NameHash []byte + IssuerKeyHash []byte + SerialNumber asn1.RawValue +} + +type responseASN1 struct { + Status asn1.Enumerated + Response responseBytes `asn1:"explicit,tag:0"` +} + +type responseBytes struct { + ResponseType asn1.ObjectIdentifier + Response []byte +} + +type basicResponse struct { + TBSResponseData responseData + SignatureAlgorithm pkix.AlgorithmIdentifier + Signature asn1.BitString + Certificates []asn1.RawValue `asn1:"explicit,tag:0,optional"` +} + +type responseData struct { + Raw asn1.RawContent + Version int `asn1:"optional,default:1,explicit,tag:0"` + RequestorName pkix.RDNSequence `asn1:"optional,explicit,tag:1"` + KeyHash []byte `asn1:"optional,explicit,tag:2"` + ProducedAt *time.Time + Responses []singleResponse +} + +type singleResponse struct { + CertID certID + Good asn1.Flag `asn1:"explicit,tag:0,optional"` + Revoked revokedInfo `asn1:"explicit,tag:1,optional"` + Unknown asn1.Flag `asn1:"explicit,tag:2,optional"` + ThisUpdate *time.Time + NextUpdate *time.Time `asn1:"explicit,tag:0,optional"` +} + +type revokedInfo struct { + RevocationTime *time.Time + Reason int `asn1:"explicit,tag:0,optional"` +} + +// This is the exposed reflection of the internal OCSP structures. + +const ( + // Good means that the certificate is valid. + Good = iota + // Revoked means that the certificate has been deliberately revoked. + Revoked = iota + // Unknown means that the OCSP responder doesn't know about the certificate. + Unknown = iota + // ServerFailed means that the OCSP responder failed to process the request. + ServerFailed = iota +) + +// Response represents an OCSP response. See RFC 2560. +type Response struct { + // Status is one of {Good, Revoked, Unknown, ServerFailed} + Status int + SerialNumber []byte + ProducedAt, ThisUpdate, NextUpdate, RevokedAt *time.Time + RevocationReason int + Certificate *x509.Certificate +} + +// ParseError results from an invalid OCSP response. +type ParseError string + +func (p ParseError) String() string { + return string(p) +} + +// ParseResponse parses an OCSP response in DER form. It only supports +// responses for a single certificate and only those using RSA signatures. +// Non-RSA responses will result in an x509.UnsupportedAlgorithmError. +// Signature errors or parse failures will result in a ParseError. +func ParseResponse(bytes []byte) (*Response, os.Error) { + var resp responseASN1 + rest, err := asn1.Unmarshal(bytes, &resp) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP response") + } + + ret := new(Response) + if resp.Status != ocspSuccess { + ret.Status = ServerFailed + return ret, nil + } + + if !resp.Response.ResponseType.Equal(idPKIXOCSPBasic) { + return nil, ParseError("bad OCSP response type") + } + + var basicResp basicResponse + rest, err = asn1.Unmarshal(resp.Response.Response, &basicResp) + if err != nil { + return nil, err + } + + if len(basicResp.Certificates) != 1 { + return nil, ParseError("OCSP response contains bad number of certificates") + } + + if len(basicResp.TBSResponseData.Responses) != 1 { + return nil, ParseError("OCSP response contains bad number of responses") + } + + ret.Certificate, err = x509.ParseCertificate(basicResp.Certificates[0].FullBytes) + if err != nil { + return nil, err + } + + if ret.Certificate.PublicKeyAlgorithm != x509.RSA || !basicResp.SignatureAlgorithm.Algorithm.Equal(idSHA1WithRSA) { + return nil, x509.UnsupportedAlgorithmError{} + } + + hashType := crypto.SHA1 + h := hashType.New() + + pub := ret.Certificate.PublicKey.(*rsa.PublicKey) + h.Write(basicResp.TBSResponseData.Raw) + digest := h.Sum() + signature := basicResp.Signature.RightAlign() + + if rsa.VerifyPKCS1v15(pub, hashType, digest, signature) != nil { + return nil, ParseError("bad OCSP signature") + } + + r := basicResp.TBSResponseData.Responses[0] + + ret.SerialNumber = r.CertID.SerialNumber.Bytes + + switch { + case bool(r.Good): + ret.Status = Good + case bool(r.Unknown): + ret.Status = Unknown + default: + ret.Status = Revoked + ret.RevokedAt = r.Revoked.RevocationTime + ret.RevocationReason = r.Revoked.Reason + } + + ret.ProducedAt = basicResp.TBSResponseData.ProducedAt + ret.ThisUpdate = r.ThisUpdate + ret.NextUpdate = r.NextUpdate + + return ret, nil +} diff --git a/src/pkg/crypto/ocsp/ocsp_test.go b/src/pkg/crypto/ocsp/ocsp_test.go new file mode 100644 index 000000000..f9889790f --- /dev/null +++ b/src/pkg/crypto/ocsp/ocsp_test.go @@ -0,0 +1,97 @@ +package ocsp + +import ( + "bytes" + "encoding/hex" + "reflect" + "testing" + "time" +) + +func TestOCSPDecode(t *testing.T) { + responseBytes, _ := hex.DecodeString(ocspResponseHex) + resp, err := ParseResponse(responseBytes) + if err != nil { + t.Error(err) + } + + expected := Response{Status: 0, SerialNumber: []byte{0x1, 0xd0, 0xfa}, RevocationReason: 0, ThisUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 15, Minute: 1, Second: 5, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}, NextUpdate: &time.Time{Year: 2010, Month: 7, Day: 7, Hour: 18, Minute: 35, Second: 17, Weekday: 0, ZoneOffset: 0, Zone: "UTC"}} + + if !reflect.DeepEqual(resp.ThisUpdate, resp.ThisUpdate) { + t.Errorf("resp.ThisUpdate: got %d, want %d", resp.ThisUpdate, expected.ThisUpdate) + } + + if !reflect.DeepEqual(resp.NextUpdate, resp.NextUpdate) { + t.Errorf("resp.NextUpdate: got %d, want %d", resp.NextUpdate, expected.NextUpdate) + } + + if resp.Status != expected.Status { + t.Errorf("resp.Status: got %d, want %d", resp.Status, expected.Status) + } + + if !bytes.Equal(resp.SerialNumber, expected.SerialNumber) { + t.Errorf("resp.SerialNumber: got %x, want %x", resp.SerialNumber, expected.SerialNumber) + } + + if resp.RevocationReason != expected.RevocationReason { + t.Errorf("resp.RevocationReason: got %d, want %d", resp.RevocationReason, expected.RevocationReason) + } +} + +// This OCSP response was taken from Thawte's public OCSP responder. +// To recreate: +// $ openssl s_client -tls1 -showcerts -servername www.google.com -connect www.google.com:443 +// Copy and paste the first certificate into /tmp/cert.crt and the second into +// /tmp/intermediate.crt +// $ openssl ocsp -issuer /tmp/intermediate.crt -cert /tmp/cert.crt -url http://ocsp.thawte.com -resp_text -respout /tmp/ocsp.der +// Then hex encode the result: +// $ python -c 'print file("/tmp/ocsp.der", "r").read().encode("hex")' + +const ocspResponseHex = "308206bc0a0100a08206b5308206b106092b0601050507300101048206a23082069e3081" + + "c9a14e304c310b300906035504061302494c31163014060355040a130d5374617274436f" + + "6d204c74642e312530230603550403131c5374617274436f6d20436c6173732031204f43" + + "5350205369676e6572180f32303130303730373137333531375a30663064303c30090605" + + "2b0e03021a050004146568874f40750f016a3475625e1f5c93e5a26d580414eb4234d098" + + "b0ab9ff41b6b08f7cc642eef0e2c45020301d0fa8000180f323031303037303731353031" + + "30355aa011180f32303130303730373138333531375a300d06092a864886f70d01010505" + + "000382010100ab557ff070d1d7cebbb5f0ec91a15c3fed22eb2e1b8244f1b84545f013a4" + + "fb46214c5e3fbfbebb8a56acc2b9db19f68fd3c3201046b3824d5ba689f99864328710cb" + + "467195eb37d84f539e49f859316b32964dc3e47e36814ce94d6c56dd02733b1d0802f7ff" + + "4eebdbbd2927dcf580f16cbc290f91e81b53cb365e7223f1d6e20a88ea064104875e0145" + + "672b20fc14829d51ca122f5f5d77d3ad6c83889c55c7dc43680ba2fe3cef8b05dbcabdc0" + + "d3e09aaf9725597f8c858c2fa38c0d6aed2e6318194420dd1a1137445d13e1c97ab47896" + + "17a4e08925f46f867b72e3a4dc1f08cb870b2b0717f7207faa0ac512e628a029aba7457a" + + "e63dcf3281e2162d9349a08204ba308204b6308204b23082039aa003020102020101300d" + + "06092a864886f70d010105050030818c310b300906035504061302494c31163014060355" + + "040a130d5374617274436f6d204c74642e312b3029060355040b13225365637572652044" + + "69676974616c204365727469666963617465205369676e696e6731383036060355040313" + + "2f5374617274436f6d20436c6173732031205072696d61727920496e7465726d65646961" + + "746520536572766572204341301e170d3037313032353030323330365a170d3132313032" + + "333030323330365a304c310b300906035504061302494c31163014060355040a130d5374" + + "617274436f6d204c74642e312530230603550403131c5374617274436f6d20436c617373" + + "2031204f435350205369676e657230820122300d06092a864886f70d0101010500038201" + + "0f003082010a0282010100b9561b4c45318717178084e96e178df2255e18ed8d8ecc7c2b" + + "7b51a6c1c2e6bf0aa3603066f132fe10ae97b50e99fa24b83fc53dd2777496387d14e1c3" + + "a9b6a4933e2ac12413d085570a95b8147414a0bc007c7bcf222446ef7f1a156d7ea1c577" + + "fc5f0facdfd42eb0f5974990cb2f5cefebceef4d1bdc7ae5c1075c5a99a93171f2b0845b" + + "4ff0864e973fcfe32f9d7511ff87a3e943410c90a4493a306b6944359340a9ca96f02b66" + + "ce67f028df2980a6aaee8d5d5d452b8b0eb93f923cc1e23fcccbdbe7ffcb114d08fa7a6a" + + "3c404f825d1a0e715935cf623a8c7b59670014ed0622f6089a9447a7a19010f7fe58f841" + + "29a2765ea367824d1c3bb2fda308530203010001a382015c30820158300c0603551d1301" + + "01ff04023000300b0603551d0f0404030203a8301e0603551d250417301506082b060105" + + "0507030906092b0601050507300105301d0603551d0e0416041445e0a36695414c5dd449" + + "bc00e33cdcdbd2343e173081a80603551d230481a030819d8014eb4234d098b0ab9ff41b" + + "6b08f7cc642eef0e2c45a18181a47f307d310b300906035504061302494c311630140603" + + "55040a130d5374617274436f6d204c74642e312b3029060355040b132253656375726520" + + "4469676974616c204365727469666963617465205369676e696e67312930270603550403" + + "13205374617274436f6d2043657274696669636174696f6e20417574686f726974798201" + + "0a30230603551d12041c301a8618687474703a2f2f7777772e737461727473736c2e636f" + + "6d2f302c06096086480186f842010d041f161d5374617274436f6d205265766f63617469" + + "6f6e20417574686f72697479300d06092a864886f70d01010505000382010100182d2215" + + "8f0fc0291324fa8574c49bb8ff2835085adcbf7b7fc4191c397ab6951328253fffe1e5ec" + + "2a7da0d50fca1a404e6968481366939e666c0a6209073eca57973e2fefa9ed1718e8176f" + + "1d85527ff522c08db702e3b2b180f1cbff05d98128252cf0f450f7dd2772f4188047f19d" + + "c85317366f94bc52d60f453a550af58e308aaab00ced33040b62bf37f5b1ab2a4f7f0f80" + + "f763bf4d707bc8841d7ad9385ee2a4244469260b6f2bf085977af9074796048ecc2f9d48" + + "a1d24ce16e41a9941568fec5b42771e118f16c106a54ccc339a4b02166445a167902e75e" + + "6d8620b0825dcd18a069b90fd851d10fa8effd409deec02860d26d8d833f304b10669b42" diff --git a/src/pkg/crypto/openpgp/Makefile b/src/pkg/crypto/openpgp/Makefile new file mode 100644 index 000000000..b46ac2bba --- /dev/null +++ b/src/pkg/crypto/openpgp/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/openpgp +GOFILES=\ + canonical_text.go\ + keys.go\ + read.go\ + write.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/armor/Makefile b/src/pkg/crypto/openpgp/armor/Makefile new file mode 100644 index 000000000..138e314e9 --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/Makefile @@ -0,0 +1,12 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/armor +GOFILES=\ + armor.go\ + encode.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/armor/armor.go b/src/pkg/crypto/openpgp/armor/armor.go new file mode 100644 index 000000000..9c4180d6d --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/armor.go @@ -0,0 +1,220 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package armor implements OpenPGP ASCII Armor, see RFC 4880. OpenPGP Armor is +// very similar to PEM except that it has an additional CRC checksum. +package armor + +import ( + "bufio" + "bytes" + "crypto/openpgp/error" + "encoding/base64" + "io" + "os" +) + +// A Block represents an OpenPGP armored structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// +// base64-encoded Bytes +// '=' base64 encoded checksum +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +// +// Since the armored data can be very large, this package presents a streaming +// interface. +type Block struct { + Type string // The type, taken from the preamble (i.e. "PGP SIGNATURE"). + Header map[string]string // Optional headers. + Body io.Reader // A Reader from which the contents can be read + lReader lineReader + oReader openpgpReader +} + +var ArmorCorrupt os.Error = error.StructuralError("armor invalid") + +const crc24Init = 0xb704ce +const crc24Poly = 0x1864cfb +const crc24Mask = 0xffffff + +// crc24 calculates the OpenPGP checksum as specified in RFC 4880, section 6.1 +func crc24(crc uint32, d []byte) uint32 { + for _, b := range d { + crc ^= uint32(b) << 16 + for i := 0; i < 8; i++ { + crc <<= 1 + if crc&0x1000000 != 0 { + crc ^= crc24Poly + } + } + } + return crc +} + +var armorStart = []byte("-----BEGIN ") +var armorEnd = []byte("-----END ") +var armorEndOfLine = []byte("-----") + +// lineReader wraps a line based reader. It watches for the end of an armor +// block and records the expected CRC value. +type lineReader struct { + in *bufio.Reader + buf []byte + eof bool + crc uint32 +} + +func (l *lineReader) Read(p []byte) (n int, err os.Error) { + if l.eof { + return 0, os.EOF + } + + if len(l.buf) > 0 { + n = copy(p, l.buf) + l.buf = l.buf[n:] + return + } + + line, isPrefix, err := l.in.ReadLine() + if err != nil { + return + } + if isPrefix { + return 0, ArmorCorrupt + } + + if len(line) == 5 && line[0] == '=' { + // This is the checksum line + var expectedBytes [3]byte + var m int + m, err = base64.StdEncoding.Decode(expectedBytes[0:], line[1:]) + if m != 3 || err != nil { + return + } + l.crc = uint32(expectedBytes[0])<<16 | + uint32(expectedBytes[1])<<8 | + uint32(expectedBytes[2]) + + line, _, err = l.in.ReadLine() + if err != nil && err != os.EOF { + return + } + if !bytes.HasPrefix(line, armorEnd) { + return 0, ArmorCorrupt + } + + l.eof = true + return 0, os.EOF + } + + if len(line) > 64 { + return 0, ArmorCorrupt + } + + n = copy(p, line) + bytesToSave := len(line) - n + if bytesToSave > 0 { + if cap(l.buf) < bytesToSave { + l.buf = make([]byte, 0, bytesToSave) + } + l.buf = l.buf[0:bytesToSave] + copy(l.buf, line[n:]) + } + + return +} + +// openpgpReader passes Read calls to the underlying base64 decoder, but keeps +// a running CRC of the resulting data and checks the CRC against the value +// found by the lineReader at EOF. +type openpgpReader struct { + lReader *lineReader + b64Reader io.Reader + currentCRC uint32 +} + +func (r *openpgpReader) Read(p []byte) (n int, err os.Error) { + n, err = r.b64Reader.Read(p) + r.currentCRC = crc24(r.currentCRC, p[:n]) + + if err == os.EOF { + if r.lReader.crc != uint32(r.currentCRC&crc24Mask) { + return 0, ArmorCorrupt + } + } + + return +} + +// Decode reads a PGP armored block from the given Reader. It will ignore +// leading garbage. If it doesn't find a block, it will return nil, os.EOF. The +// given Reader is not usable after calling this function: an arbitrary amount +// of data may have been read past the end of the block. +func Decode(in io.Reader) (p *Block, err os.Error) { + r, _ := bufio.NewReaderSize(in, 100) + var line []byte + ignoreNext := false + +TryNextBlock: + p = nil + + // Skip leading garbage + for { + ignoreThis := ignoreNext + line, ignoreNext, err = r.ReadLine() + if err != nil { + return + } + if ignoreNext || ignoreThis { + continue + } + line = bytes.TrimSpace(line) + if len(line) > len(armorStart)+len(armorEndOfLine) && bytes.HasPrefix(line, armorStart) { + break + } + } + + p = new(Block) + p.Type = string(line[len(armorStart) : len(line)-len(armorEndOfLine)]) + p.Header = make(map[string]string) + nextIsContinuation := false + var lastKey string + + // Read headers + for { + isContinuation := nextIsContinuation + line, nextIsContinuation, err = r.ReadLine() + if err != nil { + p = nil + return + } + if isContinuation { + p.Header[lastKey] += string(line) + continue + } + line = bytes.TrimSpace(line) + if len(line) == 0 { + break + } + + i := bytes.Index(line, []byte(": ")) + if i == -1 { + goto TryNextBlock + } + lastKey = string(line[:i]) + p.Header[lastKey] = string(line[i+2:]) + } + + p.lReader.in = r + p.oReader.currentCRC = crc24Init + p.oReader.lReader = &p.lReader + p.oReader.b64Reader = base64.NewDecoder(base64.StdEncoding, &p.lReader) + p.Body = &p.oReader + + return +} diff --git a/src/pkg/crypto/openpgp/armor/armor_test.go b/src/pkg/crypto/openpgp/armor/armor_test.go new file mode 100644 index 000000000..9334e94e9 --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/armor_test.go @@ -0,0 +1,95 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "bytes" + "hash/adler32" + "io/ioutil" + "testing" +) + +func TestDecodeEncode(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorExample1)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + } + expectedType := "PGP SIGNATURE" + if result.Type != expectedType { + t.Errorf("result.Type: got:%s want:%s", result.Type, expectedType) + } + if len(result.Header) != 1 { + t.Errorf("len(result.Header): got:%d want:1", len(result.Header)) + } + v, ok := result.Header["Version"] + if !ok || v != "GnuPG v1.4.10 (GNU/Linux)" { + t.Errorf("result.Header: got:%#v", result.Header) + } + + contents, err := ioutil.ReadAll(result.Body) + if err != nil { + t.Error(err) + } + + if adler32.Checksum(contents) != 0x27b144be { + t.Errorf("contents: got: %x", contents) + } + + buf = bytes.NewBuffer(nil) + w, err := Encode(buf, result.Type, result.Header) + if err != nil { + t.Error(err) + } + _, err = w.Write(contents) + if err != nil { + t.Error(err) + } + w.Close() + + if !bytes.Equal(buf.Bytes(), []byte(armorExample1)) { + t.Errorf("got: %s\nwant: %s", string(buf.Bytes()), armorExample1) + } +} + +func TestLongHeader(t *testing.T) { + buf := bytes.NewBuffer([]byte(armorLongLine)) + result, err := Decode(buf) + if err != nil { + t.Error(err) + return + } + value, ok := result.Header["Version"] + if !ok { + t.Errorf("missing Version header") + } + if value != longValueExpected { + t.Errorf("got: %s want: %s", value, longValueExpected) + } +} + +const armorExample1 = `-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.10 (GNU/Linux) + +iJwEAAECAAYFAk1Fv/0ACgkQo01+GMIMMbsYTwQAiAw+QAaNfY6WBdplZ/uMAccm +4g+81QPmTSGHnetSb6WBiY13kVzK4HQiZH8JSkmmroMLuGeJwsRTEL4wbjRyUKEt +p1xwUZDECs234F1xiG5enc5SGlRtP7foLBz9lOsjx+LEcA4sTl5/2eZR9zyFZqWW +TxRjs+fJCIFuo71xb1g= +=/teI +-----END PGP SIGNATURE-----` + +const armorLongLine = `-----BEGIN PGP SIGNATURE----- +Version: 0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz + +iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8 +kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp +cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA +byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3 +WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv +okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4= +=wfQG +-----END PGP SIGNATURE-----` + +const longValueExpected = "0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz" diff --git a/src/pkg/crypto/openpgp/armor/encode.go b/src/pkg/crypto/openpgp/armor/encode.go new file mode 100644 index 000000000..99dee375e --- /dev/null +++ b/src/pkg/crypto/openpgp/armor/encode.go @@ -0,0 +1,161 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package armor + +import ( + "encoding/base64" + "io" + "os" +) + +var armorHeaderSep = []byte(": ") +var blockEnd = []byte("\n=") +var newline = []byte("\n") +var armorEndOfLineOut = []byte("-----\n") + +// writeSlices writes its arguments to the given Writer. +func writeSlices(out io.Writer, slices ...[]byte) (err os.Error) { + for _, s := range slices { + _, err = out.Write(s) + if err != nil { + return err + } + } + return +} + +// lineBreaker breaks data across several lines, all of the same byte length +// (except possibly the last). Lines are broken with a single '\n'. +type lineBreaker struct { + lineLength int + line []byte + used int + out io.Writer + haveWritten bool +} + +func newLineBreaker(out io.Writer, lineLength int) *lineBreaker { + return &lineBreaker{ + lineLength: lineLength, + line: make([]byte, lineLength), + used: 0, + out: out, + } +} + +func (l *lineBreaker) Write(b []byte) (n int, err os.Error) { + n = len(b) + + if n == 0 { + return + } + + if l.used == 0 && l.haveWritten { + _, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + if l.used+len(b) < l.lineLength { + l.used += copy(l.line[l.used:], b) + return + } + + l.haveWritten = true + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := l.lineLength - l.used + l.used = 0 + + _, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + _, err = l.Write(b[excess:]) + return +} + +func (l *lineBreaker) Close() (err os.Error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + } + + return +} + +// encoding keeps track of a running CRC24 over the data which has been written +// to it and outputs a OpenPGP checksum when closed, followed by an armor +// trailer. +// +// It's built into a stack of io.Writers: +// encoding -> base64 encoder -> lineBreaker -> out +type encoding struct { + out io.Writer + breaker *lineBreaker + b64 io.WriteCloser + crc uint32 + blockType []byte +} + +func (e *encoding) Write(data []byte) (n int, err os.Error) { + e.crc = crc24(e.crc, data) + return e.b64.Write(data) +} + +func (e *encoding) Close() (err os.Error) { + err = e.b64.Close() + if err != nil { + return + } + e.breaker.Close() + + var checksumBytes [3]byte + checksumBytes[0] = byte(e.crc >> 16) + checksumBytes[1] = byte(e.crc >> 8) + checksumBytes[2] = byte(e.crc) + + var b64ChecksumBytes [4]byte + base64.StdEncoding.Encode(b64ChecksumBytes[:], checksumBytes[:]) + + return writeSlices(e.out, blockEnd, b64ChecksumBytes[:], newline, armorEnd, e.blockType, armorEndOfLine) +} + +// Encode returns a WriteCloser which will encode the data written to it in +// OpenPGP armor. +func Encode(out io.Writer, blockType string, headers map[string]string) (w io.WriteCloser, err os.Error) { + bType := []byte(blockType) + err = writeSlices(out, armorStart, bType, armorEndOfLineOut) + if err != nil { + return + } + + for k, v := range headers { + err = writeSlices(out, []byte(k), armorHeaderSep, []byte(v), newline) + if err != nil { + return + } + } + + _, err = out.Write(newline) + if err != nil { + return + } + + e := &encoding{ + out: out, + breaker: newLineBreaker(out, 64), + crc: crc24Init, + blockType: bType, + } + e.b64 = base64.NewEncoder(base64.StdEncoding, e.breaker) + return e, nil +} diff --git a/src/pkg/crypto/openpgp/canonical_text.go b/src/pkg/crypto/openpgp/canonical_text.go new file mode 100644 index 000000000..293eff354 --- /dev/null +++ b/src/pkg/crypto/openpgp/canonical_text.go @@ -0,0 +1,58 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "hash" + "os" +) + +// NewCanonicalTextHash reformats text written to it into the canonical +// form and then applies the hash h. See RFC 4880, section 5.2.1. +func NewCanonicalTextHash(h hash.Hash) hash.Hash { + return &canonicalTextHash{h, 0} +} + +type canonicalTextHash struct { + h hash.Hash + s int +} + +var newline = []byte{'\r', '\n'} + +func (cth *canonicalTextHash) Write(buf []byte) (int, os.Error) { + start := 0 + + for i, c := range buf { + switch cth.s { + case 0: + if c == '\r' { + cth.s = 1 + } else if c == '\n' { + cth.h.Write(buf[start:i]) + cth.h.Write(newline) + start = i + 1 + } + case 1: + cth.s = 0 + } + } + + cth.h.Write(buf[start:]) + return len(buf), nil +} + +func (cth *canonicalTextHash) Sum() []byte { + return cth.h.Sum() +} + +func (cth *canonicalTextHash) Reset() { + cth.h.Reset() + cth.s = 0 +} + +func (cth *canonicalTextHash) Size() int { + return cth.h.Size() +} diff --git a/src/pkg/crypto/openpgp/canonical_text_test.go b/src/pkg/crypto/openpgp/canonical_text_test.go new file mode 100644 index 000000000..ccf2910cc --- /dev/null +++ b/src/pkg/crypto/openpgp/canonical_text_test.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "os" + "testing" +) + +type recordingHash struct { + buf *bytes.Buffer +} + +func (r recordingHash) Write(b []byte) (n int, err os.Error) { + return r.buf.Write(b) +} + +func (r recordingHash) Sum() []byte { + return r.buf.Bytes() +} + +func (r recordingHash) Reset() { + panic("shouldn't be called") +} + +func (r recordingHash) Size() int { + panic("shouldn't be called") +} + +func testCanonicalText(t *testing.T, input, expected string) { + r := recordingHash{bytes.NewBuffer(nil)} + c := NewCanonicalTextHash(r) + c.Write([]byte(input)) + result := c.Sum() + if expected != string(result) { + t.Errorf("input: %x got: %x want: %x", input, result, expected) + } +} + +func TestCanonicalText(t *testing.T) { + testCanonicalText(t, "foo\n", "foo\r\n") + testCanonicalText(t, "foo", "foo") + testCanonicalText(t, "foo\r\n", "foo\r\n") + testCanonicalText(t, "foo\r\nbar", "foo\r\nbar") + testCanonicalText(t, "foo\r\nbar\n\n", "foo\r\nbar\r\n\r\n") +} diff --git a/src/pkg/crypto/openpgp/elgamal/Makefile b/src/pkg/crypto/openpgp/elgamal/Makefile new file mode 100644 index 000000000..f730255f8 --- /dev/null +++ b/src/pkg/crypto/openpgp/elgamal/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/elgamal +GOFILES=\ + elgamal.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/elgamal/elgamal.go b/src/pkg/crypto/openpgp/elgamal/elgamal.go new file mode 100644 index 000000000..99a6e3e1f --- /dev/null +++ b/src/pkg/crypto/openpgp/elgamal/elgamal.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elgamal implements ElGamal encryption, suitable for OpenPGP, +// as specified in "A Public-Key Cryptosystem and a Signature Scheme Based on +// Discrete Logarithms," IEEE Transactions on Information Theory, v. IT-31, +// n. 4, 1985, pp. 469-472. +// +// This form of ElGamal embeds PKCS#1 v1.5 padding, which may make it +// unsuitable for other protocols. RSA should be used in preference in any +// case. +package elgamal + +import ( + "big" + "crypto/rand" + "crypto/subtle" + "io" + "os" +) + +// PublicKey represents an ElGamal public key. +type PublicKey struct { + G, P, Y *big.Int +} + +// PrivateKey represents an ElGamal private key. +type PrivateKey struct { + PublicKey + X *big.Int +} + +// Encrypt encrypts the given message to the given public key. The result is a +// pair of integers. Errors can result from reading random, or because msg is +// too large to be encrypted to the public key. +func Encrypt(random io.Reader, pub *PublicKey, msg []byte) (c1, c2 *big.Int, err os.Error) { + pLen := (pub.P.BitLen() + 7) / 8 + if len(msg) > pLen-11 { + err = os.NewError("elgamal: message too long") + return + } + + // EM = 0x02 || PS || 0x00 || M + em := make([]byte, pLen-1) + em[0] = 2 + ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):] + err = nonZeroRandomBytes(ps, random) + if err != nil { + return + } + em[len(em)-len(msg)-1] = 0 + copy(mm, msg) + + m := new(big.Int).SetBytes(em) + + k, err := rand.Int(random, pub.P) + if err != nil { + return + } + + c1 = new(big.Int).Exp(pub.G, k, pub.P) + s := new(big.Int).Exp(pub.Y, k, pub.P) + c2 = s.Mul(s, m) + c2.Mod(c2, pub.P) + + return +} + +// Decrypt takes two integers, resulting from an ElGamal encryption, and +// returns the plaintext of the message. An error can result only if the +// ciphertext is invalid. Users should keep in mind that this is a padding +// oracle and thus, if exposed to an adaptive chosen ciphertext attack, can +// be used to break the cryptosystem. See ``Chosen Ciphertext Attacks +// Against Protocols Based on the RSA Encryption Standard PKCS #1'', Daniel +// Bleichenbacher, Advances in Cryptology (Crypto '98), +func Decrypt(priv *PrivateKey, c1, c2 *big.Int) (msg []byte, err os.Error) { + s := new(big.Int).Exp(c1, priv.X, priv.P) + s.ModInverse(s, priv.P) + s.Mul(s, c2) + s.Mod(s, priv.P) + em := s.Bytes() + + firstByteIsTwo := subtle.ConstantTimeByteEq(em[0], 2) + + // The remainder of the plaintext must be a string of non-zero random + // octets, followed by a 0, followed by the message. + // lookingForIndex: 1 iff we are still looking for the zero. + // index: the offset of the first zero byte. + var lookingForIndex, index int + lookingForIndex = 1 + + for i := 1; i < len(em); i++ { + equals0 := subtle.ConstantTimeByteEq(em[i], 0) + index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex) + } + + if firstByteIsTwo != 1 || lookingForIndex != 0 || index < 9 { + return nil, os.NewError("elgamal: decryption error") + } + return em[index+1:], nil +} + +// nonZeroRandomBytes fills the given slice with non-zero random octets. +func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) { + _, err = io.ReadFull(rand, s) + if err != nil { + return + } + + for i := 0; i < len(s); i++ { + for s[i] == 0 { + _, err = io.ReadFull(rand, s[i:i+1]) + if err != nil { + return + } + } + } + + return +} diff --git a/src/pkg/crypto/openpgp/elgamal/elgamal_test.go b/src/pkg/crypto/openpgp/elgamal/elgamal_test.go new file mode 100644 index 000000000..101121aa6 --- /dev/null +++ b/src/pkg/crypto/openpgp/elgamal/elgamal_test.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elgamal + +import ( + "big" + "bytes" + "crypto/rand" + "testing" +) + +// This is the 1024-bit MODP group from RFC 5114, section 2.1: +const primeHex = "B10B8F96A080E01DDE92DE5EAE5D54EC52C99FBCFB06A3C69A6A9DCA52D23B616073E28675A23D189838EF1E2EE652C013ECB4AEA906112324975C3CD49B83BFACCBDD7D90C4BD7098488E9C219A73724EFFD6FAE5644738FAA31A4FF55BCCC0A151AF5F0DC8B4BD45BF37DF365C1A65E68CFDA76D4DA708DF1FB2BC2E4A4371" + +const generatorHex = "A4D1CBD5C3FD34126765A442EFB99905F8104DD258AC507FD6406CFF14266D31266FEA1E5C41564B777E690F5504F213160217B4B01B886A5E91547F9E2749F4D7FBD7D3B9A92EE1909D0D2263F80A76A6A24C087A091F531DBF0A0169B6A28AD662A4D18E73AFA32D779D5918D08BC8858F4DCEF97C2A24855E6EEB22B3B2E5" + +func fromHex(hex string) *big.Int { + n, ok := new(big.Int).SetString(hex, 16) + if !ok { + panic("failed to parse hex number") + } + return n +} + +func TestEncryptDecrypt(t *testing.T) { + priv := &PrivateKey{ + PublicKey: PublicKey{ + G: fromHex(generatorHex), + P: fromHex(primeHex), + }, + X: fromHex("42"), + } + priv.Y = new(big.Int).Exp(priv.G, priv.X, priv.P) + + message := []byte("hello world") + c1, c2, err := Encrypt(rand.Reader, &priv.PublicKey, message) + if err != nil { + t.Errorf("error encrypting: %s", err) + } + message2, err := Decrypt(priv, c1, c2) + if err != nil { + t.Errorf("error decrypting: %s", err) + } + if !bytes.Equal(message2, message) { + t.Errorf("decryption failed, got: %x, want: %x", message2, message) + } +} diff --git a/src/pkg/crypto/openpgp/error/Makefile b/src/pkg/crypto/openpgp/error/Makefile new file mode 100644 index 000000000..8c370a089 --- /dev/null +++ b/src/pkg/crypto/openpgp/error/Makefile @@ -0,0 +1,11 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/error +GOFILES=\ + error.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/error/error.go b/src/pkg/crypto/openpgp/error/error.go new file mode 100644 index 000000000..3759ce161 --- /dev/null +++ b/src/pkg/crypto/openpgp/error/error.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package error contains common error types for the OpenPGP packages. +package error + +import ( + "strconv" +) + +// A StructuralError is returned when OpenPGP data is found to be syntactically +// invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "OpenPGP data invalid: " + string(s) +} + +// UnsupportedError indicates that, although the OpenPGP data is valid, it +// makes use of currently unimplemented features. +type UnsupportedError string + +func (s UnsupportedError) String() string { + return "OpenPGP feature unsupported: " + string(s) +} + +// InvalidArgumentError indicates that the caller is in error and passed an +// incorrect value. +type InvalidArgumentError string + +func (i InvalidArgumentError) String() string { + return "OpenPGP argument invalid: " + string(i) +} + +// SignatureError indicates that a syntactically valid signature failed to +// validate. +type SignatureError string + +func (b SignatureError) String() string { + return "OpenPGP signature invalid: " + string(b) +} + +type keyIncorrect int + +func (ki keyIncorrect) String() string { + return "the given key was incorrect" +} + +var KeyIncorrectError = keyIncorrect(0) + +type unknownIssuer int + +func (unknownIssuer) String() string { + return "signature make by unknown entity" +} + +var UnknownIssuerError = unknownIssuer(0) + +type UnknownPacketTypeError uint8 + +func (upte UnknownPacketTypeError) String() string { + return "unknown OpenPGP packet type: " + strconv.Itoa(int(upte)) +} diff --git a/src/pkg/crypto/openpgp/keys.go b/src/pkg/crypto/openpgp/keys.go new file mode 100644 index 000000000..c70fb7927 --- /dev/null +++ b/src/pkg/crypto/openpgp/keys.go @@ -0,0 +1,545 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/rsa" + "io" + "os" + "time" +) + +// PublicKeyType is the armor type for a PGP public key. +var PublicKeyType = "PGP PUBLIC KEY BLOCK" +// PrivateKeyType is the armor type for a PGP private key. +var PrivateKeyType = "PGP PRIVATE KEY BLOCK" + +// An Entity represents the components of an OpenPGP key: a primary public key +// (which must be a signing key), one or more identities claimed by that key, +// and zero or more subkeys, which may be encryption keys. +type Entity struct { + PrimaryKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Identities map[string]*Identity // indexed by Identity.Name + Subkeys []Subkey +} + +// An Identity represents an identity claimed by an Entity and zero or more +// assertions by other entities about that claim. +type Identity struct { + Name string // by convention, has the form "Full Name (comment) " + UserId *packet.UserId + SelfSignature *packet.Signature + Signatures []*packet.Signature +} + +// A Subkey is an additional public key in an Entity. Subkeys can be used for +// encryption. +type Subkey struct { + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Sig *packet.Signature +} + +// A Key identifies a specific public key in an Entity. This is either the +// Entity's primary key or a subkey. +type Key struct { + Entity *Entity + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + SelfSignature *packet.Signature +} + +// A KeyRing provides access to public and private keys. +type KeyRing interface { + // KeysById returns the set of keys that have the given key id. + KeysById(id uint64) []Key + // DecryptionKeys returns all private keys that are valid for + // decryption. + DecryptionKeys() []Key +} + +// primaryIdentity returns the Identity marked as primary or the first identity +// if none are so marked. +func (e *Entity) primaryIdentity() *Identity { + var firstIdentity *Identity + for _, ident := range e.Identities { + if firstIdentity == nil { + firstIdentity = ident + } + if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + return ident + } + } + return firstIdentity +} + +// encryptionKey returns the best candidate Key for encrypting a message to the +// given Entity. +func (e *Entity) encryptionKey() Key { + candidateSubkey := -1 + + for i, subkey := range e.Subkeys { + if subkey.Sig.FlagsValid && subkey.Sig.FlagEncryptCommunications && subkey.PublicKey.PubKeyAlgo.CanEncrypt() { + candidateSubkey = i + break + } + } + + i := e.primaryIdentity() + + if e.PrimaryKey.PubKeyAlgo.CanEncrypt() { + // If we don't have any candidate subkeys for encryption and + // the primary key doesn't have any usage metadata then we + // assume that the primary key is ok. Or, if the primary key is + // marked as ok to encrypt to, then we can obviously use it. + if candidateSubkey == -1 && !i.SelfSignature.FlagsValid || i.SelfSignature.FlagEncryptCommunications && i.SelfSignature.FlagsValid { + return Key{e, e.PrimaryKey, e.PrivateKey, i.SelfSignature} + } + } + + if candidateSubkey != -1 { + subkey := e.Subkeys[candidateSubkey] + return Key{e, subkey.PublicKey, subkey.PrivateKey, subkey.Sig} + } + + // This Entity appears to be signing only. + return Key{} +} + +// signingKey return the best candidate Key for signing a message with this +// Entity. +func (e *Entity) signingKey() Key { + candidateSubkey := -1 + + for i, subkey := range e.Subkeys { + if subkey.Sig.FlagsValid && subkey.Sig.FlagSign && subkey.PublicKey.PubKeyAlgo.CanSign() { + candidateSubkey = i + break + } + } + + i := e.primaryIdentity() + + // If we have no candidate subkey then we assume that it's ok to sign + // with the primary key. + if candidateSubkey == -1 || i.SelfSignature.FlagsValid && i.SelfSignature.FlagSign { + return Key{e, e.PrimaryKey, e.PrivateKey, i.SelfSignature} + } + + subkey := e.Subkeys[candidateSubkey] + return Key{e, subkey.PublicKey, subkey.PrivateKey, subkey.Sig} +} + +// An EntityList contains one or more Entities. +type EntityList []*Entity + +// KeysById returns the set of keys that have the given key id. +func (el EntityList) KeysById(id uint64) (keys []Key) { + for _, e := range el { + if e.PrimaryKey.KeyId == id { + var selfSig *packet.Signature + for _, ident := range e.Identities { + if selfSig == nil { + selfSig = ident.SelfSignature + } else if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + selfSig = ident.SelfSignature + break + } + } + keys = append(keys, Key{e, e.PrimaryKey, e.PrivateKey, selfSig}) + } + + for _, subKey := range e.Subkeys { + if subKey.PublicKey.KeyId == id { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// DecryptionKeys returns all private keys that are valid for decryption. +func (el EntityList) DecryptionKeys() (keys []Key) { + for _, e := range el { + for _, subKey := range e.Subkeys { + if subKey.PrivateKey != nil && (!subKey.Sig.FlagsValid || subKey.Sig.FlagEncryptStorage || subKey.Sig.FlagEncryptCommunications) { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// ReadArmoredKeyRing reads one or more public/private keys from an armor keyring file. +func ReadArmoredKeyRing(r io.Reader) (EntityList, os.Error) { + block, err := armor.Decode(r) + if err == os.EOF { + return nil, error.InvalidArgumentError("no armored data found") + } + if err != nil { + return nil, err + } + if block.Type != PublicKeyType && block.Type != PrivateKeyType { + return nil, error.InvalidArgumentError("expected public or private key block, got: " + block.Type) + } + + return ReadKeyRing(block.Body) +} + +// ReadKeyRing reads one or more public/private keys. Unsupported keys are +// ignored as long as at least a single valid key is found. +func ReadKeyRing(r io.Reader) (el EntityList, err os.Error) { + packets := packet.NewReader(r) + var lastUnsupportedError os.Error + + for { + var e *Entity + e, err = readEntity(packets) + if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + lastUnsupportedError = err + err = readToNextPublicKey(packets) + } + if err == os.EOF { + err = nil + break + } + if err != nil { + el = nil + break + } + } else { + el = append(el, e) + } + } + + if len(el) == 0 && err == nil { + err = lastUnsupportedError + } + return +} + +// readToNextPublicKey reads packets until the start of the entity and leaves +// the first packet of the new entity in the Reader. +func readToNextPublicKey(packets *packet.Reader) (err os.Error) { + var p packet.Packet + for { + p, err = packets.Next() + if err == os.EOF { + return + } else if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + err = nil + continue + } + return + } + + if pk, ok := p.(*packet.PublicKey); ok && !pk.IsSubkey { + packets.Unread(p) + return + } + } + + panic("unreachable") +} + +// readEntity reads an entity (public key, identities, subkeys etc) from the +// given Reader. +func readEntity(packets *packet.Reader) (*Entity, os.Error) { + e := new(Entity) + e.Identities = make(map[string]*Identity) + + p, err := packets.Next() + if err != nil { + return nil, err + } + + var ok bool + if e.PrimaryKey, ok = p.(*packet.PublicKey); !ok { + if e.PrivateKey, ok = p.(*packet.PrivateKey); !ok { + packets.Unread(p) + return nil, error.StructuralError("first packet was not a public/private key") + } else { + e.PrimaryKey = &e.PrivateKey.PublicKey + } + } + + if !e.PrimaryKey.PubKeyAlgo.CanSign() { + return nil, error.StructuralError("primary key cannot be used for signatures") + } + + var current *Identity +EachPacket: + for { + p, err := packets.Next() + if err == os.EOF { + break + } else if err != nil { + return nil, err + } + + switch pkt := p.(type) { + case *packet.UserId: + current = new(Identity) + current.Name = pkt.Id + current.UserId = pkt + e.Identities[pkt.Id] = current + + for { + p, err = packets.Next() + if err == os.EOF { + return nil, io.ErrUnexpectedEOF + } else if err != nil { + return nil, err + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("user ID packet not followed by self-signature") + } + + if (sig.SigType == packet.SigTypePositiveCert || sig.SigType == packet.SigTypeGenericCert) && sig.IssuerKeyId != nil && *sig.IssuerKeyId == e.PrimaryKey.KeyId { + if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, sig); err != nil { + return nil, error.StructuralError("user ID self-signature invalid: " + err.String()) + } + current.SelfSignature = sig + break + } + current.Signatures = append(current.Signatures, sig) + } + case *packet.Signature: + if current == nil { + return nil, error.StructuralError("signature packet found before user id packet") + } + current.Signatures = append(current.Signatures, pkt) + case *packet.PrivateKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, &pkt.PublicKey, pkt) + if err != nil { + return nil, err + } + case *packet.PublicKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, pkt, nil) + if err != nil { + return nil, err + } + default: + // we ignore unknown packets + } + } + + if len(e.Identities) == 0 { + return nil, error.StructuralError("entity without any identities") + } + + return e, nil +} + +func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) os.Error { + var subKey Subkey + subKey.PublicKey = pub + subKey.PrivateKey = priv + p, err := packets.Next() + if err == os.EOF { + return io.ErrUnexpectedEOF + } + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + var ok bool + subKey.Sig, ok = p.(*packet.Signature) + if !ok { + return error.StructuralError("subkey packet not followed by signature") + } + if subKey.Sig.SigType != packet.SigTypeSubkeyBinding { + return error.StructuralError("subkey signature with wrong type") + } + err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig) + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + e.Subkeys = append(e.Subkeys, subKey) + return nil +} + +const defaultRSAKeyBits = 2048 + +// NewEntity returns an Entity that contains a fresh RSA/RSA keypair with a +// single identity composed of the given full name, comment and email, any of +// which may be empty but must not contain any of "()<>\x00". +func NewEntity(rand io.Reader, currentTimeSecs int64, name, comment, email string) (*Entity, os.Error) { + uid := packet.NewUserId(name, comment, email) + if uid == nil { + return nil, error.InvalidArgumentError("user id field contained invalid characters") + } + signingPriv, err := rsa.GenerateKey(rand, defaultRSAKeyBits) + if err != nil { + return nil, err + } + encryptingPriv, err := rsa.GenerateKey(rand, defaultRSAKeyBits) + if err != nil { + return nil, err + } + + t := uint32(currentTimeSecs) + + e := &Entity{ + PrimaryKey: packet.NewRSAPublicKey(t, &signingPriv.PublicKey, false /* not a subkey */ ), + PrivateKey: packet.NewRSAPrivateKey(t, signingPriv, false /* not a subkey */ ), + Identities: make(map[string]*Identity), + } + isPrimaryId := true + e.Identities[uid.Id] = &Identity{ + Name: uid.Name, + UserId: uid, + SelfSignature: &packet.Signature{ + CreationTime: t, + SigType: packet.SigTypePositiveCert, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: crypto.SHA256, + IsPrimaryId: &isPrimaryId, + FlagsValid: true, + FlagSign: true, + FlagCertify: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + e.Subkeys = make([]Subkey, 1) + e.Subkeys[0] = Subkey{ + PublicKey: packet.NewRSAPublicKey(t, &encryptingPriv.PublicKey, true /* is a subkey */ ), + PrivateKey: packet.NewRSAPrivateKey(t, encryptingPriv, true /* is a subkey */ ), + Sig: &packet.Signature{ + CreationTime: t, + SigType: packet.SigTypeSubkeyBinding, + PubKeyAlgo: packet.PubKeyAlgoRSA, + Hash: crypto.SHA256, + FlagsValid: true, + FlagEncryptStorage: true, + FlagEncryptCommunications: true, + IssuerKeyId: &e.PrimaryKey.KeyId, + }, + } + + return e, nil +} + +// SerializePrivate serializes an Entity, including private key material, to +// the given Writer. For now, it must only be used on an Entity returned from +// NewEntity. +func (e *Entity) SerializePrivate(w io.Writer) (err os.Error) { + err = e.PrivateKey.Serialize(w) + if err != nil { + return + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return + } + err = ident.SelfSignature.SignUserId(ident.UserId.Id, e.PrimaryKey, e.PrivateKey) + if err != nil { + return + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return + } + } + for _, subkey := range e.Subkeys { + err = subkey.PrivateKey.Serialize(w) + if err != nil { + return + } + err = subkey.Sig.SignKey(subkey.PublicKey, e.PrivateKey) + if err != nil { + return + } + err = subkey.Sig.Serialize(w) + if err != nil { + return + } + } + return nil +} + +// Serialize writes the public part of the given Entity to w. (No private +// key material will be output). +func (e *Entity) Serialize(w io.Writer) os.Error { + err := e.PrimaryKey.Serialize(w) + if err != nil { + return err + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return err + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return err + } + for _, sig := range ident.Signatures { + err = sig.Serialize(w) + if err != nil { + return err + } + } + } + for _, subkey := range e.Subkeys { + err = subkey.PublicKey.Serialize(w) + if err != nil { + return err + } + err = subkey.Sig.Serialize(w) + if err != nil { + return err + } + } + return nil +} + +// SignIdentity adds a signature to e, from signer, attesting that identity is +// associated with e. The provided identity must already be an element of +// e.Identities and the private key of signer must have been decrypted if +// necessary. +func (e *Entity) SignIdentity(identity string, signer *Entity) os.Error { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing Entity must have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing Entity's private key must be decrypted") + } + ident, ok := e.Identities[identity] + if !ok { + return error.InvalidArgumentError("given identity string not found in Entity") + } + + sig := &packet.Signature{ + SigType: packet.SigTypeGenericCert, + PubKeyAlgo: signer.PrivateKey.PubKeyAlgo, + Hash: crypto.SHA256, + CreationTime: uint32(time.Seconds()), + IssuerKeyId: &signer.PrivateKey.KeyId, + } + if err := sig.SignKey(e.PrimaryKey, signer.PrivateKey); err != nil { + return err + } + ident.Signatures = append(ident.Signatures, sig) + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/Makefile b/src/pkg/crypto/openpgp/packet/Makefile new file mode 100644 index 000000000..0f0d94eb1 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/Makefile @@ -0,0 +1,22 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/packet +GOFILES=\ + compressed.go\ + encrypted_key.go\ + literal.go\ + one_pass_signature.go\ + packet.go\ + private_key.go\ + public_key.go\ + reader.go\ + signature.go\ + symmetrically_encrypted.go\ + symmetric_key_encrypted.go\ + userid.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/packet/compressed.go b/src/pkg/crypto/openpgp/packet/compressed.go new file mode 100644 index 000000000..1c15c24c4 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/compressed.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "compress/flate" + "compress/zlib" + "crypto/openpgp/error" + "io" + "os" + "strconv" +) + +// Compressed represents a compressed OpenPGP packet. The decompressed contents +// will contain more OpenPGP packets. See RFC 4880, section 5.6. +type Compressed struct { + Body io.Reader +} + +func (c *Compressed) parse(r io.Reader) os.Error { + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + + switch buf[0] { + case 1: + c.Body = flate.NewReader(r) + case 2: + c.Body, err = zlib.NewReader(r) + default: + err = error.UnsupportedError("unknown compression algorithm: " + strconv.Itoa(int(buf[0]))) + } + + return err +} diff --git a/src/pkg/crypto/openpgp/packet/compressed_test.go b/src/pkg/crypto/openpgp/packet/compressed_test.go new file mode 100644 index 000000000..24fe501ed --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/compressed_test.go @@ -0,0 +1,41 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "os" + "io/ioutil" + "testing" +) + +func TestCompressed(t *testing.T) { + packet, err := Read(readerFromHex(compressedHex)) + if err != nil { + t.Errorf("failed to read Compressed: %s", err) + return + } + + c, ok := packet.(*Compressed) + if !ok { + t.Error("didn't find Compressed packet") + return + } + + contents, err := ioutil.ReadAll(c.Body) + if err != nil && err != os.EOF { + t.Error(err) + return + } + + expected, _ := hex.DecodeString(compressedExpectedHex) + if !bytes.Equal(expected, contents) { + t.Errorf("got:%x want:%x", contents, expected) + } +} + +const compressedHex = "a3013b2d90c4e02b72e25f727e5e496a5e49b11e1700" +const compressedExpectedHex = "cb1062004d14c8fe636f6e74656e74732e0a" diff --git a/src/pkg/crypto/openpgp/packet/encrypted_key.go b/src/pkg/crypto/openpgp/packet/encrypted_key.go new file mode 100644 index 000000000..b4730cbc9 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/encrypted_key.go @@ -0,0 +1,168 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "crypto/openpgp/elgamal" + "crypto/openpgp/error" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "io" + "os" + "strconv" +) + +const encryptedKeyVersion = 3 + +// EncryptedKey represents a public-key encrypted session key. See RFC 4880, +// section 5.1. +type EncryptedKey struct { + KeyId uint64 + Algo PublicKeyAlgorithm + CipherFunc CipherFunction // only valid after a successful Decrypt + Key []byte // only valid after a successful Decrypt + + encryptedMPI1, encryptedMPI2 []byte +} + +func (e *EncryptedKey) parse(r io.Reader) (err os.Error) { + var buf [10]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != encryptedKeyVersion { + return error.UnsupportedError("unknown EncryptedKey version " + strconv.Itoa(int(buf[0]))) + } + e.KeyId = binary.BigEndian.Uint64(buf[1:9]) + e.Algo = PublicKeyAlgorithm(buf[9]) + switch e.Algo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + e.encryptedMPI1, _, err = readMPI(r) + case PubKeyAlgoElGamal: + e.encryptedMPI1, _, err = readMPI(r) + if err != nil { + return + } + e.encryptedMPI2, _, err = readMPI(r) + } + _, err = consumeAll(r) + return +} + +func checksumKeyMaterial(key []byte) uint16 { + var checksum uint16 + for _, v := range key { + checksum += uint16(v) + } + return checksum +} + +// Decrypt decrypts an encrypted session key with the given private key. The +// private key must have been decrypted first. +func (e *EncryptedKey) Decrypt(priv *PrivateKey) os.Error { + var err os.Error + var b []byte + + // TODO(agl): use session key decryption routines here to avoid + // padding oracle attacks. + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + b, err = rsa.DecryptPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), e.encryptedMPI1) + case PubKeyAlgoElGamal: + c1 := new(big.Int).SetBytes(e.encryptedMPI1) + c2 := new(big.Int).SetBytes(e.encryptedMPI2) + b, err = elgamal.Decrypt(priv.PrivateKey.(*elgamal.PrivateKey), c1, c2) + default: + err = error.InvalidArgumentError("cannot decrypted encrypted session key with private key of type " + strconv.Itoa(int(priv.PubKeyAlgo))) + } + + if err != nil { + return err + } + + e.CipherFunc = CipherFunction(b[0]) + e.Key = b[1 : len(b)-2] + expectedChecksum := uint16(b[len(b)-2])<<8 | uint16(b[len(b)-1]) + checksum := checksumKeyMaterial(e.Key) + if checksum != expectedChecksum { + return error.StructuralError("EncryptedKey checksum incorrect") + } + + return nil +} + +// SerializeEncryptedKey serializes an encrypted key packet to w that contains +// key, encrypted to pub. +func SerializeEncryptedKey(w io.Writer, rand io.Reader, pub *PublicKey, cipherFunc CipherFunction, key []byte) os.Error { + var buf [10]byte + buf[0] = encryptedKeyVersion + binary.BigEndian.PutUint64(buf[1:9], pub.KeyId) + buf[9] = byte(pub.PubKeyAlgo) + + keyBlock := make([]byte, 1 /* cipher type */ +len(key)+2 /* checksum */ ) + keyBlock[0] = byte(cipherFunc) + copy(keyBlock[1:], key) + checksum := checksumKeyMaterial(key) + keyBlock[1+len(key)] = byte(checksum >> 8) + keyBlock[1+len(key)+1] = byte(checksum) + + switch pub.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly: + return serializeEncryptedKeyRSA(w, rand, buf, pub.PublicKey.(*rsa.PublicKey), keyBlock) + case PubKeyAlgoElGamal: + return serializeEncryptedKeyElGamal(w, rand, buf, pub.PublicKey.(*elgamal.PublicKey), keyBlock) + case PubKeyAlgoDSA, PubKeyAlgoRSASignOnly: + return error.InvalidArgumentError("cannot encrypt to public key of type " + strconv.Itoa(int(pub.PubKeyAlgo))) + } + + return error.UnsupportedError("encrypting a key to public key of type " + strconv.Itoa(int(pub.PubKeyAlgo))) +} + +func serializeEncryptedKeyRSA(w io.Writer, rand io.Reader, header [10]byte, pub *rsa.PublicKey, keyBlock []byte) os.Error { + cipherText, err := rsa.EncryptPKCS1v15(rand, pub, keyBlock) + if err != nil { + return error.InvalidArgumentError("RSA encryption failed: " + err.String()) + } + + packetLen := 10 /* header length */ + 2 /* mpi size */ + len(cipherText) + + err = serializeHeader(w, packetTypeEncryptedKey, packetLen) + if err != nil { + return err + } + _, err = w.Write(header[:]) + if err != nil { + return err + } + return writeMPI(w, 8*uint16(len(cipherText)), cipherText) +} + +func serializeEncryptedKeyElGamal(w io.Writer, rand io.Reader, header [10]byte, pub *elgamal.PublicKey, keyBlock []byte) os.Error { + c1, c2, err := elgamal.Encrypt(rand, pub, keyBlock) + if err != nil { + return error.InvalidArgumentError("ElGamal encryption failed: " + err.String()) + } + + packetLen := 10 /* header length */ + packetLen += 2 /* mpi size */ + (c1.BitLen()+7)/8 + packetLen += 2 /* mpi size */ + (c2.BitLen()+7)/8 + + err = serializeHeader(w, packetTypeEncryptedKey, packetLen) + if err != nil { + return err + } + _, err = w.Write(header[:]) + if err != nil { + return err + } + err = writeBig(w, c1) + if err != nil { + return err + } + return writeBig(w, c2) +} diff --git a/src/pkg/crypto/openpgp/packet/encrypted_key_test.go b/src/pkg/crypto/openpgp/packet/encrypted_key_test.go new file mode 100644 index 000000000..b402245bd --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/encrypted_key_test.go @@ -0,0 +1,126 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "bytes" + "crypto/rand" + "crypto/rsa" + "fmt" + "testing" +) + +func bigFromBase10(s string) *big.Int { + b, ok := new(big.Int).SetString(s, 10) + if !ok { + panic("bigFromBase10 failed") + } + return b +} + +var encryptedKeyPub = rsa.PublicKey{ + E: 65537, + N: bigFromBase10("115804063926007623305902631768113868327816898845124614648849934718568541074358183759250136204762053879858102352159854352727097033322663029387610959884180306668628526686121021235757016368038585212410610742029286439607686208110250133174279811431933746643015923132833417396844716207301518956640020862630546868823"), +} + +var encryptedKeyRSAPriv = &rsa.PrivateKey{ + PublicKey: encryptedKeyPub, + D: bigFromBase10("32355588668219869544751561565313228297765464314098552250409557267371233892496951383426602439009993875125222579159850054973310859166139474359774543943714622292329487391199285040721944491839695981199720170366763547754915493640685849961780092241140181198779299712578774460837139360803883139311171713302987058393"), +} + +var encryptedKeyPriv = &PrivateKey{ + PublicKey: PublicKey{ + PubKeyAlgo: PubKeyAlgoRSA, + }, + PrivateKey: encryptedKeyRSAPriv, +} + +func TestDecryptingEncryptedKey(t *testing.T) { + const encryptedKeyHex = "c18c032a67d68660df41c70104005789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8" + const expectedKeyHex = "d930363f7e0308c333b9618617ea728963d8df993665ae7be1092d4926fd864b" + + p, err := Read(readerFromHex(encryptedKeyHex)) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != 0x2a67d68660df41c7 || ek.Algo != PubKeyAlgoRSA { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + err = ek.Decrypt(encryptedKeyPriv) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + if ek.CipherFunc != CipherAES256 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} + +func TestEncryptingEncryptedKey(t *testing.T) { + key := []byte{1, 2, 3, 4} + const expectedKeyHex = "01020304" + const keyId = 42 + + pub := &PublicKey{ + PublicKey: &encryptedKeyPub, + KeyId: keyId, + PubKeyAlgo: PubKeyAlgoRSAEncryptOnly, + } + + buf := new(bytes.Buffer) + err := SerializeEncryptedKey(buf, rand.Reader, pub, CipherAES128, key) + if err != nil { + t.Errorf("error writing encrypted key packet: %s", err) + } + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != keyId || ek.Algo != PubKeyAlgoRSAEncryptOnly { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + err = ek.Decrypt(encryptedKeyPriv) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + if ek.CipherFunc != CipherAES128 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} diff --git a/src/pkg/crypto/openpgp/packet/literal.go b/src/pkg/crypto/openpgp/packet/literal.go new file mode 100644 index 000000000..9411572d7 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/literal.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "encoding/binary" + "io" + "os" +) + +// LiteralData represents an encrypted file. See RFC 4880, section 5.9. +type LiteralData struct { + IsBinary bool + FileName string + Time uint32 // Unix epoch time. Either creation time or modification time. 0 means undefined. + Body io.Reader +} + +// ForEyesOnly returns whether the contents of the LiteralData have been marked +// as especially sensitive. +func (l *LiteralData) ForEyesOnly() bool { + return l.FileName == "_CONSOLE" +} + +func (l *LiteralData) parse(r io.Reader) (err os.Error) { + var buf [256]byte + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + + l.IsBinary = buf[0] == 'b' + fileNameLen := int(buf[1]) + + _, err = readFull(r, buf[:fileNameLen]) + if err != nil { + return + } + + l.FileName = string(buf[:fileNameLen]) + + _, err = readFull(r, buf[:4]) + if err != nil { + return + } + + l.Time = binary.BigEndian.Uint32(buf[:4]) + l.Body = r + return +} + +// SerializeLiteral serializes a literal data packet to w and returns a +// WriteCloser to which the data itself can be written and which MUST be closed +// on completion. The fileName is truncated to 255 bytes. +func SerializeLiteral(w io.WriteCloser, isBinary bool, fileName string, time uint32) (plaintext io.WriteCloser, err os.Error) { + var buf [4]byte + buf[0] = 't' + if isBinary { + buf[0] = 'b' + } + if len(fileName) > 255 { + fileName = fileName[:255] + } + buf[1] = byte(len(fileName)) + + inner, err := serializeStreamHeader(w, packetTypeLiteralData) + if err != nil { + return + } + + _, err = inner.Write(buf[:2]) + if err != nil { + return + } + _, err = inner.Write([]byte(fileName)) + if err != nil { + return + } + binary.BigEndian.PutUint32(buf[:], time) + _, err = inner.Write(buf[:]) + if err != nil { + return + } + + plaintext = inner + return +} diff --git a/src/pkg/crypto/openpgp/packet/one_pass_signature.go b/src/pkg/crypto/openpgp/packet/one_pass_signature.go new file mode 100644 index 000000000..ca826e4f4 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/one_pass_signature.go @@ -0,0 +1,74 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "encoding/binary" + "io" + "os" + "strconv" +) + +// OnePassSignature represents a one-pass signature packet. See RFC 4880, +// section 5.4. +type OnePassSignature struct { + SigType SignatureType + Hash crypto.Hash + PubKeyAlgo PublicKeyAlgorithm + KeyId uint64 + IsLast bool +} + +const onePassSignatureVersion = 3 + +func (ops *OnePassSignature) parse(r io.Reader) (err os.Error) { + var buf [13]byte + + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != onePassSignatureVersion { + err = error.UnsupportedError("one-pass-signature packet version " + strconv.Itoa(int(buf[0]))) + } + + var ok bool + ops.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return error.UnsupportedError("hash function: " + strconv.Itoa(int(buf[2]))) + } + + ops.SigType = SignatureType(buf[1]) + ops.PubKeyAlgo = PublicKeyAlgorithm(buf[3]) + ops.KeyId = binary.BigEndian.Uint64(buf[4:12]) + ops.IsLast = buf[12] != 0 + return +} + +// Serialize marshals the given OnePassSignature to w. +func (ops *OnePassSignature) Serialize(w io.Writer) os.Error { + var buf [13]byte + buf[0] = onePassSignatureVersion + buf[1] = uint8(ops.SigType) + var ok bool + buf[2], ok = s2k.HashToHashId(ops.Hash) + if !ok { + return error.UnsupportedError("hash type: " + strconv.Itoa(int(ops.Hash))) + } + buf[3] = uint8(ops.PubKeyAlgo) + binary.BigEndian.PutUint64(buf[4:12], ops.KeyId) + if ops.IsLast { + buf[12] = 1 + } + + if err := serializeHeader(w, packetTypeOnePassSignature, len(buf)); err != nil { + return err + } + _, err := w.Write(buf[:]) + return err +} diff --git a/src/pkg/crypto/openpgp/packet/packet.go b/src/pkg/crypto/openpgp/packet/packet.go new file mode 100644 index 000000000..1d7297e38 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/packet.go @@ -0,0 +1,483 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package packet implements parsing and serialization of OpenPGP packets, as +// specified in RFC 4880. +package packet + +import ( + "big" + "crypto/aes" + "crypto/cast5" + "crypto/cipher" + "crypto/openpgp/error" + "io" + "os" +) + +// readFull is the same as io.ReadFull except that reading zero bytes returns +// ErrUnexpectedEOF rather than EOF. +func readFull(r io.Reader, buf []byte) (n int, err os.Error) { + n, err = io.ReadFull(r, buf) + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readLength reads an OpenPGP length from r. See RFC 4880, section 4.2.2. +func readLength(r io.Reader) (length int64, isPartial bool, err os.Error) { + var buf [4]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + switch { + case buf[0] < 192: + length = int64(buf[0]) + case buf[0] < 224: + length = int64(buf[0]-192) << 8 + _, err = readFull(r, buf[0:1]) + if err != nil { + return + } + length += int64(buf[0]) + 192 + case buf[0] < 255: + length = int64(1) << (buf[0] & 0x1f) + isPartial = true + default: + _, err = readFull(r, buf[0:4]) + if err != nil { + return + } + length = int64(buf[0])<<24 | + int64(buf[1])<<16 | + int64(buf[2])<<8 | + int64(buf[3]) + } + return +} + +// partialLengthReader wraps an io.Reader and handles OpenPGP partial lengths. +// The continuation lengths are parsed and removed from the stream and EOF is +// returned at the end of the packet. See RFC 4880, section 4.2.2.4. +type partialLengthReader struct { + r io.Reader + remaining int64 + isPartial bool +} + +func (r *partialLengthReader) Read(p []byte) (n int, err os.Error) { + for r.remaining == 0 { + if !r.isPartial { + return 0, os.EOF + } + r.remaining, r.isPartial, err = readLength(r.r) + if err != nil { + return 0, err + } + } + + toRead := int64(len(p)) + if toRead > r.remaining { + toRead = r.remaining + } + + n, err = r.r.Read(p[:int(toRead)]) + r.remaining -= int64(n) + if n < int(toRead) && err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// partialLengthWriter writes a stream of data using OpenPGP partial lengths. +// See RFC 4880, section 4.2.2.4. +type partialLengthWriter struct { + w io.WriteCloser + lengthByte [1]byte +} + +func (w *partialLengthWriter) Write(p []byte) (n int, err os.Error) { + for len(p) > 0 { + for power := uint(14); power < 32; power-- { + l := 1 << power + if len(p) >= l { + w.lengthByte[0] = 224 + uint8(power) + _, err = w.w.Write(w.lengthByte[:]) + if err != nil { + return + } + var m int + m, err = w.w.Write(p[:l]) + n += m + if err != nil { + return + } + p = p[l:] + break + } + } + } + return +} + +func (w *partialLengthWriter) Close() os.Error { + w.lengthByte[0] = 0 + _, err := w.w.Write(w.lengthByte[:]) + if err != nil { + return err + } + return w.w.Close() +} + +// A spanReader is an io.LimitReader, but it returns ErrUnexpectedEOF if the +// underlying Reader returns EOF before the limit has been reached. +type spanReader struct { + r io.Reader + n int64 +} + +func (l *spanReader) Read(p []byte) (n int, err os.Error) { + if l.n <= 0 { + return 0, os.EOF + } + if int64(len(p)) > l.n { + p = p[0:l.n] + } + n, err = l.r.Read(p) + l.n -= int64(n) + if l.n > 0 && err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readHeader parses a packet header and returns an io.Reader which will return +// the contents of the packet. See RFC 4880, section 4.2. +func readHeader(r io.Reader) (tag packetType, length int64, contents io.Reader, err os.Error) { + var buf [4]byte + _, err = io.ReadFull(r, buf[:1]) + if err != nil { + return + } + if buf[0]&0x80 == 0 { + err = error.StructuralError("tag byte does not have MSB set") + return + } + if buf[0]&0x40 == 0 { + // Old format packet + tag = packetType((buf[0] & 0x3f) >> 2) + lengthType := buf[0] & 3 + if lengthType == 3 { + length = -1 + contents = r + return + } + lengthBytes := 1 << lengthType + _, err = readFull(r, buf[0:lengthBytes]) + if err != nil { + return + } + for i := 0; i < lengthBytes; i++ { + length <<= 8 + length |= int64(buf[i]) + } + contents = &spanReader{r, length} + return + } + + // New format packet + tag = packetType(buf[0] & 0x3f) + length, isPartial, err := readLength(r) + if err != nil { + return + } + if isPartial { + contents = &partialLengthReader{ + remaining: length, + isPartial: true, + r: r, + } + length = -1 + } else { + contents = &spanReader{r, length} + } + return +} + +// serializeHeader writes an OpenPGP packet header to w. See RFC 4880, section +// 4.2. +func serializeHeader(w io.Writer, ptype packetType, length int) (err os.Error) { + var buf [6]byte + var n int + + buf[0] = 0x80 | 0x40 | byte(ptype) + if length < 192 { + buf[1] = byte(length) + n = 2 + } else if length < 8384 { + length -= 192 + buf[1] = 192 + byte(length>>8) + buf[2] = byte(length) + n = 3 + } else { + buf[1] = 255 + buf[2] = byte(length >> 24) + buf[3] = byte(length >> 16) + buf[4] = byte(length >> 8) + buf[5] = byte(length) + n = 6 + } + + _, err = w.Write(buf[:n]) + return +} + +// serializeStreamHeader writes an OpenPGP packet header to w where the +// length of the packet is unknown. It returns a io.WriteCloser which can be +// used to write the contents of the packet. See RFC 4880, section 4.2. +func serializeStreamHeader(w io.WriteCloser, ptype packetType) (out io.WriteCloser, err os.Error) { + var buf [1]byte + buf[0] = 0x80 | 0x40 | byte(ptype) + _, err = w.Write(buf[:]) + if err != nil { + return + } + out = &partialLengthWriter{w: w} + return +} + +// Packet represents an OpenPGP packet. Users are expected to try casting +// instances of this interface to specific packet types. +type Packet interface { + parse(io.Reader) os.Error +} + +// consumeAll reads from the given Reader until error, returning the number of +// bytes read. +func consumeAll(r io.Reader) (n int64, err os.Error) { + var m int + var buf [1024]byte + + for { + m, err = r.Read(buf[:]) + n += int64(m) + if err == os.EOF { + err = nil + return + } + if err != nil { + return + } + } + + panic("unreachable") +} + +// packetType represents the numeric ids of the different OpenPGP packet types. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-2 +type packetType uint8 + +const ( + packetTypeEncryptedKey packetType = 1 + packetTypeSignature packetType = 2 + packetTypeSymmetricKeyEncrypted packetType = 3 + packetTypeOnePassSignature packetType = 4 + packetTypePrivateKey packetType = 5 + packetTypePublicKey packetType = 6 + packetTypePrivateSubkey packetType = 7 + packetTypeCompressed packetType = 8 + packetTypeSymmetricallyEncrypted packetType = 9 + packetTypeLiteralData packetType = 11 + packetTypeUserId packetType = 13 + packetTypePublicSubkey packetType = 14 + packetTypeSymmetricallyEncryptedMDC packetType = 18 +) + +// Read reads a single OpenPGP packet from the given io.Reader. If there is an +// error parsing a packet, the whole packet is consumed from the input. +func Read(r io.Reader) (p Packet, err os.Error) { + tag, _, contents, err := readHeader(r) + if err != nil { + return + } + + switch tag { + case packetTypeEncryptedKey: + p = new(EncryptedKey) + case packetTypeSignature: + p = new(Signature) + case packetTypeSymmetricKeyEncrypted: + p = new(SymmetricKeyEncrypted) + case packetTypeOnePassSignature: + p = new(OnePassSignature) + case packetTypePrivateKey, packetTypePrivateSubkey: + pk := new(PrivateKey) + if tag == packetTypePrivateSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypePublicKey, packetTypePublicSubkey: + pk := new(PublicKey) + if tag == packetTypePublicSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypeCompressed: + p = new(Compressed) + case packetTypeSymmetricallyEncrypted: + p = new(SymmetricallyEncrypted) + case packetTypeLiteralData: + p = new(LiteralData) + case packetTypeUserId: + p = new(UserId) + case packetTypeSymmetricallyEncryptedMDC: + se := new(SymmetricallyEncrypted) + se.MDC = true + p = se + default: + err = error.UnknownPacketTypeError(tag) + } + if p != nil { + err = p.parse(contents) + } + if err != nil { + consumeAll(contents) + } + return +} + +// SignatureType represents the different semantic meanings of an OpenPGP +// signature. See RFC 4880, section 5.2.1. +type SignatureType uint8 + +const ( + SigTypeBinary SignatureType = 0 + SigTypeText = 1 + SigTypeGenericCert = 0x10 + SigTypePersonaCert = 0x11 + SigTypeCasualCert = 0x12 + SigTypePositiveCert = 0x13 + SigTypeSubkeyBinding = 0x18 +) + +// PublicKeyAlgorithm represents the different public key system specified for +// OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-12 +type PublicKeyAlgorithm uint8 + +const ( + PubKeyAlgoRSA PublicKeyAlgorithm = 1 + PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2 + PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3 + PubKeyAlgoElGamal PublicKeyAlgorithm = 16 + PubKeyAlgoDSA PublicKeyAlgorithm = 17 +) + +// CanEncrypt returns true if it's possible to encrypt a message to a public +// key of the given type. +func (pka PublicKeyAlgorithm) CanEncrypt() bool { + switch pka { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoElGamal: + return true + } + return false +} + +// CanSign returns true if it's possible for a public key of the given type to +// sign a message. +func (pka PublicKeyAlgorithm) CanSign() bool { + switch pka { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA: + return true + } + return false +} + +// CipherFunction represents the different block ciphers specified for OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-13 +type CipherFunction uint8 + +const ( + CipherCAST5 CipherFunction = 3 + CipherAES128 CipherFunction = 7 + CipherAES192 CipherFunction = 8 + CipherAES256 CipherFunction = 9 +) + +// KeySize returns the key size, in bytes, of cipher. +func (cipher CipherFunction) KeySize() int { + switch cipher { + case CipherCAST5: + return cast5.KeySize + case CipherAES128: + return 16 + case CipherAES192: + return 24 + case CipherAES256: + return 32 + } + return 0 +} + +// blockSize returns the block size, in bytes, of cipher. +func (cipher CipherFunction) blockSize() int { + switch cipher { + case CipherCAST5: + return 8 + case CipherAES128, CipherAES192, CipherAES256: + return 16 + } + return 0 +} + +// new returns a fresh instance of the given cipher. +func (cipher CipherFunction) new(key []byte) (block cipher.Block) { + switch cipher { + case CipherCAST5: + block, _ = cast5.NewCipher(key) + case CipherAES128, CipherAES192, CipherAES256: + block, _ = aes.NewCipher(key) + } + return +} + +// readMPI reads a big integer from r. The bit length returned is the bit +// length that was specified in r. This is preserved so that the integer can be +// reserialized exactly. +func readMPI(r io.Reader) (mpi []byte, bitLength uint16, err os.Error) { + var buf [2]byte + _, err = readFull(r, buf[0:]) + if err != nil { + return + } + bitLength = uint16(buf[0])<<8 | uint16(buf[1]) + numBytes := (int(bitLength) + 7) / 8 + mpi = make([]byte, numBytes) + _, err = readFull(r, mpi) + return +} + +// mpiLength returns the length of the given *big.Int when serialized as an +// MPI. +func mpiLength(n *big.Int) (mpiLengthInBytes int) { + mpiLengthInBytes = 2 /* MPI length */ + mpiLengthInBytes += (n.BitLen() + 7) / 8 + return +} + +// writeMPI serializes a big integer to w. +func writeMPI(w io.Writer, bitLength uint16, mpiBytes []byte) (err os.Error) { + _, err = w.Write([]byte{byte(bitLength >> 8), byte(bitLength)}) + if err == nil { + _, err = w.Write(mpiBytes) + } + return +} + +// writeBig serializes a *big.Int to w. +func writeBig(w io.Writer, i *big.Int) os.Error { + return writeMPI(w, uint16(i.BitLen()), i.Bytes()) +} diff --git a/src/pkg/crypto/openpgp/packet/packet_test.go b/src/pkg/crypto/openpgp/packet/packet_test.go new file mode 100644 index 000000000..23d9978ae --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/packet_test.go @@ -0,0 +1,256 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestReadFull(t *testing.T) { + var out [4]byte + + b := bytes.NewBufferString("foo") + n, err := readFull(b, out[:3]) + if n != 3 || err != nil { + t.Errorf("full read failed n:%d err:%s", n, err) + } + + b = bytes.NewBufferString("foo") + n, err = readFull(b, out[:4]) + if n != 3 || err != io.ErrUnexpectedEOF { + t.Errorf("partial read failed n:%d err:%s", n, err) + } + + b = bytes.NewBuffer(nil) + n, err = readFull(b, out[:3]) + if n != 0 || err != io.ErrUnexpectedEOF { + t.Errorf("empty read failed n:%d err:%s", n, err) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +var readLengthTests = []struct { + hexInput string + length int64 + isPartial bool + err os.Error +}{ + {"", 0, false, io.ErrUnexpectedEOF}, + {"1f", 31, false, nil}, + {"c0", 0, false, io.ErrUnexpectedEOF}, + {"c101", 256 + 1 + 192, false, nil}, + {"e0", 1, true, nil}, + {"e1", 2, true, nil}, + {"e2", 4, true, nil}, + {"ff", 0, false, io.ErrUnexpectedEOF}, + {"ff00", 0, false, io.ErrUnexpectedEOF}, + {"ff0000", 0, false, io.ErrUnexpectedEOF}, + {"ff000000", 0, false, io.ErrUnexpectedEOF}, + {"ff00000000", 0, false, nil}, + {"ff01020304", 16909060, false, nil}, +} + +func TestReadLength(t *testing.T) { + for i, test := range readLengthTests { + length, isPartial, err := readLength(readerFromHex(test.hexInput)) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + if length != test.length || isPartial != test.isPartial { + t.Errorf("%d: bad result got:(%d,%t) want:(%d,%t)", i, length, isPartial, test.length, test.isPartial) + } + } +} + +var partialLengthReaderTests = []struct { + hexInput string + err os.Error + hexOutput string +}{ + {"e0", io.ErrUnexpectedEOF, ""}, + {"e001", io.ErrUnexpectedEOF, ""}, + {"e0010102", nil, "0102"}, + {"ff00000000", nil, ""}, + {"e10102e1030400", nil, "01020304"}, + {"e101", io.ErrUnexpectedEOF, ""}, +} + +func TestPartialLengthReader(t *testing.T) { + for i, test := range partialLengthReaderTests { + r := &partialLengthReader{readerFromHex(test.hexInput), 0, true} + out, err := ioutil.ReadAll(r) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + + got := fmt.Sprintf("%x", out) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, test.hexOutput, got) + } + } +} + +var readHeaderTests = []struct { + hexInput string + structuralError bool + unexpectedEOF bool + tag int + length int64 + hexOutput string +}{ + {"", false, false, 0, 0, ""}, + {"7f", true, false, 0, 0, ""}, + + // Old format headers + {"80", false, true, 0, 0, ""}, + {"8001", false, true, 0, 1, ""}, + {"800102", false, false, 0, 1, "02"}, + {"81000102", false, false, 0, 1, "02"}, + {"820000000102", false, false, 0, 1, "02"}, + {"860000000102", false, false, 1, 1, "02"}, + {"83010203", false, false, 0, -1, "010203"}, + + // New format headers + {"c0", false, true, 0, 0, ""}, + {"c000", false, false, 0, 0, ""}, + {"c00102", false, false, 0, 1, "02"}, + {"c0020203", false, false, 0, 2, "0203"}, + {"c00202", false, true, 0, 2, ""}, + {"c3020203", false, false, 3, 2, "0203"}, +} + +func TestReadHeader(t *testing.T) { + for i, test := range readHeaderTests { + tag, length, contents, err := readHeader(readerFromHex(test.hexInput)) + if test.structuralError { + if _, ok := err.(error.StructuralError); ok { + continue + } + t.Errorf("%d: expected StructuralError, got:%s", i, err) + continue + } + if err != nil { + if len(test.hexInput) == 0 && err == os.EOF { + continue + } + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from readHeader: %s", i, err) + } + continue + } + if int(tag) != test.tag || length != test.length { + t.Errorf("%d: got:(%d,%d) want:(%d,%d)", i, int(tag), length, test.tag, test.length) + continue + } + + body, err := ioutil.ReadAll(contents) + if err != nil { + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from contents: %s", i, err) + } + continue + } + if test.unexpectedEOF { + t.Errorf("%d: expected ErrUnexpectedEOF from contents but got no error", i) + continue + } + got := fmt.Sprintf("%x", body) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, got, test.hexOutput) + } + } +} + +func TestSerializeHeader(t *testing.T) { + tag := packetTypePublicKey + lengths := []int{0, 1, 2, 64, 192, 193, 8000, 8384, 8385, 10000} + + for _, length := range lengths { + buf := bytes.NewBuffer(nil) + serializeHeader(buf, tag, length) + tag2, length2, _, err := readHeader(buf) + if err != nil { + t.Errorf("length %d, err: %s", length, err) + } + if tag2 != tag { + t.Errorf("length %d, tag incorrect (got %d, want %d)", length, tag2, tag) + } + if int(length2) != length { + t.Errorf("length %d, length incorrect (got %d)", length, length2) + } + } +} + +func TestPartialLengths(t *testing.T) { + buf := bytes.NewBuffer(nil) + w := new(partialLengthWriter) + w.w = noOpCloser{buf} + + const maxChunkSize = 64 + + var b [maxChunkSize]byte + var n uint8 + for l := 1; l <= maxChunkSize; l++ { + for i := 0; i < l; i++ { + b[i] = n + n++ + } + m, err := w.Write(b[:l]) + if m != l { + t.Errorf("short write got: %d want: %d", m, l) + } + if err != nil { + t.Errorf("error from write: %s", err) + } + } + w.Close() + + want := (maxChunkSize * (maxChunkSize + 1)) / 2 + copyBuf := bytes.NewBuffer(nil) + r := &partialLengthReader{buf, 0, true} + m, err := io.Copy(copyBuf, r) + if m != int64(want) { + t.Errorf("short copy got: %d want: %d", m, want) + } + if err != nil { + t.Errorf("error from copy: %s", err) + } + + copyBytes := copyBuf.Bytes() + for i := 0; i < want; i++ { + if copyBytes[i] != uint8(i) { + t.Errorf("bad pattern in copy at %d", i) + break + } + } +} diff --git a/src/pkg/crypto/openpgp/packet/private_key.go b/src/pkg/crypto/openpgp/packet/private_key.go new file mode 100644 index 000000000..6f8133d98 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/private_key.go @@ -0,0 +1,301 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "bytes" + "crypto/cipher" + "crypto/dsa" + "crypto/openpgp/elgamal" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "crypto/rsa" + "crypto/sha1" + "io" + "io/ioutil" + "os" + "strconv" +) + +// PrivateKey represents a possibly encrypted private key. See RFC 4880, +// section 5.5.3. +type PrivateKey struct { + PublicKey + Encrypted bool // if true then the private key is unavailable until Decrypt has been called. + encryptedData []byte + cipher CipherFunction + s2k func(out, in []byte) + PrivateKey interface{} // An *rsa.PrivateKey. + sha1Checksum bool + iv []byte +} + +func NewRSAPrivateKey(currentTimeSecs uint32, priv *rsa.PrivateKey, isSubkey bool) *PrivateKey { + pk := new(PrivateKey) + pk.PublicKey = *NewRSAPublicKey(currentTimeSecs, &priv.PublicKey, isSubkey) + pk.PrivateKey = priv + return pk +} + +func (pk *PrivateKey) parse(r io.Reader) (err os.Error) { + err = (&pk.PublicKey).parse(r) + if err != nil { + return + } + var buf [1]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + + s2kType := buf[0] + + switch s2kType { + case 0: + pk.s2k = nil + pk.Encrypted = false + case 254, 255: + _, err = readFull(r, buf[:]) + if err != nil { + return + } + pk.cipher = CipherFunction(buf[0]) + pk.Encrypted = true + pk.s2k, err = s2k.Parse(r) + if err != nil { + return + } + if s2kType == 254 { + pk.sha1Checksum = true + } + default: + return error.UnsupportedError("deprecated s2k function in private key") + } + + if pk.Encrypted { + blockSize := pk.cipher.blockSize() + if blockSize == 0 { + return error.UnsupportedError("unsupported cipher in private key: " + strconv.Itoa(int(pk.cipher))) + } + pk.iv = make([]byte, blockSize) + _, err = readFull(r, pk.iv) + if err != nil { + return + } + } + + pk.encryptedData, err = ioutil.ReadAll(r) + if err != nil { + return + } + + if !pk.Encrypted { + return pk.parsePrivateKey(pk.encryptedData) + } + + return +} + +func mod64kHash(d []byte) uint16 { + h := uint16(0) + for i := 0; i < len(d); i += 2 { + v := uint16(d[i]) << 8 + if i+1 < len(d) { + v += uint16(d[i+1]) + } + h += v + } + return h +} + +func (pk *PrivateKey) Serialize(w io.Writer) (err os.Error) { + // TODO(agl): support encrypted private keys + buf := bytes.NewBuffer(nil) + err = pk.PublicKey.serializeWithoutHeaders(buf) + if err != nil { + return + } + buf.WriteByte(0 /* no encryption */ ) + + privateKeyBuf := bytes.NewBuffer(nil) + + switch priv := pk.PrivateKey.(type) { + case *rsa.PrivateKey: + err = serializeRSAPrivateKey(privateKeyBuf, priv) + default: + err = error.InvalidArgumentError("non-RSA private key") + } + if err != nil { + return + } + + ptype := packetTypePrivateKey + contents := buf.Bytes() + privateKeyBytes := privateKeyBuf.Bytes() + if pk.IsSubkey { + ptype = packetTypePrivateSubkey + } + err = serializeHeader(w, ptype, len(contents)+len(privateKeyBytes)+2) + if err != nil { + return + } + _, err = w.Write(contents) + if err != nil { + return + } + _, err = w.Write(privateKeyBytes) + if err != nil { + return + } + + checksum := mod64kHash(privateKeyBytes) + var checksumBytes [2]byte + checksumBytes[0] = byte(checksum >> 8) + checksumBytes[1] = byte(checksum) + _, err = w.Write(checksumBytes[:]) + + return +} + +func serializeRSAPrivateKey(w io.Writer, priv *rsa.PrivateKey) os.Error { + err := writeBig(w, priv.D) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[1]) + if err != nil { + return err + } + err = writeBig(w, priv.Primes[0]) + if err != nil { + return err + } + return writeBig(w, priv.Precomputed.Qinv) +} + +// Decrypt decrypts an encrypted private key using a passphrase. +func (pk *PrivateKey) Decrypt(passphrase []byte) os.Error { + if !pk.Encrypted { + return nil + } + + key := make([]byte, pk.cipher.KeySize()) + pk.s2k(key, passphrase) + block := pk.cipher.new(key) + cfb := cipher.NewCFBDecrypter(block, pk.iv) + + data := pk.encryptedData + cfb.XORKeyStream(data, data) + + if pk.sha1Checksum { + if len(data) < sha1.Size { + return error.StructuralError("truncated private key data") + } + h := sha1.New() + h.Write(data[:len(data)-sha1.Size]) + sum := h.Sum() + if !bytes.Equal(sum, data[len(data)-sha1.Size:]) { + return error.StructuralError("private key checksum failure") + } + data = data[:len(data)-sha1.Size] + } else { + if len(data) < 2 { + return error.StructuralError("truncated private key data") + } + var sum uint16 + for i := 0; i < len(data)-2; i++ { + sum += uint16(data[i]) + } + if data[len(data)-2] != uint8(sum>>8) || + data[len(data)-1] != uint8(sum) { + return error.StructuralError("private key checksum failure") + } + data = data[:len(data)-2] + } + + return pk.parsePrivateKey(data) +} + +func (pk *PrivateKey) parsePrivateKey(data []byte) (err os.Error) { + switch pk.PublicKey.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoRSAEncryptOnly: + return pk.parseRSAPrivateKey(data) + case PubKeyAlgoDSA: + return pk.parseDSAPrivateKey(data) + case PubKeyAlgoElGamal: + return pk.parseElGamalPrivateKey(data) + } + panic("impossible") +} + +func (pk *PrivateKey) parseRSAPrivateKey(data []byte) (err os.Error) { + rsaPub := pk.PublicKey.PublicKey.(*rsa.PublicKey) + rsaPriv := new(rsa.PrivateKey) + rsaPriv.PublicKey = *rsaPub + + buf := bytes.NewBuffer(data) + d, _, err := readMPI(buf) + if err != nil { + return + } + p, _, err := readMPI(buf) + if err != nil { + return + } + q, _, err := readMPI(buf) + if err != nil { + return + } + + rsaPriv.D = new(big.Int).SetBytes(d) + rsaPriv.Primes = make([]*big.Int, 2) + rsaPriv.Primes[0] = new(big.Int).SetBytes(p) + rsaPriv.Primes[1] = new(big.Int).SetBytes(q) + rsaPriv.Precompute() + pk.PrivateKey = rsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} + +func (pk *PrivateKey) parseDSAPrivateKey(data []byte) (err os.Error) { + dsaPub := pk.PublicKey.PublicKey.(*dsa.PublicKey) + dsaPriv := new(dsa.PrivateKey) + dsaPriv.PublicKey = *dsaPub + + buf := bytes.NewBuffer(data) + x, _, err := readMPI(buf) + if err != nil { + return + } + + dsaPriv.X = new(big.Int).SetBytes(x) + pk.PrivateKey = dsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} + +func (pk *PrivateKey) parseElGamalPrivateKey(data []byte) (err os.Error) { + pub := pk.PublicKey.PublicKey.(*elgamal.PublicKey) + priv := new(elgamal.PrivateKey) + priv.PublicKey = *pub + + buf := bytes.NewBuffer(data) + x, _, err := readMPI(buf) + if err != nil { + return + } + + priv.X = new(big.Int).SetBytes(x) + pk.PrivateKey = priv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/private_key_test.go b/src/pkg/crypto/openpgp/packet/private_key_test.go new file mode 100644 index 000000000..60eebaa6b --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/private_key_test.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +var privateKeyTests = []struct { + privateKeyHex string + creationTime uint32 +}{ + { + privKeyRSAHex, + 0x4cc349a8, + }, + { + privKeyElGamalHex, + 0x4df9ee1a, + }, +} + +func TestPrivateKeyRead(t *testing.T) { + for i, test := range privateKeyTests { + packet, err := Read(readerFromHex(test.privateKeyHex)) + if err != nil { + t.Errorf("#%d: failed to parse: %s", i, err) + continue + } + + privKey := packet.(*PrivateKey) + + if !privKey.Encrypted { + t.Errorf("#%d: private key isn't encrypted", i) + continue + } + + err = privKey.Decrypt([]byte("testing")) + if err != nil { + t.Errorf("#%d: failed to decrypt: %s", i, err) + continue + } + + if privKey.CreationTime != test.creationTime || privKey.Encrypted { + t.Errorf("#%d: bad result, got: %#v", i, privKey) + } + } +} + +// Generated with `gpg --export-secret-keys "Test Key 2"` +const privKeyRSAHex = "9501fe044cc349a8010400b70ca0010e98c090008d45d1ee8f9113bd5861fd57b88bacb7c68658747663f1e1a3b5a98f32fda6472373c024b97359cd2efc88ff60f77751adfbf6af5e615e6a1408cfad8bf0cea30b0d5f53aa27ad59089ba9b15b7ebc2777a25d7b436144027e3bcd203909f147d0e332b240cf63d3395f5dfe0df0a6c04e8655af7eacdf0011010001fe0303024a252e7d475fd445607de39a265472aa74a9320ba2dac395faa687e9e0336aeb7e9a7397e511b5afd9dc84557c80ac0f3d4d7bfec5ae16f20d41c8c84a04552a33870b930420e230e179564f6d19bb153145e76c33ae993886c388832b0fa042ddda7f133924f3854481533e0ede31d51278c0519b29abc3bf53da673e13e3e1214b52413d179d7f66deee35cac8eacb060f78379d70ef4af8607e68131ff529439668fc39c9ce6dfef8a5ac234d234802cbfb749a26107db26406213ae5c06d4673253a3cbee1fcbae58d6ab77e38d6e2c0e7c6317c48e054edadb5a40d0d48acb44643d998139a8a66bb820be1f3f80185bc777d14b5954b60effe2448a036d565c6bc0b915fcea518acdd20ab07bc1529f561c58cd044f723109b93f6fd99f876ff891d64306b5d08f48bab59f38695e9109c4dec34013ba3153488ce070268381ba923ee1eb77125b36afcb4347ec3478c8f2735b06ef17351d872e577fa95d0c397c88c71b59629a36aec" + +// Generated by `gpg --export-secret-keys` followed by a manual extraction of +// the ElGamal subkey from the packets. +const privKeyElGamalHex = "9d0157044df9ee1a100400eb8e136a58ec39b582629cdadf830bc64e0a94ed8103ca8bb247b27b11b46d1d25297ef4bcc3071785ba0c0bedfe89eabc5287fcc0edf81ab5896c1c8e4b20d27d79813c7aede75320b33eaeeaa586edc00fd1036c10133e6ba0ff277245d0d59d04b2b3421b7244aca5f4a8d870c6f1c1fbff9e1c26699a860b9504f35ca1d700030503fd1ededd3b840795be6d9ccbe3c51ee42e2f39233c432b831ddd9c4e72b7025a819317e47bf94f9ee316d7273b05d5fcf2999c3a681f519b1234bbfa6d359b4752bd9c3f77d6b6456cde152464763414ca130f4e91d91041432f90620fec0e6d6b5116076c2985d5aeaae13be492b9b329efcaf7ee25120159a0a30cd976b42d7afe030302dae7eb80db744d4960c4df930d57e87fe81412eaace9f900e6c839817a614ddb75ba6603b9417c33ea7b6c93967dfa2bcff3fa3c74a5ce2c962db65b03aece14c96cbd0038fc" diff --git a/src/pkg/crypto/openpgp/packet/public_key.go b/src/pkg/crypto/openpgp/packet/public_key.go new file mode 100644 index 000000000..e6b0ae5f3 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/public_key.go @@ -0,0 +1,393 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "crypto/dsa" + "crypto/openpgp/elgamal" + "crypto/openpgp/error" + "crypto/rsa" + "crypto/sha1" + "encoding/binary" + "fmt" + "hash" + "io" + "os" + "strconv" +) + +// PublicKey represents an OpenPGP public key. See RFC 4880, section 5.5.2. +type PublicKey struct { + CreationTime uint32 // seconds since the epoch + PubKeyAlgo PublicKeyAlgorithm + PublicKey interface{} // Either a *rsa.PublicKey or *dsa.PublicKey + Fingerprint [20]byte + KeyId uint64 + IsSubkey bool + + n, e, p, q, g, y parsedMPI +} + +func fromBig(n *big.Int) parsedMPI { + return parsedMPI{ + bytes: n.Bytes(), + bitLength: uint16(n.BitLen()), + } +} + +// NewRSAPublicKey returns a PublicKey that wraps the given rsa.PublicKey. +func NewRSAPublicKey(creationTimeSecs uint32, pub *rsa.PublicKey, isSubkey bool) *PublicKey { + pk := &PublicKey{ + CreationTime: creationTimeSecs, + PubKeyAlgo: PubKeyAlgoRSA, + PublicKey: pub, + IsSubkey: isSubkey, + n: fromBig(pub.N), + e: fromBig(big.NewInt(int64(pub.E))), + } + + pk.setFingerPrintAndKeyId() + return pk +} + +func (pk *PublicKey) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.5.2 + var buf [6]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 4 { + return error.UnsupportedError("public key version") + } + pk.CreationTime = uint32(buf[1])<<24 | uint32(buf[2])<<16 | uint32(buf[3])<<8 | uint32(buf[4]) + pk.PubKeyAlgo = PublicKeyAlgorithm(buf[5]) + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + err = pk.parseRSA(r) + case PubKeyAlgoDSA: + err = pk.parseDSA(r) + case PubKeyAlgoElGamal: + err = pk.parseElGamal(r) + default: + err = error.UnsupportedError("public key type: " + strconv.Itoa(int(pk.PubKeyAlgo))) + } + if err != nil { + return + } + + pk.setFingerPrintAndKeyId() + return +} + +func (pk *PublicKey) setFingerPrintAndKeyId() { + // RFC 4880, section 12.2 + fingerPrint := sha1.New() + pk.SerializeSignaturePrefix(fingerPrint) + pk.serializeWithoutHeaders(fingerPrint) + copy(pk.Fingerprint[:], fingerPrint.Sum()) + pk.KeyId = binary.BigEndian.Uint64(pk.Fingerprint[12:20]) +} + +// parseRSA parses RSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseRSA(r io.Reader) (err os.Error) { + pk.n.bytes, pk.n.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.e.bytes, pk.e.bitLength, err = readMPI(r) + if err != nil { + return + } + + if len(pk.e.bytes) > 3 { + err = error.UnsupportedError("large public exponent") + return + } + rsa := &rsa.PublicKey{ + N: new(big.Int).SetBytes(pk.n.bytes), + E: 0, + } + for i := 0; i < len(pk.e.bytes); i++ { + rsa.E <<= 8 + rsa.E |= int(pk.e.bytes[i]) + } + pk.PublicKey = rsa + return +} + +// parseDSA parses DSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseDSA(r io.Reader) (err os.Error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.q.bytes, pk.q.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + dsa := new(dsa.PublicKey) + dsa.P = new(big.Int).SetBytes(pk.p.bytes) + dsa.Q = new(big.Int).SetBytes(pk.q.bytes) + dsa.G = new(big.Int).SetBytes(pk.g.bytes) + dsa.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = dsa + return +} + +// parseElGamal parses ElGamal public key material from the given Reader. See +// RFC 4880, section 5.5.2. +func (pk *PublicKey) parseElGamal(r io.Reader) (err os.Error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + elgamal := new(elgamal.PublicKey) + elgamal.P = new(big.Int).SetBytes(pk.p.bytes) + elgamal.G = new(big.Int).SetBytes(pk.g.bytes) + elgamal.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = elgamal + return +} + +// SerializeSignaturePrefix writes the prefix for this public key to the given Writer. +// The prefix is used when calculating a signature over this public key. See +// RFC 4880, section 5.2.4. +func (pk *PublicKey) SerializeSignaturePrefix(h hash.Hash) { + var pLength uint16 + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + pLength += 2 + uint16(len(pk.n.bytes)) + pLength += 2 + uint16(len(pk.e.bytes)) + case PubKeyAlgoDSA: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.q.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + case PubKeyAlgoElGamal: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + default: + panic("unknown public key algorithm") + } + pLength += 6 + h.Write([]byte{0x99, byte(pLength >> 8), byte(pLength)}) + return +} + +func (pk *PublicKey) Serialize(w io.Writer) (err os.Error) { + length := 6 // 6 byte header + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + length += 2 + len(pk.n.bytes) + length += 2 + len(pk.e.bytes) + case PubKeyAlgoDSA: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.q.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + case PubKeyAlgoElGamal: + length += 2 + len(pk.p.bytes) + length += 2 + len(pk.g.bytes) + length += 2 + len(pk.y.bytes) + default: + panic("unknown public key algorithm") + } + + packetType := packetTypePublicKey + if pk.IsSubkey { + packetType = packetTypePublicSubkey + } + err = serializeHeader(w, packetType, length) + if err != nil { + return + } + return pk.serializeWithoutHeaders(w) +} + +// serializeWithoutHeaders marshals the PublicKey to w in the form of an +// OpenPGP public key packet, not including the packet header. +func (pk *PublicKey) serializeWithoutHeaders(w io.Writer) (err os.Error) { + var buf [6]byte + buf[0] = 4 + buf[1] = byte(pk.CreationTime >> 24) + buf[2] = byte(pk.CreationTime >> 16) + buf[3] = byte(pk.CreationTime >> 8) + buf[4] = byte(pk.CreationTime) + buf[5] = byte(pk.PubKeyAlgo) + + _, err = w.Write(buf[:]) + if err != nil { + return + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + return writeMPIs(w, pk.n, pk.e) + case PubKeyAlgoDSA: + return writeMPIs(w, pk.p, pk.q, pk.g, pk.y) + case PubKeyAlgoElGamal: + return writeMPIs(w, pk.p, pk.g, pk.y) + } + return error.InvalidArgumentError("bad public-key algorithm") +} + +// CanSign returns true iff this public key can generate signatures +func (pk *PublicKey) CanSign() bool { + return pk.PubKeyAlgo != PubKeyAlgoRSAEncryptOnly && pk.PubKeyAlgo != PubKeyAlgoElGamal +} + +// VerifySignature returns nil iff sig is a valid signature, made by this +// public key, of the data hashed into signed. signed is mutated by this call. +func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err os.Error) { + if !pk.CanSign() { + return error.InvalidArgumentError("public key cannot generate signatures") + } + + signed.Write(sig.HashSuffix) + hashBytes := signed.Sum() + + if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] { + return error.SignatureError("hash tag doesn't match") + } + + if pk.PubKeyAlgo != sig.PubKeyAlgo { + return error.InvalidArgumentError("public key and signature use different algorithms") + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + rsaPublicKey, _ := pk.PublicKey.(*rsa.PublicKey) + err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.RSASignature.bytes) + if err != nil { + return error.SignatureError("RSA verification failure") + } + return nil + case PubKeyAlgoDSA: + dsaPublicKey, _ := pk.PublicKey.(*dsa.PublicKey) + if !dsa.Verify(dsaPublicKey, hashBytes, new(big.Int).SetBytes(sig.DSASigR.bytes), new(big.Int).SetBytes(sig.DSASigS.bytes)) { + return error.SignatureError("DSA verification failure") + } + return nil + default: + panic("shouldn't happen") + } + panic("unreachable") +} + +// keySignatureHash returns a Hash of the message that needs to be signed for +// pk to assert a subkey relationship to signed. +func keySignatureHash(pk, signed *PublicKey, sig *Signature) (h hash.Hash, err os.Error) { + h = sig.Hash.New() + if h == nil { + return nil, error.UnsupportedError("hash function") + } + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + signed.SerializeSignaturePrefix(h) + signed.serializeWithoutHeaders(h) + return +} + +// VerifyKeySignature returns nil iff sig is a valid signature, made by this +// public key, of signed. +func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err os.Error) { + h, err := keySignatureHash(pk, signed, sig) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +// userIdSignatureHash returns a Hash of the message that needs to be signed +// to assert that pk is a valid key for id. +func userIdSignatureHash(id string, pk *PublicKey, sig *Signature) (h hash.Hash, err os.Error) { + h = sig.Hash.New() + if h == nil { + return nil, error.UnsupportedError("hash function") + } + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.serializeWithoutHeaders(h) + + var buf [5]byte + buf[0] = 0xb4 + buf[1] = byte(len(id) >> 24) + buf[2] = byte(len(id) >> 16) + buf[3] = byte(len(id) >> 8) + buf[4] = byte(len(id)) + h.Write(buf[:]) + h.Write([]byte(id)) + + return +} + +// VerifyUserIdSignature returns nil iff sig is a valid signature, made by this +// public key, of id. +func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Error) { + h, err := userIdSignatureHash(id, pk, sig) + if err != nil { + return err + } + return pk.VerifySignature(h, sig) +} + +// KeyIdString returns the public key's fingerprint in capital hex +// (e.g. "6C7EE1B8621CC013"). +func (pk *PublicKey) KeyIdString() string { + return fmt.Sprintf("%X", pk.Fingerprint[12:20]) +} + +// KeyIdShortString returns the short form of public key's fingerprint +// in capital hex, as shown by gpg --list-keys (e.g. "621CC013"). +func (pk *PublicKey) KeyIdShortString() string { + return fmt.Sprintf("%X", pk.Fingerprint[16:20]) +} + +// A parsedMPI is used to store the contents of a big integer, along with the +// bit length that was specified in the original input. This allows the MPI to +// be reserialized exactly. +type parsedMPI struct { + bytes []byte + bitLength uint16 +} + +// writeMPIs is a utility function for serializing several big integers to the +// given Writer. +func writeMPIs(w io.Writer, mpis ...parsedMPI) (err os.Error) { + for _, mpi := range mpis { + err = writeMPI(w, mpi.bitLength, mpi.bytes) + if err != nil { + return + } + } + return +} diff --git a/src/pkg/crypto/openpgp/packet/public_key_test.go b/src/pkg/crypto/openpgp/packet/public_key_test.go new file mode 100644 index 000000000..6e8bfbce6 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/public_key_test.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "testing" +) + +var pubKeyTests = []struct { + hexData string + hexFingerprint string + creationTime uint32 + pubKeyAlgo PublicKeyAlgorithm + keyId uint64 + keyIdString string + keyIdShort string +}{ + {rsaPkDataHex, rsaFingerprintHex, 0x4d3c5c10, PubKeyAlgoRSA, 0xa34d7e18c20c31bb, "A34D7E18C20C31BB", "C20C31BB"}, + {dsaPkDataHex, dsaFingerprintHex, 0x4d432f89, PubKeyAlgoDSA, 0x8e8fbe54062f19ed, "8E8FBE54062F19ED", "062F19ED"}, +} + +func TestPublicKeyRead(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + if pk.PubKeyAlgo != test.pubKeyAlgo { + t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo) + } + if pk.CreationTime != test.creationTime { + t.Errorf("#%d: bad creation time got:%x want:%x", i, pk.CreationTime, test.creationTime) + } + expectedFingerprint, _ := hex.DecodeString(test.hexFingerprint) + if !bytes.Equal(expectedFingerprint, pk.Fingerprint[:]) { + t.Errorf("#%d: bad fingerprint got:%x want:%x", i, pk.Fingerprint[:], expectedFingerprint) + } + if pk.KeyId != test.keyId { + t.Errorf("#%d: bad keyid got:%x want:%x", i, pk.KeyId, test.keyId) + } + if g, e := pk.KeyIdString(), test.keyIdString; g != e { + t.Errorf("#%d: bad KeyIdString got:%q want:%q", i, g, e) + } + if g, e := pk.KeyIdShortString(), test.keyIdShort; g != e { + t.Errorf("#%d: bad KeyIdShortString got:%q want:%q", i, g, e) + } + } +} + +func TestPublicKeySerialize(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + continue + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + continue + } + serializeBuf := bytes.NewBuffer(nil) + err = pk.Serialize(serializeBuf) + if err != nil { + t.Errorf("#%d: failed to serialize: %s", i, err) + continue + } + + packet, err = Read(serializeBuf) + if err != nil { + t.Errorf("#%d: Read error (from serialized data): %s", i, err) + continue + } + pk, ok = packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse serialized data, got: %#v", i, packet) + continue + } + } +} + +const rsaFingerprintHex = "5fb74b1d03b1e3cb31bc2f8aa34d7e18c20c31bb" + +const rsaPkDataHex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001" + +const dsaFingerprintHex = "eece4c094db002103714c63c8e8fbe54062f19ed" + +const dsaPkDataHex = "9901a2044d432f89110400cd581334f0d7a1e1bdc8b9d6d8c0baf68793632735d2bb0903224cbaa1dfbf35a60ee7a13b92643421e1eb41aa8d79bea19a115a677f6b8ba3c7818ce53a6c2a24a1608bd8b8d6e55c5090cbde09dd26e356267465ae25e69ec8bdd57c7bbb2623e4d73336f73a0a9098f7f16da2e25252130fd694c0e8070c55a812a423ae7f00a0ebf50e70c2f19c3520a551bd4b08d30f23530d3d03ff7d0bf4a53a64a09dc5e6e6e35854b7d70c882b0c60293401958b1bd9e40abec3ea05ba87cf64899299d4bd6aa7f459c201d3fbbd6c82004bdc5e8a9eb8082d12054cc90fa9d4ec251a843236a588bf49552441817436c4f43326966fe85447d4e6d0acf8fa1ef0f014730770603ad7634c3088dc52501c237328417c31c89ed70400b2f1a98b0bf42f11fefc430704bebbaa41d9f355600c3facee1e490f64208e0e094ea55e3a598a219a58500bf78ac677b670a14f4e47e9cf8eab4f368cc1ddcaa18cc59309d4cc62dd4f680e73e6cc3e1ce87a84d0925efbcb26c575c093fc42eecf45135fabf6403a25c2016e1774c0484e440a18319072c617cc97ac0a3bb0" diff --git a/src/pkg/crypto/openpgp/packet/reader.go b/src/pkg/crypto/openpgp/packet/reader.go new file mode 100644 index 000000000..5febc3bc8 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/reader.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/openpgp/error" + "io" + "os" +) + +// Reader reads packets from an io.Reader and allows packets to be 'unread' so +// that they result from the next call to Next. +type Reader struct { + q []Packet + readers []io.Reader +} + +// Next returns the most recently unread Packet, or reads another packet from +// the top-most io.Reader. Unknown packet types are skipped. +func (r *Reader) Next() (p Packet, err os.Error) { + if len(r.q) > 0 { + p = r.q[len(r.q)-1] + r.q = r.q[:len(r.q)-1] + return + } + + for len(r.readers) > 0 { + p, err = Read(r.readers[len(r.readers)-1]) + if err == nil { + return + } + if err == os.EOF { + r.readers = r.readers[:len(r.readers)-1] + continue + } + if _, ok := err.(error.UnknownPacketTypeError); !ok { + return nil, err + } + } + + return nil, os.EOF +} + +// Push causes the Reader to start reading from a new io.Reader. When an EOF +// error is seen from the new io.Reader, it is popped and the Reader continues +// to read from the next most recent io.Reader. +func (r *Reader) Push(reader io.Reader) { + r.readers = append(r.readers, reader) +} + +// Unread causes the given Packet to be returned from the next call to Next. +func (r *Reader) Unread(p Packet) { + r.q = append(r.q, p) +} + +func NewReader(r io.Reader) *Reader { + return &Reader{ + q: nil, + readers: []io.Reader{r}, + } +} diff --git a/src/pkg/crypto/openpgp/packet/signature.go b/src/pkg/crypto/openpgp/packet/signature.go new file mode 100644 index 000000000..7577e2875 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/signature.go @@ -0,0 +1,558 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/dsa" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "hash" + "io" + "os" + "strconv" +) + +// Signature represents a signature. See RFC 4880, section 5.2. +type Signature struct { + SigType SignatureType + PubKeyAlgo PublicKeyAlgorithm + Hash crypto.Hash + + // HashSuffix is extra data that is hashed in after the signed data. + HashSuffix []byte + // HashTag contains the first two bytes of the hash for fast rejection + // of bad signed data. + HashTag [2]byte + CreationTime uint32 // Unix epoch time + + RSASignature parsedMPI + DSASigR, DSASigS parsedMPI + + // rawSubpackets contains the unparsed subpackets, in order. + rawSubpackets []outputSubpacket + + // The following are optional so are nil when not included in the + // signature. + + SigLifetimeSecs, KeyLifetimeSecs *uint32 + PreferredSymmetric, PreferredHash, PreferredCompression []uint8 + IssuerKeyId *uint64 + IsPrimaryId *bool + + // FlagsValid is set if any flags were given. See RFC 4880, section + // 5.2.3.21 for details. + FlagsValid bool + FlagCertify, FlagSign, FlagEncryptCommunications, FlagEncryptStorage bool + + outSubpackets []outputSubpacket +} + +func (sig *Signature) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.2.3 + var buf [5]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + if buf[0] != 4 { + err = error.UnsupportedError("signature packet version " + strconv.Itoa(int(buf[0]))) + return + } + + _, err = readFull(r, buf[:5]) + if err != nil { + return + } + sig.SigType = SignatureType(buf[0]) + sig.PubKeyAlgo = PublicKeyAlgorithm(buf[1]) + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly, PubKeyAlgoDSA: + default: + err = error.UnsupportedError("public key algorithm " + strconv.Itoa(int(sig.PubKeyAlgo))) + return + } + + var ok bool + sig.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return error.UnsupportedError("hash function " + strconv.Itoa(int(buf[2]))) + } + + hashedSubpacketsLength := int(buf[3])<<8 | int(buf[4]) + l := 6 + hashedSubpacketsLength + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + copy(sig.HashSuffix[1:], buf[:5]) + hashedSubpackets := sig.HashSuffix[6:l] + _, err = readFull(r, hashedSubpackets) + if err != nil { + return + } + // See RFC 4880, section 5.2.4 + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = uint8(l >> 24) + trailer[3] = uint8(l >> 16) + trailer[4] = uint8(l >> 8) + trailer[5] = uint8(l) + + err = parseSignatureSubpackets(sig, hashedSubpackets, true) + if err != nil { + return + } + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + unhashedSubpacketsLength := int(buf[0])<<8 | int(buf[1]) + unhashedSubpackets := make([]byte, unhashedSubpacketsLength) + _, err = readFull(r, unhashedSubpackets) + if err != nil { + return + } + err = parseSignatureSubpackets(sig, unhashedSubpackets, false) + if err != nil { + return + } + + _, err = readFull(r, sig.HashTag[:2]) + if err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, sig.RSASignature.bitLength, err = readMPI(r) + case PubKeyAlgoDSA: + sig.DSASigR.bytes, sig.DSASigR.bitLength, err = readMPI(r) + if err == nil { + sig.DSASigS.bytes, sig.DSASigS.bitLength, err = readMPI(r) + } + default: + panic("unreachable") + } + return +} + +// parseSignatureSubpackets parses subpackets of the main signature packet. See +// RFC 4880, section 5.2.3.1. +func parseSignatureSubpackets(sig *Signature, subpackets []byte, isHashed bool) (err os.Error) { + for len(subpackets) > 0 { + subpackets, err = parseSignatureSubpacket(sig, subpackets, isHashed) + if err != nil { + return + } + } + + if sig.CreationTime == 0 { + err = error.StructuralError("no creation time in signature") + } + + return +} + +type signatureSubpacketType uint8 + +const ( + creationTimeSubpacket signatureSubpacketType = 2 + signatureExpirationSubpacket signatureSubpacketType = 3 + keyExpirySubpacket signatureSubpacketType = 9 + prefSymmetricAlgosSubpacket signatureSubpacketType = 11 + issuerSubpacket signatureSubpacketType = 16 + prefHashAlgosSubpacket signatureSubpacketType = 21 + prefCompressionSubpacket signatureSubpacketType = 22 + primaryUserIdSubpacket signatureSubpacketType = 25 + keyFlagsSubpacket signatureSubpacketType = 27 +) + +// parseSignatureSubpacket parses a single subpacket. len(subpacket) is >= 1. +func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (rest []byte, err os.Error) { + // RFC 4880, section 5.2.3.1 + var ( + length uint32 + packetType signatureSubpacketType + isCritical bool + ) + switch { + case subpacket[0] < 192: + length = uint32(subpacket[0]) + subpacket = subpacket[1:] + case subpacket[0] < 255: + if len(subpacket) < 2 { + goto Truncated + } + length = uint32(subpacket[0]-192)<<8 + uint32(subpacket[1]) + 192 + subpacket = subpacket[2:] + default: + if len(subpacket) < 5 { + goto Truncated + } + length = uint32(subpacket[1])<<24 | + uint32(subpacket[2])<<16 | + uint32(subpacket[3])<<8 | + uint32(subpacket[4]) + subpacket = subpacket[5:] + } + if length > uint32(len(subpacket)) { + goto Truncated + } + rest = subpacket[length:] + subpacket = subpacket[:length] + if len(subpacket) == 0 { + err = error.StructuralError("zero length signature subpacket") + return + } + packetType = signatureSubpacketType(subpacket[0] & 0x7f) + isCritical = subpacket[0]&0x80 == 0x80 + subpacket = subpacket[1:] + sig.rawSubpackets = append(sig.rawSubpackets, outputSubpacket{isHashed, packetType, isCritical, subpacket}) + switch packetType { + case creationTimeSubpacket: + if !isHashed { + err = error.StructuralError("signature creation time in non-hashed area") + return + } + if len(subpacket) != 4 { + err = error.StructuralError("signature creation time not four bytes") + return + } + sig.CreationTime = binary.BigEndian.Uint32(subpacket) + case signatureExpirationSubpacket: + // Signature expiration time, section 5.2.3.10 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = error.StructuralError("expiration subpacket with bad length") + return + } + sig.SigLifetimeSecs = new(uint32) + *sig.SigLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case keyExpirySubpacket: + // Key expiration time, section 5.2.3.6 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = error.StructuralError("key expiration subpacket with bad length") + return + } + sig.KeyLifetimeSecs = new(uint32) + *sig.KeyLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case prefSymmetricAlgosSubpacket: + // Preferred symmetric algorithms, section 5.2.3.7 + if !isHashed { + return + } + sig.PreferredSymmetric = make([]byte, len(subpacket)) + copy(sig.PreferredSymmetric, subpacket) + case issuerSubpacket: + // Issuer, section 5.2.3.5 + if len(subpacket) != 8 { + err = error.StructuralError("issuer subpacket with bad length") + return + } + sig.IssuerKeyId = new(uint64) + *sig.IssuerKeyId = binary.BigEndian.Uint64(subpacket) + case prefHashAlgosSubpacket: + // Preferred hash algorithms, section 5.2.3.8 + if !isHashed { + return + } + sig.PreferredHash = make([]byte, len(subpacket)) + copy(sig.PreferredHash, subpacket) + case prefCompressionSubpacket: + // Preferred compression algorithms, section 5.2.3.9 + if !isHashed { + return + } + sig.PreferredCompression = make([]byte, len(subpacket)) + copy(sig.PreferredCompression, subpacket) + case primaryUserIdSubpacket: + // Primary User ID, section 5.2.3.19 + if !isHashed { + return + } + if len(subpacket) != 1 { + err = error.StructuralError("primary user id subpacket with bad length") + return + } + sig.IsPrimaryId = new(bool) + if subpacket[0] > 0 { + *sig.IsPrimaryId = true + } + case keyFlagsSubpacket: + // Key flags, section 5.2.3.21 + if !isHashed { + return + } + if len(subpacket) == 0 { + err = error.StructuralError("empty key flags subpacket") + return + } + sig.FlagsValid = true + if subpacket[0]&1 != 0 { + sig.FlagCertify = true + } + if subpacket[0]&2 != 0 { + sig.FlagSign = true + } + if subpacket[0]&4 != 0 { + sig.FlagEncryptCommunications = true + } + if subpacket[0]&8 != 0 { + sig.FlagEncryptStorage = true + } + + default: + if isCritical { + err = error.UnsupportedError("unknown critical signature subpacket type " + strconv.Itoa(int(packetType))) + return + } + } + return + +Truncated: + err = error.StructuralError("signature subpacket truncated") + return +} + +// subpacketLengthLength returns the length, in bytes, of an encoded length value. +func subpacketLengthLength(length int) int { + if length < 192 { + return 1 + } + if length < 16320 { + return 2 + } + return 5 +} + +// serializeSubpacketLength marshals the given length into to. +func serializeSubpacketLength(to []byte, length int) int { + if length < 192 { + to[0] = byte(length) + return 1 + } + if length < 16320 { + length -= 192 + to[0] = byte(length >> 8) + to[1] = byte(length) + return 2 + } + to[0] = 255 + to[1] = byte(length >> 24) + to[2] = byte(length >> 16) + to[3] = byte(length >> 8) + to[4] = byte(length) + return 5 +} + +// subpacketsLength returns the serialized length, in bytes, of the given +// subpackets. +func subpacketsLength(subpackets []outputSubpacket, hashed bool) (length int) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + length += subpacketLengthLength(len(subpacket.contents) + 1) + length += 1 // type byte + length += len(subpacket.contents) + } + } + return +} + +// serializeSubpackets marshals the given subpackets into to. +func serializeSubpackets(to []byte, subpackets []outputSubpacket, hashed bool) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + n := serializeSubpacketLength(to, len(subpacket.contents)+1) + to[n] = byte(subpacket.subpacketType) + to = to[1+n:] + n = copy(to, subpacket.contents) + to = to[n:] + } + } + return +} + +// buildHashSuffix constructs the HashSuffix member of sig in preparation for signing. +func (sig *Signature) buildHashSuffix() (err os.Error) { + hashedSubpacketsLen := subpacketsLength(sig.outSubpackets, true) + + var ok bool + l := 6 + hashedSubpacketsLen + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + sig.HashSuffix[1] = uint8(sig.SigType) + sig.HashSuffix[2] = uint8(sig.PubKeyAlgo) + sig.HashSuffix[3], ok = s2k.HashToHashId(sig.Hash) + if !ok { + sig.HashSuffix = nil + return error.InvalidArgumentError("hash cannot be represented in OpenPGP: " + strconv.Itoa(int(sig.Hash))) + } + sig.HashSuffix[4] = byte(hashedSubpacketsLen >> 8) + sig.HashSuffix[5] = byte(hashedSubpacketsLen) + serializeSubpackets(sig.HashSuffix[6:l], sig.outSubpackets, true) + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = byte(l >> 24) + trailer[3] = byte(l >> 16) + trailer[4] = byte(l >> 8) + trailer[5] = byte(l) + return +} + +func (sig *Signature) signPrepareHash(h hash.Hash) (digest []byte, err os.Error) { + err = sig.buildHashSuffix() + if err != nil { + return + } + + h.Write(sig.HashSuffix) + digest = h.Sum() + copy(sig.HashTag[:], digest) + return +} + +// Sign signs a message with a private key. The hash, h, must contain +// the hash of the message to be signed and will be mutated by this function. +// On success, the signature is stored in sig. Call Serialize to write it out. +func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey) (err os.Error) { + sig.outSubpackets = sig.buildSubpackets() + digest, err := sig.signPrepareHash(h) + if err != nil { + return + } + + switch priv.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sig.RSASignature.bytes, err = rsa.SignPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), sig.Hash, digest) + sig.RSASignature.bitLength = uint16(8 * len(sig.RSASignature.bytes)) + case PubKeyAlgoDSA: + r, s, err := dsa.Sign(rand.Reader, priv.PrivateKey.(*dsa.PrivateKey), digest) + if err == nil { + sig.DSASigR.bytes = r.Bytes() + sig.DSASigR.bitLength = uint16(8 * len(sig.DSASigR.bytes)) + sig.DSASigS.bytes = s.Bytes() + sig.DSASigS.bitLength = uint16(8 * len(sig.DSASigS.bytes)) + } + default: + err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) + } + + return +} + +// SignUserId computes a signature from priv, asserting that pub is a valid +// key for the identity id. On success, the signature is stored in sig. Call +// Serialize to write it out. +func (sig *Signature) SignUserId(id string, pub *PublicKey, priv *PrivateKey) os.Error { + h, err := userIdSignatureHash(id, pub, sig) + if err != nil { + return nil + } + return sig.Sign(h, priv) +} + +// SignKey computes a signature from priv, asserting that pub is a subkey. On +// success, the signature is stored in sig. Call Serialize to write it out. +func (sig *Signature) SignKey(pub *PublicKey, priv *PrivateKey) os.Error { + h, err := keySignatureHash(&priv.PublicKey, pub, sig) + if err != nil { + return err + } + return sig.Sign(h, priv) +} + +// Serialize marshals sig to w. SignRSA or SignDSA must have been called first. +func (sig *Signature) Serialize(w io.Writer) (err os.Error) { + if len(sig.outSubpackets) == 0 { + sig.outSubpackets = sig.rawSubpackets + } + if sig.RSASignature.bytes == nil && sig.DSASigR.bytes == nil { + return error.InvalidArgumentError("Signature: need to call SignRSA or SignDSA before Serialize") + } + + sigLength := 0 + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + sigLength = 2 + len(sig.RSASignature.bytes) + case PubKeyAlgoDSA: + sigLength = 2 + len(sig.DSASigR.bytes) + sigLength += 2 + len(sig.DSASigS.bytes) + default: + panic("impossible") + } + + unhashedSubpacketsLen := subpacketsLength(sig.outSubpackets, false) + length := len(sig.HashSuffix) - 6 /* trailer not included */ + + 2 /* length of unhashed subpackets */ + unhashedSubpacketsLen + + 2 /* hash tag */ + sigLength + err = serializeHeader(w, packetTypeSignature, length) + if err != nil { + return + } + + _, err = w.Write(sig.HashSuffix[:len(sig.HashSuffix)-6]) + if err != nil { + return + } + + unhashedSubpackets := make([]byte, 2+unhashedSubpacketsLen) + unhashedSubpackets[0] = byte(unhashedSubpacketsLen >> 8) + unhashedSubpackets[1] = byte(unhashedSubpacketsLen) + serializeSubpackets(unhashedSubpackets[2:], sig.outSubpackets, false) + + _, err = w.Write(unhashedSubpackets) + if err != nil { + return + } + _, err = w.Write(sig.HashTag[:]) + if err != nil { + return + } + + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + err = writeMPIs(w, sig.RSASignature) + case PubKeyAlgoDSA: + err = writeMPIs(w, sig.DSASigR, sig.DSASigS) + default: + panic("impossible") + } + return +} + +// outputSubpacket represents a subpacket to be marshaled. +type outputSubpacket struct { + hashed bool // true if this subpacket is in the hashed area. + subpacketType signatureSubpacketType + isCritical bool + contents []byte +} + +func (sig *Signature) buildSubpackets() (subpackets []outputSubpacket) { + creationTime := make([]byte, 4) + creationTime[0] = byte(sig.CreationTime >> 24) + creationTime[1] = byte(sig.CreationTime >> 16) + creationTime[2] = byte(sig.CreationTime >> 8) + creationTime[3] = byte(sig.CreationTime) + subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, false, creationTime}) + + if sig.IssuerKeyId != nil { + keyId := make([]byte, 8) + binary.BigEndian.PutUint64(keyId, *sig.IssuerKeyId) + subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, false, keyId}) + } + + return +} diff --git a/src/pkg/crypto/openpgp/packet/signature_test.go b/src/pkg/crypto/openpgp/packet/signature_test.go new file mode 100644 index 000000000..c1bbde8b0 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/signature_test.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto" + "encoding/hex" + "testing" +) + +func TestSignatureRead(t *testing.T) { + packet, err := Read(readerFromHex(signatureDataHex)) + if err != nil { + t.Error(err) + return + } + sig, ok := packet.(*Signature) + if !ok || sig.SigType != SigTypeBinary || sig.PubKeyAlgo != PubKeyAlgoRSA || sig.Hash != crypto.SHA1 { + t.Errorf("failed to parse, got: %#v", packet) + } +} + +func TestSignatureReserialize(t *testing.T) { + packet, _ := Read(readerFromHex(signatureDataHex)) + sig := packet.(*Signature) + out := new(bytes.Buffer) + err := sig.Serialize(out) + if err != nil { + t.Errorf("error reserializing: %s", err) + return + } + + expected, _ := hex.DecodeString(signatureDataHex) + if !bytes.Equal(expected, out.Bytes()) { + t.Errorf("output doesn't match input (got vs expected):\n%s\n%s", hex.Dump(out.Bytes()), hex.Dump(expected)) + } +} + +const signatureDataHex = "c2c05c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e" diff --git a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go new file mode 100644 index 000000000..ad4f1d621 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted.go @@ -0,0 +1,162 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/cipher" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "io" + "os" + "strconv" +) + +// This is the largest session key that we'll support. Since no 512-bit cipher +// has even been seriously used, this is comfortably large. +const maxSessionKeySizeInBytes = 64 + +// SymmetricKeyEncrypted represents a passphrase protected session key. See RFC +// 4880, section 5.3. +type SymmetricKeyEncrypted struct { + CipherFunc CipherFunction + Encrypted bool + Key []byte // Empty unless Encrypted is false. + s2k func(out, in []byte) + encryptedKey []byte +} + +const symmetricKeyEncryptedVersion = 4 + +func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.3. + var buf [2]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != symmetricKeyEncryptedVersion { + return error.UnsupportedError("SymmetricKeyEncrypted version") + } + ske.CipherFunc = CipherFunction(buf[1]) + + if ske.CipherFunc.KeySize() == 0 { + return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(buf[1]))) + } + + ske.s2k, err = s2k.Parse(r) + if err != nil { + return + } + + encryptedKey := make([]byte, maxSessionKeySizeInBytes) + // The session key may follow. We just have to try and read to find + // out. If it exists then we limit it to maxSessionKeySizeInBytes. + n, err := readFull(r, encryptedKey) + if err != nil && err != io.ErrUnexpectedEOF { + return + } + err = nil + if n != 0 { + if n == maxSessionKeySizeInBytes { + return error.UnsupportedError("oversized encrypted session key") + } + ske.encryptedKey = encryptedKey[:n] + } + + ske.Encrypted = true + + return +} + +// Decrypt attempts to decrypt an encrypted session key. If it returns nil, +// ske.Key will contain the session key. +func (ske *SymmetricKeyEncrypted) Decrypt(passphrase []byte) os.Error { + if !ske.Encrypted { + return nil + } + + key := make([]byte, ske.CipherFunc.KeySize()) + ske.s2k(key, passphrase) + + if len(ske.encryptedKey) == 0 { + ske.Key = key + } else { + // the IV is all zeros + iv := make([]byte, ske.CipherFunc.blockSize()) + c := cipher.NewCFBDecrypter(ske.CipherFunc.new(key), iv) + c.XORKeyStream(ske.encryptedKey, ske.encryptedKey) + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + if ske.CipherFunc.blockSize() == 0 { + return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(ske.CipherFunc))) + } + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + ske.Key = ske.encryptedKey[1:] + if len(ske.Key)%ske.CipherFunc.blockSize() != 0 { + ske.Key = nil + return error.StructuralError("length of decrypted key not a multiple of block size") + } + } + + ske.Encrypted = false + return nil +} + +// SerializeSymmetricKeyEncrypted serializes a symmetric key packet to w. The +// packet contains a random session key, encrypted by a key derived from the +// given passphrase. The session key is returned and must be passed to +// SerializeSymmetricallyEncrypted. +func SerializeSymmetricKeyEncrypted(w io.Writer, rand io.Reader, passphrase []byte, cipherFunc CipherFunction) (key []byte, err os.Error) { + keySize := cipherFunc.KeySize() + if keySize == 0 { + return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(cipherFunc))) + } + + s2kBuf := new(bytes.Buffer) + keyEncryptingKey := make([]byte, keySize) + // s2k.Serialize salts and stretches the passphrase, and writes the + // resulting key to keyEncryptingKey and the s2k descriptor to s2kBuf. + err = s2k.Serialize(s2kBuf, keyEncryptingKey, rand, passphrase) + if err != nil { + return + } + s2kBytes := s2kBuf.Bytes() + + packetLength := 2 /* header */ + len(s2kBytes) + 1 /* cipher type */ + keySize + err = serializeHeader(w, packetTypeSymmetricKeyEncrypted, packetLength) + if err != nil { + return + } + + var buf [2]byte + buf[0] = symmetricKeyEncryptedVersion + buf[1] = byte(cipherFunc) + _, err = w.Write(buf[:]) + if err != nil { + return + } + _, err = w.Write(s2kBytes) + if err != nil { + return + } + + sessionKey := make([]byte, keySize) + _, err = io.ReadFull(rand, sessionKey) + if err != nil { + return + } + iv := make([]byte, cipherFunc.blockSize()) + c := cipher.NewCFBEncrypter(cipherFunc.new(keyEncryptingKey), iv) + encryptedCipherAndKey := make([]byte, keySize+1) + c.XORKeyStream(encryptedCipherAndKey, buf[1:]) + c.XORKeyStream(encryptedCipherAndKey[1:], sessionKey) + _, err = w.Write(encryptedCipherAndKey) + if err != nil { + return + } + + key = sessionKey + return +} diff --git a/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go new file mode 100644 index 000000000..823ec400d --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetric_key_encrypted_test.go @@ -0,0 +1,101 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "io/ioutil" + "os" + "testing" +) + +func TestSymmetricKeyEncrypted(t *testing.T) { + buf := readerFromHex(symmetricallyEncryptedHex) + packet, err := Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricKeyEncrypted: %s", err) + return + } + ske, ok := packet.(*SymmetricKeyEncrypted) + if !ok { + t.Error("didn't find SymmetricKeyEncrypted packet") + return + } + err = ske.Decrypt([]byte("password")) + if err != nil { + t.Error(err) + return + } + + packet, err = Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricallyEncrypted: %s", err) + return + } + se, ok := packet.(*SymmetricallyEncrypted) + if !ok { + t.Error("didn't find SymmetricallyEncrypted packet") + return + } + r, err := se.Decrypt(ske.CipherFunc, ske.Key) + if err != nil { + t.Error(err) + return + } + + contents, err := ioutil.ReadAll(r) + if err != nil && err != os.EOF { + t.Error(err) + return + } + + expectedContents, _ := hex.DecodeString(symmetricallyEncryptedContentsHex) + if !bytes.Equal(expectedContents, contents) { + t.Errorf("bad contents got:%x want:%x", contents, expectedContents) + } +} + +const symmetricallyEncryptedHex = "8c0d04030302371a0b38d884f02060c91cf97c9973b8e58e028e9501708ccfe618fb92afef7fa2d80ddadd93cf" +const symmetricallyEncryptedContentsHex = "cb1062004d14c4df636f6e74656e74732e0a" + +func TestSerializeSymmetricKeyEncrypted(t *testing.T) { + buf := bytes.NewBuffer(nil) + passphrase := []byte("testing") + cipherFunc := CipherAES128 + + key, err := SerializeSymmetricKeyEncrypted(buf, rand.Reader, passphrase, cipherFunc) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + p, err := Read(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + ske, ok := p.(*SymmetricKeyEncrypted) + if !ok { + t.Errorf("parsed a different packet type: %#v", p) + return + } + + if !ske.Encrypted { + t.Errorf("SKE not encrypted but should be") + } + if ske.CipherFunc != cipherFunc { + t.Errorf("SKE cipher function is %d (expected %d)", ske.CipherFunc, cipherFunc) + } + err = ske.Decrypt(passphrase) + if err != nil { + t.Errorf("failed to decrypt reparsed SKE: %s", err) + return + } + if !bytes.Equal(key, ske.Key) { + t.Errorf("keys don't match after Decrpyt: %x (original) vs %x (parsed)", key, ske.Key) + } +} diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go new file mode 100644 index 000000000..e33c9f3a0 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted.go @@ -0,0 +1,291 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/cipher" + "crypto/openpgp/error" + "crypto/rand" + "crypto/sha1" + "crypto/subtle" + "hash" + "io" + "os" + "strconv" +) + +// SymmetricallyEncrypted represents a symmetrically encrypted byte string. The +// encrypted contents will consist of more OpenPGP packets. See RFC 4880, +// sections 5.7 and 5.13. +type SymmetricallyEncrypted struct { + MDC bool // true iff this is a type 18 packet and thus has an embedded MAC. + contents io.Reader + prefix []byte +} + +const symmetricallyEncryptedVersion = 1 + +func (se *SymmetricallyEncrypted) parse(r io.Reader) os.Error { + if se.MDC { + // See RFC 4880, section 5.13. + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + if buf[0] != symmetricallyEncryptedVersion { + return error.UnsupportedError("unknown SymmetricallyEncrypted version") + } + } + se.contents = r + return nil +} + +// Decrypt returns a ReadCloser, from which the decrypted contents of the +// packet can be read. An incorrect key can, with high probability, be detected +// immediately and this will result in a KeyIncorrect error being returned. +func (se *SymmetricallyEncrypted) Decrypt(c CipherFunction, key []byte) (io.ReadCloser, os.Error) { + keySize := c.KeySize() + if keySize == 0 { + return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(c))) + } + if len(key) != keySize { + return nil, error.InvalidArgumentError("SymmetricallyEncrypted: incorrect key length") + } + + if se.prefix == nil { + se.prefix = make([]byte, c.blockSize()+2) + _, err := readFull(se.contents, se.prefix) + if err != nil { + return nil, err + } + } else if len(se.prefix) != c.blockSize()+2 { + return nil, error.InvalidArgumentError("can't try ciphers with different block lengths") + } + + ocfbResync := cipher.OCFBResync + if se.MDC { + // MDC packets use a different form of OCFB mode. + ocfbResync = cipher.OCFBNoResync + } + + s := cipher.NewOCFBDecrypter(c.new(key), se.prefix, ocfbResync) + if s == nil { + return nil, error.KeyIncorrectError + } + + plaintext := cipher.StreamReader{S: s, R: se.contents} + + if se.MDC { + // MDC packets have an embedded hash that we need to check. + h := sha1.New() + h.Write(se.prefix) + return &seMDCReader{in: plaintext, h: h}, nil + } + + // Otherwise, we just need to wrap plaintext so that it's a valid ReadCloser. + return seReader{plaintext}, nil +} + +// seReader wraps an io.Reader with a no-op Close method. +type seReader struct { + in io.Reader +} + +func (ser seReader) Read(buf []byte) (int, os.Error) { + return ser.in.Read(buf) +} + +func (ser seReader) Close() os.Error { + return nil +} + +const mdcTrailerSize = 1 /* tag byte */ + 1 /* length byte */ + sha1.Size + +// An seMDCReader wraps an io.Reader, maintains a running hash and keeps hold +// of the most recent 22 bytes (mdcTrailerSize). Upon EOF, those bytes form an +// MDC packet containing a hash of the previous contents which is checked +// against the running hash. See RFC 4880, section 5.13. +type seMDCReader struct { + in io.Reader + h hash.Hash + trailer [mdcTrailerSize]byte + scratch [mdcTrailerSize]byte + trailerUsed int + error bool + eof bool +} + +func (ser *seMDCReader) Read(buf []byte) (n int, err os.Error) { + if ser.error { + err = io.ErrUnexpectedEOF + return + } + if ser.eof { + err = os.EOF + return + } + + // If we haven't yet filled the trailer buffer then we must do that + // first. + for ser.trailerUsed < mdcTrailerSize { + n, err = ser.in.Read(ser.trailer[ser.trailerUsed:]) + ser.trailerUsed += n + if err == os.EOF { + if ser.trailerUsed != mdcTrailerSize { + n = 0 + err = io.ErrUnexpectedEOF + ser.error = true + return + } + ser.eof = true + n = 0 + return + } + + if err != nil { + n = 0 + return + } + } + + // If it's a short read then we read into a temporary buffer and shift + // the data into the caller's buffer. + if len(buf) <= mdcTrailerSize { + n, err = readFull(ser.in, ser.scratch[:len(buf)]) + copy(buf, ser.trailer[:n]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], ser.trailer[n:]) + copy(ser.trailer[mdcTrailerSize-n:], ser.scratch[:]) + if n < len(buf) { + ser.eof = true + err = os.EOF + } + return + } + + n, err = ser.in.Read(buf[mdcTrailerSize:]) + copy(buf, ser.trailer[:]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], buf[n:]) + + if err == os.EOF { + ser.eof = true + } + return +} + +// This is a new-format packet tag byte for a type 19 (MDC) packet. +const mdcPacketTagByte = byte(0x80) | 0x40 | 19 + +func (ser *seMDCReader) Close() os.Error { + if ser.error { + return error.SignatureError("error during reading") + } + + for !ser.eof { + // We haven't seen EOF so we need to read to the end + var buf [1024]byte + _, err := ser.Read(buf[:]) + if err == os.EOF { + break + } + if err != nil { + return error.SignatureError("error during reading") + } + } + + if ser.trailer[0] != mdcPacketTagByte || ser.trailer[1] != sha1.Size { + return error.SignatureError("MDC packet not found") + } + ser.h.Write(ser.trailer[:2]) + + final := ser.h.Sum() + if subtle.ConstantTimeCompare(final, ser.trailer[2:]) != 1 { + return error.SignatureError("hash mismatch") + } + return nil +} + +// An seMDCWriter writes through to an io.WriteCloser while maintains a running +// hash of the data written. On close, it emits an MDC packet containing the +// running hash. +type seMDCWriter struct { + w io.WriteCloser + h hash.Hash +} + +func (w *seMDCWriter) Write(buf []byte) (n int, err os.Error) { + w.h.Write(buf) + return w.w.Write(buf) +} + +func (w *seMDCWriter) Close() (err os.Error) { + var buf [mdcTrailerSize]byte + + buf[0] = mdcPacketTagByte + buf[1] = sha1.Size + w.h.Write(buf[:2]) + digest := w.h.Sum() + copy(buf[2:], digest) + + _, err = w.w.Write(buf[:]) + if err != nil { + return + } + return w.w.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err os.Error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() os.Error { + return nil +} + +// SerializeSymmetricallyEncrypted serializes a symmetrically encrypted packet +// to w and returns a WriteCloser to which the to-be-encrypted packets can be +// written. +func SerializeSymmetricallyEncrypted(w io.Writer, c CipherFunction, key []byte) (contents io.WriteCloser, err os.Error) { + if c.KeySize() != len(key) { + return nil, error.InvalidArgumentError("SymmetricallyEncrypted.Serialize: bad key length") + } + writeCloser := noOpCloser{w} + ciphertext, err := serializeStreamHeader(writeCloser, packetTypeSymmetricallyEncryptedMDC) + if err != nil { + return + } + + _, err = ciphertext.Write([]byte{symmetricallyEncryptedVersion}) + if err != nil { + return + } + + block := c.new(key) + blockSize := block.BlockSize() + iv := make([]byte, blockSize) + _, err = rand.Reader.Read(iv) + if err != nil { + return + } + s, prefix := cipher.NewOCFBEncrypter(block, iv, cipher.OCFBNoResync) + _, err = ciphertext.Write(prefix) + if err != nil { + return + } + plaintext := cipher.StreamWriter{S: s, W: ciphertext} + + h := sha1.New() + h.Write(iv) + h.Write(iv[blockSize-2:]) + contents = &seMDCWriter{w: plaintext, h: h} + return +} diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go new file mode 100644 index 000000000..1054fc2f9 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/openpgp/error" + "crypto/sha1" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err os.Error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = os.EOF + } + return +} + +func testMDCReader(t *testing.T) { + mdcPlaintext, _ := hex.DecodeString(mdcPlaintextHex) + + for stride := 1; stride < len(mdcPlaintext)/2; stride++ { + r := &testReader{data: mdcPlaintext, stride: stride} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + body, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("stride: %d, error: %s", stride, err) + continue + } + if !bytes.Equal(body, mdcPlaintext[:len(mdcPlaintext)-22]) { + t.Errorf("stride: %d: bad contents %x", stride, body) + continue + } + + err = mdcReader.Close() + if err != nil { + t.Errorf("stride: %d, error on Close: %s", stride, err) + } + } + + mdcPlaintext[15] ^= 80 + + r := &testReader{data: mdcPlaintext, stride: 2} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + _, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("corruption test, error: %s", err) + return + } + err = mdcReader.Close() + if err == nil { + t.Error("corruption: no error") + } else if _, ok := err.(*error.SignatureError); !ok { + t.Errorf("corruption: expected SignatureError, got: %s", err) + } +} + +const mdcPlaintextHex = "a302789c3b2d93c4e0eb9aba22283539b3203335af44a134afb800c849cb4c4de10200aff40b45d31432c80cb384299a0655966d6939dfdeed1dddf980" + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + c := CipherAES128 + key := make([]byte, c.KeySize()) + + w, err := SerializeSymmetricallyEncrypted(buf, c, key) + if err != nil { + t.Errorf("error from SerializeSymmetricallyEncrypted: %s", err) + return + } + + contents := []byte("hello world\n") + + w.Write(contents) + w.Close() + + p, err := Read(buf) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + se, ok := p.(*SymmetricallyEncrypted) + if !ok { + t.Errorf("didn't read a *SymmetricallyEncrypted") + return + } + + r, err := se.Decrypt(c, key) + if err != nil { + t.Errorf("error from Decrypt: %s", err) + return + } + + contentsCopy := bytes.NewBuffer(nil) + _, err = io.Copy(contentsCopy, r) + if err != nil { + t.Errorf("error from io.Copy: %s", err) + return + } + if !bytes.Equal(contentsCopy.Bytes(), contents) { + t.Errorf("contents not equal got: %x want: %x", contentsCopy.Bytes(), contents) + } +} diff --git a/src/pkg/crypto/openpgp/packet/userid.go b/src/pkg/crypto/openpgp/packet/userid.go new file mode 100644 index 000000000..0580ba3ed --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/userid.go @@ -0,0 +1,161 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "io" + "io/ioutil" + "os" + "strings" +) + +// UserId contains text that is intended to represent the name and email +// address of the key holder. See RFC 4880, section 5.11. By convention, this +// takes the form "Full Name (Comment) " +type UserId struct { + Id string // By convention, this takes the form "Full Name (Comment) " which is split out in the fields below. + + Name, Comment, Email string +} + +func hasInvalidCharacters(s string) bool { + for _, c := range s { + switch c { + case '(', ')', '<', '>', 0: + return true + } + } + return false +} + +// NewUserId returns a UserId or nil if any of the arguments contain invalid +// characters. The invalid characters are '\x00', '(', ')', '<' and '>' +func NewUserId(name, comment, email string) *UserId { + // RFC 4880 doesn't deal with the structure of userid strings; the + // name, comment and email form is just a convention. However, there's + // no convention about escaping the metacharacters and GPG just refuses + // to create user ids where, say, the name contains a '('. We mirror + // this behaviour. + + if hasInvalidCharacters(name) || hasInvalidCharacters(comment) || hasInvalidCharacters(email) { + return nil + } + + uid := new(UserId) + uid.Name, uid.Comment, uid.Email = name, comment, email + uid.Id = name + if len(comment) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "(" + uid.Id += comment + uid.Id += ")" + } + if len(email) > 0 { + if len(uid.Id) > 0 { + uid.Id += " " + } + uid.Id += "<" + uid.Id += email + uid.Id += ">" + } + return uid +} + +func (uid *UserId) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.11 + b, err := ioutil.ReadAll(r) + if err != nil { + return + } + uid.Id = string(b) + uid.Name, uid.Comment, uid.Email = parseUserId(uid.Id) + return +} + +// Serialize marshals uid to w in the form of an OpenPGP packet, including +// header. +func (uid *UserId) Serialize(w io.Writer) os.Error { + err := serializeHeader(w, packetTypeUserId, len(uid.Id)) + if err != nil { + return err + } + _, err = w.Write([]byte(uid.Id)) + return err +} + +// parseUserId extracts the name, comment and email from a user id string that +// is formatted as "Full Name (Comment) ". +func parseUserId(id string) (name, comment, email string) { + var n, c, e struct { + start, end int + } + var state int + + for offset, rune := range id { + switch state { + case 0: + // Entering name + n.start = offset + state = 1 + fallthrough + case 1: + // In name + if rune == '(' { + state = 2 + n.end = offset + } else if rune == '<' { + state = 5 + n.end = offset + } + case 2: + // Entering comment + c.start = offset + state = 3 + fallthrough + case 3: + // In comment + if rune == ')' { + state = 4 + c.end = offset + } + case 4: + // Between comment and email + if rune == '<' { + state = 5 + } + case 5: + // Entering email + e.start = offset + state = 6 + fallthrough + case 6: + // In email + if rune == '>' { + state = 7 + e.end = offset + } + default: + // After email + } + } + switch state { + case 1: + // ended in the name + n.end = len(id) + case 3: + // ended in comment + c.end = len(id) + case 6: + // ended in email + e.end = len(id) + } + + name = strings.TrimSpace(id[n.start:n.end]) + comment = strings.TrimSpace(id[c.start:c.end]) + email = strings.TrimSpace(id[e.start:e.end]) + return +} diff --git a/src/pkg/crypto/openpgp/packet/userid_test.go b/src/pkg/crypto/openpgp/packet/userid_test.go new file mode 100644 index 000000000..296819389 --- /dev/null +++ b/src/pkg/crypto/openpgp/packet/userid_test.go @@ -0,0 +1,87 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +var userIdTests = []struct { + id string + name, comment, email string +}{ + {"", "", "", ""}, + {"John Smith", "John Smith", "", ""}, + {"John Smith ()", "John Smith", "", ""}, + {"John Smith () <>", "John Smith", "", ""}, + {"(comment", "", "comment", ""}, + {"(comment)", "", "comment", ""}, + {" sdfk", "", "", "email"}, + {" John Smith ( Comment ) asdkflj < email > lksdfj", "John Smith", "Comment", "email"}, + {" John Smith < email > lksdfj", "John Smith", "", "email"}, + {"("}, + {"foo", "bar", "", "foo (bar)"}, + {"foo", "", "baz", "foo "}, + {"", "bar", "baz", "(bar) "}, + {"foo", "bar", "baz", "foo (bar) "}, +} + +func TestNewUserId(t *testing.T) { + for i, test := range newUserIdTests { + uid := NewUserId(test.name, test.comment, test.email) + if uid == nil { + t.Errorf("#%d: returned nil", i) + continue + } + if uid.Id != test.id { + t.Errorf("#%d: got '%s', want '%s'", i, uid.Id, test.id) + } + } +} + +var invalidNewUserIdTests = []struct { + name, comment, email string +}{ + {"foo(", "", ""}, + {"foo<", "", ""}, + {"", "bar)", ""}, + {"", "bar<", ""}, + {"", "", "baz>"}, + {"", "", "baz)"}, + {"", "", "baz\x00"}, +} + +func TestNewUserIdWithInvalidInput(t *testing.T) { + for i, test := range invalidNewUserIdTests { + if uid := NewUserId(test.name, test.comment, test.email); uid != nil { + t.Errorf("#%d: returned non-nil value: %#v", i, uid) + } + } +} diff --git a/src/pkg/crypto/openpgp/read.go b/src/pkg/crypto/openpgp/read.go new file mode 100644 index 000000000..d95f613c6 --- /dev/null +++ b/src/pkg/crypto/openpgp/read.go @@ -0,0 +1,415 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package openpgp implements high level operations on OpenPGP messages. +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + _ "crypto/sha256" + "hash" + "io" + "os" + "strconv" +) + +// SignatureType is the armor type for a PGP signature. +var SignatureType = "PGP SIGNATURE" + +// readArmored reads an armored block with the given type. +func readArmored(r io.Reader, expectedType string) (body io.Reader, err os.Error) { + block, err := armor.Decode(r) + if err != nil { + return + } + + if block.Type != expectedType { + return nil, error.InvalidArgumentError("expected '" + expectedType + "', got: " + block.Type) + } + + return block.Body, nil +} + +// MessageDetails contains the result of parsing an OpenPGP encrypted and/or +// signed message. +type MessageDetails struct { + IsEncrypted bool // true if the message was encrypted. + EncryptedToKeyIds []uint64 // the list of recipient key ids. + IsSymmetricallyEncrypted bool // true if a passphrase could have decrypted the message. + DecryptedWith Key // the private key used to decrypt the message, if any. + IsSigned bool // true if the message is signed. + SignedByKeyId uint64 // the key id of the signer, if any. + SignedBy *Key // the key of the signer, if available. + LiteralData *packet.LiteralData // the metadata of the contents + UnverifiedBody io.Reader // the contents of the message. + + // If IsSigned is true and SignedBy is non-zero then the signature will + // be verified as UnverifiedBody is read. The signature cannot be + // checked until the whole of UnverifiedBody is read so UnverifiedBody + // must be consumed until EOF before the data can trusted. Even if a + // message isn't signed (or the signer is unknown) the data may contain + // an authentication code that is only checked once UnverifiedBody has + // been consumed. Once EOF has been seen, the following fields are + // valid. (An authentication code failure is reported as a + // SignatureError error when reading from UnverifiedBody.) + SignatureError os.Error // nil if the signature is good. + Signature *packet.Signature // the signature packet itself. + + decrypted io.ReadCloser +} + +// A PromptFunction is used as a callback by functions that may need to decrypt +// a private key, or prompt for a passphrase. It is called with a list of +// acceptable, encrypted private keys and a boolean that indicates whether a +// passphrase is usable. It should either decrypt a private key or return a +// passphrase to try. If the decrypted private key or given passphrase isn't +// correct, the function will be called again, forever. Any error returned will +// be passed up. +type PromptFunction func(keys []Key, symmetric bool) ([]byte, os.Error) + +// A keyEnvelopePair is used to store a private key with the envelope that +// contains a symmetric key, encrypted with that key. +type keyEnvelopePair struct { + key Key + encryptedKey *packet.EncryptedKey +} + +// ReadMessage parses an OpenPGP message that may be signed and/or encrypted. +// The given KeyRing should contain both public keys (for signature +// verification) and, possibly encrypted, private keys for decrypting. +func ReadMessage(r io.Reader, keyring KeyRing, prompt PromptFunction) (md *MessageDetails, err os.Error) { + var p packet.Packet + + var symKeys []*packet.SymmetricKeyEncrypted + var pubKeys []keyEnvelopePair + var se *packet.SymmetricallyEncrypted + + packets := packet.NewReader(r) + md = new(MessageDetails) + md.IsEncrypted = true + + // The message, if encrypted, starts with a number of packets + // containing an encrypted decryption key. The decryption key is either + // encrypted to a public key, or with a passphrase. This loop + // collects these packets. +ParsePackets: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.SymmetricKeyEncrypted: + // This packet contains the decryption key encrypted with a passphrase. + md.IsSymmetricallyEncrypted = true + symKeys = append(symKeys, p) + case *packet.EncryptedKey: + // This packet contains the decryption key encrypted to a public key. + md.EncryptedToKeyIds = append(md.EncryptedToKeyIds, p.KeyId) + switch p.Algo { + case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSAEncryptOnly, packet.PubKeyAlgoElGamal: + break + default: + continue + } + var keys []Key + if p.KeyId == 0 { + keys = keyring.DecryptionKeys() + } else { + keys = keyring.KeysById(p.KeyId) + } + for _, k := range keys { + pubKeys = append(pubKeys, keyEnvelopePair{k, p}) + } + case *packet.SymmetricallyEncrypted: + se = p + break ParsePackets + case *packet.Compressed, *packet.LiteralData, *packet.OnePassSignature: + // This message isn't encrypted. + if len(symKeys) != 0 || len(pubKeys) != 0 { + return nil, error.StructuralError("key material not followed by encrypted message") + } + packets.Unread(p) + return readSignedMessage(packets, nil, keyring) + } + } + + var candidates []Key + var decrypted io.ReadCloser + + // Now that we have the list of encrypted keys we need to decrypt at + // least one of them or, if we cannot, we need to call the prompt + // function so that it can decrypt a key or give us a passphrase. +FindKey: + for { + // See if any of the keys already have a private key available + candidates = candidates[:0] + candidateFingerprints := make(map[string]bool) + + for _, pk := range pubKeys { + if pk.key.PrivateKey == nil { + continue + } + if !pk.key.PrivateKey.Encrypted { + if len(pk.encryptedKey.Key) == 0 { + pk.encryptedKey.Decrypt(pk.key.PrivateKey) + } + if len(pk.encryptedKey.Key) == 0 { + continue + } + decrypted, err = se.Decrypt(pk.encryptedKey.CipherFunc, pk.encryptedKey.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + md.DecryptedWith = pk.key + break FindKey + } + } else { + fpr := string(pk.key.PublicKey.Fingerprint[:]) + if v := candidateFingerprints[fpr]; v { + continue + } + candidates = append(candidates, pk.key) + candidateFingerprints[fpr] = true + } + } + + if len(candidates) == 0 && len(symKeys) == 0 { + return nil, error.KeyIncorrectError + } + + if prompt == nil { + return nil, error.KeyIncorrectError + } + + passphrase, err := prompt(candidates, len(symKeys) != 0) + if err != nil { + return nil, err + } + + // Try the symmetric passphrase first + if len(symKeys) != 0 && passphrase != nil { + for _, s := range symKeys { + err = s.Decrypt(passphrase) + if err == nil && !s.Encrypted { + decrypted, err = se.Decrypt(s.CipherFunc, s.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + break FindKey + } + } + + } + } + } + + md.decrypted = decrypted + packets.Push(decrypted) + return readSignedMessage(packets, md, keyring) +} + +// readSignedMessage reads a possibly signed message if mdin is non-zero then +// that structure is updated and returned. Otherwise a fresh MessageDetails is +// used. +func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err os.Error) { + if mdin == nil { + mdin = new(MessageDetails) + } + md = mdin + + var p packet.Packet + var h hash.Hash + var wrappedHash hash.Hash +FindLiteralData: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.Compressed: + packets.Push(p.Body) + case *packet.OnePassSignature: + if !p.IsLast { + return nil, error.UnsupportedError("nested signatures") + } + + h, wrappedHash, err = hashForSignature(p.Hash, p.SigType) + if err != nil { + md = nil + return + } + + md.IsSigned = true + md.SignedByKeyId = p.KeyId + keys := keyring.KeysById(p.KeyId) + for i, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + md.SignedBy = &keys[i] + break + } + case *packet.LiteralData: + md.LiteralData = p + break FindLiteralData + } + } + + if md.SignedBy != nil { + md.UnverifiedBody = &signatureCheckReader{packets, h, wrappedHash, md} + } else if md.decrypted != nil { + md.UnverifiedBody = checkReader{md} + } else { + md.UnverifiedBody = md.LiteralData.Body + } + + return md, nil +} + +// hashForSignature returns a pair of hashes that can be used to verify a +// signature. The signature may specify that the contents of the signed message +// should be preprocessed (i.e. to normalize line endings). Thus this function +// returns two hashes. The second should be used to hash the message itself and +// performs any needed preprocessing. +func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, os.Error) { + h := hashId.New() + if h == nil { + return nil, nil, error.UnsupportedError("hash not available: " + strconv.Itoa(int(hashId))) + } + + switch sigType { + case packet.SigTypeBinary: + return h, h, nil + case packet.SigTypeText: + return h, NewCanonicalTextHash(h), nil + } + + return nil, nil, error.UnsupportedError("unsupported signature type: " + strconv.Itoa(int(sigType))) +} + +// checkReader wraps an io.Reader from a LiteralData packet. When it sees EOF +// it closes the ReadCloser from any SymmetricallyEncrypted packet to trigger +// MDC checks. +type checkReader struct { + md *MessageDetails +} + +func (cr checkReader) Read(buf []byte) (n int, err os.Error) { + n, err = cr.md.LiteralData.Body.Read(buf) + if err == os.EOF { + mdcErr := cr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + return +} + +// signatureCheckReader wraps an io.Reader from a LiteralData packet and hashes +// the data as it is read. When it sees an EOF from the underlying io.Reader +// it parses and checks a trailing Signature packet and triggers any MDC checks. +type signatureCheckReader struct { + packets *packet.Reader + h, wrappedHash hash.Hash + md *MessageDetails +} + +func (scr *signatureCheckReader) Read(buf []byte) (n int, err os.Error) { + n, err = scr.md.LiteralData.Body.Read(buf) + scr.wrappedHash.Write(buf[:n]) + if err == os.EOF { + var p packet.Packet + p, scr.md.SignatureError = scr.packets.Next() + if scr.md.SignatureError != nil { + return + } + + var ok bool + if scr.md.Signature, ok = p.(*packet.Signature); !ok { + scr.md.SignatureError = error.StructuralError("LiteralData not followed by Signature") + return + } + + scr.md.SignatureError = scr.md.SignedBy.PublicKey.VerifySignature(scr.h, scr.md.Signature) + + // The SymmetricallyEncrypted packet, if any, might have an + // unsigned hash of its own. In order to check this we need to + // close that Reader. + if scr.md.decrypted != nil { + mdcErr := scr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + } + return +} + +// CheckDetachedSignature takes a signed file and a detached signature and +// returns the signer if the signature is valid. If the signer isn't know, +// UnknownIssuerError is returned. +func CheckDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + p, err := packet.Read(signature) + if err != nil { + return + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("non signature packet found") + } + + if sig.IssuerKeyId == nil { + return nil, error.StructuralError("signature doesn't have an issuer") + } + + keys := keyring.KeysById(*sig.IssuerKeyId) + if len(keys) == 0 { + return nil, error.UnknownIssuerError + } + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + + _, err = io.Copy(wrappedHash, signed) + if err != nil && err != os.EOF { + return + } + + for _, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + err = key.PublicKey.VerifySignature(h, sig) + if err == nil { + return key.Entity, nil + } + } + + if err != nil { + return + } + + return nil, error.UnknownIssuerError +} + +// CheckArmoredDetachedSignature performs the same actions as +// CheckDetachedSignature but expects the signature to be armored. +func CheckArmoredDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + body, err := readArmored(signature, SignatureType) + if err != nil { + return + } + + return CheckDetachedSignature(keyring, signed, body) +} diff --git a/src/pkg/crypto/openpgp/read_test.go b/src/pkg/crypto/openpgp/read_test.go new file mode 100644 index 000000000..4dc290ef2 --- /dev/null +++ b/src/pkg/crypto/openpgp/read_test.go @@ -0,0 +1,361 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func TestReadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestRereadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Errorf("error in initial parse: %s", err) + return + } + out := new(bytes.Buffer) + err = kring[0].Serialize(out) + if err != nil { + t.Errorf("error in serialization: %s", err) + return + } + kring, err = ReadKeyRing(out) + if err != nil { + t.Errorf("error in second parse: %s", err) + return + } + + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadPrivateKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B || kring[0].PrimaryKey == nil { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadDSAKey(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(dsaTestKeyHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0x0CCC0360 { + t.Errorf("bad parse: %#v", kring) + } +} + +func TestGetKeyById(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + keys := kring.KeysById(0xa34d7e18c20c31bb) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } + + keys = kring.KeysById(0xfd94408d4543314f) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } +} + +func checkSignedMessage(t *testing.T, signedHex, expected string) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + md, err := ReadMessage(readerFromHex(signedHex), kring, nil) + if err != nil { + t.Error(err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) != 0 || md.IsSymmetricallyEncrypted { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestSignedMessage(t *testing.T) { + checkSignedMessage(t, signedMessageHex, signedInput) +} + +func TestTextSignedMessage(t *testing.T) { + checkSignedMessage(t, signedTextMessageHex, signedTextInput) +} + +var signedEncryptedMessageTests = []struct { + keyRingHex string + messageHex string + signedByKeyId uint64 + encryptedToKeyId uint64 +}{ + { + testKeys1And2PrivateHex, + signedEncryptedMessageHex, + 0xa34d7e18c20c31bb, + 0x2a67d68660df41c7, + }, + { + dsaElGamalTestKeysHex, + signedEncryptedMessage2Hex, + 0x33af447ccd759b09, + 0xcf6a7abcd43e3673, + }, +} + +func TestSignedEncryptedMessage(t *testing.T) { + for i, test := range signedEncryptedMessageTests { + expected := "Signed and encrypted message\n" + kring, _ := ReadKeyRing(readerFromHex(test.keyRingHex)) + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if symmetric { + t.Errorf("prompt: message was marked as symmetrically encrypted") + return nil, error.KeyIncorrectError + } + + if len(keys) == 0 { + t.Error("prompt: no keys requested") + return nil, error.KeyIncorrectError + } + + err := keys[0].PrivateKey.Decrypt([]byte("passphrase")) + if err != nil { + t.Errorf("prompt: error decrypting key: %s", err) + return nil, error.KeyIncorrectError + } + + return nil, nil + } + + md, err := ReadMessage(readerFromHex(test.messageHex), kring, prompt) + if err != nil { + t.Errorf("#%d: error reading message: %s", i, err) + return + } + + if !md.IsSigned || md.SignedByKeyId != test.signedByKeyId || md.SignedBy == nil || !md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) == 0 || md.EncryptedToKeyIds[0] != test.encryptedToKeyId { + t.Errorf("#%d: bad MessageDetails: %#v", i, md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("#%d: error reading UnverifiedBody: %s", i, err) + } + if string(contents) != expected { + t.Errorf("#%d: bad UnverifiedBody got:%s want:%s", i, string(contents), expected) + } + + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("#%d: failed to validate: %s", i, md.SignatureError) + } + } +} + +func TestUnspecifiedRecipient(t *testing.T) { + expected := "Recipient unspecified\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + + md, err := ReadMessage(readerFromHex(recipientUnspecifiedHex), kring, nil) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } +} + +func TestSymmetricallyEncrypted(t *testing.T) { + expected := "Symmetrically encrypted.\n" + + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if len(keys) != 0 { + t.Errorf("prompt: len(keys) = %d (want 0)", len(keys)) + } + + if !symmetric { + t.Errorf("symmetric is not set") + } + + return []byte("password"), nil + } + + md, err := ReadMessage(readerFromHex(symmetricallyEncryptedCompressedHex), nil, prompt) + if err != nil { + t.Errorf("ReadMessage: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("ReadAll: %s", err) + } + + expectedCreationTime := uint32(1295992998) + if md.LiteralData.Time != expectedCreationTime { + t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreationTime) + } + + if string(contents) != expected { + t.Errorf("contents got: %s want: %s", string(contents), expected) + } +} + +func testDetachedSignature(t *testing.T, kring KeyRing, signature io.Reader, sigInput, tag string, expectedSignerKeyId uint64) { + signed := bytes.NewBufferString(sigInput) + signer, err := CheckDetachedSignature(kring, signed, signature) + if err != nil { + t.Errorf("%s: signature error: %s", tag, err) + return + } + if signer == nil { + t.Errorf("%s: signer is nil", tag) + return + } + if signer.PrimaryKey.KeyId != expectedSignerKeyId { + t.Errorf("%s: wrong signer got:%x want:%x", tag, signer.PrimaryKey.KeyId, expectedSignerKeyId) + } +} + +func TestDetachedSignature(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureHex), signedInput, "binary", testKey1KeyId) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureTextHex), signedInput, "text", testKey1KeyId) +} + +func TestDetachedSignatureDSA(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(dsaTestKeyHex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureDSAHex), signedInput, "binary", testKey3KeyId) +} + +func TestReadingArmoredPrivateKey(t *testing.T) { + el, err := ReadArmoredKeyRing(bytes.NewBufferString(armoredPrivateKeyBlock)) + if err != nil { + t.Error(err) + } + if len(el) != 1 { + t.Errorf("got %d entities, wanted 1\n", len(el)) + } +} + +func TestNoArmoredData(t *testing.T) { + _, err := ReadArmoredKeyRing(bytes.NewBufferString("foo")) + if _, ok := err.(error.InvalidArgumentError); !ok { + t.Errorf("error was not an InvalidArgumentError: %s", err) + } +} + +const testKey1KeyId = 0xA34D7E18C20C31BB +const testKey3KeyId = 0x338934250CCC0360 + +const signedInput = "Signed message\nline 2\nline 3\n" +const signedTextInput = "Signed message\r\nline 2\r\nline 3\r\n" + +const recipientUnspecifiedHex = "848c0300000000000000000103ff62d4d578d03cf40c3da998dfe216c074fa6ddec5e31c197c9666ba292830d91d18716a80f699f9d897389a90e6d62d0238f5f07a5248073c0f24920e4bc4a30c2d17ee4e0cae7c3d4aaa4e8dced50e3010a80ee692175fa0385f62ecca4b56ee6e9980aa3ec51b61b077096ac9e800edaf161268593eedb6cc7027ff5cb32745d250010d407a6221ae22ef18469b444f2822478c4d190b24d36371a95cb40087cdd42d9399c3d06a53c0673349bfb607927f20d1e122bde1e2bf3aa6cae6edf489629bcaa0689539ae3b718914d88ededc3b" + +const detachedSignatureHex = "889c04000102000605024d449cd1000a0910a34d7e18c20c31bb167603ff57718d09f28a519fdc7b5a68b6a3336da04df85e38c5cd5d5bd2092fa4629848a33d85b1729402a2aab39c3ac19f9d573f773cc62c264dc924c067a79dfd8a863ae06c7c8686120760749f5fd9b1e03a64d20a7df3446ddc8f0aeadeaeba7cbaee5c1e366d65b6a0c6cc749bcb912d2f15013f812795c2e29eb7f7b77f39ce77" + +const detachedSignatureTextHex = "889c04010102000605024d449d21000a0910a34d7e18c20c31bbc8c60400a24fbef7342603a41cb1165767bd18985d015fb72fe05db42db36cfb2f1d455967f1e491194fbf6cf88146222b23bf6ffbd50d17598d976a0417d3192ff9cc0034fd00f287b02e90418bbefe609484b09231e4e7a5f3562e199bf39909ab5276c4d37382fe088f6b5c3426fc1052865da8b3ab158672d58b6264b10823dc4b39" + +const detachedSignatureDSAHex = "884604001102000605024d6c4eac000a0910338934250ccc0360f18d00a087d743d6405ed7b87755476629600b8b694a39e900a0abff8126f46faf1547c1743c37b21b4ea15b8f83" + +const testKeys1And2Hex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b0020003b88d044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f0011010001889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab0020003988d044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b0020003b88d044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020003" + +const testKeys1And2PrivateHex = "9501d8044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd00110100010003ff4d91393b9a8e3430b14d6209df42f98dc927425b881f1209f319220841273a802a97c7bdb8b3a7740b3ab5866c4d1d308ad0d3a79bd1e883aacf1ac92dfe720285d10d08752a7efe3c609b1d00f17f2805b217be53999a7da7e493bfc3e9618fd17018991b8128aea70a05dbce30e4fbe626aa45775fa255dd9177aabf4df7cf0200c1ded12566e4bc2bb590455e5becfb2e2c9796482270a943343a7835de41080582c2be3caf5981aa838140e97afa40ad652a0b544f83eb1833b0957dce26e47b0200eacd6046741e9ce2ec5beb6fb5e6335457844fb09477f83b050a96be7da043e17f3a9523567ed40e7a521f818813a8b8a72209f1442844843ccc7eb9805442570200bdafe0438d97ac36e773c7162028d65844c4d463e2420aa2228c6e50dc2743c3d6c72d0d782a5173fe7be2169c8a9f4ef8a7cf3e37165e8c61b89c346cdc6c1799d2b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b00200009d01d8044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f00110100010003fd17a7490c22a79c59281fb7b20f5e6553ec0c1637ae382e8adaea295f50241037f8997cf42c1ce26417e015091451b15424b2c59eb8d4161b0975630408e394d3b00f88d4b4e18e2cc85e8251d4753a27c639c83f5ad4a571c4f19d7cd460b9b73c25ade730c99df09637bd173d8e3e981ac64432078263bb6dc30d3e974150dd0200d0ee05be3d4604d2146fb0457f31ba17c057560785aa804e8ca5530a7cd81d3440d0f4ba6851efcfd3954b7e68908fc0ba47f7ac37bf559c6c168b70d3a7c8cd0200da1c677c4bce06a068070f2b3733b0a714e88d62aa3f9a26c6f5216d48d5c2b5624144f3807c0df30be66b3268eeeca4df1fbded58faf49fc95dc3c35f134f8b01fd1396b6c0fc1b6c4f0eb8f5e44b8eace1e6073e20d0b8bc5385f86f1cf3f050f66af789f3ef1fc107b7f4421e19e0349c730c68f0a226981f4e889054fdb4dc149e8e889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab00200009501fe044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001fe030302e9030f3c783e14856063f16938530e148bc57a7aa3f3e4f90df9dceccdc779bc0835e1ad3d006e4a8d7b36d08b8e0de5a0d947254ecfbd22037e6572b426bcfdc517796b224b0036ff90bc574b5509bede85512f2eefb520fb4b02aa523ba739bff424a6fe81c5041f253f8d757e69a503d3563a104d0d49e9e890b9d0c26f96b55b743883b472caa7050c4acfd4a21f875bdf1258d88bd61224d303dc9df77f743137d51e6d5246b88c406780528fd9a3e15bab5452e5b93970d9dcc79f48b38651b9f15bfbcf6da452837e9cc70683d1bdca94507870f743e4ad902005812488dd342f836e72869afd00ce1850eea4cfa53ce10e3608e13d3c149394ee3cbd0e23d018fcbcb6e2ec5a1a22972d1d462ca05355d0d290dd2751e550d5efb38c6c89686344df64852bf4ff86638708f644e8ec6bd4af9b50d8541cb91891a431326ab2e332faa7ae86cfb6e0540aa63160c1e5cdd5a4add518b303fff0a20117c6bc77f7cfbaf36b04c865c6c2b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b00200009d01fe044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001fe030302e9030f3c783e148560f936097339ae381d63116efcf802ff8b1c9360767db5219cc987375702a4123fd8657d3e22700f23f95020d1b261eda5257e9a72f9a918e8ef22dd5b3323ae03bbc1923dd224db988cadc16acc04b120a9f8b7e84da9716c53e0334d7b66586ddb9014df604b41be1e960dcfcbc96f4ed150a1a0dd070b9eb14276b9b6be413a769a75b519a53d3ecc0c220e85cd91ca354d57e7344517e64b43b6e29823cbd87eae26e2b2e78e6dedfbb76e3e9f77bcb844f9a8932eb3db2c3f9e44316e6f5d60e9e2a56e46b72abe6b06dc9a31cc63f10023d1f5e12d2a3ee93b675c96f504af0001220991c88db759e231b3320dcedf814dcf723fd9857e3d72d66a0f2af26950b915abdf56c1596f46a325bf17ad4810d3535fb02a259b247ac3dbd4cc3ecf9c51b6c07cebb009c1506fba0a89321ec8683e3fd009a6e551d50243e2d5092fefb3321083a4bad91320dc624bd6b5dddf93553e3d53924c05bfebec1fb4bd47e89a1a889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020000" + +const dsaElGamalTestKeysHex = "9501e1044dfcb16a110400aa3e5c1a1f43dd28c2ffae8abf5cfce555ee874134d8ba0a0f7b868ce2214beddc74e5e1e21ded354a95d18acdaf69e5e342371a71fbb9093162e0c5f3427de413a7f2c157d83f5cd2f9d791256dc4f6f0e13f13c3302af27f2384075ab3021dff7a050e14854bbde0a1094174855fc02f0bae8e00a340d94a1f22b32e48485700a0cec672ac21258fb95f61de2ce1af74b2c4fa3e6703ff698edc9be22c02ae4d916e4fa223f819d46582c0516235848a77b577ea49018dcd5e9e15cff9dbb4663a1ae6dd7580fa40946d40c05f72814b0f88481207e6c0832c3bded4853ebba0a7e3bd8e8c66df33d5a537cd4acf946d1080e7a3dcea679cb2b11a72a33a2b6a9dc85f466ad2ddf4c3db6283fa645343286971e3dd700703fc0c4e290d45767f370831a90187e74e9972aae5bff488eeff7d620af0362bfb95c1a6c3413ab5d15a2e4139e5d07a54d72583914661ed6a87cce810be28a0aa8879a2dd39e52fb6fe800f4f181ac7e328f740cde3d09a05cecf9483e4cca4253e60d4429ffd679d9996a520012aad119878c941e3cf151459873bdfc2a9563472fe0303027a728f9feb3b864260a1babe83925ce794710cfd642ee4ae0e5b9d74cee49e9c67b6cd0ea5dfbb582132195a121356a1513e1bca73e5b80c58c7ccb4164453412f456c47616d616c2054657374204b65792031886204131102002205024dfcb16a021b03060b090807030206150802090a0b0416020301021e01021780000a091033af447ccd759b09fadd00a0b8fd6f5a790bad7e9f2dbb7632046dc4493588db009c087c6a9ba9f7f49fab221587a74788c00db4889ab00200009d0157044dfcb16a1004008dec3f9291205255ccff8c532318133a6840739dd68b03ba942676f9038612071447bf07d00d559c5c0875724ea16a4c774f80d8338b55fca691a0522e530e604215b467bbc9ccfd483a1da99d7bc2648b4318fdbd27766fc8bfad3fddb37c62b8ae7ccfe9577e9b8d1e77c1d417ed2c2ef02d52f4da11600d85d3229607943700030503ff506c94c87c8cab778e963b76cf63770f0a79bf48fb49d3b4e52234620fc9f7657f9f8d56c96a2b7c7826ae6b57ebb2221a3fe154b03b6637cea7e6d98e3e45d87cf8dc432f723d3d71f89c5192ac8d7290684d2c25ce55846a80c9a7823f6acd9bb29fa6cd71f20bc90eccfca20451d0c976e460e672b000df49466408d527affe0303027a728f9feb3b864260abd761730327bca2aaa4ea0525c175e92bf240682a0e83b226f97ecb2e935b62c9a133858ce31b271fa8eb41f6a1b3cd72a63025ce1a75ee4180dcc284884904181102000905024dfcb16a021b0c000a091033af447ccd759b09dd0b009e3c3e7296092c81bee5a19929462caaf2fff3ae26009e218c437a2340e7ea628149af1ec98ec091a43992b00200009501e1044dfcb1be1104009f61faa61aa43df75d128cbe53de528c4aec49ce9360c992e70c77072ad5623de0a3a6212771b66b39a30dad6781799e92608316900518ec01184a85d872365b7d2ba4bacfb5882ea3c2473d3750dc6178cc1cf82147fb58caa28b28e9f12f6d1efcb0534abed644156c91cca4ab78834268495160b2400bc422beb37d237c2300a0cac94911b6d493bda1e1fbc6feeca7cb7421d34b03fe22cec6ccb39675bb7b94a335c2b7be888fd3906a1125f33301d8aa6ec6ee6878f46f73961c8d57a3e9544d8ef2a2cbfd4d52da665b1266928cfe4cb347a58c412815f3b2d2369dec04b41ac9a71cc9547426d5ab941cccf3b18575637ccfb42df1a802df3cfe0a999f9e7109331170e3a221991bf868543960f8c816c28097e503fe319db10fb98049f3a57d7c80c420da66d56f3644371631fad3f0ff4040a19a4fedc2d07727a1b27576f75a4d28c47d8246f27071e12d7a8de62aad216ddbae6aa02efd6b8a3e2818cda48526549791ab277e447b3a36c57cefe9b592f5eab73959743fcc8e83cbefec03a329b55018b53eec196765ae40ef9e20521a603c551efe0303020950d53a146bf9c66034d00c23130cce95576a2ff78016ca471276e8227fb30b1ffbd92e61804fb0c3eff9e30b1a826ee8f3e4730b4d86273ca977b4164453412f456c47616d616c2054657374204b65792032886204131102002205024dfcb1be021b03060b090807030206150802090a0b0416020301021e01021780000a0910a86bf526325b21b22bd9009e34511620415c974750a20df5cb56b182f3b48e6600a0a9466cb1a1305a84953445f77d461593f1d42bc1b00200009d0157044dfcb1be1004009565a951da1ee87119d600c077198f1c1bceb0f7aa54552489298e41ff788fa8f0d43a69871f0f6f77ebdfb14a4260cf9fbeb65d5844b4272a1904dd95136d06c3da745dc46327dd44a0f16f60135914368c8039a34033862261806bb2c5ce1152e2840254697872c85441ccb7321431d75a747a4bfb1d2c66362b51ce76311700030503fc0ea76601c196768070b7365a200e6ddb09307f262d5f39eec467b5f5784e22abdf1aa49226f59ab37cb49969d8f5230ea65caf56015abda62604544ed526c5c522bf92bed178a078789f6c807b6d34885688024a5bed9e9f8c58d11d4b82487b44c5f470c5606806a0443b79cadb45e0f897a561a53f724e5349b9267c75ca17fe0303020950d53a146bf9c660bc5f4ce8f072465e2d2466434320c1e712272fafc20e342fe7608101580fa1a1a367e60486a7cd1246b7ef5586cf5e10b32762b710a30144f12dd17dd4884904181102000905024dfcb1be021b0c000a0910a86bf526325b21b2904c00a0b2b66b4b39ccffda1d10f3ea8d58f827e30a8b8e009f4255b2d8112a184e40cde43a34e8655ca7809370b0020000" + +const signedMessageHex = "a3019bc0cbccc0c4b8d8b74ee2108fe16ec6d3ca490cbe362d3f8333d3f352531472538b8b13d353b97232f352158c20943157c71c16064626063656269052062e4e01987e9b6fccff4b7df3a34c534b23e679cbec3bc0f8f6e64dfb4b55fe3f8efa9ce110ddb5cd79faf1d753c51aecfa669f7e7aa043436596cccc3359cb7dd6bbe9ecaa69e5989d9e57209571edc0b2fa7f57b9b79a64ee6e99ce1371395fee92fec2796f7b15a77c386ff668ee27f6d38f0baa6c438b561657377bf6acff3c5947befd7bf4c196252f1d6e5c524d0300" + +const signedTextMessageHex = "a3019bc0cbccc8c4b8d8b74ee2108fe16ec6d36a250cbece0c178233d3f352531472538b8b13d35379b97232f352158ca0b4312f57c71c1646462606365626906a062e4e019811591798ff99bf8afee860b0d8a8c2a85c3387e3bcf0bb3b17987f2bbcfab2aa526d930cbfd3d98757184df3995c9f3e7790e36e3e9779f06089d4c64e9e47dd6202cb6e9bc73c5d11bb59fbaf89d22d8dc7cf199ddf17af96e77c5f65f9bbed56f427bd8db7af37f6c9984bf9385efaf5f184f986fb3e6adb0ecfe35bbf92d16a7aa2a344fb0bc52fb7624f0200" + +const signedEncryptedMessageHex = "848c032a67d68660df41c70103ff5789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8d2c03b018bd210b1d3791e1aba74b0f1034e122ab72e760492c192383cf5e20b5628bd043272d63df9b923f147eb6091cd897553204832aba48fec54aa447547bb16305a1024713b90e77fd0065f1918271947549205af3c74891af22ee0b56cd29bfec6d6e351901cd4ab3ece7c486f1e32a792d4e474aed98ee84b3f591c7dff37b64e0ecd68fd036d517e412dcadf85840ce184ad7921ad446c4ee28db80447aea1ca8d4f574db4d4e37688158ddd19e14ee2eab4873d46947d65d14a23e788d912cf9a19624ca7352469b72a83866b7c23cb5ace3deab3c7018061b0ba0f39ed2befe27163e5083cf9b8271e3e3d52cc7ad6e2a3bd81d4c3d7022f8d" + +const signedEncryptedMessage2Hex = "85010e03cf6a7abcd43e36731003fb057f5495b79db367e277cdbe4ab90d924ddee0c0381494112ff8c1238fb0184af35d1731573b01bc4c55ecacd2aafbe2003d36310487d1ecc9ac994f3fada7f9f7f5c3a64248ab7782906c82c6ff1303b69a84d9a9529c31ecafbcdb9ba87e05439897d87e8a2a3dec55e14df19bba7f7bd316291c002ae2efd24f83f9e3441203fc081c0c23dc3092a454ca8a082b27f631abf73aca341686982e8fbda7e0e7d863941d68f3de4a755c2964407f4b5e0477b3196b8c93d551dd23c8beef7d0f03fbb1b6066f78907faf4bf1677d8fcec72651124080e0b7feae6b476e72ab207d38d90b958759fdedfc3c6c35717c9dbfc979b3cfbbff0a76d24a5e57056bb88acbd2a901ef64bc6e4db02adc05b6250ff378de81dca18c1910ab257dff1b9771b85bb9bbe0a69f5989e6d1710a35e6dfcceb7d8fb5ccea8db3932b3d9ff3fe0d327597c68b3622aec8e3716c83a6c93f497543b459b58ba504ed6bcaa747d37d2ca746fe49ae0a6ce4a8b694234e941b5159ff8bd34b9023da2814076163b86f40eed7c9472f81b551452d5ab87004a373c0172ec87ea6ce42ccfa7dbdad66b745496c4873d8019e8c28d6b3" + +const symmetricallyEncryptedCompressedHex = "8c0d04030302eb4a03808145d0d260c92f714339e13de5a79881216431925bf67ee2898ea61815f07894cd0703c50d0a76ef64d482196f47a8bc729af9b80bb6" + +const dsaTestKeyHex = "9901a2044d6c49de110400cb5ce438cf9250907ac2ba5bf6547931270b89f7c4b53d9d09f4d0213a5ef2ec1f26806d3d259960f872a4a102ef1581ea3f6d6882d15134f21ef6a84de933cc34c47cc9106efe3bd84c6aec12e78523661e29bc1a61f0aab17fa58a627fd5fd33f5149153fbe8cd70edf3d963bc287ef875270ff14b5bfdd1bca4483793923b00a0fe46d76cb6e4cbdc568435cd5480af3266d610d303fe33ae8273f30a96d4d34f42fa28ce1112d425b2e3bf7ea553d526e2db6b9255e9dc7419045ce817214d1a0056dbc8d5289956a4b1b69f20f1105124096e6a438f41f2e2495923b0f34b70642607d45559595c7fe94d7fa85fc41bf7d68c1fd509ebeaa5f315f6059a446b9369c277597e4f474a9591535354c7e7f4fd98a08aa60400b130c24ff20bdfbf683313f5daebf1c9b34b3bdadfc77f2ddd72ee1fb17e56c473664bc21d66467655dd74b9005e3a2bacce446f1920cd7017231ae447b67036c9b431b8179deacd5120262d894c26bc015bffe3d827ba7087ad9b700d2ca1f6d16cc1786581e5dd065f293c31209300f9b0afcc3f7c08dd26d0a22d87580b4db41054657374204b65792033202844534129886204131102002205024d6c49de021b03060b090807030206150802090a0b0416020301021e01021780000a0910338934250ccc03607e0400a0bdb9193e8a6b96fc2dfc108ae848914b504481f100a09c4dc148cb693293a67af24dd40d2b13a9e36794" + +const dsaTestKeyPrivateHex = "9501bb044d6c49de110400cb5ce438cf9250907ac2ba5bf6547931270b89f7c4b53d9d09f4d0213a5ef2ec1f26806d3d259960f872a4a102ef1581ea3f6d6882d15134f21ef6a84de933cc34c47cc9106efe3bd84c6aec12e78523661e29bc1a61f0aab17fa58a627fd5fd33f5149153fbe8cd70edf3d963bc287ef875270ff14b5bfdd1bca4483793923b00a0fe46d76cb6e4cbdc568435cd5480af3266d610d303fe33ae8273f30a96d4d34f42fa28ce1112d425b2e3bf7ea553d526e2db6b9255e9dc7419045ce817214d1a0056dbc8d5289956a4b1b69f20f1105124096e6a438f41f2e2495923b0f34b70642607d45559595c7fe94d7fa85fc41bf7d68c1fd509ebeaa5f315f6059a446b9369c277597e4f474a9591535354c7e7f4fd98a08aa60400b130c24ff20bdfbf683313f5daebf1c9b34b3bdadfc77f2ddd72ee1fb17e56c473664bc21d66467655dd74b9005e3a2bacce446f1920cd7017231ae447b67036c9b431b8179deacd5120262d894c26bc015bffe3d827ba7087ad9b700d2ca1f6d16cc1786581e5dd065f293c31209300f9b0afcc3f7c08dd26d0a22d87580b4d00009f592e0619d823953577d4503061706843317e4fee083db41054657374204b65792033202844534129886204131102002205024d6c49de021b03060b090807030206150802090a0b0416020301021e01021780000a0910338934250ccc03607e0400a0bdb9193e8a6b96fc2dfc108ae848914b504481f100a09c4dc148cb693293a67af24dd40d2b13a9e36794" + +const armoredPrivateKeyBlock = `-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBE2rFNoBBADFwqWQIW/DSqcB4yCQqnAFTJ27qS5AnB46ccAdw3u4Greeu3Bp +idpoHdjULy7zSKlwR1EA873dO/k/e11Ml3dlAFUinWeejWaK2ugFP6JjiieSsrKn +vWNicdCS4HTWn0X4sjl0ZiAygw6GNhqEQ3cpLeL0g8E9hnYzJKQ0LWJa0QARAQAB +AAP/TB81EIo2VYNmTq0pK1ZXwUpxCrvAAIG3hwKjEzHcbQznsjNvPUihZ+NZQ6+X +0HCfPAdPkGDCLCb6NavcSW+iNnLTrdDnSI6+3BbIONqWWdRDYJhqZCkqmG6zqSfL +IdkJgCw94taUg5BWP/AAeQrhzjChvpMQTVKQL5mnuZbUCeMCAN5qrYMP2S9iKdnk +VANIFj7656ARKt/nf4CBzxcpHTyB8+d2CtPDKCmlJP6vL8t58Jmih+kHJMvC0dzn +gr5f5+sCAOOe5gt9e0am7AvQWhdbHVfJU0TQJx+m2OiCJAqGTB1nvtBLHdJnfdC9 +TnXXQ6ZXibqLyBies/xeY2sCKL5qtTMCAKnX9+9d/5yQxRyrQUHt1NYhaXZnJbHx +q4ytu0eWz+5i68IYUSK69jJ1NWPM0T6SkqpB3KCAIv68VFm9PxqG1KmhSrQIVGVz +dCBLZXmIuAQTAQIAIgUCTasU2gIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AA +CgkQO9o98PRieSoLhgQAkLEZex02Qt7vGhZzMwuN0R22w3VwyYyjBx+fM3JFETy1 +ut4xcLJoJfIaF5ZS38UplgakHG0FQ+b49i8dMij0aZmDqGxrew1m4kBfjXw9B/v+ +eIqpODryb6cOSwyQFH0lQkXC040pjq9YqDsO5w0WYNXYKDnzRV0p4H1pweo2VDid +AdgETasU2gEEAN46UPeWRqKHvA99arOxee38fBt2CI08iiWyI8T3J6ivtFGixSqV +bRcPxYO/qLpVe5l84Nb3X71GfVXlc9hyv7CD6tcowL59hg1E/DC5ydI8K8iEpUmK +/UnHdIY5h8/kqgGxkY/T/hgp5fRQgW1ZoZxLajVlMRZ8W4tFtT0DeA+JABEBAAEA +A/0bE1jaaZKj6ndqcw86jd+QtD1SF+Cf21CWRNeLKnUds4FRRvclzTyUMuWPkUeX +TaNNsUOFqBsf6QQ2oHUBBK4VCHffHCW4ZEX2cd6umz7mpHW6XzN4DECEzOVksXtc +lUC1j4UB91DC/RNQqwX1IV2QLSwssVotPMPqhOi0ZLNY7wIA3n7DWKInxYZZ4K+6 +rQ+POsz6brEoRHwr8x6XlHenq1Oki855pSa1yXIARoTrSJkBtn5oI+f8AzrnN0BN +oyeQAwIA/7E++3HDi5aweWrViiul9cd3rcsS0dEnksPhvS0ozCJiHsq/6GFmy7J8 +QSHZPteedBnZyNp5jR+H7cIfVN3KgwH/Skq4PsuPhDq5TKK6i8Pc1WW8MA6DXTdU +nLkX7RGmMwjC0DBf7KWAlPjFaONAX3a8ndnz//fy1q7u2l9AZwrj1qa1iJ8EGAEC +AAkFAk2rFNoCGwwACgkQO9o98PRieSo2/QP/WTzr4ioINVsvN1akKuekmEMI3LAp +BfHwatufxxP1U+3Si/6YIk7kuPB9Hs+pRqCXzbvPRrI8NHZBmc8qIGthishdCYad +AHcVnXjtxrULkQFGbGvhKURLvS9WnzD/m1K2zzwxzkPTzT9/Yf06O6Mal5AdugPL +VrM0m72/jnpKo04= +=zNCn +-----END PGP PRIVATE KEY BLOCK-----` diff --git a/src/pkg/crypto/openpgp/s2k/Makefile b/src/pkg/crypto/openpgp/s2k/Makefile new file mode 100644 index 000000000..731d53431 --- /dev/null +++ b/src/pkg/crypto/openpgp/s2k/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/openpgp/s2k +GOFILES=\ + s2k.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/s2k/s2k.go b/src/pkg/crypto/openpgp/s2k/s2k.go new file mode 100644 index 000000000..da926a76e --- /dev/null +++ b/src/pkg/crypto/openpgp/s2k/s2k.go @@ -0,0 +1,180 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package s2k implements the various OpenPGP string-to-key transforms as +// specified in RFC 4800 section 3.7.1. +package s2k + +import ( + "crypto" + "crypto/openpgp/error" + "hash" + "io" + "os" + "strconv" +) + +// Simple writes to out the result of computing the Simple S2K function (RFC +// 4880, section 3.7.1.1) using the given hash and input passphrase. +func Simple(out []byte, h hash.Hash, in []byte) { + Salted(out, h, in, nil) +} + +var zero [1]byte + +// Salted writes to out the result of computing the Salted S2K function (RFC +// 4880, section 3.7.1.2) using the given hash, input passphrase and salt. +func Salted(out []byte, h hash.Hash, in []byte, salt []byte) { + done := 0 + + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + h.Write(salt) + h.Write(in) + n := copy(out[done:], h.Sum()) + done += n + } +} + +// Iterated writes to out the result of computing the Iterated and Salted S2K +// function (RFC 4880, section 3.7.1.3) using the given hash, input passphrase, +// salt and iteration count. +func Iterated(out []byte, h hash.Hash, in []byte, salt []byte, count int) { + combined := make([]byte, len(in)+len(salt)) + copy(combined, salt) + copy(combined[len(salt):], in) + + if count < len(combined) { + count = len(combined) + } + + done := 0 + for i := 0; done < len(out); i++ { + h.Reset() + for j := 0; j < i; j++ { + h.Write(zero[:]) + } + written := 0 + for written < count { + if written+len(combined) > count { + todo := count - written + h.Write(combined[:todo]) + written = count + } else { + h.Write(combined) + written += len(combined) + } + } + n := copy(out[done:], h.Sum()) + done += n + } +} + +// Parse reads a binary specification for a string-to-key transformation from r +// and returns a function which performs that transform. +func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { + var buf [9]byte + + _, err = io.ReadFull(r, buf[:2]) + if err != nil { + return + } + + hash, ok := HashIdToHash(buf[1]) + if !ok { + return nil, error.UnsupportedError("hash for S2K function: " + strconv.Itoa(int(buf[1]))) + } + h := hash.New() + if h == nil { + return nil, error.UnsupportedError("hash not available: " + strconv.Itoa(int(hash))) + } + + switch buf[0] { + case 1: + f := func(out, in []byte) { + Simple(out, h, in) + } + return f, nil + case 2: + _, err := io.ReadFull(r, buf[:8]) + if err != nil { + return + } + f := func(out, in []byte) { + Salted(out, h, in, buf[:8]) + } + return f, nil + case 3: + _, err := io.ReadFull(r, buf[:9]) + if err != nil { + return + } + count := (16 + int(buf[8]&15)) << (uint32(buf[8]>>4) + 6) + f := func(out, in []byte) { + Iterated(out, h, in, buf[:8], count) + } + return f, nil + } + + return nil, error.UnsupportedError("S2K function") +} + +// Serialize salts and stretches the given passphrase and writes the resulting +// key into key. It also serializes an S2K descriptor to w. +func Serialize(w io.Writer, key []byte, rand io.Reader, passphrase []byte) os.Error { + var buf [11]byte + buf[0] = 3 /* iterated and salted */ + buf[1], _ = HashToHashId(crypto.SHA1) + salt := buf[2:10] + if _, err := io.ReadFull(rand, salt); err != nil { + return err + } + const count = 65536 // this is the default in gpg + buf[10] = 96 // 65536 iterations + if _, err := w.Write(buf[:]); err != nil { + return err + } + + Iterated(key, crypto.SHA1.New(), passphrase, salt, count) + return nil +} + +// hashToHashIdMapping contains pairs relating OpenPGP's hash identifier with +// Go's crypto.Hash type. See RFC 4880, section 9.4. +var hashToHashIdMapping = []struct { + id byte + hash crypto.Hash +}{ + {1, crypto.MD5}, + {2, crypto.SHA1}, + {3, crypto.RIPEMD160}, + {8, crypto.SHA256}, + {9, crypto.SHA384}, + {10, crypto.SHA512}, + {11, crypto.SHA224}, +} + +// HashIdToHash returns a crypto.Hash which corresponds to the given OpenPGP +// hash id. +func HashIdToHash(id byte) (h crypto.Hash, ok bool) { + for _, m := range hashToHashIdMapping { + if m.id == id { + return m.hash, true + } + } + return 0, false +} + +// HashIdToHash returns an OpenPGP hash id which corresponds the given Hash. +func HashToHashId(h crypto.Hash) (id byte, ok bool) { + for _, m := range hashToHashIdMapping { + if m.hash == h { + return m.id, true + } + } + return 0, false +} diff --git a/src/pkg/crypto/openpgp/s2k/s2k_test.go b/src/pkg/crypto/openpgp/s2k/s2k_test.go new file mode 100644 index 000000000..ec4012c23 --- /dev/null +++ b/src/pkg/crypto/openpgp/s2k/s2k_test.go @@ -0,0 +1,118 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package s2k + +import ( + "bytes" + "crypto/sha1" + "crypto/rand" + "encoding/hex" + "testing" +) + +var saltedTests = []struct { + in, out string +}{ + {"hello", "10295ac1"}, + {"world", "ac587a5e"}, + {"foo", "4dda8077"}, + {"bar", "bd8aac6b9ea9cae04eae6a91c6133b58b5d9a61c14f355516ed9370456"}, + {"x", "f1d3f289"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "e00d7b45"}, +} + +func TestSalted(t *testing.T) { + h := sha1.New() + salt := [4]byte{1, 2, 3, 4} + + for i, test := range saltedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Salted(out, h, []byte(test.in), salt[:]) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + +var iteratedTests = []struct { + in, out string +}{ + {"hello", "83126105"}, + {"world", "6fa317f9"}, + {"foo", "8fbc35b9"}, + {"bar", "2af5a99b54f093789fd657f19bd245af7604d0f6ae06f66602a46a08ae"}, + {"x", "5a684dfe"}, + {"xxxxxxxxxxxxxxxxxxxxxxx", "18955174"}, +} + +func TestIterated(t *testing.T) { + h := sha1.New() + salt := [4]byte{4, 3, 2, 1} + + for i, test := range iteratedTests { + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + Iterated(out, h, []byte(test.in), salt[:], 31) + if !bytes.Equal(expected, out) { + t.Errorf("#%d, got: %x want: %x", i, out, expected) + } + } +} + +var parseTests = []struct { + spec, in, out string +}{ + /* Simple with SHA1 */ + {"0102", "hello", "aaf4c61d"}, + /* Salted with SHA1 */ + {"02020102030405060708", "hello", "f4f7d67e"}, + /* Iterated with SHA1 */ + {"03020102030405060708f1", "hello", "f2a57b7c"}, +} + +func TestParse(t *testing.T) { + for i, test := range parseTests { + spec, _ := hex.DecodeString(test.spec) + buf := bytes.NewBuffer(spec) + f, err := Parse(buf) + if err != nil { + t.Errorf("%d: Parse returned error: %s", i, err) + continue + } + + expected, _ := hex.DecodeString(test.out) + out := make([]byte, len(expected)) + f(out, []byte(test.in)) + if !bytes.Equal(out, expected) { + t.Errorf("%d: output got: %x want: %x", i, out, expected) + } + if testing.Short() { + break + } + } +} + +func TestSerialize(t *testing.T) { + buf := bytes.NewBuffer(nil) + key := make([]byte, 16) + passphrase := []byte("testing") + err := Serialize(buf, key, rand.Reader, passphrase) + if err != nil { + t.Errorf("failed to serialize: %s", err) + return + } + + f, err := Parse(buf) + if err != nil { + t.Errorf("failed to reparse: %s", err) + return + } + key2 := make([]byte, len(key)) + f(key2, passphrase) + if !bytes.Equal(key2, key) { + t.Errorf("keys don't match: %x (serialied) vs %x (parsed)", key, key2) + } +} diff --git a/src/pkg/crypto/openpgp/write.go b/src/pkg/crypto/openpgp/write.go new file mode 100644 index 000000000..9884472ce --- /dev/null +++ b/src/pkg/crypto/openpgp/write.go @@ -0,0 +1,308 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/openpgp/s2k" + "crypto/rand" + _ "crypto/sha256" + "hash" + "io" + "os" + "strconv" + "time" +) + +// DetachSign signs message with the private key from signer (which must +// already have been decrypted) and writes the signature to w. +func DetachSign(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeBinary) +} + +// ArmoredDetachSign signs message with the private key from signer (which +// must already have been decrypted) and writes an armored signature to w. +func ArmoredDetachSign(w io.Writer, signer *Entity, message io.Reader) (err os.Error) { + return armoredDetachSign(w, signer, message, packet.SigTypeBinary) +} + +// DetachSignText signs message (after canonicalising the line endings) with +// the private key from signer (which must already have been decrypted) and +// writes the signature to w. +func DetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeText) +} + +// ArmoredDetachSignText signs message (after canonicalising the line endings) +// with the private key from signer (which must already have been decrypted) +// and writes an armored signature to w. +func ArmoredDetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error { + return armoredDetachSign(w, signer, message, packet.SigTypeText) +} + +func armoredDetachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + out, err := armor.Encode(w, SignatureType, nil) + if err != nil { + return + } + err = detachSign(out, signer, message, sigType) + if err != nil { + return + } + return out.Close() +} + +func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing key doesn't have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing key is encrypted") + } + + sig := new(packet.Signature) + sig.SigType = sigType + sig.PubKeyAlgo = signer.PrivateKey.PubKeyAlgo + sig.Hash = crypto.SHA256 + sig.CreationTime = uint32(time.Seconds()) + sig.IssuerKeyId = &signer.PrivateKey.KeyId + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + io.Copy(wrappedHash, message) + + err = sig.Sign(h, signer.PrivateKey) + if err != nil { + return + } + + return sig.Serialize(w) +} + +// FileHints contains metadata about encrypted files. This metadata is, itself, +// encrypted. +type FileHints struct { + // IsBinary can be set to hint that the contents are binary data. + IsBinary bool + // FileName hints at the name of the file that should be written. It's + // truncated to 255 bytes if longer. It may be empty to suggest that the + // file should not be written to disk. It may be equal to "_CONSOLE" to + // suggest the data should not be written to disk. + FileName string + // EpochSeconds contains the modification time of the file, or 0 if not applicable. + EpochSeconds uint32 +} + +// SymmetricallyEncrypt acts like gpg -c: it encrypts a file with a passphrase. +// The resulting WriteCloser must be closed after the contents of the file have +// been written. +func SymmetricallyEncrypt(ciphertext io.Writer, passphrase []byte, hints *FileHints) (plaintext io.WriteCloser, err os.Error) { + if hints == nil { + hints = &FileHints{} + } + + key, err := packet.SerializeSymmetricKeyEncrypted(ciphertext, rand.Reader, passphrase, packet.CipherAES128) + if err != nil { + return + } + w, err := packet.SerializeSymmetricallyEncrypted(ciphertext, packet.CipherAES128, key) + if err != nil { + return + } + return packet.SerializeLiteral(w, hints.IsBinary, hints.FileName, hints.EpochSeconds) +} + +// intersectPreferences mutates and returns a prefix of a that contains only +// the values in the intersection of a and b. The order of a is preserved. +func intersectPreferences(a []uint8, b []uint8) (intersection []uint8) { + var j int + for _, v := range a { + for _, v2 := range b { + if v == v2 { + a[j] = v + j++ + break + } + } + } + + return a[:j] +} + +func hashToHashId(h crypto.Hash) uint8 { + v, ok := s2k.HashToHashId(h) + if !ok { + panic("tried to convert unknown hash") + } + return v +} + +// Encrypt encrypts a message to a number of recipients and, optionally, signs +// it. hints contains optional information, that is also encrypted, that aids +// the recipients in processing the message. The resulting WriteCloser must +// be closed after the contents of the file have been written. +func Encrypt(ciphertext io.Writer, to []*Entity, signed *Entity, hints *FileHints) (plaintext io.WriteCloser, err os.Error) { + var signer *packet.PrivateKey + if signed != nil { + signer = signed.signingKey().PrivateKey + if signer == nil || signer.Encrypted { + return nil, error.InvalidArgumentError("signing key must be decrypted") + } + } + + // These are the possible ciphers that we'll use for the message. + candidateCiphers := []uint8{ + uint8(packet.CipherAES128), + uint8(packet.CipherAES256), + uint8(packet.CipherCAST5), + } + // These are the possible hash functions that we'll use for the signature. + candidateHashes := []uint8{ + hashToHashId(crypto.SHA256), + hashToHashId(crypto.SHA512), + hashToHashId(crypto.SHA1), + hashToHashId(crypto.RIPEMD160), + } + // In the event that a recipient doesn't specify any supported ciphers + // or hash functions, these are the ones that we assume that every + // implementation supports. + defaultCiphers := candidateCiphers[len(candidateCiphers)-1:] + defaultHashes := candidateHashes[len(candidateHashes)-1:] + + encryptKeys := make([]Key, len(to)) + for i := range to { + encryptKeys[i] = to[i].encryptionKey() + if encryptKeys[i].PublicKey == nil { + return nil, error.InvalidArgumentError("cannot encrypt a message to key id " + strconv.Uitob64(to[i].PrimaryKey.KeyId, 16) + " because it has no encryption keys") + } + + sig := to[i].primaryIdentity().SelfSignature + + preferredSymmetric := sig.PreferredSymmetric + if len(preferredSymmetric) == 0 { + preferredSymmetric = defaultCiphers + } + preferredHashes := sig.PreferredHash + if len(preferredHashes) == 0 { + preferredHashes = defaultHashes + } + candidateCiphers = intersectPreferences(candidateCiphers, preferredSymmetric) + candidateHashes = intersectPreferences(candidateHashes, preferredHashes) + } + + if len(candidateCiphers) == 0 || len(candidateHashes) == 0 { + return nil, error.InvalidArgumentError("cannot encrypt because recipient set shares no common algorithms") + } + + cipher := packet.CipherFunction(candidateCiphers[0]) + hash, _ := s2k.HashIdToHash(candidateHashes[0]) + symKey := make([]byte, cipher.KeySize()) + if _, err := io.ReadFull(rand.Reader, symKey); err != nil { + return nil, err + } + + for _, key := range encryptKeys { + if err := packet.SerializeEncryptedKey(ciphertext, rand.Reader, key.PublicKey, cipher, symKey); err != nil { + return nil, err + } + } + + encryptedData, err := packet.SerializeSymmetricallyEncrypted(ciphertext, cipher, symKey) + if err != nil { + return + } + + if signer != nil { + ops := &packet.OnePassSignature{ + SigType: packet.SigTypeBinary, + Hash: hash, + PubKeyAlgo: signer.PubKeyAlgo, + KeyId: signer.KeyId, + IsLast: true, + } + if err := ops.Serialize(encryptedData); err != nil { + return nil, err + } + } + + if hints == nil { + hints = &FileHints{} + } + + w := encryptedData + if signer != nil { + // If we need to write a signature packet after the literal + // data then we need to stop literalData from closing + // encryptedData. + w = noOpCloser{encryptedData} + + } + literalData, err := packet.SerializeLiteral(w, hints.IsBinary, hints.FileName, hints.EpochSeconds) + if err != nil { + return nil, err + } + + if signer != nil { + return signatureWriter{encryptedData, literalData, hash, hash.New(), signer}, nil + } + return literalData, nil +} + +// signatureWriter hashes the contents of a message while passing it along to +// literalData. When closed, it closes literalData, writes a signature packet +// to encryptedData and then also closes encryptedData. +type signatureWriter struct { + encryptedData io.WriteCloser + literalData io.WriteCloser + hashType crypto.Hash + h hash.Hash + signer *packet.PrivateKey +} + +func (s signatureWriter) Write(data []byte) (int, os.Error) { + s.h.Write(data) + return s.literalData.Write(data) +} + +func (s signatureWriter) Close() os.Error { + sig := &packet.Signature{ + SigType: packet.SigTypeBinary, + PubKeyAlgo: s.signer.PubKeyAlgo, + Hash: s.hashType, + CreationTime: uint32(time.Seconds()), + IssuerKeyId: &s.signer.KeyId, + } + + if err := sig.Sign(s.h, s.signer); err != nil { + return err + } + if err := s.literalData.Close(); err != nil { + return err + } + if err := sig.Serialize(s.encryptedData); err != nil { + return err + } + return s.encryptedData.Close() +} + +// noOpCloser is like an ioutil.NopCloser, but for an io.Writer. +// TODO: we have two of these in OpenPGP packages alone. This probably needs +// to be promoted somewhere more common. +type noOpCloser struct { + w io.Writer +} + +func (c noOpCloser) Write(data []byte) (n int, err os.Error) { + return c.w.Write(data) +} + +func (c noOpCloser) Close() os.Error { + return nil +} diff --git a/src/pkg/crypto/openpgp/write_test.go b/src/pkg/crypto/openpgp/write_test.go new file mode 100644 index 000000000..c542dfa45 --- /dev/null +++ b/src/pkg/crypto/openpgp/write_test.go @@ -0,0 +1,233 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "crypto/rand" + "os" + "io" + "io/ioutil" + "testing" + "time" +) + +func TestSignDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey1KeyId) +} + +func TestSignTextDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSignText(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey1KeyId) +} + +func TestSignDetachedDSA(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(dsaTestKeyPrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check", testKey3KeyId) +} + +func TestNewEntity(t *testing.T) { + if testing.Short() { + return + } + + e, err := NewEntity(rand.Reader, time.Seconds(), "Test User", "test", "test@example.com") + if err != nil { + t.Errorf("failed to create entity: %s", err) + return + } + + w := bytes.NewBuffer(nil) + if err := e.SerializePrivate(w); err != nil { + t.Errorf("failed to serialize entity: %s", err) + return + } + serialized := w.Bytes() + + el, err := ReadKeyRing(w) + if err != nil { + t.Errorf("failed to reparse entity: %s", err) + return + } + + if len(el) != 1 { + t.Errorf("wrong number of entities found, got %d, want 1", len(el)) + } + + w = bytes.NewBuffer(nil) + if err := e.SerializePrivate(w); err != nil { + t.Errorf("failed to serialize entity second time: %s", err) + return + } + + if !bytes.Equal(w.Bytes(), serialized) { + t.Errorf("results differed") + } +} + +func TestSymmetricEncryption(t *testing.T) { + buf := new(bytes.Buffer) + plaintext, err := SymmetricallyEncrypt(buf, []byte("testing"), nil) + if err != nil { + t.Errorf("error writing headers: %s", err) + return + } + message := []byte("hello world\n") + _, err = plaintext.Write(message) + if err != nil { + t.Errorf("error writing to plaintext writer: %s", err) + } + err = plaintext.Close() + if err != nil { + t.Errorf("error closing plaintext writer: %s", err) + } + + md, err := ReadMessage(buf, nil, func(keys []Key, symmetric bool) ([]byte, os.Error) { + return []byte("testing"), nil + }) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + messageBuf := bytes.NewBuffer(nil) + _, err = io.Copy(messageBuf, md.UnverifiedBody) + if err != nil { + t.Errorf("error rereading message: %s", err) + } + if !bytes.Equal(message, messageBuf.Bytes()) { + t.Errorf("recovered message incorrect got '%s', want '%s'", messageBuf.Bytes(), message) + } +} + +var testEncryptionTests = []struct { + keyRingHex string + isSigned bool +}{ + { + testKeys1And2PrivateHex, + false, + }, + { + testKeys1And2PrivateHex, + true, + }, + { + dsaElGamalTestKeysHex, + false, + }, + { + dsaElGamalTestKeysHex, + true, + }, +} + +func TestEncryption(t *testing.T) { + for i, test := range testEncryptionTests { + kring, _ := ReadKeyRing(readerFromHex(test.keyRingHex)) + + passphrase := []byte("passphrase") + for _, entity := range kring { + if entity.PrivateKey != nil && entity.PrivateKey.Encrypted { + err := entity.PrivateKey.Decrypt(passphrase) + if err != nil { + t.Errorf("#%d: failed to decrypt key", i) + } + } + for _, subkey := range entity.Subkeys { + if subkey.PrivateKey != nil && subkey.PrivateKey.Encrypted { + err := subkey.PrivateKey.Decrypt(passphrase) + if err != nil { + t.Errorf("#%d: failed to decrypt subkey", i) + } + } + } + } + + var signed *Entity + if test.isSigned { + signed = kring[0] + } + + buf := new(bytes.Buffer) + w, err := Encrypt(buf, kring[:1], signed, nil /* no hints */ ) + if err != nil { + t.Errorf("#%d: error in Encrypt: %s", i, err) + continue + } + + const message = "testing" + _, err = w.Write([]byte(message)) + if err != nil { + t.Errorf("#%d: error writing plaintext: %s", i, err) + continue + } + err = w.Close() + if err != nil { + t.Errorf("#%d: error closing WriteCloser: %s", i, err) + continue + } + + md, err := ReadMessage(buf, kring, nil /* no prompt */ ) + if err != nil { + t.Errorf("#%d: error reading message: %s", i, err) + continue + } + + if test.isSigned { + expectedKeyId := kring[0].signingKey().PublicKey.KeyId + if md.SignedByKeyId != expectedKeyId { + t.Errorf("#%d: message signed by wrong key id, got: %d, want: %d", i, *md.SignedBy, expectedKeyId) + } + if md.SignedBy == nil { + t.Errorf("#%d: failed to find the signing Entity", i) + } + } + + plaintext, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("#%d: error reading encrypted contents: %s", i, err) + continue + } + + expectedKeyId := kring[0].encryptionKey().PublicKey.KeyId + if len(md.EncryptedToKeyIds) != 1 || md.EncryptedToKeyIds[0] != expectedKeyId { + t.Errorf("#%d: expected message to be encrypted to %v, but got %#v", i, expectedKeyId, md.EncryptedToKeyIds) + } + + if string(plaintext) != message { + t.Errorf("#%d: got: %s, want: %s", i, string(plaintext), message) + } + + if test.isSigned { + if md.SignatureError != nil { + t.Errorf("#%d: signature error: %s", i, err) + } + if md.Signature == nil { + t.Error("signature missing") + } + } + } +} diff --git a/src/pkg/crypto/rand/Makefile b/src/pkg/crypto/rand/Makefile new file mode 100644 index 000000000..d1a3d45e8 --- /dev/null +++ b/src/pkg/crypto/rand/Makefile @@ -0,0 +1,30 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/rand + +GOFILES=\ + rand.go\ + util.go\ + +GOFILES_freebsd=\ + rand_unix.go\ + +GOFILES_darwin=\ + rand_unix.go\ + +GOFILES_linux=\ + rand_unix.go\ + +GOFILES_openbsd=\ + rand_unix.go\ + +GOFILES_windows=\ + rand_windows.go\ + +GOFILES+=$(GOFILES_$(GOOS)) + +include ../../../Make.pkg diff --git a/src/pkg/crypto/rand/rand.go b/src/pkg/crypto/rand/rand.go new file mode 100644 index 000000000..42d9da0ef --- /dev/null +++ b/src/pkg/crypto/rand/rand.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rand implements a cryptographically secure +// pseudorandom number generator. +package rand + +import ( + "io" + "os" +) + +// Reader is a global, shared instance of a cryptographically +// strong pseudo-random generator. +// On Unix-like systems, Reader reads from /dev/urandom. +// On Windows systems, Reader uses the CryptGenRandom API. +var Reader io.Reader + +// Read is a helper function that calls Reader.Read. +func Read(b []byte) (n int, err os.Error) { return Reader.Read(b) } diff --git a/src/pkg/crypto/rand/rand_test.go b/src/pkg/crypto/rand/rand_test.go new file mode 100644 index 000000000..bfae7ce4f --- /dev/null +++ b/src/pkg/crypto/rand/rand_test.go @@ -0,0 +1,31 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rand + +import ( + "bytes" + "compress/flate" + "testing" +) + +func TestRead(t *testing.T) { + var n int = 4e6 + if testing.Short() { + n = 1e5 + } + b := make([]byte, n) + n, err := Read(b) + if n != len(b) || err != nil { + t.Fatalf("Read(buf) = %d, %s", n, err) + } + + var z bytes.Buffer + f := flate.NewWriter(&z, 5) + f.Write(b) + f.Close() + if z.Len() < len(b)*99/100 { + t.Fatalf("Compressed %d -> %d", len(b), z.Len()) + } +} diff --git a/src/pkg/crypto/rand/rand_unix.go b/src/pkg/crypto/rand/rand_unix.go new file mode 100644 index 000000000..3a06aa8b1 --- /dev/null +++ b/src/pkg/crypto/rand/rand_unix.go @@ -0,0 +1,125 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Unix cryptographically secure pseudorandom number +// generator. + +package rand + +import ( + "bufio" + "crypto/aes" + "io" + "os" + "sync" + "time" +) + +// Easy implementation: read from /dev/urandom. +// This is sufficient on Linux, OS X, and FreeBSD. + +func init() { Reader = &devReader{name: "/dev/urandom"} } + +// A devReader satisfies reads by reading the file named name. +type devReader struct { + name string + f io.Reader + mu sync.Mutex +} + +func (r *devReader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.f == nil { + f, err := os.Open(r.name) + if f == nil { + return 0, err + } + r.f = bufio.NewReader(f) + } + return r.f.Read(b) +} + +// Alternate pseudo-random implementation for use on +// systems without a reliable /dev/urandom. So far we +// haven't needed it. + +// newReader returns a new pseudorandom generator that +// seeds itself by reading from entropy. If entropy == nil, +// the generator seeds itself by reading from the system's +// random number generator, typically /dev/random. +// The Read method on the returned reader always returns +// the full amount asked for, or else it returns an error. +// +// The generator uses the X9.31 algorithm with AES-128, +// reseeding after every 1 MB of generated data. +func newReader(entropy io.Reader) io.Reader { + if entropy == nil { + entropy = &devReader{name: "/dev/random"} + } + return &reader{entropy: entropy} +} + +type reader struct { + mu sync.Mutex + budget int // number of bytes that can be generated + cipher *aes.Cipher + entropy io.Reader + time, seed, dst, key [aes.BlockSize]byte +} + +func (r *reader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + defer r.mu.Unlock() + n = len(b) + + for len(b) > 0 { + if r.budget == 0 { + _, err := io.ReadFull(r.entropy, r.seed[0:]) + if err != nil { + return n - len(b), err + } + _, err = io.ReadFull(r.entropy, r.key[0:]) + if err != nil { + return n - len(b), err + } + r.cipher, err = aes.NewCipher(r.key[0:]) + if err != nil { + return n - len(b), err + } + r.budget = 1 << 20 // reseed after generating 1MB + } + r.budget -= aes.BlockSize + + // ANSI X9.31 (== X9.17) algorithm, but using AES in place of 3DES. + // + // single block: + // t = encrypt(time) + // dst = encrypt(t^seed) + // seed = encrypt(t^dst) + ns := time.Nanoseconds() + r.time[0] = byte(ns >> 56) + r.time[1] = byte(ns >> 48) + r.time[2] = byte(ns >> 40) + r.time[3] = byte(ns >> 32) + r.time[4] = byte(ns >> 24) + r.time[5] = byte(ns >> 16) + r.time[6] = byte(ns >> 8) + r.time[7] = byte(ns) + r.cipher.Encrypt(r.time[0:], r.time[0:]) + for i := 0; i < aes.BlockSize; i++ { + r.dst[i] = r.time[i] ^ r.seed[i] + } + r.cipher.Encrypt(r.dst[0:], r.dst[0:]) + for i := 0; i < aes.BlockSize; i++ { + r.seed[i] = r.time[i] ^ r.dst[i] + } + r.cipher.Encrypt(r.seed[0:], r.seed[0:]) + + m := copy(b, r.dst[0:]) + b = b[m:] + } + + return n, nil +} diff --git a/src/pkg/crypto/rand/rand_windows.go b/src/pkg/crypto/rand/rand_windows.go new file mode 100755 index 000000000..0eab6b213 --- /dev/null +++ b/src/pkg/crypto/rand/rand_windows.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Windows cryptographically secure pseudorandom number +// generator. + +package rand + +import ( + "os" + "sync" + "syscall" +) + +// Implemented by using Windows CryptoAPI 2.0. + +func init() { Reader = &rngReader{} } + +// A rngReader satisfies reads by reading from the Windows CryptGenRandom API. +type rngReader struct { + prov syscall.Handle + mu sync.Mutex +} + +func (r *rngReader) Read(b []byte) (n int, err os.Error) { + r.mu.Lock() + if r.prov == 0 { + const provType = syscall.PROV_RSA_FULL + const flags = syscall.CRYPT_VERIFYCONTEXT | syscall.CRYPT_SILENT + errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags) + if errno != 0 { + r.mu.Unlock() + return 0, os.NewSyscallError("CryptAcquireContext", errno) + } + } + r.mu.Unlock() + errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0]) + if errno != 0 { + return 0, os.NewSyscallError("CryptGenRandom", errno) + } + return len(b), nil +} diff --git a/src/pkg/crypto/rand/util.go b/src/pkg/crypto/rand/util.go new file mode 100644 index 000000000..77028476e --- /dev/null +++ b/src/pkg/crypto/rand/util.go @@ -0,0 +1,80 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rand + +import ( + "big" + "io" + "os" +) + +// Prime returns a number, p, of the given size, such that p is prime +// with high probability. +func Prime(rand io.Reader, bits int) (p *big.Int, err os.Error) { + if bits < 1 { + err = os.EINVAL + } + + b := uint(bits % 8) + if b == 0 { + b = 8 + } + + bytes := make([]byte, (bits+7)/8) + p = new(big.Int) + + for { + _, err = io.ReadFull(rand, bytes) + if err != nil { + return nil, err + } + + // Clear bits in the first byte to make sure the candidate has a size <= bits. + bytes[0] &= uint8(int(1< 256 { + return nil, KeySizeError(k) + } + var c Cipher + for i := 0; i < 256; i++ { + c.s[i] = uint8(i) + } + var j uint8 = 0 + for i := 0; i < 256; i++ { + j += c.s[i] + key[i%k] + c.s[i], c.s[j] = c.s[j], c.s[i] + } + return &c, nil +} + +// XORKeyStream sets dst to the result of XORing src with the key stream. +// Dst and src may be the same slice but otherwise should not overlap. +func (c *Cipher) XORKeyStream(dst, src []byte) { + for i := range src { + c.i += 1 + c.j += c.s[c.i] + c.s[c.i], c.s[c.j] = c.s[c.j], c.s[c.i] + dst[i] = src[i] ^ c.s[c.s[c.i]+c.s[c.j]] + } +} + +// Reset zeros the key data so that it will no longer appear in the +// process's memory. +func (c *Cipher) Reset() { + for i := range c.s { + c.s[i] = 0 + } + c.i, c.j = 0, 0 +} diff --git a/src/pkg/crypto/rc4/rc4_test.go b/src/pkg/crypto/rc4/rc4_test.go new file mode 100644 index 000000000..6265d9408 --- /dev/null +++ b/src/pkg/crypto/rc4/rc4_test.go @@ -0,0 +1,59 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rc4 + +import ( + "testing" +) + +type rc4Test struct { + key, keystream []byte +} + +var golden = []rc4Test{ + // Test vectors from the original cypherpunk posting of ARC4: + // http://groups.google.com/group/sci.crypt/msg/10a300c9d21afca0?pli=1 + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + []byte{0x74, 0x94, 0xc2, 0xe7, 0x10, 0x4b, 0x08, 0x79}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xde, 0x18, 0x89, 0x41, 0xa3, 0x37, 0x5d, 0x3a}, + }, + { + []byte{0xef, 0x01, 0x23, 0x45}, + []byte{0xd6, 0xa1, 0x41, 0xa7, 0xec, 0x3c, 0x38, 0xdf, 0xbd, 0x61}, + }, + + // Test vectors from the Wikipedia page: http://en.wikipedia.org/wiki/RC4 + { + []byte{0x4b, 0x65, 0x79}, + []byte{0xeb, 0x9f, 0x77, 0x81, 0xb7, 0x34, 0xca, 0x72, 0xa7, 0x19}, + }, + { + []byte{0x57, 0x69, 0x6b, 0x69}, + []byte{0x60, 0x44, 0xdb, 0x6d, 0x41, 0xb7}, + }, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c, err := NewCipher(g.key) + if err != nil { + t.Errorf("Failed to create cipher at golden index %d", i) + return + } + keystream := make([]byte, len(g.keystream)) + c.XORKeyStream(keystream, keystream) + for j, v := range keystream { + if g.keystream[j] != v { + t.Errorf("Failed at golden index %d", i) + break + } + } + } +} diff --git a/src/pkg/crypto/ripemd160/Makefile b/src/pkg/crypto/ripemd160/Makefile new file mode 100644 index 000000000..7e529457d --- /dev/null +++ b/src/pkg/crypto/ripemd160/Makefile @@ -0,0 +1,12 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/ripemd160 +GOFILES=\ + ripemd160.go\ + ripemd160block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/ripemd160/ripemd160.go b/src/pkg/crypto/ripemd160/ripemd160.go new file mode 100644 index 000000000..5aaca59a3 --- /dev/null +++ b/src/pkg/crypto/ripemd160/ripemd160.go @@ -0,0 +1,118 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ripemd160 implements the RIPEMD-160 hash algorithm. +package ripemd160 + +// RIPEMD-160 is designed by by Hans Dobbertin, Antoon Bosselaers, and Bart +// Preneel with specifications available at: +// http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf. + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.RIPEMD160, New) +} + +// The size of the checksum in bytes. +const Size = 20 + +// The block size of the hash algorithm in bytes. +const BlockSize = 64 + +const ( + _s0 = 0x67452301 + _s1 = 0xefcdab89 + _s2 = 0x98badcfe + _s3 = 0x10325476 + _s4 = 0xc3d2e1f0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [5]uint32 // running context + x [BlockSize]byte // temporary buffer + nx int // index into x + tc uint64 // total count of bytes processed +} + +func (d *digest) Reset() { + d.s[0], d.s[1], d.s[2], d.s[3], d.s[4] = _s0, _s1, _s2, _s3, _s4 + d.nx = 0 + d.tc = 0 +} + +// New returns a new hash.Hash computing the checksum. +func New() hash.Hash { + result := new(digest) + result.Reset() + return result +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.tc += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > BlockSize-d.nx { + n = BlockSize - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == BlockSize { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + tc := d.tc + var tmp [64]byte + tmp[0] = 0x80 + if tc%64 < 56 { + d.Write(tmp[0 : 56-tc%64]) + } else { + d.Write(tmp[0 : 64+56-tc%64]) + } + + // Length in bits. + tc <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(tc >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 20) + j := 0 + for _, s := range d.s { + p[j], p[j+1], p[j+2], p[j+3] = byte(s), byte(s>>8), byte(s>>16), byte(s>>24) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/ripemd160/ripemd160_test.go b/src/pkg/crypto/ripemd160/ripemd160_test.go new file mode 100644 index 000000000..f4135f5cf --- /dev/null +++ b/src/pkg/crypto/ripemd160/ripemd160_test.go @@ -0,0 +1,64 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ripemd160 + +// Test vectors are from: +// http://homes.esat.kuleuven.be/~bosselae/ripemd160.html + +import ( + "fmt" + "io" + "testing" +) + +type mdTest struct { + out string + in string +} + +var vectors = [...]mdTest{ + {"9c1185a5c5e9fc54612808977ee8f548b2258d31", ""}, + {"0bdc9d2d256b3ee9daae347be6f4dc835a467ffe", "a"}, + {"8eb208f7e05d987a9b044a8e98c6b087f15a0bfc", "abc"}, + {"5d0689ef49d2fae572b881b123a85ffa21595f36", "message digest"}, + {"f71c27109c692c1b56bbdceb5b9d2865b3708dbc", "abcdefghijklmnopqrstuvwxyz"}, + {"12a053384a9c0c88e405a06c27dcf49ada62eb2b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"b0e20b6e3116640286ed3a87a5713079b21f5189", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + {"9b752e45573d4b39f4dbd3323cab82bf63326bfb", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, +} + +func TestVectors(t *testing.T) { + for i := 0; i < len(vectors); i++ { + tv := vectors[i] + md := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(md, tv.in) + } else { + io.WriteString(md, tv.in[0:len(tv.in)/2]) + md.Sum() + io.WriteString(md, tv.in[len(tv.in)/2:]) + } + s := fmt.Sprintf("%x", md.Sum()) + if s != tv.out { + t.Fatalf("RIPEMD-160[%d](%s) = %s, expected %s", j, tv.in, s, tv.out) + } + md.Reset() + } + } +} + +func TestMillionA(t *testing.T) { + md := New() + for i := 0; i < 100000; i++ { + io.WriteString(md, "aaaaaaaaaa") + } + out := "52783243c1697bdbe16d37f97f68f08325dc1528" + s := fmt.Sprintf("%x", md.Sum()) + if s != out { + t.Fatalf("RIPEMD-160 (1 million 'a') = %s, expected %s", s, out) + } + md.Reset() +} diff --git a/src/pkg/crypto/ripemd160/ripemd160block.go b/src/pkg/crypto/ripemd160/ripemd160block.go new file mode 100644 index 000000000..7bc8e6c48 --- /dev/null +++ b/src/pkg/crypto/ripemd160/ripemd160block.go @@ -0,0 +1,161 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// RIPEMD-160 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package ripemd160 + +// work buffer indices and roll amounts for one line +var _n = [80]uint{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8, + 3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12, + 1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2, + 4, 0, 5, 9, 7, 12, 2, 10, 14, 1, 3, 8, 11, 6, 15, 13, +} + +var _r = [80]uint{ + 11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8, + 7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12, + 11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5, + 11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12, + 9, 15, 5, 11, 6, 8, 13, 12, 5, 12, 13, 14, 11, 8, 5, 6, +} + +// same for the other parallel one +var n_ = [80]uint{ + 5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, + 6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2, + 15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13, + 8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14, + 12, 15, 10, 4, 1, 5, 8, 7, 6, 2, 13, 14, 0, 3, 9, 11, +} + +var r_ = [80]uint{ + 8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6, + 9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11, + 9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5, + 15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8, + 8, 5, 12, 9, 12, 5, 14, 6, 8, 13, 6, 5, 15, 13, 11, 11, +} + +func _Block(md *digest, p []byte) int { + n := 0 + var x [16]uint32 + var alpha, beta uint32 + for len(p) >= BlockSize { + a, b, c, d, e := md.s[0], md.s[1], md.s[2], md.s[3], md.s[4] + aa, bb, cc, dd, ee := a, b, c, d, e + j := 0 + for i := 0; i < 16; i++ { + x[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // round 1 + i := 0 + for i < 16 { + alpha = a + (b ^ c ^ d) + x[_n[i]] + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ (cc | ^dd)) + x[n_[i]] + 0x50a28be6 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 2 + for i < 32 { + alpha = a + (b&c | ^b&d) + x[_n[i]] + 0x5a827999 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&dd | cc&^dd) + x[n_[i]] + 0x5c4dd124 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 3 + for i < 48 { + alpha = a + (b | ^c ^ d) + x[_n[i]] + 0x6ed9eba1 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb | ^cc ^ dd) + x[n_[i]] + 0x6d703ef3 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 4 + for i < 64 { + alpha = a + (b&d | c&^d) + x[_n[i]] + 0x8f1bbcdc + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&cc | ^bb&dd) + x[n_[i]] + 0x7a6d76e9 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 5 + for i < 80 { + alpha = a + (b ^ (c | ^d)) + x[_n[i]] + 0xa953fd4e + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ cc ^ dd) + x[n_[i]] + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // combine results + dd += c + md.s[1] + md.s[1] = md.s[2] + d + ee + md.s[2] = md.s[3] + e + aa + md.s[3] = md.s[4] + a + bb + md.s[4] = md.s[0] + b + cc + md.s[0] = dd + + p = p[BlockSize:] + n += BlockSize + } + return n +} diff --git a/src/pkg/crypto/rsa/Makefile b/src/pkg/crypto/rsa/Makefile new file mode 100644 index 000000000..ff26ca6f2 --- /dev/null +++ b/src/pkg/crypto/rsa/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/rsa +GOFILES=\ + rsa.go\ + pkcs1v15.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/rsa/pkcs1v15.go b/src/pkg/crypto/rsa/pkcs1v15.go new file mode 100644 index 000000000..600623114 --- /dev/null +++ b/src/pkg/crypto/rsa/pkcs1v15.go @@ -0,0 +1,242 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "crypto" + "crypto/subtle" + "io" + "os" +) + +// This file implements encryption and decryption using PKCS#1 v1.5 padding. + +// EncryptPKCS1v15 encrypts the given message with RSA and the padding scheme from PKCS#1 v1.5. +// The message must be no longer than the length of the public modulus minus 11 bytes. +// WARNING: use of this function to encrypt plaintexts other than session keys +// is dangerous. Use RSA OAEP in new protocols. +func EncryptPKCS1v15(rand io.Reader, pub *PublicKey, msg []byte) (out []byte, err os.Error) { + k := (pub.N.BitLen() + 7) / 8 + if len(msg) > k-11 { + err = MessageTooLongError{} + return + } + + // EM = 0x02 || PS || 0x00 || M + em := make([]byte, k-1) + em[0] = 2 + ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):] + err = nonZeroRandomBytes(ps, rand) + if err != nil { + return + } + em[len(em)-len(msg)-1] = 0 + copy(mm, msg) + + m := new(big.Int).SetBytes(em) + c := encrypt(new(big.Int), pub, m) + out = c.Bytes() + return +} + +// DecryptPKCS1v15 decrypts a plaintext using RSA and the padding scheme from PKCS#1 v1.5. +// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks. +func DecryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (out []byte, err os.Error) { + valid, out, err := decryptPKCS1v15(rand, priv, ciphertext) + if err == nil && valid == 0 { + err = DecryptionError{} + } + + return +} + +// DecryptPKCS1v15SessionKey decrypts a session key using RSA and the padding scheme from PKCS#1 v1.5. +// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks. +// It returns an error if the ciphertext is the wrong length or if the +// ciphertext is greater than the public modulus. Otherwise, no error is +// returned. If the padding is valid, the resulting plaintext message is copied +// into key. Otherwise, key is unchanged. These alternatives occur in constant +// time. It is intended that the user of this function generate a random +// session key beforehand and continue the protocol with the resulting value. +// This will remove any possibility that an attacker can learn any information +// about the plaintext. +// See ``Chosen Ciphertext Attacks Against Protocols Based on the RSA +// Encryption Standard PKCS #1'', Daniel Bleichenbacher, Advances in Cryptology +// (Crypto '98), +func DecryptPKCS1v15SessionKey(rand io.Reader, priv *PrivateKey, ciphertext []byte, key []byte) (err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if k-(len(key)+3+8) < 0 { + err = DecryptionError{} + return + } + + valid, msg, err := decryptPKCS1v15(rand, priv, ciphertext) + if err != nil { + return + } + + valid &= subtle.ConstantTimeEq(int32(len(msg)), int32(len(key))) + subtle.ConstantTimeCopy(valid, key, msg) + return +} + +func decryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (valid int, msg []byte, err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if k < 11 { + err = DecryptionError{} + return + } + + c := new(big.Int).SetBytes(ciphertext) + m, err := decrypt(rand, priv, c) + if err != nil { + return + } + + em := leftPad(m.Bytes(), k) + firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0) + secondByteIsTwo := subtle.ConstantTimeByteEq(em[1], 2) + + // The remainder of the plaintext must be a string of non-zero random + // octets, followed by a 0, followed by the message. + // lookingForIndex: 1 iff we are still looking for the zero. + // index: the offset of the first zero byte. + var lookingForIndex, index int + lookingForIndex = 1 + + for i := 2; i < len(em); i++ { + equals0 := subtle.ConstantTimeByteEq(em[i], 0) + index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex) + } + + valid = firstByteIsZero & secondByteIsTwo & (^lookingForIndex & 1) + msg = em[index+1:] + return +} + +// nonZeroRandomBytes fills the given slice with non-zero random octets. +func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) { + _, err = io.ReadFull(rand, s) + if err != nil { + return + } + + for i := 0; i < len(s); i++ { + for s[i] == 0 { + _, err = io.ReadFull(rand, s[i:i+1]) + if err != nil { + return + } + // In tests, the PRNG may return all zeros so we do + // this to break the loop. + s[i] ^= 0x42 + } + } + + return +} + +// These are ASN1 DER structures: +// DigestInfo ::= SEQUENCE { +// digestAlgorithm AlgorithmIdentifier, +// digest OCTET STRING +// } +// For performance, we don't use the generic ASN1 encoder. Rather, we +// precompute a prefix of the digest value that makes a valid ASN1 DER string +// with the correct contents. +var hashPrefixes = map[crypto.Hash][]byte{ + crypto.MD5: {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10}, + crypto.SHA1: {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14}, + crypto.SHA256: {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}, + crypto.SHA384: {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30}, + crypto.SHA512: {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40}, + crypto.MD5SHA1: {}, // A special TLS case which doesn't use an ASN1 prefix. + crypto.RIPEMD160: {0x30, 0x20, 0x30, 0x08, 0x06, 0x06, 0x28, 0xcf, 0x06, 0x03, 0x00, 0x31, 0x04, 0x14}, +} + +// SignPKCS1v15 calculates the signature of hashed using RSASSA-PKCS1-V1_5-SIGN from RSA PKCS#1 v1.5. +// Note that hashed must be the result of hashing the input message using the +// given hash function. +func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []byte) (s []byte, err os.Error) { + hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) + if err != nil { + return + } + + tLen := len(prefix) + hashLen + k := (priv.N.BitLen() + 7) / 8 + if k < tLen+11 { + return nil, MessageTooLongError{} + } + + // EM = 0x00 || 0x01 || PS || 0x00 || T + em := make([]byte, k) + em[1] = 1 + for i := 2; i < k-tLen-1; i++ { + em[i] = 0xff + } + copy(em[k-tLen:k-hashLen], prefix) + copy(em[k-hashLen:k], hashed) + + m := new(big.Int).SetBytes(em) + c, err := decrypt(rand, priv, m) + if err == nil { + s = c.Bytes() + } + return +} + +// VerifyPKCS1v15 verifies an RSA PKCS#1 v1.5 signature. +// hashed is the result of hashing the input message using the given hash +// function and sig is the signature. A valid signature is indicated by +// returning a nil error. +func VerifyPKCS1v15(pub *PublicKey, hash crypto.Hash, hashed []byte, sig []byte) (err os.Error) { + hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) + if err != nil { + return + } + + tLen := len(prefix) + hashLen + k := (pub.N.BitLen() + 7) / 8 + if k < tLen+11 { + err = VerificationError{} + return + } + + c := new(big.Int).SetBytes(sig) + m := encrypt(new(big.Int), pub, c) + em := leftPad(m.Bytes(), k) + // EM = 0x00 || 0x01 || PS || 0x00 || T + + ok := subtle.ConstantTimeByteEq(em[0], 0) + ok &= subtle.ConstantTimeByteEq(em[1], 1) + ok &= subtle.ConstantTimeCompare(em[k-hashLen:k], hashed) + ok &= subtle.ConstantTimeCompare(em[k-tLen:k-hashLen], prefix) + ok &= subtle.ConstantTimeByteEq(em[k-tLen-1], 0) + + for i := 2; i < k-tLen-1; i++ { + ok &= subtle.ConstantTimeByteEq(em[i], 0xff) + } + + if ok != 1 { + return VerificationError{} + } + + return nil +} + +func pkcs1v15HashInfo(hash crypto.Hash, inLen int) (hashLen int, prefix []byte, err os.Error) { + hashLen = hash.Size() + if inLen != hashLen { + return 0, nil, os.NewError("input must be hashed message") + } + prefix, ok := hashPrefixes[hash] + if !ok { + return 0, nil, os.NewError("unsupported hash function") + } + return +} diff --git a/src/pkg/crypto/rsa/pkcs1v15_test.go b/src/pkg/crypto/rsa/pkcs1v15_test.go new file mode 100644 index 000000000..d69bacfd6 --- /dev/null +++ b/src/pkg/crypto/rsa/pkcs1v15_test.go @@ -0,0 +1,221 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "bytes" + "crypto" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "io" + "testing" + "testing/quick" +) + +func decodeBase64(in string) []byte { + out := make([]byte, base64.StdEncoding.DecodedLen(len(in))) + n, err := base64.StdEncoding.Decode(out, []byte(in)) + if err != nil { + return nil + } + return out[0:n] +} + +type DecryptPKCS1v15Test struct { + in, out string +} + +// These test vectors were generated with `openssl rsautl -pkcs -encrypt` +var decryptPKCS1v15Tests = []DecryptPKCS1v15Test{ + { + "gIcUIoVkD6ATMBk/u/nlCZCCWRKdkfjCgFdo35VpRXLduiKXhNz1XupLLzTXAybEq15juc+EgY5o0DHv/nt3yg==", + "x", + }, + { + "Y7TOCSqofGhkRb+jaVRLzK8xw2cSo1IVES19utzv6hwvx+M8kFsoWQm5DzBeJCZTCVDPkTpavUuEbgp8hnUGDw==", + "testing.", + }, + { + "arReP9DJtEVyV2Dg3dDp4c/PSk1O6lxkoJ8HcFupoRorBZG+7+1fDAwT1olNddFnQMjmkb8vxwmNMoTAT/BFjQ==", + "testing.\n", + }, + { + "WtaBXIoGC54+vH0NH0CHHE+dRDOsMc/6BrfFu2lEqcKL9+uDuWaf+Xj9mrbQCjjZcpQuX733zyok/jsnqe/Ftw==", + "01234567890123456789012345678901234567890123456789012", + }, +} + +func TestDecryptPKCS1v15(t *testing.T) { + for i, test := range decryptPKCS1v15Tests { + out, err := DecryptPKCS1v15(nil, rsaPrivateKey, decodeBase64(test.in)) + if err != nil { + t.Errorf("#%d error decrypting", i) + } + want := []byte(test.out) + if bytes.Compare(out, want) != 0 { + t.Errorf("#%d got:%#v want:%#v", i, out, want) + } + } +} + +func TestEncryptPKCS1v15(t *testing.T) { + random := rand.Reader + k := (rsaPrivateKey.N.BitLen() + 7) / 8 + + tryEncryptDecrypt := func(in []byte, blind bool) bool { + if len(in) > k-11 { + in = in[0 : k-11] + } + + ciphertext, err := EncryptPKCS1v15(random, &rsaPrivateKey.PublicKey, in) + if err != nil { + t.Errorf("error encrypting: %s", err) + return false + } + + var rand io.Reader + if !blind { + rand = nil + } else { + rand = random + } + plaintext, err := DecryptPKCS1v15(rand, rsaPrivateKey, ciphertext) + if err != nil { + t.Errorf("error decrypting: %s", err) + return false + } + + if bytes.Compare(plaintext, in) != 0 { + t.Errorf("output mismatch: %#v %#v", plaintext, in) + return false + } + return true + } + + config := new(quick.Config) + if testing.Short() { + config.MaxCount = 10 + } + quick.Check(tryEncryptDecrypt, config) +} + +// These test vectors were generated with `openssl rsautl -pkcs -encrypt` +var decryptPKCS1v15SessionKeyTests = []DecryptPKCS1v15Test{ + { + "e6ukkae6Gykq0fKzYwULpZehX+UPXYzMoB5mHQUDEiclRbOTqas4Y0E6nwns1BBpdvEJcilhl5zsox/6DtGsYg==", + "1234", + }, + { + "Dtis4uk/q/LQGGqGk97P59K03hkCIVFMEFZRgVWOAAhxgYpCRG0MX2adptt92l67IqMki6iVQyyt0TtX3IdtEw==", + "FAIL", + }, + { + "LIyFyCYCptPxrvTxpol8F3M7ZivlMsf53zs0vHRAv+rDIh2YsHS69ePMoPMe3TkOMZ3NupiL3takPxIs1sK+dw==", + "abcd", + }, + { + "bafnobel46bKy76JzqU/RIVOH0uAYvzUtauKmIidKgM0sMlvobYVAVQPeUQ/oTGjbIZ1v/6Gyi5AO4DtHruGdw==", + "FAIL", + }, +} + +func TestEncryptPKCS1v15SessionKey(t *testing.T) { + for i, test := range decryptPKCS1v15SessionKeyTests { + key := []byte("FAIL") + err := DecryptPKCS1v15SessionKey(nil, rsaPrivateKey, decodeBase64(test.in), key) + if err != nil { + t.Errorf("#%d error decrypting", i) + } + want := []byte(test.out) + if bytes.Compare(key, want) != 0 { + t.Errorf("#%d got:%#v want:%#v", i, key, want) + } + } +} + +func TestNonZeroRandomBytes(t *testing.T) { + random := rand.Reader + + b := make([]byte, 512) + err := nonZeroRandomBytes(b, random) + if err != nil { + t.Errorf("returned error: %s", err) + } + for _, b := range b { + if b == 0 { + t.Errorf("Zero octet found") + return + } + } +} + +type signPKCS1v15Test struct { + in, out string +} + +// These vectors have been tested with +// `openssl rsautl -verify -inkey pk -in signature | hexdump -C` +var signPKCS1v15Tests = []signPKCS1v15Test{ + {"Test.\n", "a4f3fa6ea93bcdd0c57be020c1193ecbfd6f200a3d95c409769b029578fa0e336ad9a347600e40d3ae823b8c7e6bad88cc07c1d54c3a1523cbbb6d58efc362ae"}, +} + +func TestSignPKCS1v15(t *testing.T) { + for i, test := range signPKCS1v15Tests { + h := sha1.New() + h.Write([]byte(test.in)) + digest := h.Sum() + + s, err := SignPKCS1v15(nil, rsaPrivateKey, crypto.SHA1, digest) + if err != nil { + t.Errorf("#%d %s", i, err) + } + + expected, _ := hex.DecodeString(test.out) + if bytes.Compare(s, expected) != 0 { + t.Errorf("#%d got: %x want: %x", i, s, expected) + } + } +} + +func TestVerifyPKCS1v15(t *testing.T) { + for i, test := range signPKCS1v15Tests { + h := sha1.New() + h.Write([]byte(test.in)) + digest := h.Sum() + + sig, _ := hex.DecodeString(test.out) + + err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, crypto.SHA1, digest, sig) + if err != nil { + t.Errorf("#%d %s", i, err) + } + } +} + +// In order to generate new test vectors you'll need the PEM form of this key: +// -----BEGIN RSA PRIVATE KEY----- +// MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +// fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +// /ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +// RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +// EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +// IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +// tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +// -----END RSA PRIVATE KEY----- + +var rsaPrivateKey = &PrivateKey{ + PublicKey: PublicKey{ + N: fromBase10("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), + E: 65537, + }, + D: fromBase10("7266398431328116344057699379749222532279343923819063639497049039389899328538543087657733766554155839834519529439851673014800261285757759040931985506583861"), + Primes: []*big.Int{ + fromBase10("98920366548084643601728869055592650835572950932266967461790948584315647051443"), + fromBase10("94560208308847015747498523884063394671606671904944666360068158221458669711639"), + }, +} diff --git a/src/pkg/crypto/rsa/rsa.go b/src/pkg/crypto/rsa/rsa.go new file mode 100644 index 000000000..6957659f2 --- /dev/null +++ b/src/pkg/crypto/rsa/rsa.go @@ -0,0 +1,502 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rsa implements RSA encryption as specified in PKCS#1. +package rsa + +// TODO(agl): Add support for PSS padding. + +import ( + "big" + "crypto/rand" + "crypto/subtle" + "hash" + "io" + "os" +) + +var bigZero = big.NewInt(0) +var bigOne = big.NewInt(1) + +// A PublicKey represents the public part of an RSA key. +type PublicKey struct { + N *big.Int // modulus + E int // public exponent +} + +// A PrivateKey represents an RSA key +type PrivateKey struct { + PublicKey // public part. + D *big.Int // private exponent + Primes []*big.Int // prime factors of N, has >= 2 elements. + + // Precomputed contains precomputed values that speed up private + // operations, if available. + Precomputed PrecomputedValues +} + +type PrecomputedValues struct { + Dp, Dq *big.Int // D mod (P-1) (or mod Q-1) + Qinv *big.Int // Q^-1 mod Q + + // CRTValues is used for the 3rd and subsequent primes. Due to a + // historical accident, the CRT for the first two primes is handled + // differently in PKCS#1 and interoperability is sufficiently + // important that we mirror this. + CRTValues []CRTValue +} + +// CRTValue contains the precomputed chinese remainder theorem values. +type CRTValue struct { + Exp *big.Int // D mod (prime-1). + Coeff *big.Int // R·Coeff ≡ 1 mod Prime. + R *big.Int // product of primes prior to this (inc p and q). +} + +// Validate performs basic sanity checks on the key. +// It returns nil if the key is valid, or else an os.Error describing a problem. + +func (priv *PrivateKey) Validate() os.Error { + // Check that the prime factors are actually prime. Note that this is + // just a sanity check. Since the random witnesses chosen by + // ProbablyPrime are deterministic, given the candidate number, it's + // easy for an attack to generate composites that pass this test. + for _, prime := range priv.Primes { + if !big.ProbablyPrime(prime, 20) { + return os.NewError("prime factor is composite") + } + } + + // Check that Πprimes == n. + modulus := new(big.Int).Set(bigOne) + for _, prime := range priv.Primes { + modulus.Mul(modulus, prime) + } + if modulus.Cmp(priv.N) != 0 { + return os.NewError("invalid modulus") + } + // Check that e and totient(Πprimes) are coprime. + totient := new(big.Int).Set(bigOne) + for _, prime := range priv.Primes { + pminus1 := new(big.Int).Sub(prime, bigOne) + totient.Mul(totient, pminus1) + } + e := big.NewInt(int64(priv.E)) + gcd := new(big.Int) + x := new(big.Int) + y := new(big.Int) + big.GcdInt(gcd, x, y, totient, e) + if gcd.Cmp(bigOne) != 0 { + return os.NewError("invalid public exponent E") + } + // Check that de ≡ 1 (mod totient(Πprimes)) + de := new(big.Int).Mul(priv.D, e) + de.Mod(de, totient) + if de.Cmp(bigOne) != 0 { + return os.NewError("invalid private exponent D") + } + return nil +} + +// GenerateKey generates an RSA keypair of the given bit size. +func GenerateKey(random io.Reader, bits int) (priv *PrivateKey, err os.Error) { + return GenerateMultiPrimeKey(random, 2, bits) +} + +// GenerateMultiPrimeKey generates a multi-prime RSA keypair of the given bit +// size, as suggested in [1]. Although the public keys are compatible +// (actually, indistinguishable) from the 2-prime case, the private keys are +// not. Thus it may not be possible to export multi-prime private keys in +// certain formats or to subsequently import them into other code. +// +// Table 1 in [2] suggests maximum numbers of primes for a given size. +// +// [1] US patent 4405829 (1972, expired) +// [2] http://www.cacr.math.uwaterloo.ca/techreports/2006/cacr2006-16.pdf +func GenerateMultiPrimeKey(random io.Reader, nprimes int, bits int) (priv *PrivateKey, err os.Error) { + priv = new(PrivateKey) + // Smaller public exponents lead to faster public key + // operations. Since the exponent must be coprime to + // (p-1)(q-1), the smallest possible value is 3. Some have + // suggested that a larger exponent (often 2**16+1) be used + // since previous implementation bugs[1] were avoided when this + // was the case. However, there are no current reasons not to use + // small exponents. + // [1] http://marc.info/?l=cryptography&m=115694833312008&w=2 + priv.E = 3 + + if nprimes < 2 { + return nil, os.NewError("rsa.GenerateMultiPrimeKey: nprimes must be >= 2") + } + + primes := make([]*big.Int, nprimes) + +NextSetOfPrimes: + for { + todo := bits + for i := 0; i < nprimes; i++ { + primes[i], err = rand.Prime(random, todo/(nprimes-i)) + if err != nil { + return nil, err + } + todo -= primes[i].BitLen() + } + + // Make sure that primes is pairwise unequal. + for i, prime := range primes { + for j := 0; j < i; j++ { + if prime.Cmp(primes[j]) == 0 { + continue NextSetOfPrimes + } + } + } + + n := new(big.Int).Set(bigOne) + totient := new(big.Int).Set(bigOne) + pminus1 := new(big.Int) + for _, prime := range primes { + n.Mul(n, prime) + pminus1.Sub(prime, bigOne) + totient.Mul(totient, pminus1) + } + + g := new(big.Int) + priv.D = new(big.Int) + y := new(big.Int) + e := big.NewInt(int64(priv.E)) + big.GcdInt(g, priv.D, y, e, totient) + + if g.Cmp(bigOne) == 0 { + priv.D.Add(priv.D, totient) + priv.Primes = primes + priv.N = n + + break + } + } + + priv.Precompute() + return +} + +// incCounter increments a four byte, big-endian counter. +func incCounter(c *[4]byte) { + if c[3]++; c[3] != 0 { + return + } + if c[2]++; c[2] != 0 { + return + } + if c[1]++; c[1] != 0 { + return + } + c[0]++ +} + +// mgf1XOR XORs the bytes in out with a mask generated using the MGF1 function +// specified in PKCS#1 v2.1. +func mgf1XOR(out []byte, hash hash.Hash, seed []byte) { + var counter [4]byte + + done := 0 + for done < len(out) { + hash.Write(seed) + hash.Write(counter[0:4]) + digest := hash.Sum() + hash.Reset() + + for i := 0; i < len(digest) && done < len(out); i++ { + out[done] ^= digest[i] + done++ + } + incCounter(&counter) + } +} + +// MessageTooLongError is returned when attempting to encrypt a message which +// is too large for the size of the public key. +type MessageTooLongError struct{} + +func (MessageTooLongError) String() string { + return "message too long for RSA public key size" +} + +func encrypt(c *big.Int, pub *PublicKey, m *big.Int) *big.Int { + e := big.NewInt(int64(pub.E)) + c.Exp(m, e, pub.N) + return c +} + +// EncryptOAEP encrypts the given message with RSA-OAEP. +// The message must be no longer than the length of the public modulus less +// twice the hash length plus 2. +func EncryptOAEP(hash hash.Hash, random io.Reader, pub *PublicKey, msg []byte, label []byte) (out []byte, err os.Error) { + hash.Reset() + k := (pub.N.BitLen() + 7) / 8 + if len(msg) > k-2*hash.Size()-2 { + err = MessageTooLongError{} + return + } + + hash.Write(label) + lHash := hash.Sum() + hash.Reset() + + em := make([]byte, k) + seed := em[1 : 1+hash.Size()] + db := em[1+hash.Size():] + + copy(db[0:hash.Size()], lHash) + db[len(db)-len(msg)-1] = 1 + copy(db[len(db)-len(msg):], msg) + + _, err = io.ReadFull(random, seed) + if err != nil { + return + } + + mgf1XOR(db, hash, seed) + mgf1XOR(seed, hash, db) + + m := new(big.Int) + m.SetBytes(em) + c := encrypt(new(big.Int), pub, m) + out = c.Bytes() + + if len(out) < k { + // If the output is too small, we need to left-pad with zeros. + t := make([]byte, k) + copy(t[k-len(out):], out) + out = t + } + + return +} + +// A DecryptionError represents a failure to decrypt a message. +// It is deliberately vague to avoid adaptive attacks. +type DecryptionError struct{} + +func (DecryptionError) String() string { return "RSA decryption error" } + +// A VerificationError represents a failure to verify a signature. +// It is deliberately vague to avoid adaptive attacks. +type VerificationError struct{} + +func (VerificationError) String() string { return "RSA verification error" } + +// modInverse returns ia, the inverse of a in the multiplicative group of prime +// order n. It requires that a be a member of the group (i.e. less than n). +func modInverse(a, n *big.Int) (ia *big.Int, ok bool) { + g := new(big.Int) + x := new(big.Int) + y := new(big.Int) + big.GcdInt(g, x, y, a, n) + if g.Cmp(bigOne) != 0 { + // In this case, a and n aren't coprime and we cannot calculate + // the inverse. This happens because the values of n are nearly + // prime (being the product of two primes) rather than truly + // prime. + return + } + + if x.Cmp(bigOne) < 0 { + // 0 is not the multiplicative inverse of any element so, if x + // < 1, then x is negative. + x.Add(x, n) + } + + return x, true +} + +// Precompute performs some calculations that speed up private key operations +// in the future. +func (priv *PrivateKey) Precompute() { + if priv.Precomputed.Dp != nil { + return + } + + priv.Precomputed.Dp = new(big.Int).Sub(priv.Primes[0], bigOne) + priv.Precomputed.Dp.Mod(priv.D, priv.Precomputed.Dp) + + priv.Precomputed.Dq = new(big.Int).Sub(priv.Primes[1], bigOne) + priv.Precomputed.Dq.Mod(priv.D, priv.Precomputed.Dq) + + priv.Precomputed.Qinv = new(big.Int).ModInverse(priv.Primes[1], priv.Primes[0]) + + r := new(big.Int).Mul(priv.Primes[0], priv.Primes[1]) + priv.Precomputed.CRTValues = make([]CRTValue, len(priv.Primes)-2) + for i := 2; i < len(priv.Primes); i++ { + prime := priv.Primes[i] + values := &priv.Precomputed.CRTValues[i-2] + + values.Exp = new(big.Int).Sub(prime, bigOne) + values.Exp.Mod(priv.D, values.Exp) + + values.R = new(big.Int).Set(r) + values.Coeff = new(big.Int).ModInverse(r, prime) + + r.Mul(r, prime) + } +} + +// decrypt performs an RSA decryption, resulting in a plaintext integer. If a +// random source is given, RSA blinding is used. +func decrypt(random io.Reader, priv *PrivateKey, c *big.Int) (m *big.Int, err os.Error) { + // TODO(agl): can we get away with reusing blinds? + if c.Cmp(priv.N) > 0 { + err = DecryptionError{} + return + } + + var ir *big.Int + if random != nil { + // Blinding enabled. Blinding involves multiplying c by r^e. + // Then the decryption operation performs (m^e * r^e)^d mod n + // which equals mr mod n. The factor of r can then be removed + // by multiplying by the multiplicative inverse of r. + + var r *big.Int + + for { + r, err = rand.Int(random, priv.N) + if err != nil { + return + } + if r.Cmp(bigZero) == 0 { + r = bigOne + } + var ok bool + ir, ok = modInverse(r, priv.N) + if ok { + break + } + } + bigE := big.NewInt(int64(priv.E)) + rpowe := new(big.Int).Exp(r, bigE, priv.N) + cCopy := new(big.Int).Set(c) + cCopy.Mul(cCopy, rpowe) + cCopy.Mod(cCopy, priv.N) + c = cCopy + } + + if priv.Precomputed.Dp == nil { + m = new(big.Int).Exp(c, priv.D, priv.N) + } else { + // We have the precalculated values needed for the CRT. + m = new(big.Int).Exp(c, priv.Precomputed.Dp, priv.Primes[0]) + m2 := new(big.Int).Exp(c, priv.Precomputed.Dq, priv.Primes[1]) + m.Sub(m, m2) + if m.Sign() < 0 { + m.Add(m, priv.Primes[0]) + } + m.Mul(m, priv.Precomputed.Qinv) + m.Mod(m, priv.Primes[0]) + m.Mul(m, priv.Primes[1]) + m.Add(m, m2) + + for i, values := range priv.Precomputed.CRTValues { + prime := priv.Primes[2+i] + m2.Exp(c, values.Exp, prime) + m2.Sub(m2, m) + m2.Mul(m2, values.Coeff) + m2.Mod(m2, prime) + if m2.Sign() < 0 { + m2.Add(m2, prime) + } + m2.Mul(m2, values.R) + m.Add(m, m2) + } + } + + if ir != nil { + // Unblind. + m.Mul(m, ir) + m.Mod(m, priv.N) + } + + return +} + +// DecryptOAEP decrypts ciphertext using RSA-OAEP. +// If rand != nil, DecryptOAEP uses RSA blinding to avoid timing side-channel attacks. +func DecryptOAEP(hash hash.Hash, random io.Reader, priv *PrivateKey, ciphertext []byte, label []byte) (msg []byte, err os.Error) { + k := (priv.N.BitLen() + 7) / 8 + if len(ciphertext) > k || + k < hash.Size()*2+2 { + err = DecryptionError{} + return + } + + c := new(big.Int).SetBytes(ciphertext) + + m, err := decrypt(random, priv, c) + if err != nil { + return + } + + hash.Write(label) + lHash := hash.Sum() + hash.Reset() + + // Converting the plaintext number to bytes will strip any + // leading zeros so we may have to left pad. We do this unconditionally + // to avoid leaking timing information. (Although we still probably + // leak the number of leading zeros. It's not clear that we can do + // anything about this.) + em := leftPad(m.Bytes(), k) + + firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0) + + seed := em[1 : hash.Size()+1] + db := em[hash.Size()+1:] + + mgf1XOR(seed, hash, db) + mgf1XOR(db, hash, seed) + + lHash2 := db[0:hash.Size()] + + // We have to validate the plaintext in constant time in order to avoid + // attacks like: J. Manger. A Chosen Ciphertext Attack on RSA Optimal + // Asymmetric Encryption Padding (OAEP) as Standardized in PKCS #1 + // v2.0. In J. Kilian, editor, Advances in Cryptology. + lHash2Good := subtle.ConstantTimeCompare(lHash, lHash2) + + // The remainder of the plaintext must be zero or more 0x00, followed + // by 0x01, followed by the message. + // lookingForIndex: 1 iff we are still looking for the 0x01 + // index: the offset of the first 0x01 byte + // invalid: 1 iff we saw a non-zero byte before the 0x01. + var lookingForIndex, index, invalid int + lookingForIndex = 1 + rest := db[hash.Size():] + + for i := 0; i < len(rest); i++ { + equals0 := subtle.ConstantTimeByteEq(rest[i], 0) + equals1 := subtle.ConstantTimeByteEq(rest[i], 1) + index = subtle.ConstantTimeSelect(lookingForIndex&equals1, i, index) + lookingForIndex = subtle.ConstantTimeSelect(equals1, 0, lookingForIndex) + invalid = subtle.ConstantTimeSelect(lookingForIndex&^equals0, 1, invalid) + } + + if firstByteIsZero&lHash2Good&^invalid&^lookingForIndex != 1 { + err = DecryptionError{} + return + } + + msg = rest[index+1:] + return +} + +// leftPad returns a new slice of length size. The contents of input are right +// aligned in the new slice. +func leftPad(input []byte, size int) (out []byte) { + n := len(input) + if n > size { + n = size + } + out = make([]byte, size) + copy(out[len(out)-n:], input) + return +} diff --git a/src/pkg/crypto/rsa/rsa_test.go b/src/pkg/crypto/rsa/rsa_test.go new file mode 100644 index 000000000..c36bca1cd --- /dev/null +++ b/src/pkg/crypto/rsa/rsa_test.go @@ -0,0 +1,348 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rsa + +import ( + "big" + "bytes" + "crypto/rand" + "crypto/sha1" + "testing" +) + +func TestKeyGeneration(t *testing.T) { + size := 1024 + if testing.Short() { + size = 128 + } + priv, err := GenerateKey(rand.Reader, size) + if err != nil { + t.Errorf("failed to generate key") + } + testKeyBasics(t, priv) +} + +func Test3PrimeKeyGeneration(t *testing.T) { + if testing.Short() { + return + } + + size := 768 + priv, err := GenerateMultiPrimeKey(rand.Reader, 3, size) + if err != nil { + t.Errorf("failed to generate key") + } + testKeyBasics(t, priv) +} + +func Test4PrimeKeyGeneration(t *testing.T) { + if testing.Short() { + return + } + + size := 768 + priv, err := GenerateMultiPrimeKey(rand.Reader, 4, size) + if err != nil { + t.Errorf("failed to generate key") + } + testKeyBasics(t, priv) +} + +func testKeyBasics(t *testing.T, priv *PrivateKey) { + if err := priv.Validate(); err != nil { + t.Errorf("Validate() failed: %s", err) + } + + pub := &priv.PublicKey + m := big.NewInt(42) + c := encrypt(new(big.Int), pub, m) + + m2, err := decrypt(nil, priv, c) + if err != nil { + t.Errorf("error while decrypting: %s", err) + return + } + if m.Cmp(m2) != 0 { + t.Errorf("got:%v, want:%v (%+v)", m2, m, priv) + } + + m3, err := decrypt(rand.Reader, priv, c) + if err != nil { + t.Errorf("error while decrypting (blind): %s", err) + } + if m.Cmp(m3) != 0 { + t.Errorf("(blind) got:%v, want:%v (%#v)", m3, m, priv) + } +} + +func fromBase10(base10 string) *big.Int { + i := new(big.Int) + i.SetString(base10, 10) + return i +} + +func BenchmarkRSA2048Decrypt(b *testing.B) { + b.StopTimer() + priv := &PrivateKey{ + PublicKey: PublicKey{ + N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"), + E: 3, + }, + D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"), + Primes: []*big.Int{ + fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"), + fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"), + }, + } + priv.Precompute() + + c := fromBase10("1000") + + b.StartTimer() + + for i := 0; i < b.N; i++ { + decrypt(nil, priv, c) + } +} + +func Benchmark3PrimeRSA2048Decrypt(b *testing.B) { + b.StopTimer() + priv := &PrivateKey{ + PublicKey: PublicKey{ + N: fromBase10("16346378922382193400538269749936049106320265317511766357599732575277382844051791096569333808598921852351577762718529818072849191122419410612033592401403764925096136759934497687765453905884149505175426053037420486697072448609022753683683718057795566811401938833367954642951433473337066311978821180526439641496973296037000052546108507805269279414789035461158073156772151892452251106173507240488993608650881929629163465099476849643165682709047462010581308719577053905787496296934240246311806555924593059995202856826239801816771116902778517096212527979497399966526283516447337775509777558018145573127308919204297111496233"), + E: 3, + }, + D: fromBase10("10897585948254795600358846499957366070880176878341177571733155050184921896034527397712889205732614568234385175145686545381899460748279607074689061600935843283397424506622998458510302603922766336783617368686090042765718290914099334449154829375179958369993407724946186243249568928237086215759259909861748642124071874879861299389874230489928271621259294894142840428407196932444474088857746123104978617098858619445675532587787023228852383149557470077802718705420275739737958953794088728369933811184572620857678792001136676902250566845618813972833750098806496641114644760255910789397593428910198080271317419213080834885003"), + Primes: []*big.Int{ + fromBase10("1025363189502892836833747188838978207017355117492483312747347695538428729137306368764177201532277413433182799108299960196606011786562992097313508180436744488171474690412562218914213688661311117337381958560443"), + fromBase10("3467903426626310123395340254094941045497208049900750380025518552334536945536837294961497712862519984786362199788654739924501424784631315081391467293694361474867825728031147665777546570788493758372218019373"), + fromBase10("4597024781409332673052708605078359346966325141767460991205742124888960305710298765592730135879076084498363772408626791576005136245060321874472727132746643162385746062759369754202494417496879741537284589047"), + }, + } + priv.Precompute() + + c := fromBase10("1000") + + b.StartTimer() + + for i := 0; i < b.N; i++ { + decrypt(nil, priv, c) + } +} + +type testEncryptOAEPMessage struct { + in []byte + seed []byte + out []byte +} + +type testEncryptOAEPStruct struct { + modulus string + e int + d string + msgs []testEncryptOAEPMessage +} + +func TestEncryptOAEP(t *testing.T) { + sha1 := sha1.New() + n := new(big.Int) + for i, test := range testEncryptOAEPData { + n.SetString(test.modulus, 16) + public := PublicKey{n, test.e} + + for j, message := range test.msgs { + randomSource := bytes.NewBuffer(message.seed) + out, err := EncryptOAEP(sha1, randomSource, &public, message.in, nil) + if err != nil { + t.Errorf("#%d,%d error: %s", i, j, err) + } + if bytes.Compare(out, message.out) != 0 { + t.Errorf("#%d,%d bad result: %x (want %x)", i, j, out, message.out) + } + } + } +} + +func TestDecryptOAEP(t *testing.T) { + random := rand.Reader + + sha1 := sha1.New() + n := new(big.Int) + d := new(big.Int) + for i, test := range testEncryptOAEPData { + n.SetString(test.modulus, 16) + d.SetString(test.d, 16) + private := new(PrivateKey) + private.PublicKey = PublicKey{n, test.e} + private.D = d + + for j, message := range test.msgs { + out, err := DecryptOAEP(sha1, nil, private, message.out, nil) + if err != nil { + t.Errorf("#%d,%d error: %s", i, j, err) + } else if bytes.Compare(out, message.in) != 0 { + t.Errorf("#%d,%d bad result: %#v (want %#v)", i, j, out, message.in) + } + + // Decrypt with blinding. + out, err = DecryptOAEP(sha1, random, private, message.out, nil) + if err != nil { + t.Errorf("#%d,%d (blind) error: %s", i, j, err) + } else if bytes.Compare(out, message.in) != 0 { + t.Errorf("#%d,%d (blind) bad result: %#v (want %#v)", i, j, out, message.in) + } + } + if testing.Short() { + break + } + } +} + +// testEncryptOAEPData contains a subset of the vectors from RSA's "Test vectors for RSA-OAEP". +var testEncryptOAEPData = []testEncryptOAEPStruct{ + // Key 1 + {"a8b3b284af8eb50b387034a860f146c4919f318763cd6c5598c8ae4811a1e0abc4c7e0b082d693a5e7fced675cf4668512772c0cbc64a742c6c630f533c8cc72f62ae833c40bf25842e984bb78bdbf97c0107d55bdb662f5c4e0fab9845cb5148ef7392dd3aaff93ae1e6b667bb3d4247616d4f5ba10d4cfd226de88d39f16fb", + 65537, + "53339cfdb79fc8466a655c7316aca85c55fd8f6dd898fdaf119517ef4f52e8fd8e258df93fee180fa0e4ab29693cd83b152a553d4ac4d1812b8b9fa5af0e7f55fe7304df41570926f3311f15c4d65a732c483116ee3d3d2d0af3549ad9bf7cbfb78ad884f84d5beb04724dc7369b31def37d0cf539e9cfcdd3de653729ead5d1", + []testEncryptOAEPMessage{ + // Example 1.1 + { + []byte{0x66, 0x28, 0x19, 0x4e, 0x12, 0x07, 0x3d, 0xb0, + 0x3b, 0xa9, 0x4c, 0xda, 0x9e, 0xf9, 0x53, 0x23, 0x97, + 0xd5, 0x0d, 0xba, 0x79, 0xb9, 0x87, 0x00, 0x4a, 0xfe, + 0xfe, 0x34, + }, + []byte{0x18, 0xb7, 0x76, 0xea, 0x21, 0x06, 0x9d, 0x69, + 0x77, 0x6a, 0x33, 0xe9, 0x6b, 0xad, 0x48, 0xe1, 0xdd, + 0xa0, 0xa5, 0xef, + }, + []byte{0x35, 0x4f, 0xe6, 0x7b, 0x4a, 0x12, 0x6d, 0x5d, + 0x35, 0xfe, 0x36, 0xc7, 0x77, 0x79, 0x1a, 0x3f, 0x7b, + 0xa1, 0x3d, 0xef, 0x48, 0x4e, 0x2d, 0x39, 0x08, 0xaf, + 0xf7, 0x22, 0xfa, 0xd4, 0x68, 0xfb, 0x21, 0x69, 0x6d, + 0xe9, 0x5d, 0x0b, 0xe9, 0x11, 0xc2, 0xd3, 0x17, 0x4f, + 0x8a, 0xfc, 0xc2, 0x01, 0x03, 0x5f, 0x7b, 0x6d, 0x8e, + 0x69, 0x40, 0x2d, 0xe5, 0x45, 0x16, 0x18, 0xc2, 0x1a, + 0x53, 0x5f, 0xa9, 0xd7, 0xbf, 0xc5, 0xb8, 0xdd, 0x9f, + 0xc2, 0x43, 0xf8, 0xcf, 0x92, 0x7d, 0xb3, 0x13, 0x22, + 0xd6, 0xe8, 0x81, 0xea, 0xa9, 0x1a, 0x99, 0x61, 0x70, + 0xe6, 0x57, 0xa0, 0x5a, 0x26, 0x64, 0x26, 0xd9, 0x8c, + 0x88, 0x00, 0x3f, 0x84, 0x77, 0xc1, 0x22, 0x70, 0x94, + 0xa0, 0xd9, 0xfa, 0x1e, 0x8c, 0x40, 0x24, 0x30, 0x9c, + 0xe1, 0xec, 0xcc, 0xb5, 0x21, 0x00, 0x35, 0xd4, 0x7a, + 0xc7, 0x2e, 0x8a, + }, + }, + // Example 1.2 + { + []byte{0x75, 0x0c, 0x40, 0x47, 0xf5, 0x47, 0xe8, 0xe4, + 0x14, 0x11, 0x85, 0x65, 0x23, 0x29, 0x8a, 0xc9, 0xba, + 0xe2, 0x45, 0xef, 0xaf, 0x13, 0x97, 0xfb, 0xe5, 0x6f, + 0x9d, 0xd5, + }, + []byte{0x0c, 0xc7, 0x42, 0xce, 0x4a, 0x9b, 0x7f, 0x32, + 0xf9, 0x51, 0xbc, 0xb2, 0x51, 0xef, 0xd9, 0x25, 0xfe, + 0x4f, 0xe3, 0x5f, + }, + []byte{0x64, 0x0d, 0xb1, 0xac, 0xc5, 0x8e, 0x05, 0x68, + 0xfe, 0x54, 0x07, 0xe5, 0xf9, 0xb7, 0x01, 0xdf, 0xf8, + 0xc3, 0xc9, 0x1e, 0x71, 0x6c, 0x53, 0x6f, 0xc7, 0xfc, + 0xec, 0x6c, 0xb5, 0xb7, 0x1c, 0x11, 0x65, 0x98, 0x8d, + 0x4a, 0x27, 0x9e, 0x15, 0x77, 0xd7, 0x30, 0xfc, 0x7a, + 0x29, 0x93, 0x2e, 0x3f, 0x00, 0xc8, 0x15, 0x15, 0x23, + 0x6d, 0x8d, 0x8e, 0x31, 0x01, 0x7a, 0x7a, 0x09, 0xdf, + 0x43, 0x52, 0xd9, 0x04, 0xcd, 0xeb, 0x79, 0xaa, 0x58, + 0x3a, 0xdc, 0xc3, 0x1e, 0xa6, 0x98, 0xa4, 0xc0, 0x52, + 0x83, 0xda, 0xba, 0x90, 0x89, 0xbe, 0x54, 0x91, 0xf6, + 0x7c, 0x1a, 0x4e, 0xe4, 0x8d, 0xc7, 0x4b, 0xbb, 0xe6, + 0x64, 0x3a, 0xef, 0x84, 0x66, 0x79, 0xb4, 0xcb, 0x39, + 0x5a, 0x35, 0x2d, 0x5e, 0xd1, 0x15, 0x91, 0x2d, 0xf6, + 0x96, 0xff, 0xe0, 0x70, 0x29, 0x32, 0x94, 0x6d, 0x71, + 0x49, 0x2b, 0x44, + }, + }, + // Example 1.3 + { + []byte{0xd9, 0x4a, 0xe0, 0x83, 0x2e, 0x64, 0x45, 0xce, + 0x42, 0x33, 0x1c, 0xb0, 0x6d, 0x53, 0x1a, 0x82, 0xb1, + 0xdb, 0x4b, 0xaa, 0xd3, 0x0f, 0x74, 0x6d, 0xc9, 0x16, + 0xdf, 0x24, 0xd4, 0xe3, 0xc2, 0x45, 0x1f, 0xff, 0x59, + 0xa6, 0x42, 0x3e, 0xb0, 0xe1, 0xd0, 0x2d, 0x4f, 0xe6, + 0x46, 0xcf, 0x69, 0x9d, 0xfd, 0x81, 0x8c, 0x6e, 0x97, + 0xb0, 0x51, + }, + []byte{0x25, 0x14, 0xdf, 0x46, 0x95, 0x75, 0x5a, 0x67, + 0xb2, 0x88, 0xea, 0xf4, 0x90, 0x5c, 0x36, 0xee, 0xc6, + 0x6f, 0xd2, 0xfd, + }, + []byte{0x42, 0x37, 0x36, 0xed, 0x03, 0x5f, 0x60, 0x26, + 0xaf, 0x27, 0x6c, 0x35, 0xc0, 0xb3, 0x74, 0x1b, 0x36, + 0x5e, 0x5f, 0x76, 0xca, 0x09, 0x1b, 0x4e, 0x8c, 0x29, + 0xe2, 0xf0, 0xbe, 0xfe, 0xe6, 0x03, 0x59, 0x5a, 0xa8, + 0x32, 0x2d, 0x60, 0x2d, 0x2e, 0x62, 0x5e, 0x95, 0xeb, + 0x81, 0xb2, 0xf1, 0xc9, 0x72, 0x4e, 0x82, 0x2e, 0xca, + 0x76, 0xdb, 0x86, 0x18, 0xcf, 0x09, 0xc5, 0x34, 0x35, + 0x03, 0xa4, 0x36, 0x08, 0x35, 0xb5, 0x90, 0x3b, 0xc6, + 0x37, 0xe3, 0x87, 0x9f, 0xb0, 0x5e, 0x0e, 0xf3, 0x26, + 0x85, 0xd5, 0xae, 0xc5, 0x06, 0x7c, 0xd7, 0xcc, 0x96, + 0xfe, 0x4b, 0x26, 0x70, 0xb6, 0xea, 0xc3, 0x06, 0x6b, + 0x1f, 0xcf, 0x56, 0x86, 0xb6, 0x85, 0x89, 0xaa, 0xfb, + 0x7d, 0x62, 0x9b, 0x02, 0xd8, 0xf8, 0x62, 0x5c, 0xa3, + 0x83, 0x36, 0x24, 0xd4, 0x80, 0x0f, 0xb0, 0x81, 0xb1, + 0xcf, 0x94, 0xeb, + }, + }, + }, + }, + // Key 10 + {"ae45ed5601cec6b8cc05f803935c674ddbe0d75c4c09fd7951fc6b0caec313a8df39970c518bffba5ed68f3f0d7f22a4029d413f1ae07e4ebe9e4177ce23e7f5404b569e4ee1bdcf3c1fb03ef113802d4f855eb9b5134b5a7c8085adcae6fa2fa1417ec3763be171b0c62b760ede23c12ad92b980884c641f5a8fac26bdad4a03381a22fe1b754885094c82506d4019a535a286afeb271bb9ba592de18dcf600c2aeeae56e02f7cf79fc14cf3bdc7cd84febbbf950ca90304b2219a7aa063aefa2c3c1980e560cd64afe779585b6107657b957857efde6010988ab7de417fc88d8f384c4e6e72c3f943e0c31c0c4a5cc36f879d8a3ac9d7d59860eaada6b83bb", + 65537, + "056b04216fe5f354ac77250a4b6b0c8525a85c59b0bd80c56450a22d5f438e596a333aa875e291dd43f48cb88b9d5fc0d499f9fcd1c397f9afc070cd9e398c8d19e61db7c7410a6b2675dfbf5d345b804d201add502d5ce2dfcb091ce9997bbebe57306f383e4d588103f036f7e85d1934d152a323e4a8db451d6f4a5b1b0f102cc150e02feee2b88dea4ad4c1baccb24d84072d14e1d24a6771f7408ee30564fb86d4393a34bcf0b788501d193303f13a2284b001f0f649eaf79328d4ac5c430ab4414920a9460ed1b7bc40ec653e876d09abc509ae45b525190116a0c26101848298509c1c3bf3a483e7274054e15e97075036e989f60932807b5257751e79", + []testEncryptOAEPMessage{ + // Example 10.1 + { + []byte{0x8b, 0xba, 0x6b, 0xf8, 0x2a, 0x6c, 0x0f, 0x86, + 0xd5, 0xf1, 0x75, 0x6e, 0x97, 0x95, 0x68, 0x70, 0xb0, + 0x89, 0x53, 0xb0, 0x6b, 0x4e, 0xb2, 0x05, 0xbc, 0x16, + 0x94, 0xee, + }, + []byte{0x47, 0xe1, 0xab, 0x71, 0x19, 0xfe, 0xe5, 0x6c, + 0x95, 0xee, 0x5e, 0xaa, 0xd8, 0x6f, 0x40, 0xd0, 0xaa, + 0x63, 0xbd, 0x33, + }, + []byte{0x53, 0xea, 0x5d, 0xc0, 0x8c, 0xd2, 0x60, 0xfb, + 0x3b, 0x85, 0x85, 0x67, 0x28, 0x7f, 0xa9, 0x15, 0x52, + 0xc3, 0x0b, 0x2f, 0xeb, 0xfb, 0xa2, 0x13, 0xf0, 0xae, + 0x87, 0x70, 0x2d, 0x06, 0x8d, 0x19, 0xba, 0xb0, 0x7f, + 0xe5, 0x74, 0x52, 0x3d, 0xfb, 0x42, 0x13, 0x9d, 0x68, + 0xc3, 0xc5, 0xaf, 0xee, 0xe0, 0xbf, 0xe4, 0xcb, 0x79, + 0x69, 0xcb, 0xf3, 0x82, 0xb8, 0x04, 0xd6, 0xe6, 0x13, + 0x96, 0x14, 0x4e, 0x2d, 0x0e, 0x60, 0x74, 0x1f, 0x89, + 0x93, 0xc3, 0x01, 0x4b, 0x58, 0xb9, 0xb1, 0x95, 0x7a, + 0x8b, 0xab, 0xcd, 0x23, 0xaf, 0x85, 0x4f, 0x4c, 0x35, + 0x6f, 0xb1, 0x66, 0x2a, 0xa7, 0x2b, 0xfc, 0xc7, 0xe5, + 0x86, 0x55, 0x9d, 0xc4, 0x28, 0x0d, 0x16, 0x0c, 0x12, + 0x67, 0x85, 0xa7, 0x23, 0xeb, 0xee, 0xbe, 0xff, 0x71, + 0xf1, 0x15, 0x94, 0x44, 0x0a, 0xae, 0xf8, 0x7d, 0x10, + 0x79, 0x3a, 0x87, 0x74, 0xa2, 0x39, 0xd4, 0xa0, 0x4c, + 0x87, 0xfe, 0x14, 0x67, 0xb9, 0xda, 0xf8, 0x52, 0x08, + 0xec, 0x6c, 0x72, 0x55, 0x79, 0x4a, 0x96, 0xcc, 0x29, + 0x14, 0x2f, 0x9a, 0x8b, 0xd4, 0x18, 0xe3, 0xc1, 0xfd, + 0x67, 0x34, 0x4b, 0x0c, 0xd0, 0x82, 0x9d, 0xf3, 0xb2, + 0xbe, 0xc6, 0x02, 0x53, 0x19, 0x62, 0x93, 0xc6, 0xb3, + 0x4d, 0x3f, 0x75, 0xd3, 0x2f, 0x21, 0x3d, 0xd4, 0x5c, + 0x62, 0x73, 0xd5, 0x05, 0xad, 0xf4, 0xcc, 0xed, 0x10, + 0x57, 0xcb, 0x75, 0x8f, 0xc2, 0x6a, 0xee, 0xfa, 0x44, + 0x12, 0x55, 0xed, 0x4e, 0x64, 0xc1, 0x99, 0xee, 0x07, + 0x5e, 0x7f, 0x16, 0x64, 0x61, 0x82, 0xfd, 0xb4, 0x64, + 0x73, 0x9b, 0x68, 0xab, 0x5d, 0xaf, 0xf0, 0xe6, 0x3e, + 0x95, 0x52, 0x01, 0x68, 0x24, 0xf0, 0x54, 0xbf, 0x4d, + 0x3c, 0x8c, 0x90, 0xa9, 0x7b, 0xb6, 0xb6, 0x55, 0x32, + 0x84, 0xeb, 0x42, 0x9f, 0xcc, + }, + }, + }, + }, +} diff --git a/src/pkg/crypto/sha1/Makefile b/src/pkg/crypto/sha1/Makefile new file mode 100644 index 000000000..81ac38c0b --- /dev/null +++ b/src/pkg/crypto/sha1/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/sha1 +GOFILES=\ + sha1.go\ + sha1block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/sha1/sha1.go b/src/pkg/crypto/sha1/sha1.go new file mode 100644 index 000000000..788d1ff55 --- /dev/null +++ b/src/pkg/crypto/sha1/sha1.go @@ -0,0 +1,119 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174. +package sha1 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.SHA1, New) +} + +// The size of a SHA1 checksum in bytes. +const Size = 20 + +const ( + _Chunk = 64 + _Init0 = 0x67452301 + _Init1 = 0xEFCDAB89 + _Init2 = 0x98BADCFE + _Init3 = 0x10325476 + _Init4 = 0xC3D2E1F0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [5]uint32 + x [_Chunk]byte + nx int + len uint64 +} + +func (d *digest) Reset() { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA1 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 20) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 24) + p[j+1] = byte(s >> 16) + p[j+2] = byte(s >> 8) + p[j+3] = byte(s >> 0) + j += 4 + } + return p +} diff --git a/src/pkg/crypto/sha1/sha1_test.go b/src/pkg/crypto/sha1/sha1_test.go new file mode 100644 index 000000000..2712fe35e --- /dev/null +++ b/src/pkg/crypto/sha1/sha1_test.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA1 hash algorithm. See RFC 3174. + +package sha1 + +import ( + "fmt" + "io" + "testing" +) + +type sha1Test struct { + out string + in string +} + +var golden = []sha1Test{ + {"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""}, + {"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"}, + {"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"}, + {"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"}, + {"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"}, + {"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"}, + {"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"}, + {"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"}, + {"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"}, + {"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"}, + {"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"}, + {"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."}, + {"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."}, + {"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."}, + {"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."}, + {"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."}, + {"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size: a.out: bad magic"}, + {"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail. -Mark Horton"}, + {"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."}, + {"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"}, + {"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/sha1/sha1block.go b/src/pkg/crypto/sha1/sha1block.go new file mode 100644 index 000000000..b5d32af70 --- /dev/null +++ b/src/pkg/crypto/sha1/sha1block.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA1 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha1 + +const ( + _K0 = 0x5A827999 + _K1 = 0x6ED9EBA1 + _K2 = 0x8F1BBCDC + _K3 = 0xCA62C1D6 +) + +func _Block(dig *digest, p []byte) int { + var w [80]uint32 + + n := 0 + h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] + for len(p) >= _Chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + for i := 16; i < 80; i++ { + tmp := w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16] + w[i] = tmp<<1 | tmp>>(32-1) + } + + a, b, c, d, e := h0, h1, h2, h3, h4 + + // Each of the four 20-iteration rounds + // differs only in the computation of f and + // the choice of K (_K0, _K1, etc). + for i := 0; i < 20; i++ { + f := b&c | (^b)&d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K0 + a, b, c, d, e = t, a, b30, c, d + } + for i := 20; i < 40; i++ { + f := b ^ c ^ d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K1 + a, b, c, d, e = t, a, b30, c, d + } + for i := 40; i < 60; i++ { + f := b&c | b&d | c&d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K2 + a, b, c, d, e = t, a, b30, c, d + } + for i := 60; i < 80; i++ { + f := b ^ c ^ d + a5 := a<<5 | a>>(32-5) + b30 := b<<30 | b>>(32-30) + t := a5 + f + e + w[i] + _K3 + a, b, c, d, e = t, a, b30, c, d + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4 + return n +} diff --git a/src/pkg/crypto/sha256/Makefile b/src/pkg/crypto/sha256/Makefile new file mode 100644 index 000000000..97fe4d8e6 --- /dev/null +++ b/src/pkg/crypto/sha256/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/sha256 +GOFILES=\ + sha256.go\ + sha256block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/sha256/sha256.go b/src/pkg/crypto/sha256/sha256.go new file mode 100644 index 000000000..a2c058d18 --- /dev/null +++ b/src/pkg/crypto/sha256/sha256.go @@ -0,0 +1,166 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha256 implements the SHA224 and SHA256 hash algorithms as defined +// in FIPS 180-2. +package sha256 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.SHA224, New224) + crypto.RegisterHash(crypto.SHA256, New) +} + +// The size of a SHA256 checksum in bytes. +const Size = 32 + +// The size of a SHA224 checksum in bytes. +const Size224 = 28 + +const ( + _Chunk = 64 + _Init0 = 0x6A09E667 + _Init1 = 0xBB67AE85 + _Init2 = 0x3C6EF372 + _Init3 = 0xA54FF53A + _Init4 = 0x510E527F + _Init5 = 0x9B05688C + _Init6 = 0x1F83D9AB + _Init7 = 0x5BE0CD19 + _Init0_224 = 0xC1059ED8 + _Init1_224 = 0x367CD507 + _Init2_224 = 0x3070DD17 + _Init3_224 = 0xF70E5939 + _Init4_224 = 0xFFC00B31 + _Init5_224 = 0x68581511 + _Init6_224 = 0x64F98FA7 + _Init7_224 = 0xBEFA4FA4 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint32 + x [_Chunk]byte + nx int + len uint64 + is224 bool // mark if this digest is SHA-224 +} + +func (d *digest) Reset() { + if !d.is224 { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.h[5] = _Init5 + d.h[6] = _Init6 + d.h[7] = _Init7 + } else { + d.h[0] = _Init0_224 + d.h[1] = _Init1_224 + d.h[2] = _Init2_224 + d.h[3] = _Init3_224 + d.h[4] = _Init4_224 + d.h[5] = _Init5_224 + d.h[6] = _Init6_224 + d.h[7] = _Init7_224 + } + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA256 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +// New224 returns a new hash.Hash computing the SHA224 checksum. +func New224() hash.Hash { + d := new(digest) + d.is224 = true + d.Reset() + return d +} + +func (d *digest) Size() int { + if !d.is224 { + return Size + } + return Size224 +} + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + len := d.len + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 32) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 24) + p[j+1] = byte(s >> 16) + p[j+2] = byte(s >> 8) + p[j+3] = byte(s >> 0) + j += 4 + } + if d.is224 { + return p[0:28] + } + return p +} diff --git a/src/pkg/crypto/sha256/sha256_test.go b/src/pkg/crypto/sha256/sha256_test.go new file mode 100644 index 000000000..42a3fa7a0 --- /dev/null +++ b/src/pkg/crypto/sha256/sha256_test.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA256 hash algorithm. See FIPS 180-2. + +package sha256 + +import ( + "fmt" + "io" + "testing" +) + +type sha256Test struct { + out string + in string +} + +var golden = []sha256Test{ + {"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ""}, + {"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", "a"}, + {"fb8e20fc2e4c3f248c60c39bd652f3c1347298bb977b8b4d5903b85055620603", "ab"}, + {"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc"}, + {"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", "abcd"}, + {"36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c", "abcde"}, + {"bef57ec7f53a6d40beb640a780a639c83bc29ac8a9816f1fc6c5c6dcd93c4721", "abcdef"}, + {"7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a", "abcdefg"}, + {"9c56cc51b374c3ba189210d5b6d4bf57790d351c96c47c02190ecf1e430635ab", "abcdefgh"}, + {"19cc02f26df43cc571bc9ed7b0c4d29224a3ec229529221725ef76d021c8326f", "abcdefghi"}, + {"72399361da6a7754fec986dca5b7cbaf1c810a28ded4abaf56b2106d06cb78b0", "abcdefghij"}, + {"a144061c271f152da4d151034508fed1c138b8c976339de229c3bb6d4bbb4fce", "Discard medicine more than two years old."}, + {"6dae5caa713a10ad04b46028bf6dad68837c581616a1589a265a11288d4bb5c4", "He who has a shady past knows that nice guys finish last."}, + {"ae7a702a9509039ddbf29f0765e70d0001177914b86459284dab8b348c2dce3f", "I wouldn't marry him with a ten foot pole."}, + {"6748450b01c568586715291dfa3ee018da07d36bb7ea6f180c1af6270215c64f", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"14b82014ad2b11f661b5ae6a99b75105c2ffac278cd071cd6c05832793635774", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"7102cfd76e2e324889eece5d6c41921b1e142a4ac5a2692be78803097f6a48d8", "Nepal premier won't resign."}, + {"23b1018cd81db1d67983c5f7417c44da9deb582459e378d7a068552ea649dc9f", "For every action there is an equal and opposite government program."}, + {"8001f190dfb527261c4cfcab70c98e8097a7a1922129bc4096950e57c7999a5a", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"8c87deb65505c3993eb24b7a150c4155e82eee6960cf0c3a8114ff736d69cad5", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"bfb0a67a19cdec3646498b2e0f751bddc41bba4b7f30081b0b932aad214d16d7", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"7f9a0b9bf56332e19f5a0ec1ad9c1425a153da1c624868fda44561d6b74daf36", "size: a.out: bad magic"}, + {"b13f81b8aad9e3666879af19886140904f7f429ef083286195982a7588858cfc", "The major problem is with sendmail. -Mark Horton"}, + {"b26c38d61519e894480c70c8374ea35aa0ad05b2ae3d6674eec5f52a69305ed4", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"049d5e26d4f10222cd841a119e38bd8d2e0d1129728688449575d4ff42b842c1", "If the enemy is within range, then so are you."}, + {"0e116838e3cc1c1a14cd045397e29b4d087aa11b0853fc69ec82e90330d60949", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"4f7d8eb5bcf11de2a56b971021a444aa4eafd6ecd0f307b5109e4e776cd0fe46", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"61c0cc4c4bd8406d5120b3fb4ebc31ce87667c162f29468b3c779675a85aebce", "C is as portable as Stonehedge!!"}, + {"1fb2eb3688093c4a3f80cd87a5547e2ce940a4f923243a79a2a1e242220693ac", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"395585ce30617b62c80b93e8208ce866d4edc811a177fdb4b82d3911d8696423", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"4f9b189a13d030838269dce846b16a1ce9ce81fe63e65de2f636863336a98fe6", "How can you write a big system without C++? -Paul Glick"}, +} + +var golden224 = []sha256Test{ + {"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f", ""}, + {"abd37534c7d9a2efb9465de931cd7055ffdb8879563ae98078d6d6d5", "a"}, + {"db3cda86d4429a1d39c148989566b38f7bda0156296bd364ba2f878b", "ab"}, + {"23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", "abc"}, + {"a76654d8e3550e9a2d67a0eeb6c67b220e5885eddd3fde135806e601", "abcd"}, + {"bdd03d560993e675516ba5a50638b6531ac2ac3d5847c61916cfced6", "abcde"}, + {"7043631cb415556a275a4ebecb802c74ee9f6153908e1792a90b6a98", "abcdef"}, + {"d1884e711701ad81abe0c77a3b0ea12e19ba9af64077286c72fc602d", "abcdefg"}, + {"17eb7d40f0356f8598e89eafad5f6c759b1f822975d9c9b737c8a517", "abcdefgh"}, + {"aeb35915346c584db820d2de7af3929ffafef9222a9bcb26516c7334", "abcdefghi"}, + {"d35e1e5af29ddb0d7e154357df4ad9842afee527c689ee547f753188", "abcdefghij"}, + {"19297f1cef7ddc8a7e947f5c5a341e10f7245045e425db67043988d7", "Discard medicine more than two years old."}, + {"0f10c2eb436251f777fbbd125e260d36aecf180411726c7c885f599a", "He who has a shady past knows that nice guys finish last."}, + {"4d1842104919f314cad8a3cd20b3cba7e8ed3e7abed62b57441358f6", "I wouldn't marry him with a ten foot pole."}, + {"a8ba85c6fe0c48fbffc72bbb2f03fcdbc87ae2dc7a56804d1590fb3b", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"5543fbab26e67e8885b1a852d567d1cb8b9bfe42e0899584c50449a9", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"65ca107390f5da9efa05d28e57b221657edc7e43a9a18fb15b053ddb", "Nepal premier won't resign."}, + {"84953962be366305a9cc9b5cd16ed019edc37ac96c0deb3e12cca116", "For every action there is an equal and opposite government program."}, + {"35a189ce987151dfd00b3577583cc6a74b9869eecf894459cb52038d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"2fc333713983edfd4ef2c0da6fb6d6415afb94987c91e4069eb063e6", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"cbe32d38d577a1b355960a4bc3c659c2dc4670859a19777a875842c4", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"a2dc118ce959e027576413a7b440c875cdc8d40df9141d6ef78a57e1", "size: a.out: bad magic"}, + {"d10787e24052bcff26dc484787a54ed819e4e4511c54890ee977bf81", "The major problem is with sendmail. -Mark Horton"}, + {"62efcf16ab8a893acdf2f348aaf06b63039ff1bf55508c830532c9fb", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"3e9b7e4613c59f58665104c5fa86c272db5d3a2ff30df5bb194a5c99", "If the enemy is within range, then so are you."}, + {"5999c208b8bdf6d471bb7c359ac5b829e73a8211dff686143a4e7f18", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"3b2d67ff54eabc4ef737b14edf87c64280ef582bcdf2a6d56908b405", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"d0733595d20e4d3d6b5c565a445814d1bbb2fd08b9a3b8ffb97930c6", "C is as portable as Stonehedge!!"}, + {"43fb8aeed8a833175c9295c1165415f98c866ef08a4922959d673507", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"ec18e66e93afc4fb1604bc2baedbfd20b44c43d76e65c0996d7851c6", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"86ed2eaa9c75ba98396e5c9fb2f679ecf0ea2ed1e0ee9ceecb4a9332", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha256[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } + for i := 0; i < len(golden224); i++ { + g := golden224[i] + c := New224() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha224[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/sha256/sha256block.go b/src/pkg/crypto/sha256/sha256block.go new file mode 100644 index 000000000..7b0f55444 --- /dev/null +++ b/src/pkg/crypto/sha256/sha256block.go @@ -0,0 +1,129 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA256 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha256 + +var _K = []uint32{ + 0x428a2f98, + 0x71374491, + 0xb5c0fbcf, + 0xe9b5dba5, + 0x3956c25b, + 0x59f111f1, + 0x923f82a4, + 0xab1c5ed5, + 0xd807aa98, + 0x12835b01, + 0x243185be, + 0x550c7dc3, + 0x72be5d74, + 0x80deb1fe, + 0x9bdc06a7, + 0xc19bf174, + 0xe49b69c1, + 0xefbe4786, + 0x0fc19dc6, + 0x240ca1cc, + 0x2de92c6f, + 0x4a7484aa, + 0x5cb0a9dc, + 0x76f988da, + 0x983e5152, + 0xa831c66d, + 0xb00327c8, + 0xbf597fc7, + 0xc6e00bf3, + 0xd5a79147, + 0x06ca6351, + 0x14292967, + 0x27b70a85, + 0x2e1b2138, + 0x4d2c6dfc, + 0x53380d13, + 0x650a7354, + 0x766a0abb, + 0x81c2c92e, + 0x92722c85, + 0xa2bfe8a1, + 0xa81a664b, + 0xc24b8b70, + 0xc76c51a3, + 0xd192e819, + 0xd6990624, + 0xf40e3585, + 0x106aa070, + 0x19a4c116, + 0x1e376c08, + 0x2748774c, + 0x34b0bcb5, + 0x391c0cb3, + 0x4ed8aa4a, + 0x5b9cca4f, + 0x682e6ff3, + 0x748f82ee, + 0x78a5636f, + 0x84c87814, + 0x8cc70208, + 0x90befffa, + 0xa4506ceb, + 0xbef9a3f7, + 0xc67178f2, +} + +func _Block(dig *digest, p []byte) int { + var w [64]uint32 + n := 0 + h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] + for len(p) >= _Chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + for i := 16; i < 64; i++ { + t1 := (w[i-2]>>17 | w[i-2]<<(32-17)) ^ (w[i-2]>>19 | w[i-2]<<(32-19)) ^ (w[i-2] >> 10) + + t2 := (w[i-15]>>7 | w[i-15]<<(32-7)) ^ (w[i-15]>>18 | w[i-15]<<(32-18)) ^ (w[i-15] >> 3) + + w[i] = t1 + w[i-7] + t2 + w[i-16] + } + + a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7 + + for i := 0; i < 64; i++ { + t1 := h + ((e>>6 | e<<(32-6)) ^ (e>>11 | e<<(32-11)) ^ (e>>25 | e<<(32-25))) + ((e & f) ^ (^e & g)) + _K[i] + w[i] + + t2 := ((a>>2 | a<<(32-2)) ^ (a>>13 | a<<(32-13)) ^ (a>>22 | a<<(32-22))) + ((a & b) ^ (a & c) ^ (b & c)) + + h = g + g = f + f = e + e = d + t1 + d = c + c = b + b = a + a = t1 + t2 + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + h5 += f + h6 += g + h7 += h + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7 + return n +} diff --git a/src/pkg/crypto/sha512/Makefile b/src/pkg/crypto/sha512/Makefile new file mode 100644 index 000000000..2f7633fa3 --- /dev/null +++ b/src/pkg/crypto/sha512/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/sha512 +GOFILES=\ + sha512.go\ + sha512block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/sha512/sha512.go b/src/pkg/crypto/sha512/sha512.go new file mode 100644 index 000000000..78f5fe26f --- /dev/null +++ b/src/pkg/crypto/sha512/sha512.go @@ -0,0 +1,170 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha512 implements the SHA384 and SHA512 hash algorithms as defined +// in FIPS 180-2. +package sha512 + +import ( + "crypto" + "hash" + "os" +) + +func init() { + crypto.RegisterHash(crypto.SHA384, New384) + crypto.RegisterHash(crypto.SHA512, New) +} + +// The size of a SHA512 checksum in bytes. +const Size = 64 + +// The size of a SHA384 checksum in bytes. +const Size384 = 48 + +const ( + _Chunk = 128 + _Init0 = 0x6a09e667f3bcc908 + _Init1 = 0xbb67ae8584caa73b + _Init2 = 0x3c6ef372fe94f82b + _Init3 = 0xa54ff53a5f1d36f1 + _Init4 = 0x510e527fade682d1 + _Init5 = 0x9b05688c2b3e6c1f + _Init6 = 0x1f83d9abfb41bd6b + _Init7 = 0x5be0cd19137e2179 + _Init0_384 = 0xcbbb9d5dc1059ed8 + _Init1_384 = 0x629a292a367cd507 + _Init2_384 = 0x9159015a3070dd17 + _Init3_384 = 0x152fecd8f70e5939 + _Init4_384 = 0x67332667ffc00b31 + _Init5_384 = 0x8eb44a8768581511 + _Init6_384 = 0xdb0c2e0d64f98fa7 + _Init7_384 = 0x47b5481dbefa4fa4 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint64 + x [_Chunk]byte + nx int + len uint64 + is384 bool // mark if this digest is SHA-384 +} + +func (d *digest) Reset() { + if !d.is384 { + d.h[0] = _Init0 + d.h[1] = _Init1 + d.h[2] = _Init2 + d.h[3] = _Init3 + d.h[4] = _Init4 + d.h[5] = _Init5 + d.h[6] = _Init6 + d.h[7] = _Init7 + } else { + d.h[0] = _Init0_384 + d.h[1] = _Init1_384 + d.h[2] = _Init2_384 + d.h[3] = _Init3_384 + d.h[4] = _Init4_384 + d.h[5] = _Init5_384 + d.h[6] = _Init6_384 + d.h[7] = _Init7_384 + } + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA512 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +// New384 returns a new hash.Hash computing the SHA384 checksum. +func New384() hash.Hash { + d := new(digest) + d.is384 = true + d.Reset() + return d +} + +func (d *digest) Size() int { + if !d.is384 { + return Size + } + return Size384 +} + +func (d *digest) Write(p []byte) (nn int, err os.Error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > _Chunk-d.nx { + n = _Chunk - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == _Chunk { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum() []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + + // Padding. Add a 1 bit and 0 bits until 112 bytes mod 128. + len := d.len + var tmp [128]byte + tmp[0] = 0x80 + if len%128 < 112 { + d.Write(tmp[0 : 112-len%128]) + } else { + d.Write(tmp[0 : 128+112-len%128]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 16; i++ { + tmp[i] = byte(len >> (120 - 8*i)) + } + d.Write(tmp[0:16]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + p := make([]byte, 64) + j := 0 + for _, s := range d.h { + p[j+0] = byte(s >> 56) + p[j+1] = byte(s >> 48) + p[j+2] = byte(s >> 40) + p[j+3] = byte(s >> 32) + p[j+4] = byte(s >> 24) + p[j+5] = byte(s >> 16) + p[j+6] = byte(s >> 8) + p[j+7] = byte(s >> 0) + j += 8 + } + if d.is384 { + return p[0:48] + } + return p +} diff --git a/src/pkg/crypto/sha512/sha512_test.go b/src/pkg/crypto/sha512/sha512_test.go new file mode 100644 index 000000000..dd116dc17 --- /dev/null +++ b/src/pkg/crypto/sha512/sha512_test.go @@ -0,0 +1,125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA512 hash algorithm. See FIPS 180-2. + +package sha512 + +import ( + "fmt" + "io" + "testing" +) + +type sha512Test struct { + out string + in string +} + +var golden = []sha512Test{ + {"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""}, + {"1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75", "a"}, + {"2d408a0717ec188158278a796c689044361dc6fdde28d6f04973b80896e1823975cdbf12eb63f9e0591328ee235d80e9b5bf1aa6a44f4617ff3caf6400eb172d", "ab"}, + {"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"}, + {"d8022f2060ad6efd297ab73dcc5355c9b214054b0d1776a136a669d26a7d3b14f73aa0d0ebff19ee333368f0164b6419a96da49e3e481753e7e96b716bdccb6f", "abcd"}, + {"878ae65a92e86cac011a570d4c30a7eaec442b85ce8eca0c2952b5e3cc0628c2e79d889ad4d5c7c626986d452dd86374b6ffaa7cd8b67665bef2289a5c70b0a1", "abcde"}, + {"e32ef19623e8ed9d267f657a81944b3d07adbb768518068e88435745564e8d4150a0a703be2a7d88b61e3d390c2bb97e2d4c311fdc69d6b1267f05f59aa920e7", "abcdef"}, + {"d716a4188569b68ab1b6dfac178e570114cdf0ea3a1cc0e31486c3e41241bc6a76424e8c37ab26f096fc85ef9886c8cb634187f4fddff645fb099f1ff54c6b8c", "abcdefg"}, + {"a3a8c81bc97c2560010d7389bc88aac974a104e0e2381220c6e084c4dccd1d2d17d4f86db31c2a851dc80e6681d74733c55dcd03dd96f6062cdda12a291ae6ce", "abcdefgh"}, + {"f22d51d25292ca1d0f68f69aedc7897019308cc9db46efb75a03dd494fc7f126c010e8ade6a00a0c1a5f1b75d81e0ed5a93ce98dc9b833db7839247b1d9c24fe", "abcdefghi"}, + {"ef6b97321f34b1fea2169a7db9e1960b471aa13302a988087357c520be957ca119c3ba68e6b4982c019ec89de3865ccf6a3cda1fe11e59f98d99f1502c8b9745", "abcdefghij"}, + {"2210d99af9c8bdecda1b4beff822136753d8342505ddce37f1314e2cdbb488c6016bdaa9bd2ffa513dd5de2e4b50f031393d8ab61f773b0e0130d7381e0f8a1d", "Discard medicine more than two years old."}, + {"a687a8985b4d8d0a24f115fe272255c6afaf3909225838546159c1ed685c211a203796ae8ecc4c81a5b6315919b3a64f10713da07e341fcdbb08541bf03066ce", "He who has a shady past knows that nice guys finish last."}, + {"8ddb0392e818b7d585ab22769a50df660d9f6d559cca3afc5691b8ca91b8451374e42bcdabd64589ed7c91d85f626596228a5c8572677eb98bc6b624befb7af8", "I wouldn't marry him with a ten foot pole."}, + {"26ed8f6ca7f8d44b6a8a54ae39640fa8ad5c673f70ee9ce074ba4ef0d483eea00bab2f61d8695d6b34df9c6c48ae36246362200ed820448bdc03a720366a87c6", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"e5a14bf044be69615aade89afcf1ab0389d5fc302a884d403579d1386a2400c089b0dbb387ed0f463f9ee342f8244d5a38cfbc0e819da9529fbff78368c9a982", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"420a1faa48919e14651bed45725abe0f7a58e0f099424c4e5a49194946e38b46c1f8034b18ef169b2e31050d1648e0b982386595f7df47da4b6fd18e55333015", "Nepal premier won't resign."}, + {"d926a863beadb20134db07683535c72007b0e695045876254f341ddcccde132a908c5af57baa6a6a9c63e6649bba0c213dc05fadcf9abccea09f23dcfb637fbe", "For every action there is an equal and opposite government program."}, + {"9a98dd9bb67d0da7bf83da5313dff4fd60a4bac0094f1b05633690ffa7f6d61de9a1d4f8617937d560833a9aaa9ccafe3fd24db418d0e728833545cadd3ad92d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"d7fde2d2351efade52f4211d3746a0780a26eec3df9b2ed575368a8a1c09ec452402293a8ea4eceb5a4f60064ea29b13cdd86918cd7a4faf366160b009804107", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"b0f35ffa2697359c33a56f5c0cf715c7aeed96da9905ca2698acadb08fbc9e669bf566b6bd5d61a3e86dc22999bcc9f2224e33d1d4f32a228cf9d0349e2db518", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"3d2e5f91778c9e66f7e061293aaa8a8fc742dd3b2e4f483772464b1144189b49273e610e5cccd7a81a19ca1fa70f16b10f1a100a4d8c1372336be8484c64b311", "size: a.out: bad magic"}, + {"b2f68ff58ac015efb1c94c908b0d8c2bf06f491e4de8e6302c49016f7f8a33eac3e959856c7fddbc464de618701338a4b46f76dbfaf9a1e5262b5f40639771c7", "The major problem is with sendmail. -Mark Horton"}, + {"d8c92db5fdf52cf8215e4df3b4909d29203ff4d00e9ad0b64a6a4e04dec5e74f62e7c35c7fb881bd5de95442123df8f57a489b0ae616bd326f84d10021121c57", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"19a9f8dc0a233e464e8566ad3ca9b91e459a7b8c4780985b015776e1bf239a19bc233d0556343e2b0a9bc220900b4ebf4f8bdf89ff8efeaf79602d6849e6f72e", "If the enemy is within range, then so are you."}, + {"00b4c41f307bde87301cdc5b5ab1ae9a592e8ecbb2021dd7bc4b34e2ace60741cc362560bec566ba35178595a91932b8d5357e2c9cec92d393b0fa7831852476", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"91eccc3d5375fd026e4d6787874b1dce201cecd8a27dbded5065728cb2d09c58a3d467bb1faf353bf7ba567e005245d5321b55bc344f7c07b91cb6f26c959be7", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"fabbbe22180f1f137cfdc9556d2570e775d1ae02a597ded43a72a40f9b485d500043b7be128fb9fcd982b83159a0d99aa855a9e7cc4240c00dc01a9bdf8218d7", "C is as portable as Stonehedge!!"}, + {"2ecdec235c1fa4fc2a154d8fba1dddb8a72a1ad73838b51d792331d143f8b96a9f6fcb0f34d7caa351fe6d88771c4f105040e0392f06e0621689d33b2f3ba92e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"7ad681f6f96f82f7abfa7ecc0334e8fa16d3dc1cdc45b60b7af43fe4075d2357c0c1d60e98350f1afb1f2fe7a4d7cd2ad55b88e458e06b73c40b437331f5dab4", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"833f9248ab4a3b9e5131f745fda1ffd2dd435b30e965957e78291c7ab73605fd1912b0794e5c233ab0a12d205a39778d19b83515d6a47003f19cdee51d98c7e0", "How can you write a big system without C++? -Paul Glick"}, +} + +var golden384 = []sha512Test{ + {"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", ""}, + {"54a59b9f22b0b80880d8427e548b7c23abd873486e1f035dce9cd697e85175033caa88e6d57bc35efae0b5afd3145f31", "a"}, + {"c7be03ba5bcaa384727076db0018e99248e1a6e8bd1b9ef58a9ec9dd4eeebb3f48b836201221175befa74ddc3d35afdd", "ab"}, + {"cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", "abc"}, + {"1165b3406ff0b52a3d24721f785462ca2276c9f454a116c2b2ba20171a7905ea5a026682eb659c4d5f115c363aa3c79b", "abcd"}, + {"4c525cbeac729eaf4b4665815bc5db0c84fe6300068a727cf74e2813521565abc0ec57a37ee4d8be89d097c0d2ad52f0", "abcde"}, + {"c6a4c65b227e7387b9c3e839d44869c4cfca3ef583dea64117859b808c1e3d8ae689e1e314eeef52a6ffe22681aa11f5", "abcdef"}, + {"9f11fc131123f844c1226f429b6a0a6af0525d9f40f056c7fc16cdf1b06bda08e302554417a59fa7dcf6247421959d22", "abcdefg"}, + {"9000cd7cada59d1d2eb82912f7f24e5e69cc5517f68283b005fa27c285b61e05edf1ad1a8a9bded6fd29eb87d75ad806", "abcdefgh"}, + {"ef54915b60cf062b8dd0c29ae3cad69abe6310de63ac081f46ef019c5c90897caefd79b796cfa81139788a260ded52df", "abcdefghi"}, + {"a12070030a02d86b0ddacd0d3a5b598344513d0a051e7355053e556a0055489c1555399b03342845c4adde2dc44ff66c", "abcdefghij"}, + {"86f58ec2d74d1b7f8eb0c2ff0967316699639e8d4eb129de54bdf34c96cdbabe200d052149f2dd787f43571ba74670d4", "Discard medicine more than two years old."}, + {"ae4a2b639ca9bfa04b1855d5a05fe7f230994f790891c6979103e2605f660c4c1262a48142dcbeb57a1914ba5f7c3fa7", "He who has a shady past knows that nice guys finish last."}, + {"40ae213df6436eca952aa6841886fcdb82908ef1576a99c8f49bb9dd5023169f7c53035abdda0b54c302f4974e2105e7", "I wouldn't marry him with a ten foot pole."}, + {"e7cf8b873c9bc950f06259aa54309f349cefa72c00d597aebf903e6519a50011dfe355afff064a10701c705693848df9", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"c3d4f0f4047181c7d39d34703365f7bf70207183caf2c2f6145f04da895ef69124d9cdeb635da636c3a474e61024e29b", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"a097aab567e167d5cf93676ed73252a69f9687cb3179bb2d27c9878119e94bf7b7c4b58dc90582edfaf66e11388ed714", "Nepal premier won't resign."}, + {"5026ca45c41fc64712eb65065da92f6467541c78f8966d3fe2c8e3fb769a3ec14215f819654b47bd64f7f0eac17184f3", "For every action there is an equal and opposite government program."}, + {"ac1cc0f5ac8d5f5514a7b738ac322b7fb52a161b449c3672e9b6a6ad1a5e4b26b001cf3bad24c56598676ca17d4b445a", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"722d10c5de371ec0c8c4b5247ac8a5f1d240d68c73f8da13d8b25f0166d6f309bf9561979a111a0049405771d201941a", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"dc2d3ea18bfa10549c63bf2b75b39b5167a80c12aff0e05443168ea87ff149fb0eda5e0bd234eb5d48c7d02ffc5807f1", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"1d67c969e2a945ae5346d2139760261504d4ba164c522443afe19ef3e29b152a4c52445489cfc9d7215e5a450e8e1e4e", "size: a.out: bad magic"}, + {"5ff8e075e465646e7b73ef36d812c6e9f7d60fa6ea0e533e5569b4f73cde53cdd2cc787f33540af57cca3fe467d32fe0", "The major problem is with sendmail. -Mark Horton"}, + {"5bd0a997a67c9ae1979a894eb0cde403dde003c9b6f2c03cf21925c42ff4e1176e6df1ca005381612ef18457b9b7ec3b", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"1eee6da33e7e54fc5be52ae23b94b16ba4d2a947ae4505c6a3edfc7401151ea5205ac01b669b56f27d8ef7f175ed7762", "If the enemy is within range, then so are you."}, + {"76b06e9dea66bfbb1a96029426dc0dfd7830bd297eb447ff5358d94a87cd00c88b59df2493fef56ecbb5231073892ea9", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"12acaf21452cff586143e3f5db0bfdf7802c057e1adf2a619031c4e1b0ccc4208cf6cef8fe722bbaa2fb46a30d9135d8", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"0fc23d7f4183efd186f0bc4fc5db867e026e2146b06cb3d52f4bdbd57d1740122caa853b41868b197b2ac759db39df88", "C is as portable as Stonehedge!!"}, + {"bc805578a7f85d34a86a32976e1c34fe65cf815186fbef76f46ef99cda10723f971f3f1464d488243f5e29db7488598d", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"b23918399a12ebf4431559eec3813eaf7412e875fd7464f16d581e473330842d2e96c6be49a7ce3f9bb0b8bc0fcbe0fe", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"1764b700eb1ead52a2fc33cc28975c2180f1b8faa5038d94cffa8d78154aab16e91dd787e7b0303948ebed62561542c8", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha512[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } + for i := 0; i < len(golden384); i++ { + g := golden384[i] + c := New384() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum() + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum()) + if s != g.out { + t.Fatalf("sha384[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} diff --git a/src/pkg/crypto/sha512/sha512block.go b/src/pkg/crypto/sha512/sha512block.go new file mode 100644 index 000000000..6b7506287 --- /dev/null +++ b/src/pkg/crypto/sha512/sha512block.go @@ -0,0 +1,144 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// SHA512 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package sha512 + +var _K = []uint64{ + 0x428a2f98d728ae22, + 0x7137449123ef65cd, + 0xb5c0fbcfec4d3b2f, + 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, + 0x59f111f1b605d019, + 0x923f82a4af194f9b, + 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, + 0x12835b0145706fbe, + 0x243185be4ee4b28c, + 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, + 0x80deb1fe3b1696b1, + 0x9bdc06a725c71235, + 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, + 0xefbe4786384f25e3, + 0x0fc19dc68b8cd5b5, + 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, + 0x4a7484aa6ea6e483, + 0x5cb0a9dcbd41fbd4, + 0x76f988da831153b5, + 0x983e5152ee66dfab, + 0xa831c66d2db43210, + 0xb00327c898fb213f, + 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, + 0xd5a79147930aa725, + 0x06ca6351e003826f, + 0x142929670a0e6e70, + 0x27b70a8546d22ffc, + 0x2e1b21385c26c926, + 0x4d2c6dfc5ac42aed, + 0x53380d139d95b3df, + 0x650a73548baf63de, + 0x766a0abb3c77b2a8, + 0x81c2c92e47edaee6, + 0x92722c851482353b, + 0xa2bfe8a14cf10364, + 0xa81a664bbc423001, + 0xc24b8b70d0f89791, + 0xc76c51a30654be30, + 0xd192e819d6ef5218, + 0xd69906245565a910, + 0xf40e35855771202a, + 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, + 0x1e376c085141ab53, + 0x2748774cdf8eeb99, + 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, + 0x4ed8aa4ae3418acb, + 0x5b9cca4f7763e373, + 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, + 0x78a5636f43172f60, + 0x84c87814a1f0ab72, + 0x8cc702081a6439ec, + 0x90befffa23631e28, + 0xa4506cebde82bde9, + 0xbef9a3f7b2c67915, + 0xc67178f2e372532b, + 0xca273eceea26619c, + 0xd186b8c721c0c207, + 0xeada7dd6cde0eb1e, + 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, + 0x0a637dc5a2c898a6, + 0x113f9804bef90dae, + 0x1b710b35131c471b, + 0x28db77f523047d84, + 0x32caab7b40c72493, + 0x3c9ebe0a15c9bebc, + 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, + 0x597f299cfc657e2a, + 0x5fcb6fab3ad6faec, + 0x6c44198c4a475817, +} + +func _Block(dig *digest, p []byte) int { + var w [80]uint64 + n := 0 + h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] + for len(p) >= _Chunk { + for i := 0; i < 16; i++ { + j := i * 8 + w[i] = uint64(p[j])<<56 | uint64(p[j+1])<<48 | uint64(p[j+2])<<40 | uint64(p[j+3])<<32 | + uint64(p[j+4])<<24 | uint64(p[j+5])<<16 | uint64(p[j+6])<<8 | uint64(p[j+7]) + } + for i := 16; i < 80; i++ { + t1 := (w[i-2]>>19 | w[i-2]<<(64-19)) ^ (w[i-2]>>61 | w[i-2]<<(64-61)) ^ (w[i-2] >> 6) + + t2 := (w[i-15]>>1 | w[i-15]<<(64-1)) ^ (w[i-15]>>8 | w[i-15]<<(64-8)) ^ (w[i-15] >> 7) + + w[i] = t1 + w[i-7] + t2 + w[i-16] + } + + a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7 + + for i := 0; i < 80; i++ { + t1 := h + ((e>>14 | e<<(64-14)) ^ (e>>18 | e<<(64-18)) ^ (e>>41 | e<<(64-41))) + ((e & f) ^ (^e & g)) + _K[i] + w[i] + + t2 := ((a>>28 | a<<(64-28)) ^ (a>>34 | a<<(64-34)) ^ (a>>39 | a<<(64-39))) + ((a & b) ^ (a & c) ^ (b & c)) + + h = g + g = f + f = e + e = d + t1 + d = c + c = b + b = a + a = t1 + t2 + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + h5 += f + h6 += g + h7 += h + + p = p[_Chunk:] + n += _Chunk + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7 + return n +} diff --git a/src/pkg/crypto/subtle/Makefile b/src/pkg/crypto/subtle/Makefile new file mode 100644 index 000000000..08d8bbfa0 --- /dev/null +++ b/src/pkg/crypto/subtle/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/subtle +GOFILES=\ + constant_time.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/subtle/constant_time.go b/src/pkg/crypto/subtle/constant_time.go new file mode 100644 index 000000000..57dbe9db5 --- /dev/null +++ b/src/pkg/crypto/subtle/constant_time.go @@ -0,0 +1,57 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package subtle implements functions that are often useful in cryptographic +// code but require careful thought to use correctly. +package subtle + +// ConstantTimeCompare returns 1 iff the two equal length slices, x +// and y, have equal contents. The time taken is a function of the length of +// the slices and is independent of the contents. +func ConstantTimeCompare(x, y []byte) int { + var v byte + + for i := 0; i < len(x); i++ { + v |= x[i] ^ y[i] + } + + return ConstantTimeByteEq(v, 0) +} + +// ConstantTimeSelect returns x if v is 1 and y if v is 0. +// Its behavior is undefined if v takes any other value. +func ConstantTimeSelect(v, x, y int) int { return ^(v-1)&x | (v-1)&y } + +// ConstantTimeByteEq returns 1 if x == y and 0 otherwise. +func ConstantTimeByteEq(x, y uint8) int { + z := ^(x ^ y) + z &= z >> 4 + z &= z >> 2 + z &= z >> 1 + + return int(z) +} + +// ConstantTimeEq returns 1 if x == y and 0 otherwise. +func ConstantTimeEq(x, y int32) int { + z := ^(x ^ y) + z &= z >> 16 + z &= z >> 8 + z &= z >> 4 + z &= z >> 2 + z &= z >> 1 + + return int(z & 1) +} + +// ConstantTimeCopy copies the contents of y into x iff v == 1. If v == 0, x is left unchanged. +// Its behavior is undefined if v takes any other value. +func ConstantTimeCopy(v int, x, y []byte) { + xmask := byte(v - 1) + ymask := byte(^(v - 1)) + for i := 0; i < len(x); i++ { + x[i] = x[i]&xmask | y[i]&ymask + } + return +} diff --git a/src/pkg/crypto/subtle/constant_time_test.go b/src/pkg/crypto/subtle/constant_time_test.go new file mode 100644 index 000000000..adab8e2e8 --- /dev/null +++ b/src/pkg/crypto/subtle/constant_time_test.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package subtle + +import ( + "testing" + "testing/quick" +) + +type TestConstantTimeCompareStruct struct { + a, b []byte + out int +} + +var testConstantTimeCompareData = []TestConstantTimeCompareStruct{ + {[]byte{}, []byte{}, 1}, + {[]byte{0x11}, []byte{0x11}, 1}, + {[]byte{0x12}, []byte{0x11}, 0}, +} + +func TestConstantTimeCompare(t *testing.T) { + for i, test := range testConstantTimeCompareData { + if r := ConstantTimeCompare(test.a, test.b); r != test.out { + t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) + } + } +} + +type TestConstantTimeByteEqStruct struct { + a, b uint8 + out int +} + +var testConstandTimeByteEqData = []TestConstantTimeByteEqStruct{ + {0, 0, 1}, + {0, 1, 0}, + {1, 0, 0}, + {0xff, 0xff, 1}, + {0xff, 0xfe, 0}, +} + +func byteEq(a, b uint8) int { + if a == b { + return 1 + } + return 0 +} + +func TestConstantTimeByteEq(t *testing.T) { + for i, test := range testConstandTimeByteEqData { + if r := ConstantTimeByteEq(test.a, test.b); r != test.out { + t.Errorf("#%d bad result (got %x, want %x)", i, r, test.out) + } + } + err := quick.CheckEqual(ConstantTimeByteEq, byteEq, nil) + if err != nil { + t.Error(err) + } +} + +func eq(a, b int32) int { + if a == b { + return 1 + } + return 0 +} + +func TestConstantTimeEq(t *testing.T) { + err := quick.CheckEqual(ConstantTimeEq, eq, nil) + if err != nil { + t.Error(err) + } +} + +func makeCopy(v int, x, y []byte) []byte { + if len(x) > len(y) { + x = x[0:len(y)] + } else { + y = y[0:len(x)] + } + if v == 1 { + copy(x, y) + } + return x +} + +func constantTimeCopyWrapper(v int, x, y []byte) []byte { + if len(x) > len(y) { + x = x[0:len(y)] + } else { + y = y[0:len(x)] + } + v &= 1 + ConstantTimeCopy(v, x, y) + return x +} + +func TestConstantTimeCopy(t *testing.T) { + err := quick.CheckEqual(constantTimeCopyWrapper, makeCopy, nil) + if err != nil { + t.Error(err) + } +} diff --git a/src/pkg/crypto/tls/Makefile b/src/pkg/crypto/tls/Makefile new file mode 100644 index 000000000..000314be5 --- /dev/null +++ b/src/pkg/crypto/tls/Makefile @@ -0,0 +1,20 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/tls +GOFILES=\ + alert.go\ + cipher_suites.go\ + common.go\ + conn.go\ + handshake_client.go\ + handshake_messages.go\ + handshake_server.go\ + key_agreement.go\ + prf.go\ + tls.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/tls/alert.go b/src/pkg/crypto/tls/alert.go new file mode 100644 index 000000000..3b9e0e241 --- /dev/null +++ b/src/pkg/crypto/tls/alert.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import "strconv" + +type alert uint8 + +const ( + // alert level + alertLevelWarning = 1 + alertLevelError = 2 +) + +const ( + alertCloseNotify alert = 0 + alertUnexpectedMessage alert = 10 + alertBadRecordMAC alert = 20 + alertDecryptionFailed alert = 21 + alertRecordOverflow alert = 22 + alertDecompressionFailure alert = 30 + alertHandshakeFailure alert = 40 + alertBadCertificate alert = 42 + alertUnsupportedCertificate alert = 43 + alertCertificateRevoked alert = 44 + alertCertificateExpired alert = 45 + alertCertificateUnknown alert = 46 + alertIllegalParameter alert = 47 + alertUnknownCA alert = 48 + alertAccessDenied alert = 49 + alertDecodeError alert = 50 + alertDecryptError alert = 51 + alertProtocolVersion alert = 70 + alertInsufficientSecurity alert = 71 + alertInternalError alert = 80 + alertUserCanceled alert = 90 + alertNoRenegotiation alert = 100 +) + +var alertText = map[alert]string{ + alertCloseNotify: "close notify", + alertUnexpectedMessage: "unexpected message", + alertBadRecordMAC: "bad record MAC", + alertDecryptionFailed: "decryption failed", + alertRecordOverflow: "record overflow", + alertDecompressionFailure: "decompression failure", + alertHandshakeFailure: "handshake failure", + alertBadCertificate: "bad certificate", + alertUnsupportedCertificate: "unsupported certificate", + alertCertificateRevoked: "revoked certificate", + alertCertificateExpired: "expired certificate", + alertCertificateUnknown: "unknown certificate", + alertIllegalParameter: "illegal parameter", + alertUnknownCA: "unknown certificate authority", + alertAccessDenied: "access denied", + alertDecodeError: "error decoding message", + alertDecryptError: "error decrypting message", + alertProtocolVersion: "protocol version not supported", + alertInsufficientSecurity: "insufficient security level", + alertInternalError: "internal error", + alertUserCanceled: "user canceled", + alertNoRenegotiation: "no renegotiation", +} + +func (e alert) String() string { + s, ok := alertText[e] + if ok { + return s + } + return "alert(" + strconv.Itoa(int(e)) + ")" +} diff --git a/src/pkg/crypto/tls/cipher_suites.go b/src/pkg/crypto/tls/cipher_suites.go new file mode 100644 index 000000000..bc7b0d32f --- /dev/null +++ b/src/pkg/crypto/tls/cipher_suites.go @@ -0,0 +1,102 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rc4" + "crypto/x509" + "hash" + "os" +) + +// a keyAgreement implements the client and server side of a TLS key agreement +// protocol by generating and processing key exchange messages. +type keyAgreement interface { + // On the server side, the first two methods are called in order. + + // In the case that the key agreement protocol doesn't use a + // ServerKeyExchange message, generateServerKeyExchange can return nil, + // nil. + generateServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) + processClientKeyExchange(*Config, *clientKeyExchangeMsg) ([]byte, os.Error) + + // On the client side, the next two methods are called in order. + + // This method may not be called if the server doesn't send a + // ServerKeyExchange message. + processServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg, *x509.Certificate, *serverKeyExchangeMsg) os.Error + generateClientKeyExchange(*Config, *clientHelloMsg, *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) +} + +// A cipherSuite is a specific combination of key agreement, cipher and MAC +// function. All cipher suites currently assume RSA key agreement. +type cipherSuite struct { + // the lengths, in bytes, of the key material needed for each component. + keyLen int + macLen int + ivLen int + ka func() keyAgreement + // If elliptic is set, a server will only consider this ciphersuite if + // the ClientHello indicated that the client supports an elliptic curve + // and point format that we can handle. + elliptic bool + cipher func(key, iv []byte, isRead bool) interface{} + mac func(macKey []byte) hash.Hash +} + +var cipherSuites = map[uint16]*cipherSuite{ + TLS_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, rsaKA, false, cipherRC4, hmacSHA1}, + TLS_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, rsaKA, false, cipherAES, hmacSHA1}, + TLS_ECDHE_RSA_WITH_RC4_128_SHA: &cipherSuite{16, 20, 0, ecdheRSAKA, true, cipherRC4, hmacSHA1}, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: &cipherSuite{16, 20, 16, ecdheRSAKA, true, cipherAES, hmacSHA1}, +} + +func cipherRC4(key, iv []byte, isRead bool) interface{} { + cipher, _ := rc4.NewCipher(key) + return cipher +} + +func cipherAES(key, iv []byte, isRead bool) interface{} { + block, _ := aes.NewCipher(key) + if isRead { + return cipher.NewCBCDecrypter(block, iv) + } + return cipher.NewCBCEncrypter(block, iv) +} + +func hmacSHA1(key []byte) hash.Hash { + return hmac.NewSHA1(key) +} + +func rsaKA() keyAgreement { + return rsaKeyAgreement{} +} + +func ecdheRSAKA() keyAgreement { + return new(ecdheRSAKeyAgreement) +} + +// mutualCipherSuite returns a cipherSuite and its id given a list of supported +// ciphersuites and the id requested by the peer. +func mutualCipherSuite(have []uint16, want uint16) (suite *cipherSuite, id uint16) { + for _, id := range have { + if id == want { + return cipherSuites[id], id + } + } + return +} + +// A list of the possible cipher suite ids. Taken from +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml +const ( + TLS_RSA_WITH_RC4_128_SHA uint16 = 0x0005 + TLS_RSA_WITH_AES_128_CBC_SHA uint16 = 0x002f + TLS_ECDHE_RSA_WITH_RC4_128_SHA uint16 = 0xc011 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA uint16 = 0xc013 +) diff --git a/src/pkg/crypto/tls/common.go b/src/pkg/crypto/tls/common.go new file mode 100644 index 000000000..3efac9c13 --- /dev/null +++ b/src/pkg/crypto/tls/common.go @@ -0,0 +1,267 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "io" + "io/ioutil" + "sync" + "time" +) + +const ( + maxPlaintext = 16384 // maximum plaintext payload length + maxCiphertext = 16384 + 2048 // maximum ciphertext payload length + recordHeaderLen = 5 // record header length + maxHandshake = 65536 // maximum handshake we support (protocol max is 16 MB) + + minVersion = 0x0301 // minimum supported version - TLS 1.0 + maxVersion = 0x0301 // maximum supported version - TLS 1.0 +) + +// TLS record types. +type recordType uint8 + +const ( + recordTypeChangeCipherSpec recordType = 20 + recordTypeAlert recordType = 21 + recordTypeHandshake recordType = 22 + recordTypeApplicationData recordType = 23 +) + +// TLS handshake message types. +const ( + typeClientHello uint8 = 1 + typeServerHello uint8 = 2 + typeCertificate uint8 = 11 + typeServerKeyExchange uint8 = 12 + typeCertificateRequest uint8 = 13 + typeServerHelloDone uint8 = 14 + typeCertificateVerify uint8 = 15 + typeClientKeyExchange uint8 = 16 + typeFinished uint8 = 20 + typeCertificateStatus uint8 = 22 + typeNextProtocol uint8 = 67 // Not IANA assigned +) + +// TLS compression types. +const ( + compressionNone uint8 = 0 +) + +// TLS extension numbers +var ( + extensionServerName uint16 = 0 + extensionStatusRequest uint16 = 5 + extensionSupportedCurves uint16 = 10 + extensionSupportedPoints uint16 = 11 + extensionNextProtoNeg uint16 = 13172 // not IANA assigned +) + +// TLS Elliptic Curves +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-8 +var ( + curveP256 uint16 = 23 + curveP384 uint16 = 24 + curveP521 uint16 = 25 +) + +// TLS Elliptic Curve Point Formats +// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-9 +var ( + pointFormatUncompressed uint8 = 0 +) + +// TLS CertificateStatusType (RFC 3546) +const ( + statusTypeOCSP uint8 = 1 +) + +// Certificate types (for certificateRequestMsg) +const ( + certTypeRSASign = 1 // A certificate containing an RSA key + certTypeDSSSign = 2 // A certificate containing a DSA key + certTypeRSAFixedDH = 3 // A certificate containing a static DH key + certTypeDSSFixedDH = 4 // A certificate containing a static DH key + // Rest of these are reserved by the TLS spec +) + +// ConnectionState records basic TLS details about the connection. +type ConnectionState struct { + HandshakeComplete bool + CipherSuite uint16 + NegotiatedProtocol string + NegotiatedProtocolIsMutual bool + + // the certificate chain that was presented by the other side + PeerCertificates []*x509.Certificate + // the verified certificate chains built from PeerCertificates. + VerifiedChains [][]*x509.Certificate +} + +// A Config structure is used to configure a TLS client or server. After one +// has been passed to a TLS function it must not be modified. +type Config struct { + // Rand provides the source of entropy for nonces and RSA blinding. + // If Rand is nil, TLS uses the cryptographic random reader in package + // crypto/rand. + Rand io.Reader + + // Time returns the current time as the number of seconds since the epoch. + // If Time is nil, TLS uses the system time.Seconds. + Time func() int64 + + // Certificates contains one or more certificate chains + // to present to the other side of the connection. + // Server configurations must include at least one certificate. + Certificates []Certificate + + // RootCAs defines the set of root certificate authorities + // that clients use when verifying server certificates. + // If RootCAs is nil, TLS uses the host's root CA set. + RootCAs *x509.CertPool + + // NextProtos is a list of supported, application level protocols. + NextProtos []string + + // ServerName is included in the client's handshake to support virtual + // hosting. + ServerName string + + // AuthenticateClient controls whether a server will request a certificate + // from the client. It does not require that the client send a + // certificate nor does it require that the certificate sent be + // anything more than self-signed. + AuthenticateClient bool + + // CipherSuites is a list of supported cipher suites. If CipherSuites + // is nil, TLS uses a list of suites supported by the implementation. + CipherSuites []uint16 +} + +func (c *Config) rand() io.Reader { + r := c.Rand + if r == nil { + return rand.Reader + } + return r +} + +func (c *Config) time() int64 { + t := c.Time + if t == nil { + t = time.Seconds + } + return t() +} + +func (c *Config) rootCAs() *x509.CertPool { + s := c.RootCAs + if s == nil { + s = defaultRoots() + } + return s +} + +func (c *Config) cipherSuites() []uint16 { + s := c.CipherSuites + if s == nil { + s = defaultCipherSuites() + } + return s +} + +// A Certificate is a chain of one or more certificates, leaf first. +type Certificate struct { + Certificate [][]byte + PrivateKey *rsa.PrivateKey + // OCSPStaple contains an optional OCSP response which will be served + // to clients that request it. + OCSPStaple []byte +} + +// A TLS record. +type record struct { + contentType recordType + major, minor uint8 + payload []byte +} + +type handshakeMessage interface { + marshal() []byte + unmarshal([]byte) bool +} + +// mutualVersion returns the protocol version to use given the advertised +// version of the peer. +func mutualVersion(vers uint16) (uint16, bool) { + if vers < minVersion { + return 0, false + } + if vers > maxVersion { + vers = maxVersion + } + return vers, true +} + +var emptyConfig Config + +func defaultConfig() *Config { + return &emptyConfig +} + +// Possible certificate files; stop after finding one. +// On OS X we should really be using the Directory Services keychain +// but that requires a lot of Mach goo to get at. Instead we use +// the same root set that curl uses. +var certFiles = []string{ + "/etc/ssl/certs/ca-certificates.crt", // Linux etc + "/usr/share/curl/curl-ca-bundle.crt", // OS X +} + +var once sync.Once + +func defaultRoots() *x509.CertPool { + once.Do(initDefaults) + return varDefaultRoots +} + +func defaultCipherSuites() []uint16 { + once.Do(initDefaults) + return varDefaultCipherSuites +} + +func initDefaults() { + initDefaultRoots() + initDefaultCipherSuites() +} + +var varDefaultRoots *x509.CertPool + +func initDefaultRoots() { + roots := x509.NewCertPool() + for _, file := range certFiles { + data, err := ioutil.ReadFile(file) + if err == nil { + roots.AppendCertsFromPEM(data) + break + } + } + varDefaultRoots = roots +} + +var varDefaultCipherSuites []uint16 + +func initDefaultCipherSuites() { + varDefaultCipherSuites = make([]uint16, len(cipherSuites)) + i := 0 + for id := range cipherSuites { + varDefaultCipherSuites[i] = id + i++ + } +} diff --git a/src/pkg/crypto/tls/conn.go b/src/pkg/crypto/tls/conn.go new file mode 100644 index 000000000..fac65afd9 --- /dev/null +++ b/src/pkg/crypto/tls/conn.go @@ -0,0 +1,799 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TLS low level connection and record layer + +package tls + +import ( + "bytes" + "crypto/cipher" + "crypto/subtle" + "crypto/x509" + "hash" + "io" + "net" + "os" + "sync" +) + +// A Conn represents a secured connection. +// It implements the net.Conn interface. +type Conn struct { + // constant + conn net.Conn + isClient bool + + // constant after handshake; protected by handshakeMutex + handshakeMutex sync.Mutex // handshakeMutex < in.Mutex, out.Mutex, errMutex + vers uint16 // TLS version + haveVers bool // version has been negotiated + config *Config // configuration passed to constructor + handshakeComplete bool + cipherSuite uint16 + ocspResponse []byte // stapled OCSP response + peerCertificates []*x509.Certificate + // verifiedChains contains the certificate chains that we built, as + // opposed to the ones presented by the server. + verifiedChains [][]*x509.Certificate + + clientProtocol string + clientProtocolFallback bool + + // first permanent error + errMutex sync.Mutex + err os.Error + + // input/output + in, out halfConn // in.Mutex < out.Mutex + rawInput *block // raw input, right off the wire + input *block // application data waiting to be read + hand bytes.Buffer // handshake data waiting to be read + + tmp [16]byte +} + +func (c *Conn) setError(err os.Error) os.Error { + c.errMutex.Lock() + defer c.errMutex.Unlock() + + if c.err == nil { + c.err = err + } + return err +} + +func (c *Conn) error() os.Error { + c.errMutex.Lock() + defer c.errMutex.Unlock() + + return c.err +} + +// Access to net.Conn methods. +// Cannot just embed net.Conn because that would +// export the struct field too. + +// LocalAddr returns the local network address. +func (c *Conn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +// RemoteAddr returns the remote network address. +func (c *Conn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +// SetTimeout sets the read deadline associated with the connection. +// There is no write deadline. +func (c *Conn) SetTimeout(nsec int64) os.Error { + return c.conn.SetTimeout(nsec) +} + +// SetReadTimeout sets the time (in nanoseconds) that +// Read will wait for data before returning os.EAGAIN. +// Setting nsec == 0 (the default) disables the deadline. +func (c *Conn) SetReadTimeout(nsec int64) os.Error { + return c.conn.SetReadTimeout(nsec) +} + +// SetWriteTimeout exists to satisfy the net.Conn interface +// but is not implemented by TLS. It always returns an error. +func (c *Conn) SetWriteTimeout(nsec int64) os.Error { + return os.NewError("TLS does not support SetWriteTimeout") +} + +// A halfConn represents one direction of the record layer +// connection, either sending or receiving. +type halfConn struct { + sync.Mutex + cipher interface{} // cipher algorithm + mac hash.Hash // MAC algorithm + seq [8]byte // 64-bit sequence number + bfree *block // list of free blocks + + nextCipher interface{} // next encryption state + nextMac hash.Hash // next MAC algorithm +} + +// prepareCipherSpec sets the encryption and MAC states +// that a subsequent changeCipherSpec will use. +func (hc *halfConn) prepareCipherSpec(cipher interface{}, mac hash.Hash) { + hc.nextCipher = cipher + hc.nextMac = mac +} + +// changeCipherSpec changes the encryption and MAC states +// to the ones previously passed to prepareCipherSpec. +func (hc *halfConn) changeCipherSpec() os.Error { + if hc.nextCipher == nil { + return alertInternalError + } + hc.cipher = hc.nextCipher + hc.mac = hc.nextMac + hc.nextCipher = nil + hc.nextMac = nil + return nil +} + +// incSeq increments the sequence number. +func (hc *halfConn) incSeq() { + for i := 7; i >= 0; i-- { + hc.seq[i]++ + if hc.seq[i] != 0 { + return + } + } + + // Not allowed to let sequence number wrap. + // Instead, must renegotiate before it does. + // Not likely enough to bother. + panic("TLS: sequence number wraparound") +} + +// resetSeq resets the sequence number to zero. +func (hc *halfConn) resetSeq() { + for i := range hc.seq { + hc.seq[i] = 0 + } +} + +// removePadding returns an unpadded slice, in constant time, which is a prefix +// of the input. It also returns a byte which is equal to 255 if the padding +// was valid and 0 otherwise. See RFC 2246, section 6.2.3.2 +func removePadding(payload []byte) ([]byte, byte) { + if len(payload) < 1 { + return payload, 0 + } + + paddingLen := payload[len(payload)-1] + t := uint(len(payload)-1) - uint(paddingLen) + // if len(payload) >= (paddingLen - 1) then the MSB of t is zero + good := byte(int32(^t) >> 31) + + toCheck := 255 // the maximum possible padding length + // The length of the padded data is public, so we can use an if here + if toCheck+1 > len(payload) { + toCheck = len(payload) - 1 + } + + for i := 0; i < toCheck; i++ { + t := uint(paddingLen) - uint(i) + // if i <= paddingLen then the MSB of t is zero + mask := byte(int32(^t) >> 31) + b := payload[len(payload)-1-i] + good &^= mask&paddingLen ^ mask&b + } + + // We AND together the bits of good and replicate the result across + // all the bits. + good &= good << 4 + good &= good << 2 + good &= good << 1 + good = uint8(int8(good) >> 7) + + toRemove := good&paddingLen + 1 + return payload[:len(payload)-int(toRemove)], good +} + +func roundUp(a, b int) int { + return a + (b-a%b)%b +} + +// decrypt checks and strips the mac and decrypts the data in b. +func (hc *halfConn) decrypt(b *block) (bool, alert) { + // pull out payload + payload := b.data[recordHeaderLen:] + + macSize := 0 + if hc.mac != nil { + macSize = hc.mac.Size() + } + + paddingGood := byte(255) + + // decrypt + if hc.cipher != nil { + switch c := hc.cipher.(type) { + case cipher.Stream: + c.XORKeyStream(payload, payload) + case cipher.BlockMode: + blockSize := c.BlockSize() + + if len(payload)%blockSize != 0 || len(payload) < roundUp(macSize+1, blockSize) { + return false, alertBadRecordMAC + } + + c.CryptBlocks(payload, payload) + payload, paddingGood = removePadding(payload) + b.resize(recordHeaderLen + len(payload)) + + // note that we still have a timing side-channel in the + // MAC check, below. An attacker can align the record + // so that a correct padding will cause one less hash + // block to be calculated. Then they can iteratively + // decrypt a record by breaking each byte. See + // "Password Interception in a SSL/TLS Channel", Brice + // Canvel et al. + // + // However, our behavior matches OpenSSL, so we leak + // only as much as they do. + default: + panic("unknown cipher type") + } + } + + // check, strip mac + if hc.mac != nil { + if len(payload) < macSize { + return false, alertBadRecordMAC + } + + // strip mac off payload, b.data + n := len(payload) - macSize + b.data[3] = byte(n >> 8) + b.data[4] = byte(n) + b.resize(recordHeaderLen + n) + remoteMAC := payload[n:] + + hc.mac.Reset() + hc.mac.Write(hc.seq[0:]) + hc.incSeq() + hc.mac.Write(b.data) + + if subtle.ConstantTimeCompare(hc.mac.Sum(), remoteMAC) != 1 || paddingGood != 255 { + return false, alertBadRecordMAC + } + } + + return true, 0 +} + +// padToBlockSize calculates the needed padding block, if any, for a payload. +// On exit, prefix aliases payload and extends to the end of the last full +// block of payload. finalBlock is a fresh slice which contains the contents of +// any suffix of payload as well as the needed padding to make finalBlock a +// full block. +func padToBlockSize(payload []byte, blockSize int) (prefix, finalBlock []byte) { + overrun := len(payload) % blockSize + paddingLen := blockSize - overrun + prefix = payload[:len(payload)-overrun] + finalBlock = make([]byte, blockSize) + copy(finalBlock, payload[len(payload)-overrun:]) + for i := overrun; i < blockSize; i++ { + finalBlock[i] = byte(paddingLen - 1) + } + return +} + +// encrypt encrypts and macs the data in b. +func (hc *halfConn) encrypt(b *block) (bool, alert) { + // mac + if hc.mac != nil { + hc.mac.Reset() + hc.mac.Write(hc.seq[0:]) + hc.incSeq() + hc.mac.Write(b.data) + mac := hc.mac.Sum() + n := len(b.data) + b.resize(n + len(mac)) + copy(b.data[n:], mac) + } + + payload := b.data[recordHeaderLen:] + + // encrypt + if hc.cipher != nil { + switch c := hc.cipher.(type) { + case cipher.Stream: + c.XORKeyStream(payload, payload) + case cipher.BlockMode: + prefix, finalBlock := padToBlockSize(payload, c.BlockSize()) + b.resize(recordHeaderLen + len(prefix) + len(finalBlock)) + c.CryptBlocks(b.data[recordHeaderLen:], prefix) + c.CryptBlocks(b.data[recordHeaderLen+len(prefix):], finalBlock) + default: + panic("unknown cipher type") + } + } + + // update length to include MAC and any block padding needed. + n := len(b.data) - recordHeaderLen + b.data[3] = byte(n >> 8) + b.data[4] = byte(n) + + return true, 0 +} + +// A block is a simple data buffer. +type block struct { + data []byte + off int // index for Read + link *block +} + +// resize resizes block to be n bytes, growing if necessary. +func (b *block) resize(n int) { + if n > cap(b.data) { + b.reserve(n) + } + b.data = b.data[0:n] +} + +// reserve makes sure that block contains a capacity of at least n bytes. +func (b *block) reserve(n int) { + if cap(b.data) >= n { + return + } + m := cap(b.data) + if m == 0 { + m = 1024 + } + for m < n { + m *= 2 + } + data := make([]byte, len(b.data), m) + copy(data, b.data) + b.data = data +} + +// readFromUntil reads from r into b until b contains at least n bytes +// or else returns an error. +func (b *block) readFromUntil(r io.Reader, n int) os.Error { + // quick case + if len(b.data) >= n { + return nil + } + + // read until have enough. + b.reserve(n) + for { + m, err := r.Read(b.data[len(b.data):cap(b.data)]) + b.data = b.data[0 : len(b.data)+m] + if len(b.data) >= n { + break + } + if err != nil { + return err + } + } + return nil +} + +func (b *block) Read(p []byte) (n int, err os.Error) { + n = copy(p, b.data[b.off:]) + b.off += n + return +} + +// newBlock allocates a new block, from hc's free list if possible. +func (hc *halfConn) newBlock() *block { + b := hc.bfree + if b == nil { + return new(block) + } + hc.bfree = b.link + b.link = nil + b.resize(0) + return b +} + +// freeBlock returns a block to hc's free list. +// The protocol is such that each side only has a block or two on +// its free list at a time, so there's no need to worry about +// trimming the list, etc. +func (hc *halfConn) freeBlock(b *block) { + b.link = hc.bfree + hc.bfree = b +} + +// splitBlock splits a block after the first n bytes, +// returning a block with those n bytes and a +// block with the remainder. the latter may be nil. +func (hc *halfConn) splitBlock(b *block, n int) (*block, *block) { + if len(b.data) <= n { + return b, nil + } + bb := hc.newBlock() + bb.resize(len(b.data) - n) + copy(bb.data, b.data[n:]) + b.data = b.data[0:n] + return b, bb +} + +// readRecord reads the next TLS record from the connection +// and updates the record layer state. +// c.in.Mutex <= L; c.input == nil. +func (c *Conn) readRecord(want recordType) os.Error { + // Caller must be in sync with connection: + // handshake data if handshake not yet completed, + // else application data. (We don't support renegotiation.) + switch want { + default: + return c.sendAlert(alertInternalError) + case recordTypeHandshake, recordTypeChangeCipherSpec: + if c.handshakeComplete { + return c.sendAlert(alertInternalError) + } + case recordTypeApplicationData: + if !c.handshakeComplete { + return c.sendAlert(alertInternalError) + } + } + +Again: + if c.rawInput == nil { + c.rawInput = c.in.newBlock() + } + b := c.rawInput + + // Read header, payload. + if err := b.readFromUntil(c.conn, recordHeaderLen); err != nil { + // RFC suggests that EOF without an alertCloseNotify is + // an error, but popular web sites seem to do this, + // so we can't make it an error. + // if err == os.EOF { + // err = io.ErrUnexpectedEOF + // } + if e, ok := err.(net.Error); !ok || !e.Temporary() { + c.setError(err) + } + return err + } + typ := recordType(b.data[0]) + vers := uint16(b.data[1])<<8 | uint16(b.data[2]) + n := int(b.data[3])<<8 | int(b.data[4]) + if c.haveVers && vers != c.vers { + return c.sendAlert(alertProtocolVersion) + } + if n > maxCiphertext { + return c.sendAlert(alertRecordOverflow) + } + if err := b.readFromUntil(c.conn, recordHeaderLen+n); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if e, ok := err.(net.Error); !ok || !e.Temporary() { + c.setError(err) + } + return err + } + + // Process message. + b, c.rawInput = c.in.splitBlock(b, recordHeaderLen+n) + b.off = recordHeaderLen + if ok, err := c.in.decrypt(b); !ok { + return c.sendAlert(err) + } + data := b.data[b.off:] + if len(data) > maxPlaintext { + c.sendAlert(alertRecordOverflow) + c.in.freeBlock(b) + return c.error() + } + + switch typ { + default: + c.sendAlert(alertUnexpectedMessage) + + case recordTypeAlert: + if len(data) != 2 { + c.sendAlert(alertUnexpectedMessage) + break + } + if alert(data[1]) == alertCloseNotify { + c.setError(os.EOF) + break + } + switch data[0] { + case alertLevelWarning: + // drop on the floor + c.in.freeBlock(b) + goto Again + case alertLevelError: + c.setError(&net.OpError{Op: "remote error", Error: alert(data[1])}) + default: + c.sendAlert(alertUnexpectedMessage) + } + + case recordTypeChangeCipherSpec: + if typ != want || len(data) != 1 || data[0] != 1 { + c.sendAlert(alertUnexpectedMessage) + break + } + err := c.in.changeCipherSpec() + if err != nil { + c.sendAlert(err.(alert)) + } + + case recordTypeApplicationData: + if typ != want { + c.sendAlert(alertUnexpectedMessage) + break + } + c.input = b + b = nil + + case recordTypeHandshake: + // TODO(rsc): Should at least pick off connection close. + if typ != want { + return c.sendAlert(alertNoRenegotiation) + } + c.hand.Write(data) + } + + if b != nil { + c.in.freeBlock(b) + } + return c.error() +} + +// sendAlert sends a TLS alert message. +// c.out.Mutex <= L. +func (c *Conn) sendAlertLocked(err alert) os.Error { + c.tmp[0] = alertLevelError + if err == alertNoRenegotiation { + c.tmp[0] = alertLevelWarning + } + c.tmp[1] = byte(err) + c.writeRecord(recordTypeAlert, c.tmp[0:2]) + // closeNotify is a special case in that it isn't an error: + if err != alertCloseNotify { + return c.setError(&net.OpError{Op: "local error", Error: err}) + } + return nil +} + +// sendAlert sends a TLS alert message. +// L < c.out.Mutex. +func (c *Conn) sendAlert(err alert) os.Error { + c.out.Lock() + defer c.out.Unlock() + return c.sendAlertLocked(err) +} + +// writeRecord writes a TLS record with the given type and payload +// to the connection and updates the record layer state. +// c.out.Mutex <= L. +func (c *Conn) writeRecord(typ recordType, data []byte) (n int, err os.Error) { + b := c.out.newBlock() + for len(data) > 0 { + m := len(data) + if m > maxPlaintext { + m = maxPlaintext + } + b.resize(recordHeaderLen + m) + b.data[0] = byte(typ) + vers := c.vers + if vers == 0 { + vers = maxVersion + } + b.data[1] = byte(vers >> 8) + b.data[2] = byte(vers) + b.data[3] = byte(m >> 8) + b.data[4] = byte(m) + copy(b.data[recordHeaderLen:], data) + c.out.encrypt(b) + _, err = c.conn.Write(b.data) + if err != nil { + break + } + n += m + data = data[m:] + } + c.out.freeBlock(b) + + if typ == recordTypeChangeCipherSpec { + err = c.out.changeCipherSpec() + if err != nil { + // Cannot call sendAlert directly, + // because we already hold c.out.Mutex. + c.tmp[0] = alertLevelError + c.tmp[1] = byte(err.(alert)) + c.writeRecord(recordTypeAlert, c.tmp[0:2]) + c.err = &net.OpError{Op: "local error", Error: err} + return n, c.err + } + } + return +} + +// readHandshake reads the next handshake message from +// the record layer. +// c.in.Mutex < L; c.out.Mutex < L. +func (c *Conn) readHandshake() (interface{}, os.Error) { + for c.hand.Len() < 4 { + if c.err != nil { + return nil, c.err + } + c.readRecord(recordTypeHandshake) + } + + data := c.hand.Bytes() + n := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + if n > maxHandshake { + c.sendAlert(alertInternalError) + return nil, c.err + } + for c.hand.Len() < 4+n { + if c.err != nil { + return nil, c.err + } + c.readRecord(recordTypeHandshake) + } + data = c.hand.Next(4 + n) + var m handshakeMessage + switch data[0] { + case typeClientHello: + m = new(clientHelloMsg) + case typeServerHello: + m = new(serverHelloMsg) + case typeCertificate: + m = new(certificateMsg) + case typeCertificateRequest: + m = new(certificateRequestMsg) + case typeCertificateStatus: + m = new(certificateStatusMsg) + case typeServerKeyExchange: + m = new(serverKeyExchangeMsg) + case typeServerHelloDone: + m = new(serverHelloDoneMsg) + case typeClientKeyExchange: + m = new(clientKeyExchangeMsg) + case typeCertificateVerify: + m = new(certificateVerifyMsg) + case typeNextProtocol: + m = new(nextProtoMsg) + case typeFinished: + m = new(finishedMsg) + default: + c.sendAlert(alertUnexpectedMessage) + return nil, alertUnexpectedMessage + } + + // The handshake message unmarshallers + // expect to be able to keep references to data, + // so pass in a fresh copy that won't be overwritten. + data = append([]byte(nil), data...) + + if !m.unmarshal(data) { + c.sendAlert(alertUnexpectedMessage) + return nil, alertUnexpectedMessage + } + return m, nil +} + +// Write writes data to the connection. +func (c *Conn) Write(b []byte) (n int, err os.Error) { + if err = c.Handshake(); err != nil { + return + } + + c.out.Lock() + defer c.out.Unlock() + + if !c.handshakeComplete { + return 0, alertInternalError + } + if c.err != nil { + return 0, c.err + } + return c.writeRecord(recordTypeApplicationData, b) +} + +// Read can be made to time out and return err == os.EAGAIN +// after a fixed time limit; see SetTimeout and SetReadTimeout. +func (c *Conn) Read(b []byte) (n int, err os.Error) { + if err = c.Handshake(); err != nil { + return + } + + c.in.Lock() + defer c.in.Unlock() + + for c.input == nil && c.err == nil { + if err := c.readRecord(recordTypeApplicationData); err != nil { + // Soft error, like EAGAIN + return 0, err + } + } + if c.err != nil { + return 0, c.err + } + n, err = c.input.Read(b) + if c.input.off >= len(c.input.data) { + c.in.freeBlock(c.input) + c.input = nil + } + return n, nil +} + +// Close closes the connection. +func (c *Conn) Close() os.Error { + if err := c.Handshake(); err != nil { + return err + } + return c.sendAlert(alertCloseNotify) +} + +// Handshake runs the client or server handshake +// protocol if it has not yet been run. +// Most uses of this package need not call Handshake +// explicitly: the first Read or Write will call it automatically. +func (c *Conn) Handshake() os.Error { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if err := c.error(); err != nil { + return err + } + if c.handshakeComplete { + return nil + } + if c.isClient { + return c.clientHandshake() + } + return c.serverHandshake() +} + +// ConnectionState returns basic TLS details about the connection. +func (c *Conn) ConnectionState() ConnectionState { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + var state ConnectionState + state.HandshakeComplete = c.handshakeComplete + if c.handshakeComplete { + state.NegotiatedProtocol = c.clientProtocol + state.NegotiatedProtocolIsMutual = !c.clientProtocolFallback + state.CipherSuite = c.cipherSuite + state.PeerCertificates = c.peerCertificates + state.VerifiedChains = c.verifiedChains + } + + return state +} + +// OCSPResponse returns the stapled OCSP response from the TLS server, if +// any. (Only valid for client connections.) +func (c *Conn) OCSPResponse() []byte { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + + return c.ocspResponse +} + +// VerifyHostname checks that the peer certificate chain is valid for +// connecting to host. If so, it returns nil; if not, it returns an os.Error +// describing the problem. +func (c *Conn) VerifyHostname(host string) os.Error { + c.handshakeMutex.Lock() + defer c.handshakeMutex.Unlock() + if !c.isClient { + return os.NewError("VerifyHostname called on TLS server connection") + } + if !c.handshakeComplete { + return os.NewError("TLS handshake has not yet been performed") + } + return c.peerCertificates[0].VerifyHostname(host) +} diff --git a/src/pkg/crypto/tls/conn_test.go b/src/pkg/crypto/tls/conn_test.go new file mode 100644 index 000000000..f44a50bed --- /dev/null +++ b/src/pkg/crypto/tls/conn_test.go @@ -0,0 +1,52 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "testing" +) + +func TestRoundUp(t *testing.T) { + if roundUp(0, 16) != 0 || + roundUp(1, 16) != 16 || + roundUp(15, 16) != 16 || + roundUp(16, 16) != 16 || + roundUp(17, 16) != 32 { + t.Error("roundUp broken") + } +} + +var paddingTests = []struct { + in []byte + good bool + expectedLen int +}{ + {[]byte{1, 2, 3, 4, 0}, true, 4}, + {[]byte{1, 2, 3, 4, 0, 1}, false, 0}, + {[]byte{1, 2, 3, 4, 99, 99}, false, 0}, + {[]byte{1, 2, 3, 4, 1, 1}, true, 4}, + {[]byte{1, 2, 3, 2, 2, 2}, true, 3}, + {[]byte{1, 2, 3, 3, 3, 3}, true, 2}, + {[]byte{1, 2, 3, 4, 3, 3}, false, 0}, + {[]byte{1, 4, 4, 4, 4, 4}, true, 1}, + {[]byte{5, 5, 5, 5, 5, 5}, true, 0}, + {[]byte{6, 6, 6, 6, 6, 6}, false, 0}, +} + +func TestRemovePadding(t *testing.T) { + for i, test := range paddingTests { + payload, good := removePadding(test.in) + expectedGood := byte(255) + if !test.good { + expectedGood = 0 + } + if good != expectedGood { + t.Errorf("#%d: wrong validity, want:%d got:%d", i, expectedGood, good) + } + if good == 255 && len(payload) != test.expectedLen { + t.Errorf("#%d: got %d, want %d", i, len(payload), test.expectedLen) + } + } +} diff --git a/src/pkg/crypto/tls/generate_cert.go b/src/pkg/crypto/tls/generate_cert.go new file mode 100644 index 000000000..41206e276 --- /dev/null +++ b/src/pkg/crypto/tls/generate_cert.go @@ -0,0 +1,72 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Generate a self-signed X.509 certificate for a TLS server. Outputs to +// 'cert.pem' and 'key.pem' and will overwrite existing files. + +package main + +import ( + "big" + "crypto/x509/pkix" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "flag" + "log" + "os" + "time" +) + +var hostName *string = flag.String("host", "127.0.0.1", "Hostname to generate a certificate for") + +func main() { + flag.Parse() + + priv, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + log.Fatalf("failed to generate private key: %s", err) + return + } + + now := time.Seconds() + + template := x509.Certificate{ + SerialNumber: new(big.Int).SetInt64(0), + Subject: pkix.Name{ + CommonName: *hostName, + Organization: []string{"Acme Co"}, + }, + NotBefore: time.SecondsToUTC(now - 300), + NotAfter: time.SecondsToUTC(now + 60*60*24*365), // valid for 1 year. + + SubjectKeyId: []byte{1, 2, 3, 4}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + log.Fatalf("Failed to create certificate: %s", err) + return + } + + certOut, err := os.Create("cert.pem") + if err != nil { + log.Fatalf("failed to open cert.pem for writing: %s", err) + return + } + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + log.Print("written cert.pem\n") + + keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + log.Print("failed to open key.pem for writing:", err) + return + } + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + log.Print("written key.pem\n") +} diff --git a/src/pkg/crypto/tls/handshake_client.go b/src/pkg/crypto/tls/handshake_client.go new file mode 100644 index 000000000..15604cea7 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_client.go @@ -0,0 +1,315 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto" + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "io" + "os" +) + +func (c *Conn) clientHandshake() os.Error { + finishedHash := newFinishedHash() + + if c.config == nil { + c.config = defaultConfig() + } + + hello := &clientHelloMsg{ + vers: maxVersion, + cipherSuites: c.config.cipherSuites(), + compressionMethods: []uint8{compressionNone}, + random: make([]byte, 32), + ocspStapling: true, + serverName: c.config.ServerName, + supportedCurves: []uint16{curveP256, curveP384, curveP521}, + supportedPoints: []uint8{pointFormatUncompressed}, + nextProtoNeg: len(c.config.NextProtos) > 0, + } + + t := uint32(c.config.time()) + hello.random[0] = byte(t >> 24) + hello.random[1] = byte(t >> 16) + hello.random[2] = byte(t >> 8) + hello.random[3] = byte(t) + _, err := io.ReadFull(c.config.rand(), hello.random[4:]) + if err != nil { + c.sendAlert(alertInternalError) + return os.NewError("short read from Rand") + } + + finishedHash.Write(hello.marshal()) + c.writeRecord(recordTypeHandshake, hello.marshal()) + + msg, err := c.readHandshake() + if err != nil { + return err + } + serverHello, ok := msg.(*serverHelloMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(serverHello.marshal()) + + vers, ok := mutualVersion(serverHello.vers) + if !ok { + return c.sendAlert(alertProtocolVersion) + } + c.vers = vers + c.haveVers = true + + if serverHello.compressionMethod != compressionNone { + return c.sendAlert(alertUnexpectedMessage) + } + + if !hello.nextProtoNeg && serverHello.nextProtoNeg { + c.sendAlert(alertHandshakeFailure) + return os.NewError("server advertised unrequested NPN") + } + + suite, suiteId := mutualCipherSuite(c.config.cipherSuites(), serverHello.cipherSuite) + if suite == nil { + return c.sendAlert(alertHandshakeFailure) + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + certMsg, ok := msg.(*certificateMsg) + if !ok || len(certMsg.certificates) == 0 { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(certMsg.marshal()) + + certs := make([]*x509.Certificate, len(certMsg.certificates)) + for i, asn1Data := range certMsg.certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("failed to parse certificate from server: " + err.String()) + } + certs[i] = cert + } + + // If we don't have a root CA set configured then anything is accepted. + // TODO(rsc): Find certificates for OS X 10.6. + if c.config.RootCAs != nil { + opts := x509.VerifyOptions{ + Roots: c.config.RootCAs, + CurrentTime: c.config.time(), + DNSName: c.config.ServerName, + Intermediates: x509.NewCertPool(), + } + + for i, cert := range certs { + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + c.verifiedChains, err = certs[0].Verify(opts) + if err != nil { + c.sendAlert(alertBadCertificate) + return err + } + } + + if _, ok := certs[0].PublicKey.(*rsa.PublicKey); !ok { + return c.sendAlert(alertUnsupportedCertificate) + } + + c.peerCertificates = certs + + if serverHello.ocspStapling { + msg, err = c.readHandshake() + if err != nil { + return err + } + cs, ok := msg.(*certificateStatusMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(cs.marshal()) + + if cs.statusType == statusTypeOCSP { + c.ocspResponse = cs.response + } + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + + keyAgreement := suite.ka() + + skx, ok := msg.(*serverKeyExchangeMsg) + if ok { + finishedHash.Write(skx.marshal()) + err = keyAgreement.processServerKeyExchange(c.config, hello, serverHello, certs[0], skx) + if err != nil { + c.sendAlert(alertUnexpectedMessage) + return err + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + } + + transmitCert := false + certReq, ok := msg.(*certificateRequestMsg) + if ok { + // We only accept certificates with RSA keys. + rsaAvail := false + for _, certType := range certReq.certificateTypes { + if certType == certTypeRSASign { + rsaAvail = true + break + } + } + + // For now, only send a certificate back if the server gives us an + // empty list of certificateAuthorities. + // + // RFC 4346 on the certificateAuthorities field: + // A list of the distinguished names of acceptable certificate + // authorities. These distinguished names may specify a desired + // distinguished name for a root CA or for a subordinate CA; thus, + // this message can be used to describe both known roots and a + // desired authorization space. If the certificate_authorities + // list is empty then the client MAY send any certificate of the + // appropriate ClientCertificateType, unless there is some + // external arrangement to the contrary. + if rsaAvail && len(certReq.certificateAuthorities) == 0 { + transmitCert = true + } + + finishedHash.Write(certReq.marshal()) + + msg, err = c.readHandshake() + if err != nil { + return err + } + } + + shd, ok := msg.(*serverHelloDoneMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(shd.marshal()) + + var cert *x509.Certificate + if transmitCert { + certMsg = new(certificateMsg) + if len(c.config.Certificates) > 0 { + cert, err = x509.ParseCertificate(c.config.Certificates[0].Certificate[0]) + if err == nil && cert.PublicKeyAlgorithm == x509.RSA { + certMsg.certificates = c.config.Certificates[0].Certificate + } else { + cert = nil + } + } + finishedHash.Write(certMsg.marshal()) + c.writeRecord(recordTypeHandshake, certMsg.marshal()) + } + + preMasterSecret, ckx, err := keyAgreement.generateClientKeyExchange(c.config, hello, certs[0]) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + if ckx != nil { + finishedHash.Write(ckx.marshal()) + c.writeRecord(recordTypeHandshake, ckx.marshal()) + } + + if cert != nil { + certVerify := new(certificateVerifyMsg) + var digest [36]byte + copy(digest[0:16], finishedHash.serverMD5.Sum()) + copy(digest[16:36], finishedHash.serverSHA1.Sum()) + signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, crypto.MD5SHA1, digest[0:]) + if err != nil { + return c.sendAlert(alertInternalError) + } + certVerify.signature = signed + + finishedHash.Write(certVerify.marshal()) + c.writeRecord(recordTypeHandshake, certVerify.marshal()) + } + + masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := + keysFromPreMasterSecret10(preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen) + + clientCipher := suite.cipher(clientKey, clientIV, false /* not for reading */ ) + clientHash := suite.mac(clientMAC) + c.out.prepareCipherSpec(clientCipher, clientHash) + c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) + + if serverHello.nextProtoNeg { + nextProto := new(nextProtoMsg) + proto, fallback := mutualProtocol(c.config.NextProtos, serverHello.nextProtos) + nextProto.proto = proto + c.clientProtocol = proto + c.clientProtocolFallback = fallback + + finishedHash.Write(nextProto.marshal()) + c.writeRecord(recordTypeHandshake, nextProto.marshal()) + } + + finished := new(finishedMsg) + finished.verifyData = finishedHash.clientSum(masterSecret) + finishedHash.Write(finished.marshal()) + c.writeRecord(recordTypeHandshake, finished.marshal()) + + serverCipher := suite.cipher(serverKey, serverIV, true /* for reading */ ) + serverHash := suite.mac(serverMAC) + c.in.prepareCipherSpec(serverCipher, serverHash) + c.readRecord(recordTypeChangeCipherSpec) + if c.err != nil { + return c.err + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + serverFinished, ok := msg.(*finishedMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + verify := finishedHash.serverSum(masterSecret) + if len(verify) != len(serverFinished.verifyData) || + subtle.ConstantTimeCompare(verify, serverFinished.verifyData) != 1 { + return c.sendAlert(alertHandshakeFailure) + } + + c.handshakeComplete = true + c.cipherSuite = suiteId + return nil +} + +// mutualProtocol finds the mutual Next Protocol Negotiation protocol given the +// set of client and server supported protocols. The set of client supported +// protocols must not be empty. It returns the resulting protocol and flag +// indicating if the fallback case was reached. +func mutualProtocol(clientProtos, serverProtos []string) (string, bool) { + for _, s := range serverProtos { + for _, c := range clientProtos { + if s == c { + return s, false + } + } + } + + return clientProtos[0], true +} diff --git a/src/pkg/crypto/tls/handshake_client_test.go b/src/pkg/crypto/tls/handshake_client_test.go new file mode 100644 index 000000000..3f91c7acf --- /dev/null +++ b/src/pkg/crypto/tls/handshake_client_test.go @@ -0,0 +1,211 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "bytes" + "flag" + "io" + "net" + "testing" +) + +func testClientScript(t *testing.T, name string, clientScript [][]byte, config *Config) { + c, s := net.Pipe() + cli := Client(c, config) + go func() { + cli.Write([]byte("hello\n")) + cli.Close() + }() + + defer c.Close() + for i, b := range clientScript { + if i%2 == 1 { + s.Write(b) + continue + } + bb := make([]byte, len(b)) + _, err := io.ReadFull(s, bb) + if err != nil { + t.Fatalf("%s #%d: %s", name, i, err) + } + if !bytes.Equal(b, bb) { + t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) + } + } +} + +func TestHandshakeClientRC4(t *testing.T) { + testClientScript(t, "RC4", rc4ClientScript, testConfig) +} + +var connect = flag.Bool("connect", false, "connect to a TLS server on :10443") + +func TestRunClient(t *testing.T) { + if !*connect { + return + } + + testConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_RC4_128_SHA} + + conn, err := Dial("tcp", "127.0.0.1:10443", testConfig) + if err != nil { + t.Fatal(err) + } + + conn.Write([]byte("hello\n")) + conn.Close() +} + +// Script of interaction with gnutls implementation. +// The values for this test are obtained by building and running in client mode: +// % gotest -test.run "TestRunClient" -connect +// and then: +// % gnutls-serv -p 10443 --debug 100 --x509keyfile key.pem --x509certfile cert.pem -a > /tmp/log 2>&1 +// % python parse-gnutls-cli-debug-log.py < /tmp/log +// +// Where key.pem is: +// -----BEGIN RSA PRIVATE KEY----- +// MIIBPAIBAAJBAJ+zw4Qnlf8SMVIPFe9GEcStgOY2Ww/dgNdhjeD8ckUJNP5VZkVD +// TGiXav6ooKXfX3j/7tdkuD8Ey2//Kv7+ue0CAwEAAQJAN6W31vDEP2DjdqhzCDDu +// OA4NACqoiFqyblo7yc2tM4h4xMbC3Yx5UKMN9ZkCtX0gzrz6DyF47bdKcWBzNWCj +// gQIhANEoojVt7hq+SQ6MCN6FTAysGgQf56Q3TYoJMoWvdiXVAiEAw3e3rc+VJpOz +// rHuDo6bgpjUAAXM+v3fcpsfZSNO6V7kCIQCtbVjanpUwvZkMI9by02oUk9taki3b +// PzPfAfNPYAbCJQIhAJXNQDWyqwn/lGmR11cqY2y9nZ1+5w3yHGatLrcDnQHxAiEA +// vnlEGo8K85u+KwIOimM48ZG8oTk7iFdkqLJR1utT3aU= +// -----END RSA PRIVATE KEY----- +// +// and cert.pem is: +// -----BEGIN CERTIFICATE----- +// MIIBoDCCAUoCAQAwDQYJKoZIhvcNAQEEBQAwYzELMAkGA1UEBhMCQVUxEzARBgNV +// BAgTClF1ZWVuc2xhbmQxGjAYBgNVBAoTEUNyeXB0U29mdCBQdHkgTHRkMSMwIQYD +// VQQDExpTZXJ2ZXIgdGVzdCBjZXJ0ICg1MTIgYml0KTAeFw05NzA5MDkwMzQxMjZa +// Fw05NzEwMDkwMzQxMjZaMF4xCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0 +// YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxFzAVBgNVBAMT +// DkVyaWMgdGhlIFlvdW5nMFEwCQYFKw4DAgwFAANEAAJBALVEqPODnpI4rShlY8S7 +// tB713JNvabvn6Gned7zylwLLiXQAo/PAT6mfdWPTyCX9RlId/Aroh1ou893BA32Q +// sggwDQYJKoZIhvcNAQEEBQADQQCU5SSgapJSdRXJoX+CpCvFy+JVh9HpSjCpSNKO +// 19raHv98hKAUJuP9HyM+SUsffO6mAIgitUaqW8/wDMePhEC3 +// -----END CERTIFICATE----- +var rc4ClientScript = [][]byte{ + { + 0x16, 0x03, 0x01, 0x00, 0x4a, 0x01, 0x00, 0x00, + 0x46, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, + 0x01, 0x00, 0x00, 0x1b, 0x00, 0x05, 0x00, 0x05, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, + 0x08, 0x00, 0x06, 0x00, 0x17, 0x00, 0x18, 0x00, + 0x19, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x4a, 0x02, 0x00, 0x00, + 0x46, 0x03, 0x01, 0x4d, 0x0a, 0x56, 0x16, 0xb5, + 0x91, 0xd1, 0xcb, 0x80, 0x4d, 0xc7, 0x46, 0xf3, + 0x37, 0x0c, 0xef, 0xea, 0x64, 0x11, 0x14, 0x56, + 0x97, 0x9b, 0xc5, 0x67, 0x08, 0xb7, 0x13, 0xea, + 0xf8, 0xc9, 0xb3, 0x20, 0xe2, 0xfc, 0x41, 0xf6, + 0x96, 0x90, 0x9d, 0x43, 0x9b, 0xe9, 0x6e, 0xf8, + 0x41, 0x16, 0xcc, 0xf3, 0xc7, 0xde, 0xda, 0x5a, + 0xa1, 0x33, 0x69, 0xe2, 0xde, 0x5b, 0xaf, 0x2a, + 0x92, 0xe7, 0xd4, 0xa0, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x01, 0x01, 0xf7, 0x0b, 0x00, 0x01, 0xf3, + 0x00, 0x01, 0xf0, 0x00, 0x01, 0xed, 0x30, 0x82, + 0x01, 0xe9, 0x30, 0x82, 0x01, 0x52, 0x02, 0x01, + 0x06, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, + 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x04, 0x05, 0x00, + 0x30, 0x5b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, + 0x55, 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, + 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, + 0x13, 0x0a, 0x51, 0x75, 0x65, 0x65, 0x6e, 0x73, + 0x6c, 0x61, 0x6e, 0x64, 0x31, 0x1a, 0x30, 0x18, + 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x11, 0x43, + 0x72, 0x79, 0x70, 0x74, 0x53, 0x6f, 0x66, 0x74, + 0x20, 0x50, 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, + 0x31, 0x1b, 0x30, 0x19, 0x06, 0x03, 0x55, 0x04, + 0x03, 0x13, 0x12, 0x54, 0x65, 0x73, 0x74, 0x20, + 0x43, 0x41, 0x20, 0x28, 0x31, 0x30, 0x32, 0x34, + 0x20, 0x62, 0x69, 0x74, 0x29, 0x30, 0x1e, 0x17, + 0x0d, 0x30, 0x30, 0x31, 0x30, 0x31, 0x36, 0x32, + 0x32, 0x33, 0x31, 0x30, 0x33, 0x5a, 0x17, 0x0d, + 0x30, 0x33, 0x30, 0x31, 0x31, 0x34, 0x32, 0x32, + 0x33, 0x31, 0x30, 0x33, 0x5a, 0x30, 0x63, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x51, + 0x75, 0x65, 0x65, 0x6e, 0x73, 0x6c, 0x61, 0x6e, + 0x64, 0x31, 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x11, 0x43, 0x72, 0x79, 0x70, + 0x74, 0x53, 0x6f, 0x66, 0x74, 0x20, 0x50, 0x74, + 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x23, 0x30, + 0x21, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x1a, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x74, + 0x65, 0x73, 0x74, 0x20, 0x63, 0x65, 0x72, 0x74, + 0x20, 0x28, 0x35, 0x31, 0x32, 0x20, 0x62, 0x69, + 0x74, 0x29, 0x30, 0x5c, 0x30, 0x0d, 0x06, 0x09, + 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, + 0x01, 0x05, 0x00, 0x03, 0x4b, 0x00, 0x30, 0x48, + 0x02, 0x41, 0x00, 0x9f, 0xb3, 0xc3, 0x84, 0x27, + 0x95, 0xff, 0x12, 0x31, 0x52, 0x0f, 0x15, 0xef, + 0x46, 0x11, 0xc4, 0xad, 0x80, 0xe6, 0x36, 0x5b, + 0x0f, 0xdd, 0x80, 0xd7, 0x61, 0x8d, 0xe0, 0xfc, + 0x72, 0x45, 0x09, 0x34, 0xfe, 0x55, 0x66, 0x45, + 0x43, 0x4c, 0x68, 0x97, 0x6a, 0xfe, 0xa8, 0xa0, + 0xa5, 0xdf, 0x5f, 0x78, 0xff, 0xee, 0xd7, 0x64, + 0xb8, 0x3f, 0x04, 0xcb, 0x6f, 0xff, 0x2a, 0xfe, + 0xfe, 0xb9, 0xed, 0x02, 0x03, 0x01, 0x00, 0x01, + 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, + 0xf7, 0x0d, 0x01, 0x01, 0x04, 0x05, 0x00, 0x03, + 0x81, 0x81, 0x00, 0x93, 0xd2, 0x0a, 0xc5, 0x41, + 0xe6, 0x5a, 0xa9, 0x86, 0xf9, 0x11, 0x87, 0xe4, + 0xdb, 0x45, 0xe2, 0xc5, 0x95, 0x78, 0x1a, 0x6c, + 0x80, 0x6d, 0x73, 0x1f, 0xb4, 0x6d, 0x44, 0xa3, + 0xba, 0x86, 0x88, 0xc8, 0x58, 0xcd, 0x1c, 0x06, + 0x35, 0x6c, 0x44, 0x62, 0x88, 0xdf, 0xe4, 0xf6, + 0x64, 0x61, 0x95, 0xef, 0x4a, 0xa6, 0x7f, 0x65, + 0x71, 0xd7, 0x6b, 0x88, 0x39, 0xf6, 0x32, 0xbf, + 0xac, 0x93, 0x67, 0x69, 0x51, 0x8c, 0x93, 0xec, + 0x48, 0x5f, 0xc9, 0xb1, 0x42, 0xf9, 0x55, 0xd2, + 0x7e, 0x4e, 0xf4, 0xf2, 0x21, 0x6b, 0x90, 0x57, + 0xe6, 0xd7, 0x99, 0x9e, 0x41, 0xca, 0x80, 0xbf, + 0x1a, 0x28, 0xa2, 0xca, 0x5b, 0x50, 0x4a, 0xed, + 0x84, 0xe7, 0x82, 0xc7, 0xd2, 0xcf, 0x36, 0x9e, + 0x6a, 0x67, 0xb9, 0x88, 0xa7, 0xf3, 0x8a, 0xd0, + 0x04, 0xf8, 0xe8, 0xc6, 0x17, 0xe3, 0xc5, 0x29, + 0xbc, 0x17, 0xf1, 0x16, 0x03, 0x01, 0x00, 0x04, + 0x0e, 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x46, 0x10, 0x00, 0x00, + 0x42, 0x00, 0x40, 0x87, 0xa1, 0x1f, 0x14, 0xe1, + 0xfb, 0x91, 0xac, 0x58, 0x2e, 0xf3, 0x71, 0xce, + 0x01, 0x85, 0x2c, 0xc7, 0xfe, 0x84, 0x87, 0x82, + 0xb7, 0x57, 0xdb, 0x37, 0x4d, 0x46, 0x83, 0x67, + 0x52, 0x82, 0x51, 0x01, 0x95, 0x23, 0x68, 0x69, + 0x6b, 0xd0, 0xa7, 0xa7, 0xe5, 0x88, 0xd0, 0x47, + 0x71, 0xb8, 0xd2, 0x03, 0x05, 0x25, 0x56, 0x5c, + 0x10, 0x08, 0xc6, 0x9b, 0xd4, 0x67, 0xcd, 0x28, + 0xbe, 0x9c, 0x48, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x24, 0xc1, 0xb8, + 0xd3, 0x7f, 0xc5, 0xc2, 0x5a, 0x1d, 0x6d, 0x5b, + 0x2d, 0x5c, 0x82, 0x87, 0xc2, 0x6f, 0x0d, 0x63, + 0x7b, 0x72, 0x2b, 0xda, 0x69, 0xc4, 0xfe, 0x3c, + 0x84, 0xa1, 0x5a, 0x62, 0x38, 0x37, 0xc6, 0x54, + 0x25, 0x2a, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x24, 0xea, 0x88, 0x9c, 0x00, 0xf6, + 0x35, 0xb8, 0x42, 0x7f, 0x15, 0x17, 0x76, 0x5e, + 0x4b, 0x24, 0xcb, 0x7e, 0xa0, 0x7b, 0xc3, 0x70, + 0x52, 0x0a, 0x88, 0x2a, 0x7a, 0x45, 0x59, 0x90, + 0x59, 0xac, 0xc6, 0xb5, 0x56, 0x55, 0x96, + }, +} diff --git a/src/pkg/crypto/tls/handshake_messages.go b/src/pkg/crypto/tls/handshake_messages.go new file mode 100644 index 000000000..6645adce4 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_messages.go @@ -0,0 +1,904 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +type clientHelloMsg struct { + raw []byte + vers uint16 + random []byte + sessionId []byte + cipherSuites []uint16 + compressionMethods []uint8 + nextProtoNeg bool + serverName string + ocspStapling bool + supportedCurves []uint16 + supportedPoints []uint8 +} + +func (m *clientHelloMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + length := 2 + 32 + 1 + len(m.sessionId) + 2 + len(m.cipherSuites)*2 + 1 + len(m.compressionMethods) + numExtensions := 0 + extensionsLength := 0 + if m.nextProtoNeg { + numExtensions++ + } + if m.ocspStapling { + extensionsLength += 1 + 2 + 2 + numExtensions++ + } + if len(m.serverName) > 0 { + extensionsLength += 5 + len(m.serverName) + numExtensions++ + } + if len(m.supportedCurves) > 0 { + extensionsLength += 2 + 2*len(m.supportedCurves) + numExtensions++ + } + if len(m.supportedPoints) > 0 { + extensionsLength += 1 + len(m.supportedPoints) + numExtensions++ + } + if numExtensions > 0 { + extensionsLength += 4 * numExtensions + length += 2 + extensionsLength + } + + x := make([]byte, 4+length) + x[0] = typeClientHello + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(m.vers >> 8) + x[5] = uint8(m.vers) + copy(x[6:38], m.random) + x[38] = uint8(len(m.sessionId)) + copy(x[39:39+len(m.sessionId)], m.sessionId) + y := x[39+len(m.sessionId):] + y[0] = uint8(len(m.cipherSuites) >> 7) + y[1] = uint8(len(m.cipherSuites) << 1) + for i, suite := range m.cipherSuites { + y[2+i*2] = uint8(suite >> 8) + y[3+i*2] = uint8(suite) + } + z := y[2+len(m.cipherSuites)*2:] + z[0] = uint8(len(m.compressionMethods)) + copy(z[1:], m.compressionMethods) + + z = z[1+len(m.compressionMethods):] + if numExtensions > 0 { + z[0] = byte(extensionsLength >> 8) + z[1] = byte(extensionsLength) + z = z[2:] + } + if m.nextProtoNeg { + z[0] = byte(extensionNextProtoNeg >> 8) + z[1] = byte(extensionNextProtoNeg) + // The length is always 0 + z = z[4:] + } + if len(m.serverName) > 0 { + z[0] = byte(extensionServerName >> 8) + z[1] = byte(extensionServerName) + l := len(m.serverName) + 5 + z[2] = byte(l >> 8) + z[3] = byte(l) + z = z[4:] + + // RFC 3546, section 3.1 + // + // struct { + // NameType name_type; + // select (name_type) { + // case host_name: HostName; + // } name; + // } ServerName; + // + // enum { + // host_name(0), (255) + // } NameType; + // + // opaque HostName<1..2^16-1>; + // + // struct { + // ServerName server_name_list<1..2^16-1> + // } ServerNameList; + + z[0] = byte((len(m.serverName) + 3) >> 8) + z[1] = byte(len(m.serverName) + 3) + z[3] = byte(len(m.serverName) >> 8) + z[4] = byte(len(m.serverName)) + copy(z[5:], []byte(m.serverName)) + z = z[l:] + } + if m.ocspStapling { + // RFC 4366, section 3.6 + z[0] = byte(extensionStatusRequest >> 8) + z[1] = byte(extensionStatusRequest) + z[2] = 0 + z[3] = 5 + z[4] = 1 // OCSP type + // Two zero valued uint16s for the two lengths. + z = z[9:] + } + if len(m.supportedCurves) > 0 { + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + z[0] = byte(extensionSupportedCurves >> 8) + z[1] = byte(extensionSupportedCurves) + l := 2 + 2*len(m.supportedCurves) + z[2] = byte(l >> 8) + z[3] = byte(l) + l -= 2 + z[4] = byte(l >> 8) + z[5] = byte(l) + z = z[6:] + for _, curve := range m.supportedCurves { + z[0] = byte(curve >> 8) + z[1] = byte(curve) + z = z[2:] + } + } + if len(m.supportedPoints) > 0 { + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + z[0] = byte(extensionSupportedPoints >> 8) + z[1] = byte(extensionSupportedPoints) + l := 1 + len(m.supportedPoints) + z[2] = byte(l >> 8) + z[3] = byte(l) + l-- + z[4] = byte(l) + z = z[5:] + for _, pointFormat := range m.supportedPoints { + z[0] = byte(pointFormat) + z = z[1:] + } + } + + m.raw = x + + return x +} + +func (m *clientHelloMsg) unmarshal(data []byte) bool { + if len(data) < 42 { + return false + } + m.raw = data + m.vers = uint16(data[4])<<8 | uint16(data[5]) + m.random = data[6:38] + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return false + } + m.sessionId = data[39 : 39+sessionIdLen] + data = data[39+sessionIdLen:] + if len(data) < 2 { + return false + } + // cipherSuiteLen is the number of bytes of cipher suite numbers. Since + // they are uint16s, the number must be even. + cipherSuiteLen := int(data[0])<<8 | int(data[1]) + if cipherSuiteLen%2 == 1 || len(data) < 2+cipherSuiteLen { + return false + } + numCipherSuites := cipherSuiteLen / 2 + m.cipherSuites = make([]uint16, numCipherSuites) + for i := 0; i < numCipherSuites; i++ { + m.cipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i]) + } + data = data[2+cipherSuiteLen:] + if len(data) < 1 { + return false + } + compressionMethodsLen := int(data[0]) + if len(data) < 1+compressionMethodsLen { + return false + } + m.compressionMethods = data[1 : 1+compressionMethodsLen] + + data = data[1+compressionMethodsLen:] + + m.nextProtoNeg = false + m.serverName = "" + m.ocspStapling = false + + if len(data) == 0 { + // ClientHello is optionally followed by extension data + return true + } + if len(data) < 2 { + return false + } + + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if extensionsLength != len(data) { + return false + } + + for len(data) != 0 { + if len(data) < 4 { + return false + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return false + } + + switch extension { + case extensionServerName: + if length < 2 { + return false + } + numNames := int(data[0])<<8 | int(data[1]) + d := data[2:] + for i := 0; i < numNames; i++ { + if len(d) < 3 { + return false + } + nameType := d[0] + nameLen := int(d[1])<<8 | int(d[2]) + d = d[3:] + if len(d) < nameLen { + return false + } + if nameType == 0 { + m.serverName = string(d[0:nameLen]) + break + } + d = d[nameLen:] + } + case extensionNextProtoNeg: + if length > 0 { + return false + } + m.nextProtoNeg = true + case extensionStatusRequest: + m.ocspStapling = length > 0 && data[0] == statusTypeOCSP + case extensionSupportedCurves: + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + if length < 2 { + return false + } + l := int(data[0])<<8 | int(data[1]) + if l%2 == 1 || length != l+2 { + return false + } + numCurves := l / 2 + m.supportedCurves = make([]uint16, numCurves) + d := data[2:] + for i := 0; i < numCurves; i++ { + m.supportedCurves[i] = uint16(d[0])<<8 | uint16(d[1]) + d = d[2:] + } + case extensionSupportedPoints: + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + if length < 1 { + return false + } + l := int(data[0]) + if length != l+1 { + return false + } + m.supportedPoints = make([]uint8, l) + copy(m.supportedPoints, data[1:]) + } + data = data[length:] + } + + return true +} + +type serverHelloMsg struct { + raw []byte + vers uint16 + random []byte + sessionId []byte + cipherSuite uint16 + compressionMethod uint8 + nextProtoNeg bool + nextProtos []string + ocspStapling bool +} + +func (m *serverHelloMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + length := 38 + len(m.sessionId) + numExtensions := 0 + extensionsLength := 0 + + nextProtoLen := 0 + if m.nextProtoNeg { + numExtensions++ + for _, v := range m.nextProtos { + nextProtoLen += len(v) + } + nextProtoLen += len(m.nextProtos) + extensionsLength += nextProtoLen + } + if m.ocspStapling { + numExtensions++ + } + if numExtensions > 0 { + extensionsLength += 4 * numExtensions + length += 2 + extensionsLength + } + + x := make([]byte, 4+length) + x[0] = typeServerHello + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(m.vers >> 8) + x[5] = uint8(m.vers) + copy(x[6:38], m.random) + x[38] = uint8(len(m.sessionId)) + copy(x[39:39+len(m.sessionId)], m.sessionId) + z := x[39+len(m.sessionId):] + z[0] = uint8(m.cipherSuite >> 8) + z[1] = uint8(m.cipherSuite) + z[2] = uint8(m.compressionMethod) + + z = z[3:] + if numExtensions > 0 { + z[0] = byte(extensionsLength >> 8) + z[1] = byte(extensionsLength) + z = z[2:] + } + if m.nextProtoNeg { + z[0] = byte(extensionNextProtoNeg >> 8) + z[1] = byte(extensionNextProtoNeg) + z[2] = byte(nextProtoLen >> 8) + z[3] = byte(nextProtoLen) + z = z[4:] + + for _, v := range m.nextProtos { + l := len(v) + if l > 255 { + l = 255 + } + z[0] = byte(l) + copy(z[1:], []byte(v[0:l])) + z = z[1+l:] + } + } + if m.ocspStapling { + z[0] = byte(extensionStatusRequest >> 8) + z[1] = byte(extensionStatusRequest) + z = z[4:] + } + + m.raw = x + + return x +} + +func (m *serverHelloMsg) unmarshal(data []byte) bool { + if len(data) < 42 { + return false + } + m.raw = data + m.vers = uint16(data[4])<<8 | uint16(data[5]) + m.random = data[6:38] + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return false + } + m.sessionId = data[39 : 39+sessionIdLen] + data = data[39+sessionIdLen:] + if len(data) < 3 { + return false + } + m.cipherSuite = uint16(data[0])<<8 | uint16(data[1]) + m.compressionMethod = data[2] + data = data[3:] + + m.nextProtoNeg = false + m.nextProtos = nil + m.ocspStapling = false + + if len(data) == 0 { + // ServerHello is optionally followed by extension data + return true + } + if len(data) < 2 { + return false + } + + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if len(data) != extensionsLength { + return false + } + + for len(data) != 0 { + if len(data) < 4 { + return false + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return false + } + + switch extension { + case extensionNextProtoNeg: + m.nextProtoNeg = true + d := data + for len(d) > 0 { + l := int(d[0]) + d = d[1:] + if l == 0 || l > len(d) { + return false + } + m.nextProtos = append(m.nextProtos, string(d[0:l])) + d = d[l:] + } + case extensionStatusRequest: + if length > 0 { + return false + } + m.ocspStapling = true + } + data = data[length:] + } + + return true +} + +type certificateMsg struct { + raw []byte + certificates [][]byte +} + +func (m *certificateMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + var i int + for _, slice := range m.certificates { + i += len(slice) + } + + length := 3 + 3*len(m.certificates) + i + x = make([]byte, 4+length) + x[0] = typeCertificate + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + certificateOctets := length - 3 + x[4] = uint8(certificateOctets >> 16) + x[5] = uint8(certificateOctets >> 8) + x[6] = uint8(certificateOctets) + + y := x[7:] + for _, slice := range m.certificates { + y[0] = uint8(len(slice) >> 16) + y[1] = uint8(len(slice) >> 8) + y[2] = uint8(len(slice)) + copy(y[3:], slice) + y = y[3+len(slice):] + } + + m.raw = x + return +} + +func (m *certificateMsg) unmarshal(data []byte) bool { + if len(data) < 7 { + return false + } + + m.raw = data + certsLen := uint32(data[4])<<16 | uint32(data[5])<<8 | uint32(data[6]) + if uint32(len(data)) != certsLen+7 { + return false + } + + numCerts := 0 + d := data[7:] + for certsLen > 0 { + if len(d) < 4 { + return false + } + certLen := uint32(d[0])<<24 | uint32(d[1])<<8 | uint32(d[2]) + if uint32(len(d)) < 3+certLen { + return false + } + d = d[3+certLen:] + certsLen -= 3 + certLen + numCerts++ + } + + m.certificates = make([][]byte, numCerts) + d = data[7:] + for i := 0; i < numCerts; i++ { + certLen := uint32(d[0])<<24 | uint32(d[1])<<8 | uint32(d[2]) + m.certificates[i] = d[3 : 3+certLen] + d = d[3+certLen:] + } + + return true +} + +type serverKeyExchangeMsg struct { + raw []byte + key []byte +} + +func (m *serverKeyExchangeMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + length := len(m.key) + x := make([]byte, length+4) + x[0] = typeServerKeyExchange + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + copy(x[4:], m.key) + + m.raw = x + return x +} + +func (m *serverKeyExchangeMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 4 { + return false + } + m.key = data[4:] + return true +} + +type certificateStatusMsg struct { + raw []byte + statusType uint8 + response []byte +} + +func (m *certificateStatusMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + + var x []byte + if m.statusType == statusTypeOCSP { + x = make([]byte, 4+4+len(m.response)) + x[0] = typeCertificateStatus + l := len(m.response) + 4 + x[1] = byte(l >> 16) + x[2] = byte(l >> 8) + x[3] = byte(l) + x[4] = statusTypeOCSP + + l -= 4 + x[5] = byte(l >> 16) + x[6] = byte(l >> 8) + x[7] = byte(l) + copy(x[8:], m.response) + } else { + x = []byte{typeCertificateStatus, 0, 0, 1, m.statusType} + } + + m.raw = x + return x +} + +func (m *certificateStatusMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 5 { + return false + } + m.statusType = data[4] + + m.response = nil + if m.statusType == statusTypeOCSP { + if len(data) < 8 { + return false + } + respLen := uint32(data[5])<<16 | uint32(data[6])<<8 | uint32(data[7]) + if uint32(len(data)) != 4+4+respLen { + return false + } + m.response = data[8:] + } + return true +} + +type serverHelloDoneMsg struct{} + +func (m *serverHelloDoneMsg) marshal() []byte { + x := make([]byte, 4) + x[0] = typeServerHelloDone + return x +} + +func (m *serverHelloDoneMsg) unmarshal(data []byte) bool { + return len(data) == 4 +} + +type clientKeyExchangeMsg struct { + raw []byte + ciphertext []byte +} + +func (m *clientKeyExchangeMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + length := len(m.ciphertext) + x := make([]byte, length+4) + x[0] = typeClientKeyExchange + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + copy(x[4:], m.ciphertext) + + m.raw = x + return x +} + +func (m *clientKeyExchangeMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) < 4 { + return false + } + l := int(data[1])<<16 | int(data[2])<<8 | int(data[3]) + if l != len(data)-4 { + return false + } + m.ciphertext = data[4:] + return true +} + +type finishedMsg struct { + raw []byte + verifyData []byte +} + +func (m *finishedMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + x = make([]byte, 16) + x[0] = typeFinished + x[3] = 12 + copy(x[4:], m.verifyData) + m.raw = x + return +} + +func (m *finishedMsg) unmarshal(data []byte) bool { + m.raw = data + if len(data) != 4+12 { + return false + } + m.verifyData = data[4:] + return true +} + +type nextProtoMsg struct { + raw []byte + proto string +} + +func (m *nextProtoMsg) marshal() []byte { + if m.raw != nil { + return m.raw + } + l := len(m.proto) + if l > 255 { + l = 255 + } + + padding := 32 - (l+2)%32 + length := l + padding + 2 + x := make([]byte, length+4) + x[0] = typeNextProtocol + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + y := x[4:] + y[0] = byte(l) + copy(y[1:], []byte(m.proto[0:l])) + y = y[1+l:] + y[0] = byte(padding) + + m.raw = x + + return x +} + +func (m *nextProtoMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 5 { + return false + } + data = data[4:] + protoLen := int(data[0]) + data = data[1:] + if len(data) < protoLen { + return false + } + m.proto = string(data[0:protoLen]) + data = data[protoLen:] + + if len(data) < 1 { + return false + } + paddingLen := int(data[0]) + data = data[1:] + if len(data) != paddingLen { + return false + } + + return true +} + +type certificateRequestMsg struct { + raw []byte + certificateTypes []byte + certificateAuthorities [][]byte +} + +func (m *certificateRequestMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + // See http://tools.ietf.org/html/rfc4346#section-7.4.4 + length := 1 + len(m.certificateTypes) + 2 + for _, ca := range m.certificateAuthorities { + length += 2 + len(ca) + } + + x = make([]byte, 4+length) + x[0] = typeCertificateRequest + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + + x[4] = uint8(len(m.certificateTypes)) + + copy(x[5:], m.certificateTypes) + y := x[5+len(m.certificateTypes):] + + numCA := len(m.certificateAuthorities) + y[0] = uint8(numCA >> 8) + y[1] = uint8(numCA) + y = y[2:] + for _, ca := range m.certificateAuthorities { + y[0] = uint8(len(ca) >> 8) + y[1] = uint8(len(ca)) + y = y[2:] + copy(y, ca) + y = y[len(ca):] + } + + m.raw = x + + return +} + +func (m *certificateRequestMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 5 { + return false + } + + length := uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) + if uint32(len(data))-4 != length { + return false + } + + numCertTypes := int(data[4]) + data = data[5:] + if numCertTypes == 0 || len(data) <= numCertTypes { + return false + } + + m.certificateTypes = make([]byte, numCertTypes) + if copy(m.certificateTypes, data) != numCertTypes { + return false + } + + data = data[numCertTypes:] + if len(data) < 2 { + return false + } + + numCAs := uint16(data[0])<<16 | uint16(data[1]) + data = data[2:] + + m.certificateAuthorities = make([][]byte, numCAs) + for i := uint16(0); i < numCAs; i++ { + if len(data) < 2 { + return false + } + caLen := uint16(data[0])<<16 | uint16(data[1]) + + data = data[2:] + if len(data) < int(caLen) { + return false + } + + ca := make([]byte, caLen) + copy(ca, data) + m.certificateAuthorities[i] = ca + data = data[caLen:] + } + + if len(data) > 0 { + return false + } + + return true +} + +type certificateVerifyMsg struct { + raw []byte + signature []byte +} + +func (m *certificateVerifyMsg) marshal() (x []byte) { + if m.raw != nil { + return m.raw + } + + // See http://tools.ietf.org/html/rfc4346#section-7.4.8 + siglength := len(m.signature) + length := 2 + siglength + x = make([]byte, 4+length) + x[0] = typeCertificateVerify + x[1] = uint8(length >> 16) + x[2] = uint8(length >> 8) + x[3] = uint8(length) + x[4] = uint8(siglength >> 8) + x[5] = uint8(siglength) + copy(x[6:], m.signature) + + m.raw = x + + return +} + +func (m *certificateVerifyMsg) unmarshal(data []byte) bool { + m.raw = data + + if len(data) < 6 { + return false + } + + length := uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) + if uint32(len(data))-4 != length { + return false + } + + siglength := int(data[4])<<8 + int(data[5]) + if len(data)-6 != siglength { + return false + } + + m.signature = data[6:] + + return true +} diff --git a/src/pkg/crypto/tls/handshake_messages_test.go b/src/pkg/crypto/tls/handshake_messages_test.go new file mode 100644 index 000000000..23f729dd9 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_messages_test.go @@ -0,0 +1,206 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "rand" + "reflect" + "testing" + "testing/quick" +) + +var tests = []interface{}{ + &clientHelloMsg{}, + &serverHelloMsg{}, + + &certificateMsg{}, + &certificateRequestMsg{}, + &certificateVerifyMsg{}, + &certificateStatusMsg{}, + &clientKeyExchangeMsg{}, + &finishedMsg{}, + &nextProtoMsg{}, +} + +type testMessage interface { + marshal() []byte + unmarshal([]byte) bool +} + +func TestMarshalUnmarshal(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for i, iface := range tests { + ty := reflect.ValueOf(iface).Type() + + n := 100 + if testing.Short() { + n = 5 + } + for j := 0; j < n; j++ { + v, ok := quick.Value(ty, rand) + if !ok { + t.Errorf("#%d: failed to create value", i) + break + } + + m1 := v.Interface().(testMessage) + marshaled := m1.marshal() + m2 := iface.(testMessage) + if !m2.unmarshal(marshaled) { + t.Errorf("#%d failed to unmarshal %#v %x", i, m1, marshaled) + break + } + m2.marshal() // to fill any marshal cache in the message + + if !reflect.DeepEqual(m1, m2) { + t.Errorf("#%d got:%#v want:%#v %x", i, m2, m1, marshaled) + break + } + + if i >= 2 { + // The first two message types (ClientHello and + // ServerHello) are allowed to have parsable + // prefixes because the extension data is + // optional. + for j := 0; j < len(marshaled); j++ { + if m2.unmarshal(marshaled[0:j]) { + t.Errorf("#%d unmarshaled a prefix of length %d of %#v", i, j, m1) + break + } + } + } + } + } +} + +func TestFuzz(t *testing.T) { + rand := rand.New(rand.NewSource(0)) + for _, iface := range tests { + m := iface.(testMessage) + + for j := 0; j < 1000; j++ { + len := rand.Intn(100) + bytes := randomBytes(len, rand) + // This just looks for crashes due to bounds errors etc. + m.unmarshal(bytes) + } + } +} + +func randomBytes(n int, rand *rand.Rand) []byte { + r := make([]byte, n) + for i := 0; i < n; i++ { + r[i] = byte(rand.Int31()) + } + return r +} + +func randomString(n int, rand *rand.Rand) string { + b := randomBytes(n, rand) + return string(b) +} + +func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &clientHelloMsg{} + m.vers = uint16(rand.Intn(65536)) + m.random = randomBytes(32, rand) + m.sessionId = randomBytes(rand.Intn(32), rand) + m.cipherSuites = make([]uint16, rand.Intn(63)+1) + for i := 0; i < len(m.cipherSuites); i++ { + m.cipherSuites[i] = uint16(rand.Int31()) + } + m.compressionMethods = randomBytes(rand.Intn(63)+1, rand) + if rand.Intn(10) > 5 { + m.nextProtoNeg = true + } + if rand.Intn(10) > 5 { + m.serverName = randomString(rand.Intn(255), rand) + } + m.ocspStapling = rand.Intn(10) > 5 + m.supportedPoints = randomBytes(rand.Intn(5)+1, rand) + m.supportedCurves = make([]uint16, rand.Intn(5)+1) + for i := range m.supportedCurves { + m.supportedCurves[i] = uint16(rand.Intn(30000)) + } + + return reflect.ValueOf(m) +} + +func (*serverHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &serverHelloMsg{} + m.vers = uint16(rand.Intn(65536)) + m.random = randomBytes(32, rand) + m.sessionId = randomBytes(rand.Intn(32), rand) + m.cipherSuite = uint16(rand.Int31()) + m.compressionMethod = uint8(rand.Intn(256)) + + if rand.Intn(10) > 5 { + m.nextProtoNeg = true + + n := rand.Intn(10) + m.nextProtos = make([]string, n) + for i := 0; i < n; i++ { + m.nextProtos[i] = randomString(20, rand) + } + } + + return reflect.ValueOf(m) +} + +func (*certificateMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateMsg{} + numCerts := rand.Intn(20) + m.certificates = make([][]byte, numCerts) + for i := 0; i < numCerts; i++ { + m.certificates[i] = randomBytes(rand.Intn(10)+1, rand) + } + return reflect.ValueOf(m) +} + +func (*certificateRequestMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateRequestMsg{} + m.certificateTypes = randomBytes(rand.Intn(5)+1, rand) + numCAs := rand.Intn(100) + m.certificateAuthorities = make([][]byte, numCAs) + for i := 0; i < numCAs; i++ { + m.certificateAuthorities[i] = randomBytes(rand.Intn(15)+1, rand) + } + return reflect.ValueOf(m) +} + +func (*certificateVerifyMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateVerifyMsg{} + m.signature = randomBytes(rand.Intn(15)+1, rand) + return reflect.ValueOf(m) +} + +func (*certificateStatusMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &certificateStatusMsg{} + if rand.Intn(10) > 5 { + m.statusType = statusTypeOCSP + m.response = randomBytes(rand.Intn(10)+1, rand) + } else { + m.statusType = 42 + } + return reflect.ValueOf(m) +} + +func (*clientKeyExchangeMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &clientKeyExchangeMsg{} + m.ciphertext = randomBytes(rand.Intn(1000)+1, rand) + return reflect.ValueOf(m) +} + +func (*finishedMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &finishedMsg{} + m.verifyData = randomBytes(12, rand) + return reflect.ValueOf(m) +} + +func (*nextProtoMsg) Generate(rand *rand.Rand, size int) reflect.Value { + m := &nextProtoMsg{} + m.proto = randomString(rand.Intn(255), rand) + return reflect.ValueOf(m) +} diff --git a/src/pkg/crypto/tls/handshake_server.go b/src/pkg/crypto/tls/handshake_server.go new file mode 100644 index 000000000..44a324041 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_server.go @@ -0,0 +1,298 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto" + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "io" + "os" +) + +func (c *Conn) serverHandshake() os.Error { + config := c.config + msg, err := c.readHandshake() + if err != nil { + return err + } + clientHello, ok := msg.(*clientHelloMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + vers, ok := mutualVersion(clientHello.vers) + if !ok { + return c.sendAlert(alertProtocolVersion) + } + c.vers = vers + c.haveVers = true + + finishedHash := newFinishedHash() + finishedHash.Write(clientHello.marshal()) + + hello := new(serverHelloMsg) + + supportedCurve := false +Curves: + for _, curve := range clientHello.supportedCurves { + switch curve { + case curveP256, curveP384, curveP521: + supportedCurve = true + break Curves + } + } + + supportedPointFormat := false + for _, pointFormat := range clientHello.supportedPoints { + if pointFormat == pointFormatUncompressed { + supportedPointFormat = true + break + } + } + + ellipticOk := supportedCurve && supportedPointFormat + + var suite *cipherSuite + var suiteId uint16 +FindCipherSuite: + for _, id := range clientHello.cipherSuites { + for _, supported := range config.cipherSuites() { + if id == supported { + suite = cipherSuites[id] + // Don't select a ciphersuite which we can't + // support for this client. + if suite.elliptic && !ellipticOk { + continue + } + suiteId = id + break FindCipherSuite + } + } + } + + foundCompression := false + // We only support null compression, so check that the client offered it. + for _, compression := range clientHello.compressionMethods { + if compression == compressionNone { + foundCompression = true + break + } + } + + if suite == nil || !foundCompression { + return c.sendAlert(alertHandshakeFailure) + } + + hello.vers = vers + hello.cipherSuite = suiteId + t := uint32(config.time()) + hello.random = make([]byte, 32) + hello.random[0] = byte(t >> 24) + hello.random[1] = byte(t >> 16) + hello.random[2] = byte(t >> 8) + hello.random[3] = byte(t) + _, err = io.ReadFull(config.rand(), hello.random[4:]) + if err != nil { + return c.sendAlert(alertInternalError) + } + hello.compressionMethod = compressionNone + if clientHello.nextProtoNeg { + hello.nextProtoNeg = true + hello.nextProtos = config.NextProtos + } + if clientHello.ocspStapling && len(config.Certificates[0].OCSPStaple) > 0 { + hello.ocspStapling = true + } + + finishedHash.Write(hello.marshal()) + c.writeRecord(recordTypeHandshake, hello.marshal()) + + if len(config.Certificates) == 0 { + return c.sendAlert(alertInternalError) + } + + certMsg := new(certificateMsg) + certMsg.certificates = config.Certificates[0].Certificate + finishedHash.Write(certMsg.marshal()) + c.writeRecord(recordTypeHandshake, certMsg.marshal()) + + if hello.ocspStapling { + certStatus := new(certificateStatusMsg) + certStatus.statusType = statusTypeOCSP + certStatus.response = config.Certificates[0].OCSPStaple + finishedHash.Write(certStatus.marshal()) + c.writeRecord(recordTypeHandshake, certStatus.marshal()) + } + + keyAgreement := suite.ka() + + skx, err := keyAgreement.generateServerKeyExchange(config, clientHello, hello) + if err != nil { + c.sendAlert(alertHandshakeFailure) + return err + } + if skx != nil { + finishedHash.Write(skx.marshal()) + c.writeRecord(recordTypeHandshake, skx.marshal()) + } + + if config.AuthenticateClient { + // Request a client certificate + certReq := new(certificateRequestMsg) + certReq.certificateTypes = []byte{certTypeRSASign} + // An empty list of certificateAuthorities signals to + // the client that it may send any certificate in response + // to our request. + + finishedHash.Write(certReq.marshal()) + c.writeRecord(recordTypeHandshake, certReq.marshal()) + } + + helloDone := new(serverHelloDoneMsg) + finishedHash.Write(helloDone.marshal()) + c.writeRecord(recordTypeHandshake, helloDone.marshal()) + + var pub *rsa.PublicKey + if config.AuthenticateClient { + // Get client certificate + msg, err = c.readHandshake() + if err != nil { + return err + } + certMsg, ok = msg.(*certificateMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(certMsg.marshal()) + + certs := make([]*x509.Certificate, len(certMsg.certificates)) + for i, asn1Data := range certMsg.certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("could not parse client's certificate: " + err.String()) + } + certs[i] = cert + } + + // TODO(agl): do better validation of certs: max path length, name restrictions etc. + for i := 1; i < len(certs); i++ { + if err := certs[i-1].CheckSignatureFrom(certs[i]); err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("could not validate certificate signature: " + err.String()) + } + } + + if len(certs) > 0 { + key, ok := certs[0].PublicKey.(*rsa.PublicKey) + if !ok { + return c.sendAlert(alertUnsupportedCertificate) + } + pub = key + c.peerCertificates = certs + } + } + + // Get client key exchange + msg, err = c.readHandshake() + if err != nil { + return err + } + ckx, ok := msg.(*clientKeyExchangeMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(ckx.marshal()) + + // If we received a client cert in response to our certificate request message, + // the client will send us a certificateVerifyMsg immediately after the + // clientKeyExchangeMsg. This message is a MD5SHA1 digest of all preceding + // handshake-layer messages that is signed using the private key corresponding + // to the client's certificate. This allows us to verify that the client is in + // possession of the private key of the certificate. + if len(c.peerCertificates) > 0 { + msg, err = c.readHandshake() + if err != nil { + return err + } + certVerify, ok := msg.(*certificateVerifyMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + digest := make([]byte, 36) + copy(digest[0:16], finishedHash.serverMD5.Sum()) + copy(digest[16:36], finishedHash.serverSHA1.Sum()) + err = rsa.VerifyPKCS1v15(pub, crypto.MD5SHA1, digest, certVerify.signature) + if err != nil { + c.sendAlert(alertBadCertificate) + return os.NewError("could not validate signature of connection nonces: " + err.String()) + } + + finishedHash.Write(certVerify.marshal()) + } + + preMasterSecret, err := keyAgreement.processClientKeyExchange(config, ckx) + if err != nil { + c.sendAlert(alertHandshakeFailure) + return err + } + + masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV := + keysFromPreMasterSecret10(preMasterSecret, clientHello.random, hello.random, suite.macLen, suite.keyLen, suite.ivLen) + + clientCipher := suite.cipher(clientKey, clientIV, true /* for reading */ ) + clientHash := suite.mac(clientMAC) + c.in.prepareCipherSpec(clientCipher, clientHash) + c.readRecord(recordTypeChangeCipherSpec) + if err := c.error(); err != nil { + return err + } + + if hello.nextProtoNeg { + msg, err = c.readHandshake() + if err != nil { + return err + } + nextProto, ok := msg.(*nextProtoMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + finishedHash.Write(nextProto.marshal()) + c.clientProtocol = nextProto.proto + } + + msg, err = c.readHandshake() + if err != nil { + return err + } + clientFinished, ok := msg.(*finishedMsg) + if !ok { + return c.sendAlert(alertUnexpectedMessage) + } + + verify := finishedHash.clientSum(masterSecret) + if len(verify) != len(clientFinished.verifyData) || + subtle.ConstantTimeCompare(verify, clientFinished.verifyData) != 1 { + return c.sendAlert(alertHandshakeFailure) + } + + finishedHash.Write(clientFinished.marshal()) + + serverCipher := suite.cipher(serverKey, serverIV, false /* not for reading */ ) + serverHash := suite.mac(serverMAC) + c.out.prepareCipherSpec(serverCipher, serverHash) + c.writeRecord(recordTypeChangeCipherSpec, []byte{1}) + + finished := new(finishedMsg) + finished.verifyData = finishedHash.serverSum(masterSecret) + c.writeRecord(recordTypeHandshake, finished.marshal()) + + c.handshakeComplete = true + c.cipherSuite = suiteId + + return nil +} diff --git a/src/pkg/crypto/tls/handshake_server_test.go b/src/pkg/crypto/tls/handshake_server_test.go new file mode 100644 index 000000000..b77646e43 --- /dev/null +++ b/src/pkg/crypto/tls/handshake_server_test.go @@ -0,0 +1,517 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "big" + "bytes" + "crypto/rsa" + "encoding/hex" + "flag" + "io" + "net" + "os" + "testing" +) + +type zeroSource struct{} + +func (zeroSource) Read(b []byte) (n int, err os.Error) { + for i := range b { + b[i] = 0 + } + + return len(b), nil +} + +var testConfig *Config + +func init() { + testConfig = new(Config) + testConfig.Time = func() int64 { return 0 } + testConfig.Rand = zeroSource{} + testConfig.Certificates = make([]Certificate, 1) + testConfig.Certificates[0].Certificate = [][]byte{testCertificate} + testConfig.Certificates[0].PrivateKey = testPrivateKey + testConfig.CipherSuites = []uint16{TLS_RSA_WITH_RC4_128_SHA} +} + +func testClientHelloFailure(t *testing.T, m handshakeMessage, expected os.Error) { + // Create in-memory network connection, + // send message to server. Should return + // expected error. + c, s := net.Pipe() + go func() { + cli := Client(c, testConfig) + if ch, ok := m.(*clientHelloMsg); ok { + cli.vers = ch.vers + } + cli.writeRecord(recordTypeHandshake, m.marshal()) + c.Close() + }() + err := Server(s, testConfig).Handshake() + s.Close() + if e, ok := err.(*net.OpError); !ok || e.Error != expected { + t.Errorf("Got error: %s; expected: %s", err, expected) + } +} + +func TestSimpleError(t *testing.T) { + testClientHelloFailure(t, &serverHelloDoneMsg{}, alertUnexpectedMessage) +} + +var badProtocolVersions = []uint16{0x0000, 0x0005, 0x0100, 0x0105, 0x0200, 0x0205, 0x0300} + +func TestRejectBadProtocolVersion(t *testing.T) { + for _, v := range badProtocolVersions { + testClientHelloFailure(t, &clientHelloMsg{vers: v}, alertProtocolVersion) + } +} + +func TestNoSuiteOverlap(t *testing.T) { + clientHello := &clientHelloMsg{nil, 0x0301, nil, nil, []uint16{0xff00}, []uint8{0}, false, "", false, nil, nil} + testClientHelloFailure(t, clientHello, alertHandshakeFailure) + +} + +func TestNoCompressionOverlap(t *testing.T) { + clientHello := &clientHelloMsg{nil, 0x0301, nil, nil, []uint16{TLS_RSA_WITH_RC4_128_SHA}, []uint8{0xff}, false, "", false, nil, nil} + testClientHelloFailure(t, clientHello, alertHandshakeFailure) +} + +func TestAlertForwarding(t *testing.T) { + c, s := net.Pipe() + go func() { + Client(c, testConfig).sendAlert(alertUnknownCA) + c.Close() + }() + + err := Server(s, testConfig).Handshake() + s.Close() + if e, ok := err.(*net.OpError); !ok || e.Error != os.Error(alertUnknownCA) { + t.Errorf("Got error: %s; expected: %s", err, alertUnknownCA) + } +} + +func TestClose(t *testing.T) { + c, s := net.Pipe() + go c.Close() + + err := Server(s, testConfig).Handshake() + s.Close() + if err != os.EOF { + t.Errorf("Got error: %s; expected: %s", err, os.EOF) + } +} + +func testServerScript(t *testing.T, name string, serverScript [][]byte, config *Config) { + c, s := net.Pipe() + srv := Server(s, config) + go func() { + srv.Write([]byte("hello, world\n")) + srv.Close() + }() + + defer c.Close() + for i, b := range serverScript { + if i%2 == 0 { + c.Write(b) + continue + } + bb := make([]byte, len(b)) + _, err := io.ReadFull(c, bb) + if err != nil { + t.Fatalf("%s #%d: %s", name, i, err) + } + if !bytes.Equal(b, bb) { + t.Fatalf("%s #%d: mismatch on read: got:%x want:%x", name, i, bb, b) + } + } +} + +func TestHandshakeServerRC4(t *testing.T) { + testServerScript(t, "RC4", rc4ServerScript, testConfig) +} + +func TestHandshakeServerAES(t *testing.T) { + aesConfig := new(Config) + *aesConfig = *testConfig + aesConfig.CipherSuites = []uint16{TLS_RSA_WITH_AES_128_CBC_SHA} + testServerScript(t, "AES", aesServerScript, aesConfig) +} + +var serve = flag.Bool("serve", false, "run a TLS server on :10443") + +func TestRunServer(t *testing.T) { + if !*serve { + return + } + + l, err := Listen("tcp", ":10443", testConfig) + if err != nil { + t.Fatal(err) + } + + for { + c, err := l.Accept() + if err != nil { + break + } + _, err = c.Write([]byte("hello, world\n")) + if err != nil { + t.Errorf("error from TLS: %s", err) + break + } + c.Close() + } +} + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +func fromHex(s string) []byte { + b, _ := hex.DecodeString(s) + return b +} + +var testCertificate = fromHex("308202b030820219a00302010202090085b0bba48a7fb8ca300d06092a864886f70d01010505003045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c7464301e170d3130303432343039303933385a170d3131303432343039303933385a3045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c746430819f300d06092a864886f70d010101050003818d0030818902818100bb79d6f517b5e5bf4610d0dc69bee62b07435ad0032d8a7a4385b71452e7a5654c2c78b8238cb5b482e5de1f953b7e62a52ca533d6fe125c7a56fcf506bffa587b263fb5cd04d3d0c921964ac7f4549f5abfef427100fe1899077f7e887d7df10439c4a22edb51c97ce3c04c3b326601cfafb11db8719a1ddbdb896baeda2d790203010001a381a73081a4301d0603551d0e04160414b1ade2855acfcb28db69ce2369ded3268e18883930750603551d23046e306c8014b1ade2855acfcb28db69ce2369ded3268e188839a149a4473045310b3009060355040613024155311330110603550408130a536f6d652d53746174653121301f060355040a1318496e7465726e6574205769646769747320507479204c746482090085b0bba48a7fb8ca300c0603551d13040530030101ff300d06092a864886f70d010105050003818100086c4524c76bb159ab0c52ccf2b014d7879d7a6475b55a9566e4c52b8eae12661feb4f38b36e60d392fdf74108b52513b1187a24fb301dbaed98b917ece7d73159db95d31d78ea50565cd5825a2d5a5f33c4b6d8c97590968c0f5298b5cd981f89205ff2a01ca31b9694dda9fd57e970e8266d71999b266e3850296c90a7bdd9") + +var testPrivateKey = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: bigFromString("131650079503776001033793877885499001334664249354723305978524647182322416328664556247316495448366990052837680518067798333412266673813370895702118944398081598789828837447552603077848001020611640547221687072142537202428102790818451901395596882588063427854225330436740647715202971973145151161964464812406232198521"), + E: 65537, + }, + D: bigFromString("29354450337804273969007277378287027274721892607543397931919078829901848876371746653677097639302788129485893852488285045793268732234230875671682624082413996177431586734171663258657462237320300610850244186316880055243099640544518318093544057213190320837094958164973959123058337475052510833916491060913053867729"), + Primes: []*big.Int{ + bigFromString("11969277782311800166562047708379380720136961987713178380670422671426759650127150688426177829077494755200794297055316163155755835813760102405344560929062149"), + bigFromString("10998999429884441391899182616418192492905073053684657075974935218461686523870125521822756579792315215543092255516093840728890783887287417039645833477273829"), + }, +} + +// Script of interaction with gnutls implementation. +// The values for this test are obtained by building and running in server mode: +// % gotest -test.run "TestRunServer" -serve +// and then: +// % gnutls-cli --insecure --debug 100 -p 10443 localhost > /tmp/log 2>&1 +// % python parse-gnutls-cli-debug-log.py < /tmp/log +var rc4ServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, + 0x7b, 0x03, 0x02, 0x4d, 0x08, 0x1f, 0x5a, 0x7a, + 0x0a, 0x92, 0x2f, 0xf0, 0x73, 0x16, 0x3a, 0x88, + 0x14, 0x85, 0x4c, 0x98, 0x15, 0x7b, 0x65, 0xe0, + 0x78, 0xd0, 0xed, 0xd0, 0xf3, 0x65, 0x20, 0xeb, + 0x80, 0xd1, 0x0b, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x1e, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0xff, + 0x01, 0x00, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0x3c, 0x13, 0xd7, 0x12, 0xc1, + 0x6a, 0xf0, 0x3f, 0x8c, 0xa1, 0x35, 0x5d, 0xc5, + 0x89, 0x1e, 0x9e, 0xcd, 0x32, 0xc7, 0x9e, 0xe6, + 0xae, 0xd5, 0xf1, 0xbf, 0x70, 0xd7, 0xa9, 0xef, + 0x2c, 0x4c, 0xf4, 0x22, 0xbc, 0x17, 0x17, 0xaa, + 0x05, 0xf3, 0x9f, 0x80, 0xf2, 0xe9, 0x82, 0x2f, + 0x2a, 0x15, 0x54, 0x0d, 0x16, 0x0e, 0x77, 0x4c, + 0x28, 0x3c, 0x03, 0x2d, 0x2d, 0xd7, 0xc8, 0x64, + 0xd9, 0x59, 0x4b, 0x1c, 0xf4, 0xde, 0xff, 0x2f, + 0xbc, 0x94, 0xaf, 0x18, 0x26, 0x37, 0xce, 0x4f, + 0x84, 0x74, 0x2e, 0x45, 0x66, 0x7c, 0x0c, 0x54, + 0x46, 0x36, 0x5f, 0x65, 0x21, 0x7b, 0x83, 0x8c, + 0x6d, 0x76, 0xcd, 0x0d, 0x9f, 0xda, 0x1c, 0xa4, + 0x6e, 0xfe, 0xb1, 0xf7, 0x09, 0x0d, 0xfb, 0x74, + 0x66, 0x34, 0x99, 0x89, 0x7f, 0x5f, 0x77, 0x87, + 0x4a, 0x66, 0x4b, 0xa9, 0x59, 0x57, 0xe3, 0x56, + 0x0d, 0xdd, 0xd8, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x24, 0xc0, 0x4e, + 0xd3, 0x0f, 0xb5, 0xc0, 0x57, 0xa6, 0x18, 0x80, + 0x80, 0x6b, 0x49, 0xfe, 0xbd, 0x3a, 0x7a, 0x2c, + 0xef, 0x70, 0xb5, 0x1c, 0xd2, 0xdf, 0x5f, 0x78, + 0x5a, 0xd8, 0x4f, 0xa0, 0x95, 0xb4, 0xb3, 0xb5, + 0xaa, 0x3b, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x24, 0x9d, 0xc9, 0xda, 0xdf, 0xeb, + 0xc8, 0xdb, 0xf8, 0x94, 0xa5, 0xef, 0xd5, 0xfc, + 0x89, 0x01, 0x64, 0x30, 0x77, 0x5a, 0x18, 0x4b, + 0x16, 0x79, 0x9c, 0xf6, 0xf5, 0x09, 0x22, 0x12, + 0x4c, 0x3e, 0xa8, 0x8e, 0x91, 0xa5, 0x24, + }, +} + +var aesServerScript = [][]byte{ + { + 0x16, 0x03, 0x02, 0x00, 0x7f, 0x01, 0x00, 0x00, + 0x7b, 0x03, 0x02, 0x4d, 0x08, 0x2d, 0x0b, 0xb3, + 0x57, 0x85, 0x71, 0x4b, 0xfb, 0x34, 0xab, 0x16, + 0xd4, 0x92, 0x50, 0x81, 0x16, 0x95, 0x11, 0x28, + 0x1a, 0xcb, 0xff, 0x09, 0x4d, 0x23, 0xa6, 0xfe, + 0x2e, 0xbb, 0x78, 0x00, 0x00, 0x34, 0x00, 0x33, + 0x00, 0x45, 0x00, 0x39, 0x00, 0x88, 0x00, 0x16, + 0x00, 0x32, 0x00, 0x44, 0x00, 0x38, 0x00, 0x87, + 0x00, 0x13, 0x00, 0x66, 0x00, 0x90, 0x00, 0x91, + 0x00, 0x8f, 0x00, 0x8e, 0x00, 0x2f, 0x00, 0x41, + 0x00, 0x35, 0x00, 0x84, 0x00, 0x0a, 0x00, 0x05, + 0x00, 0x04, 0x00, 0x8c, 0x00, 0x8d, 0x00, 0x8b, + 0x00, 0x8a, 0x01, 0x00, 0x00, 0x1e, 0x00, 0x09, + 0x00, 0x03, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x0c, 0x00, 0x00, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0xff, + 0x01, 0x00, 0x01, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, + 0x26, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x16, + 0x03, 0x01, 0x02, 0xbe, 0x0b, 0x00, 0x02, 0xba, + 0x00, 0x02, 0xb7, 0x00, 0x02, 0xb4, 0x30, 0x82, + 0x02, 0xb0, 0x30, 0x82, 0x02, 0x19, 0xa0, 0x03, + 0x02, 0x01, 0x02, 0x02, 0x09, 0x00, 0x85, 0xb0, + 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, 0xca, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x45, 0x31, + 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, + 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, + 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, + 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, + 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, + 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, + 0x20, 0x4c, 0x74, 0x64, 0x30, 0x1e, 0x17, 0x0d, + 0x31, 0x30, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, + 0x30, 0x39, 0x33, 0x38, 0x5a, 0x17, 0x0d, 0x31, + 0x31, 0x30, 0x34, 0x32, 0x34, 0x30, 0x39, 0x30, + 0x39, 0x33, 0x38, 0x5a, 0x30, 0x45, 0x31, 0x0b, + 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, + 0x03, 0x55, 0x04, 0x08, 0x13, 0x0a, 0x53, 0x6f, + 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x04, + 0x0a, 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, + 0x4c, 0x74, 0x64, 0x30, 0x81, 0x9f, 0x30, 0x0d, + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, + 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, + 0xbb, 0x79, 0xd6, 0xf5, 0x17, 0xb5, 0xe5, 0xbf, + 0x46, 0x10, 0xd0, 0xdc, 0x69, 0xbe, 0xe6, 0x2b, + 0x07, 0x43, 0x5a, 0xd0, 0x03, 0x2d, 0x8a, 0x7a, + 0x43, 0x85, 0xb7, 0x14, 0x52, 0xe7, 0xa5, 0x65, + 0x4c, 0x2c, 0x78, 0xb8, 0x23, 0x8c, 0xb5, 0xb4, + 0x82, 0xe5, 0xde, 0x1f, 0x95, 0x3b, 0x7e, 0x62, + 0xa5, 0x2c, 0xa5, 0x33, 0xd6, 0xfe, 0x12, 0x5c, + 0x7a, 0x56, 0xfc, 0xf5, 0x06, 0xbf, 0xfa, 0x58, + 0x7b, 0x26, 0x3f, 0xb5, 0xcd, 0x04, 0xd3, 0xd0, + 0xc9, 0x21, 0x96, 0x4a, 0xc7, 0xf4, 0x54, 0x9f, + 0x5a, 0xbf, 0xef, 0x42, 0x71, 0x00, 0xfe, 0x18, + 0x99, 0x07, 0x7f, 0x7e, 0x88, 0x7d, 0x7d, 0xf1, + 0x04, 0x39, 0xc4, 0xa2, 0x2e, 0xdb, 0x51, 0xc9, + 0x7c, 0xe3, 0xc0, 0x4c, 0x3b, 0x32, 0x66, 0x01, + 0xcf, 0xaf, 0xb1, 0x1d, 0xb8, 0x71, 0x9a, 0x1d, + 0xdb, 0xdb, 0x89, 0x6b, 0xae, 0xda, 0x2d, 0x79, + 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, 0x81, 0xa7, + 0x30, 0x81, 0xa4, 0x30, 0x1d, 0x06, 0x03, 0x55, + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xb1, 0xad, + 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, 0x69, + 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, 0x18, + 0x88, 0x39, 0x30, 0x75, 0x06, 0x03, 0x55, 0x1d, + 0x23, 0x04, 0x6e, 0x30, 0x6c, 0x80, 0x14, 0xb1, + 0xad, 0xe2, 0x85, 0x5a, 0xcf, 0xcb, 0x28, 0xdb, + 0x69, 0xce, 0x23, 0x69, 0xde, 0xd3, 0x26, 0x8e, + 0x18, 0x88, 0x39, 0xa1, 0x49, 0xa4, 0x47, 0x30, + 0x45, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, + 0x04, 0x06, 0x13, 0x02, 0x41, 0x55, 0x31, 0x13, + 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, 0x13, + 0x0a, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x06, + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x18, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x20, 0x57, + 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x82, 0x09, + 0x00, 0x85, 0xb0, 0xbb, 0xa4, 0x8a, 0x7f, 0xb8, + 0xca, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, + 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x03, 0x81, + 0x81, 0x00, 0x08, 0x6c, 0x45, 0x24, 0xc7, 0x6b, + 0xb1, 0x59, 0xab, 0x0c, 0x52, 0xcc, 0xf2, 0xb0, + 0x14, 0xd7, 0x87, 0x9d, 0x7a, 0x64, 0x75, 0xb5, + 0x5a, 0x95, 0x66, 0xe4, 0xc5, 0x2b, 0x8e, 0xae, + 0x12, 0x66, 0x1f, 0xeb, 0x4f, 0x38, 0xb3, 0x6e, + 0x60, 0xd3, 0x92, 0xfd, 0xf7, 0x41, 0x08, 0xb5, + 0x25, 0x13, 0xb1, 0x18, 0x7a, 0x24, 0xfb, 0x30, + 0x1d, 0xba, 0xed, 0x98, 0xb9, 0x17, 0xec, 0xe7, + 0xd7, 0x31, 0x59, 0xdb, 0x95, 0xd3, 0x1d, 0x78, + 0xea, 0x50, 0x56, 0x5c, 0xd5, 0x82, 0x5a, 0x2d, + 0x5a, 0x5f, 0x33, 0xc4, 0xb6, 0xd8, 0xc9, 0x75, + 0x90, 0x96, 0x8c, 0x0f, 0x52, 0x98, 0xb5, 0xcd, + 0x98, 0x1f, 0x89, 0x20, 0x5f, 0xf2, 0xa0, 0x1c, + 0xa3, 0x1b, 0x96, 0x94, 0xdd, 0xa9, 0xfd, 0x57, + 0xe9, 0x70, 0xe8, 0x26, 0x6d, 0x71, 0x99, 0x9b, + 0x26, 0x6e, 0x38, 0x50, 0x29, 0x6c, 0x90, 0xa7, + 0xbd, 0xd9, 0x16, 0x03, 0x01, 0x00, 0x04, 0x0e, + 0x00, 0x00, 0x00, + }, + + { + 0x16, 0x03, 0x01, 0x00, 0x86, 0x10, 0x00, 0x00, + 0x82, 0x00, 0x80, 0x71, 0x9c, 0xe7, 0x23, 0xfc, + 0xb9, 0x19, 0x29, 0x82, 0xbf, 0xef, 0x08, 0xf7, + 0x99, 0x36, 0xc3, 0x4c, 0x6f, 0x05, 0xd2, 0x8b, + 0x62, 0x2b, 0x19, 0x9b, 0x7f, 0xc0, 0xcc, 0x48, + 0x30, 0x5f, 0xcd, 0xc3, 0x70, 0x55, 0x53, 0x73, + 0xfa, 0x79, 0x74, 0xf3, 0xa3, 0x76, 0x9f, 0xa1, + 0x7f, 0x98, 0xc2, 0xc0, 0xe3, 0xc5, 0xa0, 0x31, + 0x2f, 0xa6, 0xe8, 0x1e, 0x61, 0x46, 0xb3, 0x9b, + 0x4b, 0x16, 0xf1, 0x2d, 0xc7, 0x63, 0x7f, 0x79, + 0x22, 0x30, 0xd1, 0xf2, 0xfc, 0x77, 0x98, 0x0a, + 0x16, 0x11, 0x63, 0x71, 0x7f, 0x70, 0xef, 0x16, + 0xbb, 0x39, 0x87, 0x34, 0xac, 0x49, 0xbd, 0x07, + 0x67, 0xcb, 0x9c, 0xcc, 0xde, 0xef, 0xb1, 0xe0, + 0xdb, 0x01, 0xb5, 0x35, 0xa9, 0xb3, 0x10, 0x0c, + 0x4b, 0xee, 0xb3, 0x4e, 0xfd, 0xbe, 0x15, 0x27, + 0xf0, 0x46, 0xb2, 0x38, 0xba, 0x5f, 0xcc, 0x89, + 0xec, 0x29, 0x82, 0x14, 0x03, 0x01, 0x00, 0x01, + 0x01, 0x16, 0x03, 0x01, 0x00, 0x30, 0x3c, 0xfb, + 0xa4, 0x12, 0xcb, 0x00, 0xf9, 0x57, 0x7e, 0x9b, + 0xc9, 0xdc, 0x0c, 0xba, 0x9a, 0x81, 0x62, 0xfb, + 0x26, 0x13, 0x53, 0xfe, 0xaa, 0xcc, 0x82, 0xbb, + 0xb6, 0x67, 0x7f, 0x39, 0xbe, 0x4d, 0xbb, 0xc0, + 0x6c, 0x24, 0x31, 0x83, 0xa5, 0x50, 0x3a, 0x75, + 0x32, 0x64, 0xb5, 0xdb, 0xbe, 0x0a, + }, + + { + 0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03, + 0x01, 0x00, 0x30, 0x43, 0x24, 0x42, 0x55, 0x08, + 0xe4, 0xc2, 0x15, 0xc9, 0xdb, 0x71, 0x69, 0xee, + 0x09, 0xc5, 0x1c, 0xfd, 0x46, 0x10, 0xa0, 0x68, + 0x21, 0xf2, 0x48, 0xac, 0x6c, 0xc0, 0x2b, 0x62, + 0x07, 0x8f, 0x48, 0x33, 0x0a, 0x6b, 0x62, 0x28, + 0x2e, 0x2c, 0xad, 0xcb, 0x34, 0x85, 0xca, 0x2e, + 0xcd, 0x84, 0xf0, + }, +} diff --git a/src/pkg/crypto/tls/key_agreement.go b/src/pkg/crypto/tls/key_agreement.go new file mode 100644 index 000000000..a40d18fd9 --- /dev/null +++ b/src/pkg/crypto/tls/key_agreement.go @@ -0,0 +1,245 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "big" + "crypto" + "crypto/elliptic" + "crypto/md5" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "io" + "os" +) + +// rsaKeyAgreement implements the standard TLS key agreement where the client +// encrypts the pre-master secret to the server's public key. +type rsaKeyAgreement struct{} + +func (ka rsaKeyAgreement) generateServerKeyExchange(config *Config, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) { + return nil, nil +} + +func (ka rsaKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { + preMasterSecret := make([]byte, 48) + _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) + if err != nil { + return nil, err + } + + if len(ckx.ciphertext) < 2 { + return nil, os.NewError("bad ClientKeyExchange") + } + ciphertextLen := int(ckx.ciphertext[0])<<8 | int(ckx.ciphertext[1]) + if ciphertextLen != len(ckx.ciphertext)-2 { + return nil, os.NewError("bad ClientKeyExchange") + } + ciphertext := ckx.ciphertext[2:] + + err = rsa.DecryptPKCS1v15SessionKey(config.rand(), config.Certificates[0].PrivateKey, ciphertext, preMasterSecret) + if err != nil { + return nil, err + } + // We don't check the version number in the premaster secret. For one, + // by checking it, we would leak information about the validity of the + // encrypted pre-master secret. Secondly, it provides only a small + // benefit against a downgrade attack and some implementations send the + // wrong version anyway. See the discussion at the end of section + // 7.4.7.1 of RFC 4346. + return preMasterSecret, nil +} + +func (ka rsaKeyAgreement) processServerKeyExchange(config *Config, clientHello *clientHelloMsg, serverHello *serverHelloMsg, cert *x509.Certificate, skx *serverKeyExchangeMsg) os.Error { + return os.NewError("unexpected ServerKeyExchange") +} + +func (ka rsaKeyAgreement) generateClientKeyExchange(config *Config, clientHello *clientHelloMsg, cert *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) { + preMasterSecret := make([]byte, 48) + preMasterSecret[0] = byte(clientHello.vers >> 8) + preMasterSecret[1] = byte(clientHello.vers) + _, err := io.ReadFull(config.rand(), preMasterSecret[2:]) + if err != nil { + return nil, nil, err + } + + encrypted, err := rsa.EncryptPKCS1v15(config.rand(), cert.PublicKey.(*rsa.PublicKey), preMasterSecret) + if err != nil { + return nil, nil, err + } + ckx := new(clientKeyExchangeMsg) + ckx.ciphertext = make([]byte, len(encrypted)+2) + ckx.ciphertext[0] = byte(len(encrypted) >> 8) + ckx.ciphertext[1] = byte(len(encrypted)) + copy(ckx.ciphertext[2:], encrypted) + return preMasterSecret, ckx, nil +} + +// md5SHA1Hash implements TLS 1.0's hybrid hash function which consists of the +// concatenation of an MD5 and SHA1 hash. +func md5SHA1Hash(slices ...[]byte) []byte { + md5sha1 := make([]byte, md5.Size+sha1.Size) + hmd5 := md5.New() + for _, slice := range slices { + hmd5.Write(slice) + } + copy(md5sha1, hmd5.Sum()) + + hsha1 := sha1.New() + for _, slice := range slices { + hsha1.Write(slice) + } + copy(md5sha1[md5.Size:], hsha1.Sum()) + return md5sha1 +} + +// ecdheRSAKeyAgreement implements a TLS key agreement where the server +// generates a ephemeral EC public/private key pair and signs it. The +// pre-master secret is then calculated using ECDH. +type ecdheRSAKeyAgreement struct { + privateKey []byte + curve *elliptic.Curve + x, y *big.Int +} + +func (ka *ecdheRSAKeyAgreement) generateServerKeyExchange(config *Config, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, os.Error) { + var curveid uint16 + +Curve: + for _, c := range clientHello.supportedCurves { + switch c { + case curveP256: + ka.curve = elliptic.P256() + curveid = c + break Curve + case curveP384: + ka.curve = elliptic.P384() + curveid = c + break Curve + case curveP521: + ka.curve = elliptic.P521() + curveid = c + break Curve + } + } + + var x, y *big.Int + var err os.Error + ka.privateKey, x, y, err = ka.curve.GenerateKey(config.rand()) + if err != nil { + return nil, err + } + ecdhePublic := ka.curve.Marshal(x, y) + + // http://tools.ietf.org/html/rfc4492#section-5.4 + serverECDHParams := make([]byte, 1+2+1+len(ecdhePublic)) + serverECDHParams[0] = 3 // named curve + serverECDHParams[1] = byte(curveid >> 8) + serverECDHParams[2] = byte(curveid) + serverECDHParams[3] = byte(len(ecdhePublic)) + copy(serverECDHParams[4:], ecdhePublic) + + md5sha1 := md5SHA1Hash(clientHello.random, hello.random, serverECDHParams) + sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, crypto.MD5SHA1, md5sha1) + if err != nil { + return nil, os.NewError("failed to sign ECDHE parameters: " + err.String()) + } + + skx := new(serverKeyExchangeMsg) + skx.key = make([]byte, len(serverECDHParams)+2+len(sig)) + copy(skx.key, serverECDHParams) + k := skx.key[len(serverECDHParams):] + k[0] = byte(len(sig) >> 8) + k[1] = byte(len(sig)) + copy(k[2:], sig) + + return skx, nil +} + +func (ka *ecdheRSAKeyAgreement) processClientKeyExchange(config *Config, ckx *clientKeyExchangeMsg) ([]byte, os.Error) { + if len(ckx.ciphertext) == 0 || int(ckx.ciphertext[0]) != len(ckx.ciphertext)-1 { + return nil, os.NewError("bad ClientKeyExchange") + } + x, y := ka.curve.Unmarshal(ckx.ciphertext[1:]) + if x == nil { + return nil, os.NewError("bad ClientKeyExchange") + } + x, _ = ka.curve.ScalarMult(x, y, ka.privateKey) + preMasterSecret := make([]byte, (ka.curve.BitSize+7)>>3) + xBytes := x.Bytes() + copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) + + return preMasterSecret, nil +} + +var errServerKeyExchange = os.NewError("invalid ServerKeyExchange") + +func (ka *ecdheRSAKeyAgreement) processServerKeyExchange(config *Config, clientHello *clientHelloMsg, serverHello *serverHelloMsg, cert *x509.Certificate, skx *serverKeyExchangeMsg) os.Error { + if len(skx.key) < 4 { + return errServerKeyExchange + } + if skx.key[0] != 3 { // named curve + return os.NewError("server selected unsupported curve") + } + curveid := uint16(skx.key[1])<<8 | uint16(skx.key[2]) + + switch curveid { + case curveP256: + ka.curve = elliptic.P256() + case curveP384: + ka.curve = elliptic.P384() + case curveP521: + ka.curve = elliptic.P521() + default: + return os.NewError("server selected unsupported curve") + } + + publicLen := int(skx.key[3]) + if publicLen+4 > len(skx.key) { + return errServerKeyExchange + } + ka.x, ka.y = ka.curve.Unmarshal(skx.key[4 : 4+publicLen]) + if ka.x == nil { + return errServerKeyExchange + } + serverECDHParams := skx.key[:4+publicLen] + + sig := skx.key[4+publicLen:] + if len(sig) < 2 { + return errServerKeyExchange + } + sigLen := int(sig[0])<<8 | int(sig[1]) + if sigLen+2 != len(sig) { + return errServerKeyExchange + } + sig = sig[2:] + + md5sha1 := md5SHA1Hash(clientHello.random, serverHello.random, serverECDHParams) + return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), crypto.MD5SHA1, md5sha1, sig) +} + +func (ka *ecdheRSAKeyAgreement) generateClientKeyExchange(config *Config, clientHello *clientHelloMsg, cert *x509.Certificate) ([]byte, *clientKeyExchangeMsg, os.Error) { + if ka.curve == nil { + return nil, nil, os.NewError("missing ServerKeyExchange message") + } + priv, mx, my, err := ka.curve.GenerateKey(config.rand()) + if err != nil { + return nil, nil, err + } + x, _ := ka.curve.ScalarMult(ka.x, ka.y, priv) + preMasterSecret := make([]byte, (ka.curve.BitSize+7)>>3) + xBytes := x.Bytes() + copy(preMasterSecret[len(preMasterSecret)-len(xBytes):], xBytes) + + serialized := ka.curve.Marshal(mx, my) + + ckx := new(clientKeyExchangeMsg) + ckx.ciphertext = make([]byte, 1+len(serialized)) + ckx.ciphertext[0] = byte(len(serialized)) + copy(ckx.ciphertext[1:], serialized) + + return preMasterSecret, ckx, nil +} diff --git a/src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py b/src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py new file mode 100644 index 000000000..c03eaa6ea --- /dev/null +++ b/src/pkg/crypto/tls/parse-gnutls-cli-debug-log.py @@ -0,0 +1,55 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This code is used to parse the debug log from gnutls-cli and generate a +# script of the handshake. This script is included in handshake_server_test.go. +# See the comments there for details. + +import sys + +blocks = [] + +READ = 1 +WRITE = 2 + +currentBlockType = 0 +currentBlock = [] +for line in sys.stdin.readlines(): + line = line[:-1] + if line.startswith("|<7>| WRITE: "): + if currentBlockType != WRITE: + if len(currentBlock) > 0: + blocks.append(currentBlock) + currentBlock = [] + currentBlockType = WRITE + elif line.startswith("|<7>| READ: "): + if currentBlockType != READ: + if len(currentBlock) > 0: + blocks.append(currentBlock) + currentBlock = [] + currentBlockType = READ + elif line.startswith("|<7>| 0"): + line = line[13:] + line = line.strip() + bs = line.split() + for b in bs: + currentBlock.append(int(b, 16)) + +if len(currentBlock) > 0: + blocks.append(currentBlock) + +for block in blocks: + sys.stdout.write("\t{\n") + + i = 0 + for b in block: + if i % 8 == 0: + sys.stdout.write("\t\t") + sys.stdout.write("0x%02x," % b) + if i % 8 == 7: + sys.stdout.write("\n") + else: + sys.stdout.write(" ") + i += 1 + sys.stdout.write("\n\t},\n\n") diff --git a/src/pkg/crypto/tls/prf.go b/src/pkg/crypto/tls/prf.go new file mode 100644 index 000000000..478cf65f9 --- /dev/null +++ b/src/pkg/crypto/tls/prf.go @@ -0,0 +1,153 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "hash" + "os" +) + +// Split a premaster secret in two as specified in RFC 4346, section 5. +func splitPreMasterSecret(secret []byte) (s1, s2 []byte) { + s1 = secret[0 : (len(secret)+1)/2] + s2 = secret[len(secret)/2:] + return +} + +// pHash implements the P_hash function, as defined in RFC 4346, section 5. +func pHash(result, secret, seed []byte, hash func() hash.Hash) { + h := hmac.New(hash, secret) + h.Write(seed) + a := h.Sum() + + j := 0 + for j < len(result) { + h.Reset() + h.Write(a) + h.Write(seed) + b := h.Sum() + todo := len(b) + if j+todo > len(result) { + todo = len(result) - j + } + copy(result[j:j+todo], b) + j += todo + + h.Reset() + h.Write(a) + a = h.Sum() + } +} + +// pRF10 implements the TLS 1.0 pseudo-random function, as defined in RFC 2246, section 5. +func pRF10(result, secret, label, seed []byte) { + hashSHA1 := sha1.New + hashMD5 := md5.New + + labelAndSeed := make([]byte, len(label)+len(seed)) + copy(labelAndSeed, label) + copy(labelAndSeed[len(label):], seed) + + s1, s2 := splitPreMasterSecret(secret) + pHash(result, s1, labelAndSeed, hashMD5) + result2 := make([]byte, len(result)) + pHash(result2, s2, labelAndSeed, hashSHA1) + + for i, b := range result2 { + result[i] ^= b + } +} + +const ( + tlsRandomLength = 32 // Length of a random nonce in TLS 1.1. + masterSecretLength = 48 // Length of a master secret in TLS 1.1. + finishedVerifyLength = 12 // Length of verify_data in a Finished message. +) + +var masterSecretLabel = []byte("master secret") +var keyExpansionLabel = []byte("key expansion") +var clientFinishedLabel = []byte("client finished") +var serverFinishedLabel = []byte("server finished") + +// keysFromPreMasterSecret generates the connection keys from the pre master +// secret, given the lengths of the MAC key, cipher key and IV, as defined in +// RFC 2246, section 6.3. +func keysFromPreMasterSecret10(preMasterSecret, clientRandom, serverRandom []byte, macLen, keyLen, ivLen int) (masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV []byte) { + var seed [tlsRandomLength * 2]byte + copy(seed[0:len(clientRandom)], clientRandom) + copy(seed[len(clientRandom):], serverRandom) + masterSecret = make([]byte, masterSecretLength) + pRF10(masterSecret, preMasterSecret, masterSecretLabel, seed[0:]) + + copy(seed[0:len(clientRandom)], serverRandom) + copy(seed[len(serverRandom):], clientRandom) + + n := 2*macLen + 2*keyLen + 2*ivLen + keyMaterial := make([]byte, n) + pRF10(keyMaterial, masterSecret, keyExpansionLabel, seed[0:]) + clientMAC = keyMaterial[:macLen] + keyMaterial = keyMaterial[macLen:] + serverMAC = keyMaterial[:macLen] + keyMaterial = keyMaterial[macLen:] + clientKey = keyMaterial[:keyLen] + keyMaterial = keyMaterial[keyLen:] + serverKey = keyMaterial[:keyLen] + keyMaterial = keyMaterial[keyLen:] + clientIV = keyMaterial[:ivLen] + keyMaterial = keyMaterial[ivLen:] + serverIV = keyMaterial[:ivLen] + return +} + +// A finishedHash calculates the hash of a set of handshake messages suitable +// for including in a Finished message. +type finishedHash struct { + clientMD5 hash.Hash + clientSHA1 hash.Hash + serverMD5 hash.Hash + serverSHA1 hash.Hash +} + +func newFinishedHash() finishedHash { + return finishedHash{md5.New(), sha1.New(), md5.New(), sha1.New()} +} + +func (h finishedHash) Write(msg []byte) (n int, err os.Error) { + h.clientMD5.Write(msg) + h.clientSHA1.Write(msg) + h.serverMD5.Write(msg) + h.serverSHA1.Write(msg) + return len(msg), nil +} + +// finishedSum calculates the contents of the verify_data member of a Finished +// message given the MD5 and SHA1 hashes of a set of handshake messages. +func finishedSum(md5, sha1, label, masterSecret []byte) []byte { + seed := make([]byte, len(md5)+len(sha1)) + copy(seed, md5) + copy(seed[len(md5):], sha1) + out := make([]byte, finishedVerifyLength) + pRF10(out, masterSecret, label, seed) + return out +} + +// clientSum returns the contents of the verify_data member of a client's +// Finished message. +func (h finishedHash) clientSum(masterSecret []byte) []byte { + md5 := h.clientMD5.Sum() + sha1 := h.clientSHA1.Sum() + return finishedSum(md5, sha1, clientFinishedLabel, masterSecret) +} + +// serverSum returns the contents of the verify_data member of a server's +// Finished message. +func (h finishedHash) serverSum(masterSecret []byte) []byte { + md5 := h.serverMD5.Sum() + sha1 := h.serverSHA1.Sum() + return finishedSum(md5, sha1, serverFinishedLabel, masterSecret) +} diff --git a/src/pkg/crypto/tls/prf_test.go b/src/pkg/crypto/tls/prf_test.go new file mode 100644 index 000000000..f8c4acb9d --- /dev/null +++ b/src/pkg/crypto/tls/prf_test.go @@ -0,0 +1,104 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "encoding/hex" + "testing" +) + +type testSplitPreMasterSecretTest struct { + in, out1, out2 string +} + +var testSplitPreMasterSecretTests = []testSplitPreMasterSecretTest{ + {"", "", ""}, + {"00", "00", "00"}, + {"0011", "00", "11"}, + {"001122", "0011", "1122"}, + {"00112233", "0011", "2233"}, +} + +func TestSplitPreMasterSecret(t *testing.T) { + for i, test := range testSplitPreMasterSecretTests { + in, _ := hex.DecodeString(test.in) + out1, out2 := splitPreMasterSecret(in) + s1 := hex.EncodeToString(out1) + s2 := hex.EncodeToString(out2) + if s1 != test.out1 || s2 != test.out2 { + t.Errorf("#%d: got: (%s, %s) want: (%s, %s)", i, s1, s2, test.out1, test.out2) + } + } +} + +type testKeysFromTest struct { + preMasterSecret string + clientRandom, serverRandom string + masterSecret string + clientMAC, serverMAC string + clientKey, serverKey string + macLen, keyLen int +} + +func TestKeysFromPreMasterSecret(t *testing.T) { + for i, test := range testKeysFromTests { + in, _ := hex.DecodeString(test.preMasterSecret) + clientRandom, _ := hex.DecodeString(test.clientRandom) + serverRandom, _ := hex.DecodeString(test.serverRandom) + master, clientMAC, serverMAC, clientKey, serverKey, _, _ := keysFromPreMasterSecret10(in, clientRandom, serverRandom, test.macLen, test.keyLen, 0) + masterString := hex.EncodeToString(master) + clientMACString := hex.EncodeToString(clientMAC) + serverMACString := hex.EncodeToString(serverMAC) + clientKeyString := hex.EncodeToString(clientKey) + serverKeyString := hex.EncodeToString(serverKey) + if masterString != test.masterSecret || + clientMACString != test.clientMAC || + serverMACString != test.serverMAC || + clientKeyString != test.clientKey || + serverKeyString != test.serverKey { + t.Errorf("#%d: got: (%s, %s, %s, %s, %s) want: (%s, %s, %s, %s %s)", i, masterString, clientMACString, serverMACString, clientKeyString, serverMACString, test.masterSecret, test.clientMAC, test.serverMAC, test.clientKey, test.serverKey) + } + } +} + +// These test vectors were generated from GnuTLS using `gnutls-cli --insecure -d 9 ` +var testKeysFromTests = []testKeysFromTest{ + { + "0302cac83ad4b1db3b9ab49ad05957de2a504a634a386fc600889321e1a971f57479466830ac3e6f468e87f5385fa0c5", + "4ae66303755184a3917fcb44880605fcc53baa01912b22ed94473fc69cebd558", + "4ae663020ec16e6bb5130be918cfcafd4d765979a3136a5d50c593446e4e44db", + "3d851bab6e5556e959a16bc36d66cfae32f672bfa9ecdef6096cbb1b23472df1da63dbbd9827606413221d149ed08ceb", + "805aaa19b3d2c0a0759a4b6c9959890e08480119", + "2d22f9fe519c075c16448305ceee209fc24ad109", + "d50b5771244f850cd8117a9ccafe2cf1", + "e076e33206b30507a85c32855acd0919", + 20, + 16, + }, + { + "03023f7527316bc12cbcd69e4b9e8275d62c028f27e65c745cfcddc7ce01bd3570a111378b63848127f1c36e5f9e4890", + "4ae66364b5ea56b20ce4e25555aed2d7e67f42788dd03f3fee4adae0459ab106", + "4ae66363ab815cbf6a248b87d6b556184e945e9b97fbdf247858b0bdafacfa1c", + "7d64be7c80c59b740200b4b9c26d0baaa1c5ae56705acbcf2307fe62beb4728c19392c83f20483801cce022c77645460", + "97742ed60a0554ca13f04f97ee193177b971e3b0", + "37068751700400e03a8477a5c7eec0813ab9e0dc", + "207cddbc600d2a200abac6502053ee5c", + "df3f94f6e1eacc753b815fe16055cd43", + 20, + 16, + }, + { + "832d515f1d61eebb2be56ba0ef79879efb9b527504abb386fb4310ed5d0e3b1f220d3bb6b455033a2773e6d8bdf951d278a187482b400d45deb88a5d5a6bb7d6a7a1decc04eb9ef0642876cd4a82d374d3b6ff35f0351dc5d411104de431375355addc39bfb1f6329fb163b0bc298d658338930d07d313cd980a7e3d9196cac1", + "4ae663b2ee389c0de147c509d8f18f5052afc4aaf9699efe8cb05ece883d3a5e", + "4ae664d503fd4cff50cfc1fb8fc606580f87b0fcdac9554ba0e01d785bdf278e", + "1aff2e7a2c4279d0126f57a65a77a8d9d0087cf2733366699bec27eb53d5740705a8574bb1acc2abbe90e44f0dd28d6c", + "3c7647c93c1379a31a609542aa44e7f117a70085", + "0d73102994be74a575a3ead8532590ca32a526d4", + "ac7581b0b6c10d85bbd905ffbf36c65e", + "ff07edde49682b45466bd2e39464b306", + 20, + 16, + }, +} diff --git a/src/pkg/crypto/tls/tls.go b/src/pkg/crypto/tls/tls.go new file mode 100644 index 000000000..4f0859fee --- /dev/null +++ b/src/pkg/crypto/tls/tls.go @@ -0,0 +1,181 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tls partially implements the TLS 1.1 protocol, as specified in RFC +// 4346. +package tls + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "net" + "os" + "strings" +) + +// Server returns a new TLS server side connection +// using conn as the underlying transport. +// The configuration config must be non-nil and must have +// at least one certificate. +func Server(conn net.Conn, config *Config) *Conn { + return &Conn{conn: conn, config: config} +} + +// Client returns a new TLS client side connection +// using conn as the underlying transport. +// Client interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Client(conn net.Conn, config *Config) *Conn { + return &Conn{conn: conn, config: config, isClient: true} +} + +// A Listener implements a network listener (net.Listener) for TLS connections. +type Listener struct { + listener net.Listener + config *Config +} + +// Accept waits for and returns the next incoming TLS connection. +// The returned connection c is a *tls.Conn. +func (l *Listener) Accept() (c net.Conn, err os.Error) { + c, err = l.listener.Accept() + if err != nil { + return + } + c = Server(c, l.config) + return +} + +// Close closes the listener. +func (l *Listener) Close() os.Error { return l.listener.Close() } + +// Addr returns the listener's network address. +func (l *Listener) Addr() net.Addr { return l.listener.Addr() } + +// NewListener creates a Listener which accepts connections from an inner +// Listener and wraps each connection with Server. +// The configuration config must be non-nil and must have +// at least one certificate. +func NewListener(listener net.Listener, config *Config) (l *Listener) { + l = new(Listener) + l.listener = listener + l.config = config + return +} + +// Listen creates a TLS listener accepting connections on the +// given network address using net.Listen. +// The configuration config must be non-nil and must have +// at least one certificate. +func Listen(network, laddr string, config *Config) (*Listener, os.Error) { + if config == nil || len(config.Certificates) == 0 { + return nil, os.NewError("tls.Listen: no certificates in configuration") + } + l, err := net.Listen(network, laddr) + if err != nil { + return nil, err + } + return NewListener(l, config), nil +} + +// Dial connects to the given network address using net.Dial +// and then initiates a TLS handshake, returning the resulting +// TLS connection. +// Dial interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Dial(network, addr string, config *Config) (*Conn, os.Error) { + raddr := addr + c, err := net.Dial(network, raddr) + if err != nil { + return nil, err + } + + colonPos := strings.LastIndex(raddr, ":") + if colonPos == -1 { + colonPos = len(raddr) + } + hostname := raddr[:colonPos] + + if config == nil { + config = defaultConfig() + } + if config.ServerName != "" { + // Make a copy to avoid polluting argument or default. + c := *config + c.ServerName = hostname + config = &c + } + conn := Client(c, config) + if err = conn.Handshake(); err != nil { + c.Close() + return nil, err + } + return conn, nil +} + +// LoadX509KeyPair reads and parses a public/private key pair from a pair of +// files. The files must contain PEM encoded data. +func LoadX509KeyPair(certFile string, keyFile string) (cert Certificate, err os.Error) { + certPEMBlock, err := ioutil.ReadFile(certFile) + if err != nil { + return + } + keyPEMBlock, err := ioutil.ReadFile(keyFile) + if err != nil { + return + } + return X509KeyPair(certPEMBlock, keyPEMBlock) +} + +// X509KeyPair parses a public/private key pair from a pair of +// PEM encoded data. +func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err os.Error) { + var certDERBlock *pem.Block + for { + certDERBlock, certPEMBlock = pem.Decode(certPEMBlock) + if certDERBlock == nil { + break + } + if certDERBlock.Type == "CERTIFICATE" { + cert.Certificate = append(cert.Certificate, certDERBlock.Bytes) + } + } + + if len(cert.Certificate) == 0 { + err = os.NewError("crypto/tls: failed to parse certificate PEM data") + return + } + + keyDERBlock, _ := pem.Decode(keyPEMBlock) + if keyDERBlock == nil { + err = os.NewError("crypto/tls: failed to parse key PEM data") + return + } + + key, err := x509.ParsePKCS1PrivateKey(keyDERBlock.Bytes) + if err != nil { + err = os.NewError("crypto/tls: failed to parse key: " + err.String()) + return + } + + cert.PrivateKey = key + + // We don't need to parse the public key for TLS, but we so do anyway + // to check that it looks sane and matches the private key. + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return + } + + if x509Cert.PublicKeyAlgorithm != x509.RSA || x509Cert.PublicKey.(*rsa.PublicKey).N.Cmp(key.PublicKey.N) != 0 { + err = os.NewError("crypto/tls: private key does not match public key") + return + } + + return +} diff --git a/src/pkg/crypto/twofish/Makefile b/src/pkg/crypto/twofish/Makefile new file mode 100644 index 000000000..aec61659d --- /dev/null +++ b/src/pkg/crypto/twofish/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/twofish +GOFILES=\ + twofish.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/twofish/twofish.go b/src/pkg/crypto/twofish/twofish.go new file mode 100644 index 000000000..2e537c606 --- /dev/null +++ b/src/pkg/crypto/twofish/twofish.go @@ -0,0 +1,358 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package twofish implements Bruce Schneier's Twofish encryption algorithm. +package twofish + +// Twofish is defined in http://www.schneier.com/paper-twofish-paper.pdf [TWOFISH] + +// This code is a port of the LibTom C implementation. +// See http://libtom.org/?page=features&newsitems=5&whatfile=crypt. +// LibTomCrypt is free for all purposes under the public domain. +// It was heavily inspired by the go blowfish package. + +import ( + "os" + "strconv" +) + +// BlockSize is the constant block size of Twofish. +const BlockSize = 16 + +const mdsPolynomial = 0x169 // x^8 + x^6 + x^5 + x^3 + 1, see [TWOFISH] 4.2 +const rsPolynomial = 0x14d // x^8 + x^6 + x^3 + x^2 + 1, see [TWOFISH] 4.3 + +// A Cipher is an instance of Twofish encryption using a particular key. +type Cipher struct { + s [4][256]uint32 + k [40]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/twofish: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a Cipher. +// The key argument should be the Twofish key, 16, 24 or 32 bytes. +func NewCipher(key []byte) (*Cipher, os.Error) { + keylen := len(key) + + if keylen != 16 && keylen != 24 && keylen != 32 { + return nil, KeySizeError(keylen) + } + + // k is the number of 64 bit words in key + k := keylen / 8 + + // Create the S[..] words + var S [4 * 4]byte + for i := 0; i < k; i++ { + // Computes [y0 y1 y2 y3] = rs . [x0 x1 x2 x3 x4 x5 x6 x7] + for j, rsRow := range rs { + for k, rsVal := range rsRow { + S[4*i+j] ^= gfMult(key[8*i+k], rsVal, rsPolynomial) + } + } + } + + // Calculate subkeys + c := new(Cipher) + var tmp [4]byte + for i := byte(0); i < 20; i++ { + // A = h(p * 2x, Me) + for j := range tmp { + tmp[j] = 2 * i + } + A := h(tmp[:], key, 0) + + // B = rolc(h(p * (2x + 1), Mo), 8) + for j := range tmp { + tmp[j] = 2*i + 1 + } + B := h(tmp[:], key, 1) + B = rol(B, 8) + + c.k[2*i] = A + B + + // K[2i+1] = (A + 2B) <<< 9 + c.k[2*i+1] = rol(2*B+A, 9) + } + + // Calculate sboxes + switch k { + case 2: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][byte(i)]^S[0]]^S[4]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][byte(i)]^S[1]]^S[5]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][byte(i)]^S[2]]^S[6]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][byte(i)]^S[3]]^S[7]], 3) + } + case 3: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[1]]^S[5]]^S[9]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[3]]^S[7]]^S[11]], 3) + } + default: + for i := range c.s[0] { + c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]]^S[12]], 0) + c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[1]]^S[5]]^S[9]]^S[13]], 1) + c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]]^S[14]], 2) + c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][sbox[1][byte(i)]^S[3]]^S[7]]^S[11]]^S[15]], 3) + } + } + + return c, nil +} + +// Reset zeros the key data, so that it will no longer appear in the process's +// memory. +func (c *Cipher) Reset() { + for i := range c.k { + c.k[i] = 0 + } + for i := range c.s { + for j := 0; j < 256; j++ { + c.s[i][j] = 0 + } + } +} + +// BlockSize returns the Twofish block size, 16 bytes. +func (c *Cipher) BlockSize() int { return BlockSize } + +// store32l stores src in dst in little-endian form. +func store32l(dst []byte, src uint32) { + dst[0] = byte(src) + dst[1] = byte(src >> 8) + dst[2] = byte(src >> 16) + dst[3] = byte(src >> 24) + return +} + +// load32l reads a little-endian uint32 from src. +func load32l(src []byte) uint32 { + return uint32(src[0]) | uint32(src[1])<<8 | uint32(src[2])<<16 | uint32(src[3])<<24 +} + +// rol returns x after a left circular rotation of y bits. +func rol(x, y uint32) uint32 { + return (x << (y & 31)) | (x >> (32 - (y & 31))) +} + +// ror returns x after a right circular rotation of y bits. +func ror(x, y uint32) uint32 { + return (x >> (y & 31)) | (x << (32 - (y & 31))) +} + +// The RS matrix. See [TWOFISH] 4.3 +var rs = [4][8]byte{ + {0x01, 0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E}, + {0xA4, 0x56, 0x82, 0xF3, 0x1E, 0xC6, 0x68, 0xE5}, + {0x02, 0xA1, 0xFC, 0xC1, 0x47, 0xAE, 0x3D, 0x19}, + {0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E, 0x03}, +} + +// sbox tables +var sbox = [2][256]byte{ + { + 0xa9, 0x67, 0xb3, 0xe8, 0x04, 0xfd, 0xa3, 0x76, 0x9a, 0x92, 0x80, 0x78, 0xe4, 0xdd, 0xd1, 0x38, + 0x0d, 0xc6, 0x35, 0x98, 0x18, 0xf7, 0xec, 0x6c, 0x43, 0x75, 0x37, 0x26, 0xfa, 0x13, 0x94, 0x48, + 0xf2, 0xd0, 0x8b, 0x30, 0x84, 0x54, 0xdf, 0x23, 0x19, 0x5b, 0x3d, 0x59, 0xf3, 0xae, 0xa2, 0x82, + 0x63, 0x01, 0x83, 0x2e, 0xd9, 0x51, 0x9b, 0x7c, 0xa6, 0xeb, 0xa5, 0xbe, 0x16, 0x0c, 0xe3, 0x61, + 0xc0, 0x8c, 0x3a, 0xf5, 0x73, 0x2c, 0x25, 0x0b, 0xbb, 0x4e, 0x89, 0x6b, 0x53, 0x6a, 0xb4, 0xf1, + 0xe1, 0xe6, 0xbd, 0x45, 0xe2, 0xf4, 0xb6, 0x66, 0xcc, 0x95, 0x03, 0x56, 0xd4, 0x1c, 0x1e, 0xd7, + 0xfb, 0xc3, 0x8e, 0xb5, 0xe9, 0xcf, 0xbf, 0xba, 0xea, 0x77, 0x39, 0xaf, 0x33, 0xc9, 0x62, 0x71, + 0x81, 0x79, 0x09, 0xad, 0x24, 0xcd, 0xf9, 0xd8, 0xe5, 0xc5, 0xb9, 0x4d, 0x44, 0x08, 0x86, 0xe7, + 0xa1, 0x1d, 0xaa, 0xed, 0x06, 0x70, 0xb2, 0xd2, 0x41, 0x7b, 0xa0, 0x11, 0x31, 0xc2, 0x27, 0x90, + 0x20, 0xf6, 0x60, 0xff, 0x96, 0x5c, 0xb1, 0xab, 0x9e, 0x9c, 0x52, 0x1b, 0x5f, 0x93, 0x0a, 0xef, + 0x91, 0x85, 0x49, 0xee, 0x2d, 0x4f, 0x8f, 0x3b, 0x47, 0x87, 0x6d, 0x46, 0xd6, 0x3e, 0x69, 0x64, + 0x2a, 0xce, 0xcb, 0x2f, 0xfc, 0x97, 0x05, 0x7a, 0xac, 0x7f, 0xd5, 0x1a, 0x4b, 0x0e, 0xa7, 0x5a, + 0x28, 0x14, 0x3f, 0x29, 0x88, 0x3c, 0x4c, 0x02, 0xb8, 0xda, 0xb0, 0x17, 0x55, 0x1f, 0x8a, 0x7d, + 0x57, 0xc7, 0x8d, 0x74, 0xb7, 0xc4, 0x9f, 0x72, 0x7e, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, + 0x6e, 0x50, 0xde, 0x68, 0x65, 0xbc, 0xdb, 0xf8, 0xc8, 0xa8, 0x2b, 0x40, 0xdc, 0xfe, 0x32, 0xa4, + 0xca, 0x10, 0x21, 0xf0, 0xd3, 0x5d, 0x0f, 0x00, 0x6f, 0x9d, 0x36, 0x42, 0x4a, 0x5e, 0xc1, 0xe0, + }, + { + 0x75, 0xf3, 0xc6, 0xf4, 0xdb, 0x7b, 0xfb, 0xc8, 0x4a, 0xd3, 0xe6, 0x6b, 0x45, 0x7d, 0xe8, 0x4b, + 0xd6, 0x32, 0xd8, 0xfd, 0x37, 0x71, 0xf1, 0xe1, 0x30, 0x0f, 0xf8, 0x1b, 0x87, 0xfa, 0x06, 0x3f, + 0x5e, 0xba, 0xae, 0x5b, 0x8a, 0x00, 0xbc, 0x9d, 0x6d, 0xc1, 0xb1, 0x0e, 0x80, 0x5d, 0xd2, 0xd5, + 0xa0, 0x84, 0x07, 0x14, 0xb5, 0x90, 0x2c, 0xa3, 0xb2, 0x73, 0x4c, 0x54, 0x92, 0x74, 0x36, 0x51, + 0x38, 0xb0, 0xbd, 0x5a, 0xfc, 0x60, 0x62, 0x96, 0x6c, 0x42, 0xf7, 0x10, 0x7c, 0x28, 0x27, 0x8c, + 0x13, 0x95, 0x9c, 0xc7, 0x24, 0x46, 0x3b, 0x70, 0xca, 0xe3, 0x85, 0xcb, 0x11, 0xd0, 0x93, 0xb8, + 0xa6, 0x83, 0x20, 0xff, 0x9f, 0x77, 0xc3, 0xcc, 0x03, 0x6f, 0x08, 0xbf, 0x40, 0xe7, 0x2b, 0xe2, + 0x79, 0x0c, 0xaa, 0x82, 0x41, 0x3a, 0xea, 0xb9, 0xe4, 0x9a, 0xa4, 0x97, 0x7e, 0xda, 0x7a, 0x17, + 0x66, 0x94, 0xa1, 0x1d, 0x3d, 0xf0, 0xde, 0xb3, 0x0b, 0x72, 0xa7, 0x1c, 0xef, 0xd1, 0x53, 0x3e, + 0x8f, 0x33, 0x26, 0x5f, 0xec, 0x76, 0x2a, 0x49, 0x81, 0x88, 0xee, 0x21, 0xc4, 0x1a, 0xeb, 0xd9, + 0xc5, 0x39, 0x99, 0xcd, 0xad, 0x31, 0x8b, 0x01, 0x18, 0x23, 0xdd, 0x1f, 0x4e, 0x2d, 0xf9, 0x48, + 0x4f, 0xf2, 0x65, 0x8e, 0x78, 0x5c, 0x58, 0x19, 0x8d, 0xe5, 0x98, 0x57, 0x67, 0x7f, 0x05, 0x64, + 0xaf, 0x63, 0xb6, 0xfe, 0xf5, 0xb7, 0x3c, 0xa5, 0xce, 0xe9, 0x68, 0x44, 0xe0, 0x4d, 0x43, 0x69, + 0x29, 0x2e, 0xac, 0x15, 0x59, 0xa8, 0x0a, 0x9e, 0x6e, 0x47, 0xdf, 0x34, 0x35, 0x6a, 0xcf, 0xdc, + 0x22, 0xc9, 0xc0, 0x9b, 0x89, 0xd4, 0xed, 0xab, 0x12, 0xa2, 0x0d, 0x52, 0xbb, 0x02, 0x2f, 0xa9, + 0xd7, 0x61, 0x1e, 0xb4, 0x50, 0x04, 0xf6, 0xc2, 0x16, 0x25, 0x86, 0x56, 0x55, 0x09, 0xbe, 0x91, + }, +} + +// gfMult returns a·b in GF(2^8)/p +func gfMult(a, b byte, p uint32) byte { + B := [2]uint32{0, uint32(b)} + P := [2]uint32{0, p} + var result uint32 + + // branchless GF multiplier + for i := 0; i < 7; i++ { + result ^= B[a&1] + a >>= 1 + B[1] = P[B[1]>>7] ^ (B[1] << 1) + } + result ^= B[a&1] + return byte(result) +} + +// mdsColumnMult calculates y{col} where [y0 y1 y2 y3] = MDS · [x0] +func mdsColumnMult(in byte, col int) uint32 { + mul01 := in + mul5B := gfMult(in, 0x5B, mdsPolynomial) + mulEF := gfMult(in, 0xEF, mdsPolynomial) + + switch col { + case 0: + return uint32(mul01) | uint32(mul5B)<<8 | uint32(mulEF)<<16 | uint32(mulEF)<<24 + case 1: + return uint32(mulEF) | uint32(mulEF)<<8 | uint32(mul5B)<<16 | uint32(mul01)<<24 + case 2: + return uint32(mul5B) | uint32(mulEF)<<8 | uint32(mul01)<<16 | uint32(mulEF)<<24 + case 3: + return uint32(mul5B) | uint32(mul01)<<8 | uint32(mulEF)<<16 | uint32(mul5B)<<24 + } + + panic("unreachable") +} + +// h implements the S-box generation function. See [TWOFISH] 4.3.5 +func h(in, key []byte, offset int) uint32 { + var y [4]byte + for x := range y { + y[x] = in[x] + } + switch len(key) / 8 { + case 4: + y[0] = sbox[1][y[0]] ^ key[4*(6+offset)+0] + y[1] = sbox[0][y[1]] ^ key[4*(6+offset)+1] + y[2] = sbox[0][y[2]] ^ key[4*(6+offset)+2] + y[3] = sbox[1][y[3]] ^ key[4*(6+offset)+3] + fallthrough + case 3: + y[0] = sbox[1][y[0]] ^ key[4*(4+offset)+0] + y[1] = sbox[1][y[1]] ^ key[4*(4+offset)+1] + y[2] = sbox[0][y[2]] ^ key[4*(4+offset)+2] + y[3] = sbox[0][y[3]] ^ key[4*(4+offset)+3] + fallthrough + case 2: + y[0] = sbox[1][sbox[0][sbox[0][y[0]]^key[4*(2+offset)+0]]^key[4*(0+offset)+0]] + y[1] = sbox[0][sbox[0][sbox[1][y[1]]^key[4*(2+offset)+1]]^key[4*(0+offset)+1]] + y[2] = sbox[1][sbox[1][sbox[0][y[2]]^key[4*(2+offset)+2]]^key[4*(0+offset)+2]] + y[3] = sbox[0][sbox[1][sbox[1][y[3]]^key[4*(2+offset)+3]]^key[4*(0+offset)+3]] + } + // [y0 y1 y2 y3] = MDS . [x0 x1 x2 x3] + var mdsMult uint32 + for i := range y { + mdsMult ^= mdsColumnMult(y[i], i) + } + return mdsMult +} + +// Encrypt encrypts a 16-byte block from src to dst, which may overlap. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { + S1 := c.s[0] + S2 := c.s[1] + S3 := c.s[2] + S4 := c.s[3] + + // Load input + ia := load32l(src[0:4]) + ib := load32l(src[4:8]) + ic := load32l(src[8:12]) + id := load32l(src[12:16]) + + // Pre-whitening + ia ^= c.k[0] + ib ^= c.k[1] + ic ^= c.k[2] + id ^= c.k[3] + + for i := 0; i < 8; i++ { + k := c.k[8+i*4 : 12+i*4] + t2 := S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] + t1 := S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 + ic = ror(ic^(t1+k[0]), 1) + id = rol(id, 1) ^ (t2 + t1 + k[1]) + + t2 = S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] + t1 = S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 + ia = ror(ia^(t1+k[2]), 1) + ib = rol(ib, 1) ^ (t2 + t1 + k[3]) + } + + // Output with "undo last swap" + ta := ic ^ c.k[4] + tb := id ^ c.k[5] + tc := ia ^ c.k[6] + td := ib ^ c.k[7] + + store32l(dst[0:4], ta) + store32l(dst[4:8], tb) + store32l(dst[8:12], tc) + store32l(dst[12:16], td) +} + +// Decrypt decrypts a 16-byte block from src to dst, which may overlap. +func (c *Cipher) Decrypt(dst, src []byte) { + S1 := c.s[0] + S2 := c.s[1] + S3 := c.s[2] + S4 := c.s[3] + + // Load input + ta := load32l(src[0:4]) + tb := load32l(src[4:8]) + tc := load32l(src[8:12]) + td := load32l(src[12:16]) + + // Undo undo final swap + ia := tc ^ c.k[6] + ib := td ^ c.k[7] + ic := ta ^ c.k[4] + id := tb ^ c.k[5] + + for i := 8; i > 0; i-- { + k := c.k[4+i*4 : 8+i*4] + t2 := S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] + t1 := S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 + ia = rol(ia, 1) ^ (t1 + k[2]) + ib = ror(ib^(t2+t1+k[3]), 1) + + t2 = S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] + t1 = S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 + ic = rol(ic, 1) ^ (t1 + k[0]) + id = ror(id^(t2+t1+k[1]), 1) + } + + // Undo pre-whitening + ia ^= c.k[0] + ib ^= c.k[1] + ic ^= c.k[2] + id ^= c.k[3] + + store32l(dst[0:4], ia) + store32l(dst[4:8], ib) + store32l(dst[8:12], ic) + store32l(dst[12:16], id) +} diff --git a/src/pkg/crypto/twofish/twofish_test.go b/src/pkg/crypto/twofish/twofish_test.go new file mode 100644 index 000000000..303081f3f --- /dev/null +++ b/src/pkg/crypto/twofish/twofish_test.go @@ -0,0 +1,129 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package twofish + +import ( + "bytes" + "testing" +) + +var qbox = [2][4][16]byte{ + { + {0x8, 0x1, 0x7, 0xD, 0x6, 0xF, 0x3, 0x2, 0x0, 0xB, 0x5, 0x9, 0xE, 0xC, 0xA, 0x4}, + {0xE, 0xC, 0xB, 0x8, 0x1, 0x2, 0x3, 0x5, 0xF, 0x4, 0xA, 0x6, 0x7, 0x0, 0x9, 0xD}, + {0xB, 0xA, 0x5, 0xE, 0x6, 0xD, 0x9, 0x0, 0xC, 0x8, 0xF, 0x3, 0x2, 0x4, 0x7, 0x1}, + {0xD, 0x7, 0xF, 0x4, 0x1, 0x2, 0x6, 0xE, 0x9, 0xB, 0x3, 0x0, 0x8, 0x5, 0xC, 0xA}, + }, + { + {0x2, 0x8, 0xB, 0xD, 0xF, 0x7, 0x6, 0xE, 0x3, 0x1, 0x9, 0x4, 0x0, 0xA, 0xC, 0x5}, + {0x1, 0xE, 0x2, 0xB, 0x4, 0xC, 0x3, 0x7, 0x6, 0xD, 0xA, 0x5, 0xF, 0x9, 0x0, 0x8}, + {0x4, 0xC, 0x7, 0x5, 0x1, 0x6, 0x9, 0xA, 0x0, 0xE, 0xD, 0x8, 0x2, 0xB, 0x3, 0xF}, + {0xB, 0x9, 0x5, 0x1, 0xC, 0x3, 0xD, 0xE, 0x6, 0x4, 0x7, 0xF, 0x2, 0x0, 0x8, 0xA}, + }, +} + +// genSbox generates the variable sbox +func genSbox(qi int, x byte) byte { + a0, b0 := x/16, x%16 + for i := 0; i < 2; i++ { + a1 := a0 ^ b0 + b1 := (a0 ^ ((b0 << 3) | (b0 >> 1)) ^ (a0 << 3)) & 15 + a0 = qbox[qi][2*i][a1] + b0 = qbox[qi][2*i+1][b1] + } + return (b0 << 4) + a0 +} + +func TestSbox(t *testing.T) { + for n := range sbox { + for m := range sbox[n] { + if genSbox(n, byte(m)) != sbox[n][m] { + t.Errorf("#%d|%d: sbox value = %d want %d", n, m, sbox[n][m], genSbox(n, byte(m))) + } + } + } +} + +var testVectors = []struct { + key []byte + dec []byte + enc []byte +}{ + // These tests are extracted from LibTom + { + []byte{0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A}, + []byte{0xD4, 0x91, 0xDB, 0x16, 0xE7, 0xB1, 0xC3, 0x9E, 0x86, 0xCB, 0x08, 0x6B, 0x78, 0x9F, 0x54, 0x19}, + []byte{0x01, 0x9F, 0x98, 0x09, 0xDE, 0x17, 0x11, 0x85, 0x8F, 0xAA, 0xC3, 0xA3, 0xBA, 0x20, 0xFB, 0xC3}, + }, + { + []byte{0x88, 0xB2, 0xB2, 0x70, 0x6B, 0x10, 0x5E, 0x36, 0xB4, 0x46, 0xBB, 0x6D, 0x73, 0x1A, 0x1E, 0x88, + 0xEF, 0xA7, 0x1F, 0x78, 0x89, 0x65, 0xBD, 0x44}, + []byte{0x39, 0xDA, 0x69, 0xD6, 0xBA, 0x49, 0x97, 0xD5, 0x85, 0xB6, 0xDC, 0x07, 0x3C, 0xA3, 0x41, 0xB2}, + []byte{0x18, 0x2B, 0x02, 0xD8, 0x14, 0x97, 0xEA, 0x45, 0xF9, 0xDA, 0xAC, 0xDC, 0x29, 0x19, 0x3A, 0x65}, + }, + { + []byte{0xD4, 0x3B, 0xB7, 0x55, 0x6E, 0xA3, 0x2E, 0x46, 0xF2, 0xA2, 0x82, 0xB7, 0xD4, 0x5B, 0x4E, 0x0D, + 0x57, 0xFF, 0x73, 0x9D, 0x4D, 0xC9, 0x2C, 0x1B, 0xD7, 0xFC, 0x01, 0x70, 0x0C, 0xC8, 0x21, 0x6F}, + []byte{0x90, 0xAF, 0xE9, 0x1B, 0xB2, 0x88, 0x54, 0x4F, 0x2C, 0x32, 0xDC, 0x23, 0x9B, 0x26, 0x35, 0xE6}, + []byte{0x6C, 0xB4, 0x56, 0x1C, 0x40, 0xBF, 0x0A, 0x97, 0x05, 0x93, 0x1C, 0xB6, 0xD4, 0x08, 0xE7, 0xFA}, + }, + // These test are derived from http://www.schneier.com/code/ecb_ival.txt + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x9F, 0x58, 0x9F, 0x5C, 0xF6, 0x12, 0x2C, 0x32, 0xB6, 0xBF, 0xEC, 0x2F, 0x2A, 0xE8, 0xC3, 0x5A}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + }, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xCF, 0xD1, 0xD2, 0xE5, 0xA9, 0xBE, 0x9C, 0xDF, 0x50, 0x1F, 0x13, 0xB8, 0x92, 0xBD, 0x22, 0x48}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + }, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x37, 0x52, 0x7B, 0xE0, 0x05, 0x23, 0x34, 0xB8, 0x9F, 0x0C, 0xFC, 0xCA, 0xE8, 0x7C, 0xFA, 0x20}, + }, +} + +func TestCipher(t *testing.T) { + for n, tt := range testVectors { + // Test if the plaintext (dec) is encrypts to the given + // ciphertext (enc) using the given key. Test also if enc can + // be decrypted again into dec. + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("#%d: NewCipher: %v", n, err) + return + } + + buf := make([]byte, 16) + c.Encrypt(buf, tt.dec) + if !bytes.Equal(buf, tt.enc) { + t.Errorf("#%d: encrypt = %x want %x", n, buf, tt.enc) + } + c.Decrypt(buf, tt.enc) + if !bytes.Equal(buf, tt.dec) { + t.Errorf("#%d: decrypt = %x want %x", n, buf, tt.dec) + } + + // Test that 16 zero bytes, encrypted 1000 times then decrypted + // 1000 times results in zero bytes again. + zero := make([]byte, 16) + buf = make([]byte, 16) + for i := 0; i < 1000; i++ { + c.Encrypt(buf, buf) + } + for i := 0; i < 1000; i++ { + c.Decrypt(buf, buf) + } + if !bytes.Equal(buf, zero) { + t.Errorf("#%d: encrypt/decrypt 1000: have %x want %x", n, buf, zero) + } + } +} diff --git a/src/pkg/crypto/x509/Makefile b/src/pkg/crypto/x509/Makefile new file mode 100644 index 000000000..14ffd095f --- /dev/null +++ b/src/pkg/crypto/x509/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/x509 +GOFILES=\ + cert_pool.go\ + verify.go\ + x509.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/x509/cert_pool.go b/src/pkg/crypto/x509/cert_pool.go new file mode 100644 index 000000000..16cd92efc --- /dev/null +++ b/src/pkg/crypto/x509/cert_pool.go @@ -0,0 +1,106 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "crypto/x509/pkix" + "encoding/pem" + "strings" +) + +// Roots is a set of certificates. +type CertPool struct { + bySubjectKeyId map[string][]int + byName map[string][]int + certs []*Certificate +} + +// NewCertPool returns a new, empty CertPool. +func NewCertPool() *CertPool { + return &CertPool{ + make(map[string][]int), + make(map[string][]int), + nil, + } +} + +func nameToKey(name *pkix.Name) string { + return strings.Join(name.Country, ",") + "/" + strings.Join(name.Organization, ",") + "/" + strings.Join(name.OrganizationalUnit, ",") + "/" + name.CommonName +} + +// findVerifiedParents attempts to find certificates in s which have signed the +// given certificate. If no such certificate can be found or the signature +// doesn't match, it returns nil. +func (s *CertPool) findVerifiedParents(cert *Certificate) (parents []int) { + var candidates []int + + if len(cert.AuthorityKeyId) > 0 { + candidates = s.bySubjectKeyId[string(cert.AuthorityKeyId)] + } + if len(candidates) == 0 { + candidates = s.byName[nameToKey(&cert.Issuer)] + } + + for _, c := range candidates { + if cert.CheckSignatureFrom(s.certs[c]) == nil { + parents = append(parents, c) + } + } + + return +} + +// AddCert adds a certificate to a pool. +func (s *CertPool) AddCert(cert *Certificate) { + if cert == nil { + panic("adding nil Certificate to CertPool") + } + + // Check that the certificate isn't being added twice. + for _, c := range s.certs { + if c.Equal(cert) { + return + } + } + + n := len(s.certs) + s.certs = append(s.certs, cert) + + if len(cert.SubjectKeyId) > 0 { + keyId := string(cert.SubjectKeyId) + s.bySubjectKeyId[keyId] = append(s.bySubjectKeyId[keyId], n) + } + name := nameToKey(&cert.Subject) + s.byName[name] = append(s.byName[name], n) +} + +// AppendCertsFromPEM attempts to parse a series of PEM encoded root +// certificates. It appends any certificates found to s and returns true if any +// certificates were successfully parsed. +// +// On many Linux systems, /etc/ssl/cert.pem will contains the system wide set +// of root CAs in a format suitable for this function. +func (s *CertPool) AppendCertsFromPEM(pemCerts []byte) (ok bool) { + for len(pemCerts) > 0 { + var block *pem.Block + block, pemCerts = pem.Decode(pemCerts) + if block == nil { + break + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + + cert, err := ParseCertificate(block.Bytes) + if err != nil { + continue + } + + s.AddCert(cert) + ok = true + } + + return +} diff --git a/src/pkg/crypto/x509/pkix/Makefile b/src/pkg/crypto/x509/pkix/Makefile new file mode 100644 index 000000000..e29b74c01 --- /dev/null +++ b/src/pkg/crypto/x509/pkix/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=crypto/x509/pkix +GOFILES=\ + pkix.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/crypto/x509/pkix/pkix.go b/src/pkg/crypto/x509/pkix/pkix.go new file mode 100644 index 000000000..266fd557a --- /dev/null +++ b/src/pkg/crypto/x509/pkix/pkix.go @@ -0,0 +1,167 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pkix contains shared, low level structures used for ASN.1 parsing +// and serialization of X.509 certificates, CRL and OCSP. +package pkix + +import ( + "asn1" + "big" + "time" +) + +// AlgorithmIdentifier represents the ASN.1 structure of the same name. See RFC +// 5280, section 4.1.1.2. +type AlgorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier + Parameters asn1.RawValue `asn1:"optional"` +} + +type RDNSequence []RelativeDistinguishedNameSET + +type RelativeDistinguishedNameSET []AttributeTypeAndValue + +type AttributeTypeAndValue struct { + Type asn1.ObjectIdentifier + Value interface{} +} + +// Extension represents the ASN.1 structure of the same name. See RFC +// 5280, section 4.2. +type Extension struct { + Id asn1.ObjectIdentifier + Critical bool `asn1:"optional"` + Value []byte +} + +// Name represents an X.509 distinguished name. This only includes the common +// elements of a DN. Additional elements in the name are ignored. +type Name struct { + Country, Organization, OrganizationalUnit []string + Locality, Province []string + StreetAddress, PostalCode []string + SerialNumber, CommonName string +} + +func (n *Name) FillFromRDNSequence(rdns *RDNSequence) { + for _, rdn := range *rdns { + if len(rdn) == 0 { + continue + } + atv := rdn[0] + value, ok := atv.Value.(string) + if !ok { + continue + } + + t := atv.Type + if len(t) == 4 && t[0] == 2 && t[1] == 5 && t[2] == 4 { + switch t[3] { + case 3: + n.CommonName = value + case 5: + n.SerialNumber = value + case 6: + n.Country = append(n.Country, value) + case 7: + n.Locality = append(n.Locality, value) + case 8: + n.Province = append(n.Province, value) + case 9: + n.StreetAddress = append(n.StreetAddress, value) + case 10: + n.Organization = append(n.Organization, value) + case 11: + n.OrganizationalUnit = append(n.OrganizationalUnit, value) + case 17: + n.PostalCode = append(n.PostalCode, value) + } + } + } +} + +var ( + oidCountry = []int{2, 5, 4, 6} + oidOrganization = []int{2, 5, 4, 10} + oidOrganizationalUnit = []int{2, 5, 4, 11} + oidCommonName = []int{2, 5, 4, 3} + oidSerialNumber = []int{2, 5, 4, 5} + oidLocality = []int{2, 5, 4, 7} + oidProvince = []int{2, 5, 4, 8} + oidStreetAddress = []int{2, 5, 4, 9} + oidPostalCode = []int{2, 5, 4, 17} +) + +// appendRDNs appends a relativeDistinguishedNameSET to the given RDNSequence +// and returns the new value. The relativeDistinguishedNameSET contains an +// attributeTypeAndValue for each of the given values. See RFC 5280, A.1, and +// search for AttributeTypeAndValue. +func appendRDNs(in RDNSequence, values []string, oid asn1.ObjectIdentifier) RDNSequence { + if len(values) == 0 { + return in + } + + s := make([]AttributeTypeAndValue, len(values)) + for i, value := range values { + s[i].Type = oid + s[i].Value = value + } + + return append(in, s) +} + +func (n Name) ToRDNSequence() (ret RDNSequence) { + ret = appendRDNs(ret, n.Country, oidCountry) + ret = appendRDNs(ret, n.Organization, oidOrganization) + ret = appendRDNs(ret, n.OrganizationalUnit, oidOrganizationalUnit) + ret = appendRDNs(ret, n.Locality, oidLocality) + ret = appendRDNs(ret, n.Province, oidProvince) + ret = appendRDNs(ret, n.StreetAddress, oidStreetAddress) + ret = appendRDNs(ret, n.PostalCode, oidPostalCode) + if len(n.CommonName) > 0 { + ret = appendRDNs(ret, []string{n.CommonName}, oidCommonName) + } + if len(n.SerialNumber) > 0 { + ret = appendRDNs(ret, []string{n.SerialNumber}, oidSerialNumber) + } + + return ret +} + +// CertificateList represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. Use Certificate.CheckCRLSignature to verify the +// signature. +type CertificateList struct { + TBSCertList TBSCertificateList + SignatureAlgorithm AlgorithmIdentifier + SignatureValue asn1.BitString +} + +// HasExpired returns true iff currentTimeSeconds is past the expiry time of +// certList. +func (certList *CertificateList) HasExpired(currentTimeSeconds int64) bool { + return certList.TBSCertList.NextUpdate.Seconds() <= currentTimeSeconds +} + +// TBSCertificateList represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. +type TBSCertificateList struct { + Raw asn1.RawContent + Version int `asn1:"optional,default:2"` + Signature AlgorithmIdentifier + Issuer RDNSequence + ThisUpdate *time.Time + NextUpdate *time.Time + RevokedCertificates []RevokedCertificate `asn1:"optional"` + Extensions []Extension `asn1:"tag:0,optional,explicit"` +} + +// RevokedCertificate represents the ASN.1 structure of the same name. See RFC +// 5280, section 5.1. +type RevokedCertificate struct { + SerialNumber *big.Int + RevocationTime *time.Time + Extensions []Extension `asn1:"optional"` +} diff --git a/src/pkg/crypto/x509/verify.go b/src/pkg/crypto/x509/verify.go new file mode 100644 index 000000000..4c0fecccb --- /dev/null +++ b/src/pkg/crypto/x509/verify.go @@ -0,0 +1,244 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "os" + "strings" + "time" +) + +type InvalidReason int + +const ( + // NotAuthorizedToSign results when a certificate is signed by another + // which isn't marked as a CA certificate. + NotAuthorizedToSign InvalidReason = iota + // Expired results when a certificate has expired, based on the time + // given in the VerifyOptions. + Expired + // CANotAuthorizedForThisName results when an intermediate or root + // certificate has a name constraint which doesn't include the name + // being checked. + CANotAuthorizedForThisName +) + +// CertificateInvalidError results when an odd error occurs. Users of this +// library probably want to handle all these errors uniformly. +type CertificateInvalidError struct { + Cert *Certificate + Reason InvalidReason +} + +func (e CertificateInvalidError) String() string { + switch e.Reason { + case NotAuthorizedToSign: + return "x509: certificate is not authorized to sign other other certificates" + case Expired: + return "x509: certificate has expired or is not yet valid" + case CANotAuthorizedForThisName: + return "x509: a root or intermediate certificate is not authorized to sign in this domain" + } + return "x509: unknown error" +} + +// HostnameError results when the set of authorized names doesn't match the +// requested name. +type HostnameError struct { + Certificate *Certificate + Host string +} + +func (h HostnameError) String() string { + var valid string + c := h.Certificate + if len(c.DNSNames) > 0 { + valid = strings.Join(c.DNSNames, ", ") + } else { + valid = c.Subject.CommonName + } + return "certificate is valid for " + valid + ", not " + h.Host +} + +// UnknownAuthorityError results when the certificate issuer is unknown +type UnknownAuthorityError struct { + cert *Certificate +} + +func (e UnknownAuthorityError) String() string { + return "x509: certificate signed by unknown authority" +} + +// VerifyOptions contains parameters for Certificate.Verify. It's a structure +// because other PKIX verification APIs have ended up needing many options. +type VerifyOptions struct { + DNSName string + Intermediates *CertPool + Roots *CertPool + CurrentTime int64 // if 0, the current system time is used. +} + +const ( + leafCertificate = iota + intermediateCertificate + rootCertificate +) + +// isValid performs validity checks on the c. +func (c *Certificate) isValid(certType int, opts *VerifyOptions) os.Error { + if opts.CurrentTime < c.NotBefore.Seconds() || + opts.CurrentTime > c.NotAfter.Seconds() { + return CertificateInvalidError{c, Expired} + } + + if len(c.PermittedDNSDomains) > 0 { + for _, domain := range c.PermittedDNSDomains { + if opts.DNSName == domain || + (strings.HasSuffix(opts.DNSName, domain) && + len(opts.DNSName) >= 1+len(domain) && + opts.DNSName[len(opts.DNSName)-len(domain)-1] == '.') { + continue + } + + return CertificateInvalidError{c, CANotAuthorizedForThisName} + } + } + + // KeyUsage status flags are ignored. From Engineering Security, Peter + // Gutmann: A European government CA marked its signing certificates as + // being valid for encryption only, but no-one noticed. Another + // European CA marked its signature keys as not being valid for + // signatures. A different CA marked its own trusted root certificate + // as being invalid for certificate signing. Another national CA + // distributed a certificate to be used to encrypt data for the + // country’s tax authority that was marked as only being usable for + // digital signatures but not for encryption. Yet another CA reversed + // the order of the bit flags in the keyUsage due to confusion over + // encoding endianness, essentially setting a random keyUsage in + // certificates that it issued. Another CA created a self-invalidating + // certificate by adding a certificate policy statement stipulating + // that the certificate had to be used strictly as specified in the + // keyUsage, and a keyUsage containing a flag indicating that the RSA + // encryption key could only be used for Diffie-Hellman key agreement. + + if certType == intermediateCertificate && (!c.BasicConstraintsValid || !c.IsCA) { + return CertificateInvalidError{c, NotAuthorizedToSign} + } + + return nil +} + +// Verify attempts to verify c by building one or more chains from c to a +// certificate in opts.roots, using certificates in opts.Intermediates if +// needed. If successful, it returns one or chains where the first element of +// the chain is c and the last element is from opts.Roots. +// +// WARNING: this doesn't do any revocation checking. +func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err os.Error) { + if opts.CurrentTime == 0 { + opts.CurrentTime = time.Seconds() + } + err = c.isValid(leafCertificate, &opts) + if err != nil { + return + } + if len(opts.DNSName) > 0 { + err = c.VerifyHostname(opts.DNSName) + if err != nil { + return + } + } + return c.buildChains(make(map[int][][]*Certificate), []*Certificate{c}, &opts) +} + +func appendToFreshChain(chain []*Certificate, cert *Certificate) []*Certificate { + n := make([]*Certificate, len(chain)+1) + copy(n, chain) + n[len(chain)] = cert + return n +} + +func (c *Certificate) buildChains(cache map[int][][]*Certificate, currentChain []*Certificate, opts *VerifyOptions) (chains [][]*Certificate, err os.Error) { + for _, rootNum := range opts.Roots.findVerifiedParents(c) { + root := opts.Roots.certs[rootNum] + err = root.isValid(rootCertificate, opts) + if err != nil { + continue + } + chains = append(chains, appendToFreshChain(currentChain, root)) + } + +nextIntermediate: + for _, intermediateNum := range opts.Intermediates.findVerifiedParents(c) { + intermediate := opts.Intermediates.certs[intermediateNum] + for _, cert := range currentChain { + if cert == intermediate { + continue nextIntermediate + } + } + err = intermediate.isValid(intermediateCertificate, opts) + if err != nil { + continue + } + var childChains [][]*Certificate + childChains, ok := cache[intermediateNum] + if !ok { + childChains, err = intermediate.buildChains(cache, appendToFreshChain(currentChain, intermediate), opts) + cache[intermediateNum] = childChains + } + chains = append(chains, childChains...) + } + + if len(chains) > 0 { + err = nil + } + + if len(chains) == 0 && err == nil { + err = UnknownAuthorityError{c} + } + + return +} + +func matchHostnames(pattern, host string) bool { + if len(pattern) == 0 || len(host) == 0 { + return false + } + + patternParts := strings.Split(pattern, ".") + hostParts := strings.Split(host, ".") + + if len(patternParts) != len(hostParts) { + return false + } + + for i, patternPart := range patternParts { + if patternPart == "*" { + continue + } + if patternPart != hostParts[i] { + return false + } + } + + return true +} + +// VerifyHostname returns nil if c is a valid certificate for the named host. +// Otherwise it returns an os.Error describing the mismatch. +func (c *Certificate) VerifyHostname(h string) os.Error { + if len(c.DNSNames) > 0 { + for _, match := range c.DNSNames { + if matchHostnames(match, h) { + return nil + } + } + // If Subject Alt Name is given, we ignore the common name. + } else if matchHostnames(c.Subject.CommonName, h) { + return nil + } + + return HostnameError{c, h} +} diff --git a/src/pkg/crypto/x509/verify_test.go b/src/pkg/crypto/x509/verify_test.go new file mode 100644 index 000000000..111f60eb1 --- /dev/null +++ b/src/pkg/crypto/x509/verify_test.go @@ -0,0 +1,391 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "encoding/pem" + "os" + "strings" + "testing" +) + +type verifyTest struct { + leaf string + intermediates []string + roots []string + currentTime int64 + dnsName string + + errorCallback func(*testing.T, int, os.Error) bool + expectedChains [][]string +} + +var verifyTests = []verifyTest{ + { + leaf: googleLeaf, + intermediates: []string{thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.google.com", + + expectedChains: [][]string{ + []string{"Google", "Thawte", "VeriSign"}, + }, + }, + { + leaf: googleLeaf, + intermediates: []string{thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.example.com", + + errorCallback: expectHostnameError, + }, + { + leaf: googleLeaf, + intermediates: []string{thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1, + dnsName: "www.example.com", + + errorCallback: expectExpired, + }, + { + leaf: googleLeaf, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.google.com", + + errorCallback: expectAuthorityUnknown, + }, + { + leaf: googleLeaf, + intermediates: []string{verisignRoot, thawteIntermediate}, + roots: []string{verisignRoot}, + currentTime: 1302726541, + dnsName: "www.google.com", + + expectedChains: [][]string{ + []string{"Google", "Thawte", "VeriSign"}, + }, + }, + { + leaf: dnssecExpLeaf, + intermediates: []string{startComIntermediate}, + roots: []string{startComRoot}, + currentTime: 1302726541, + + expectedChains: [][]string{ + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + }, + }, + { + leaf: dnssecExpLeaf, + intermediates: []string{startComIntermediate, startComRoot}, + roots: []string{startComRoot}, + currentTime: 1302726541, + + expectedChains: [][]string{ + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, + }, + }, +} + +func expectHostnameError(t *testing.T, i int, err os.Error) (ok bool) { + if _, ok := err.(HostnameError); !ok { + t.Errorf("#%d: error was not a HostnameError: %s", i, err) + return false + } + return true +} + +func expectExpired(t *testing.T, i int, err os.Error) (ok bool) { + if inval, ok := err.(CertificateInvalidError); !ok || inval.Reason != Expired { + t.Errorf("#%d: error was not Expired: %s", i, err) + return false + } + return true +} + +func expectAuthorityUnknown(t *testing.T, i int, err os.Error) (ok bool) { + if _, ok := err.(UnknownAuthorityError); !ok { + t.Errorf("#%d: error was not UnknownAuthorityError: %s", i, err) + return false + } + return true +} + +func certificateFromPEM(pemBytes string) (*Certificate, os.Error) { + block, _ := pem.Decode([]byte(pemBytes)) + if block == nil { + return nil, os.NewError("failed to decode PEM") + } + return ParseCertificate(block.Bytes) +} + +func TestVerify(t *testing.T) { + for i, test := range verifyTests { + opts := VerifyOptions{ + Roots: NewCertPool(), + Intermediates: NewCertPool(), + DNSName: test.dnsName, + CurrentTime: test.currentTime, + } + + for j, root := range test.roots { + ok := opts.Roots.AppendCertsFromPEM([]byte(root)) + if !ok { + t.Errorf("#%d: failed to parse root #%d", i, j) + return + } + } + + for j, intermediate := range test.intermediates { + ok := opts.Intermediates.AppendCertsFromPEM([]byte(intermediate)) + if !ok { + t.Errorf("#%d: failed to parse intermediate #%d", i, j) + return + } + } + + leaf, err := certificateFromPEM(test.leaf) + if err != nil { + t.Errorf("#%d: failed to parse leaf: %s", i, err) + return + } + + chains, err := leaf.Verify(opts) + + if test.errorCallback == nil && err != nil { + t.Errorf("#%d: unexpected error: %s", i, err) + } + if test.errorCallback != nil { + if !test.errorCallback(t, i, err) { + return + } + } + + if len(chains) != len(test.expectedChains) { + t.Errorf("#%d: wanted %d chains, got %d", i, len(test.expectedChains), len(chains)) + } + + // We check that each returned chain matches a chain from + // expectedChains but an entry in expectedChains can't match + // two chains. + seenChains := make([]bool, len(chains)) + NextOutputChain: + for _, chain := range chains { + TryNextExpected: + for j, expectedChain := range test.expectedChains { + if seenChains[j] { + continue + } + if len(chain) != len(expectedChain) { + continue + } + for k, cert := range chain { + if strings.Index(nameToKey(&cert.Subject), expectedChain[k]) == -1 { + continue TryNextExpected + } + } + // we matched + seenChains[j] = true + continue NextOutputChain + } + t.Errorf("#%d: No expected chain matched %s", i, chainToDebugString(chain)) + } + } +} + +func chainToDebugString(chain []*Certificate) string { + var chainStr string + for _, cert := range chain { + if len(chainStr) > 0 { + chainStr += " -> " + } + chainStr += nameToKey(&cert.Subject) + } + return chainStr +} + +const verisignRoot = `-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE +BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is +I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do +lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc +AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k +-----END CERTIFICATE----- +` + +const thawteIntermediate = `-----BEGIN CERTIFICATE----- +MIIDIzCCAoygAwIBAgIEMAAAAjANBgkqhkiG9w0BAQUFADBfMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xNzA1BgNVBAsTLkNsYXNzIDMgUHVi +bGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQwNTEzMDAw +MDAwWhcNMTQwNTEyMjM1OTU5WjBMMQswCQYDVQQGEwJaQTElMCMGA1UEChMcVGhh +d3RlIENvbnN1bHRpbmcgKFB0eSkgTHRkLjEWMBQGA1UEAxMNVGhhd3RlIFNHQyBD +QTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1NNn0I0Vf67NMf59HZGhPwtx +PKzMyGT7Y/wySweUvW+Aui/hBJPAM/wJMyPpC3QrccQDxtLN4i/1CWPN/0ilAL/g +5/OIty0y3pg25gqtAHvEZEo7hHUD8nCSfQ5i9SGraTaEMXWQ+L/HbIgbBpV8yeWo +3nWhLHpo39XKHIdYYBkCAwEAAaOB/jCB+zASBgNVHRMBAf8ECDAGAQH/AgEAMAsG +A1UdDwQEAwIBBjARBglghkgBhvhCAQEEBAMCAQYwKAYDVR0RBCEwH6QdMBsxGTAX +BgNVBAMTEFByaXZhdGVMYWJlbDMtMTUwMQYDVR0fBCowKDAmoCSgIoYgaHR0cDov +L2NybC52ZXJpc2lnbi5jb20vcGNhMy5jcmwwMgYIKwYBBQUHAQEEJjAkMCIGCCsG +AQUFBzABhhZodHRwOi8vb2NzcC50aGF3dGUuY29tMDQGA1UdJQQtMCsGCCsGAQUF +BwMBBggrBgEFBQcDAgYJYIZIAYb4QgQBBgpghkgBhvhFAQgBMA0GCSqGSIb3DQEB +BQUAA4GBAFWsY+reod3SkF+fC852vhNRj5PZBSvIG3dLrWlQoe7e3P3bB+noOZTc +q3J5Lwa/q4FwxKjt6lM07e8eU9kGx1Yr0Vz00YqOtCuxN5BICEIlxT6Ky3/rbwTR +bcV0oveifHtgPHfNDs5IAn8BL7abN+AqKjbc1YXWrOU/VG+WHgWv +-----END CERTIFICATE----- +` + +const googleLeaf = `-----BEGIN CERTIFICATE----- +MIIDITCCAoqgAwIBAgIQL9+89q6RUm0PmqPfQDQ+mjANBgkqhkiG9w0BAQUFADBM +MQswCQYDVQQGEwJaQTElMCMGA1UEChMcVGhhd3RlIENvbnN1bHRpbmcgKFB0eSkg +THRkLjEWMBQGA1UEAxMNVGhhd3RlIFNHQyBDQTAeFw0wOTEyMTgwMDAwMDBaFw0x +MTEyMTgyMzU5NTlaMGgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlh +MRYwFAYDVQQHFA1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKFApHb29nbGUgSW5jMRcw +FQYDVQQDFA53d3cuZ29vZ2xlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC +gYEA6PmGD5D6htffvXImttdEAoN4c9kCKO+IRTn7EOh8rqk41XXGOOsKFQebg+jN +gtXj9xVoRaELGYW84u+E593y17iYwqG7tcFR39SDAqc9BkJb4SLD3muFXxzW2k6L +05vuuWciKh0R73mkszeK9P4Y/bz5RiNQl/Os/CRGK1w7t0UCAwEAAaOB5zCB5DAM +BgNVHRMBAf8EAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9jcmwudGhhd3Rl +LmNvbS9UaGF3dGVTR0NDQS5jcmwwKAYDVR0lBCEwHwYIKwYBBQUHAwEGCCsGAQUF +BwMCBglghkgBhvhCBAEwcgYIKwYBBQUHAQEEZjBkMCIGCCsGAQUFBzABhhZodHRw +Oi8vb2NzcC50aGF3dGUuY29tMD4GCCsGAQUFBzAChjJodHRwOi8vd3d3LnRoYXd0 +ZS5jb20vcmVwb3NpdG9yeS9UaGF3dGVfU0dDX0NBLmNydDANBgkqhkiG9w0BAQUF +AAOBgQCfQ89bxFApsb/isJr/aiEdLRLDLE5a+RLizrmCUi3nHX4adpaQedEkUjh5 +u2ONgJd8IyAPkU0Wueru9G2Jysa9zCRo1kNbzipYvzwY4OA8Ys+WAi0oR1A04Se6 +z5nRUP8pJcA2NhUzUnC+MY+f6H/nEQyNv4SgQhqAibAxWEEHXw== +-----END CERTIFICATE-----` + +const dnssecExpLeaf = `-----BEGIN CERTIFICATE----- +MIIGzTCCBbWgAwIBAgIDAdD6MA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTAwNzA0MTQ1MjQ1 +WhcNMTEwNzA1MTA1NzA0WjCBwTEgMB4GA1UEDRMXMjIxMTM3LWxpOWE5dHhJRzZM +NnNyVFMxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVQZXJzb25hIE5vdCBWYWxpZGF0 +ZWQxKTAnBgNVBAsTIFN0YXJ0Q29tIEZyZWUgQ2VydGlmaWNhdGUgTWVtYmVyMRsw +GQYDVQQDExJ3d3cuZG5zc2VjLWV4cC5vcmcxKDAmBgkqhkiG9w0BCQEWGWhvc3Rt +YXN0ZXJAZG5zc2VjLWV4cC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDEdF/22vaxrPbqpgVYMWi+alfpzBctpbfLBdPGuqOazJdCT0NbWcK8/+B4 +X6OlSOURNIlwLzhkmwVsWdVv6dVSaN7d4yI/fJkvgfDB9+au+iBJb6Pcz8ULBfe6 +D8HVvqKdORp6INzHz71z0sghxrQ0EAEkoWAZLh+kcn2ZHdcmZaBNUfjmGbyU6PRt +RjdqoP+owIaC1aktBN7zl4uO7cRjlYFdusINrh2kPP02KAx2W84xjxX1uyj6oS6e +7eBfvcwe8czW/N1rbE0CoR7h9+HnIrjnVG9RhBiZEiw3mUmF++Up26+4KTdRKbu3 ++BL4yMpfd66z0+zzqu+HkvyLpFn5AgMBAAGjggL/MIIC+zAJBgNVHRMEAjAAMAsG +A1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQUy04I5guM +drzfh2JQaXhgV86+4jUwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8OLEUw +LQYDVR0RBCYwJIISd3d3LmRuc3NlYy1leHAub3Jngg5kbnNzZWMtZXhwLm9yZzCC +AUIGA1UdIASCATkwggE1MIIBMQYLKwYBBAGBtTcBAgIwggEgMC4GCCsGAQUFBwIB +FiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMDQGCCsGAQUFBwIB +FihodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9pbnRlcm1lZGlhdGUucGRmMIG3Bggr +BgEFBQcCAjCBqjAUFg1TdGFydENvbSBMdGQuMAMCAQEagZFMaW1pdGVkIExpYWJp +bGl0eSwgc2VlIHNlY3Rpb24gKkxlZ2FsIExpbWl0YXRpb25zKiBvZiB0aGUgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUG9saWN5IGF2YWlsYWJsZSBh +dCBodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMGEGA1UdHwRaMFgw +KqAooCaGJGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL2NydDEtY3JsLmNybDAqoCig +JoYkaHR0cDovL2NybC5zdGFydHNzbC5jb20vY3J0MS1jcmwuY3JsMIGOBggrBgEF +BQcBAQSBgTB/MDkGCCsGAQUFBzABhi1odHRwOi8vb2NzcC5zdGFydHNzbC5jb20v +c3ViL2NsYXNzMS9zZXJ2ZXIvY2EwQgYIKwYBBQUHMAKGNmh0dHA6Ly93d3cuc3Rh +cnRzc2wuY29tL2NlcnRzL3N1Yi5jbGFzczEuc2VydmVyLmNhLmNydDAjBgNVHRIE +HDAahhhodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS8wDQYJKoZIhvcNAQEFBQADggEB +ACXj6SB59KRJPenn6gUdGEqcta97U769SATyiQ87i9er64qLwvIGLMa3o2Rcgl2Y +kghUeyLdN/EXyFBYA8L8uvZREPoc7EZukpT/ZDLXy9i2S0jkOxvF2fD/XLbcjGjM +iEYG1/6ASw0ri9C0k4oDDoJLCoeH9++yqF7SFCCMcDkJqiAGXNb4euDpa8vCCtEQ +CSS+ObZbfkreRt3cNCf5LfCXe9OsTnCfc8Cuq81c0oLaG+SmaLUQNBuToq8e9/Zm ++b+/a3RVjxmkV5OCcGVBxsXNDn54Q6wsdw0TBMcjwoEndzpLS7yWgFbbkq5ZiGpw +Qibb2+CfKuQ+WFV1GkVQmVA= +-----END CERTIFICATE-----` + +const startComIntermediate = `-----BEGIN CERTIFICATE----- +MIIGNDCCBBygAwIBAgIBGDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjA1NDE3WhcNMTcxMDI0MjA1NDE3WjCB +jDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzApBgNVBAsT +IlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNVBAMTL1N0 +YXJ0Q29tIENsYXNzIDEgUHJpbWFyeSBJbnRlcm1lZGlhdGUgU2VydmVyIENBMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtonGrO8JUngHrJJj0PREGBiE +gFYfka7hh/oyULTTRwbw5gdfcA4Q9x3AzhA2NIVaD5Ksg8asWFI/ujjo/OenJOJA +pgh2wJJuniptTT9uYSAK21ne0n1jsz5G/vohURjXzTCm7QduO3CHtPn66+6CPAVv +kvek3AowHpNz/gfK11+AnSJYUq4G2ouHI2mw5CrY6oPSvfNx23BaKA+vWjhwRRI/ +ME3NO68X5Q/LoKldSKqxYVDLNM08XMML6BDAjJvwAwNi/rJsPnIO7hxDKslIDlc5 +xDEhyBDBLIf+VJVSH1I8MRKbf+fAoKVZ1eKPPvDVqOHXcDGpxLPPr21TLwb0pwID +AQABo4IBrTCCAakwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFOtCNNCYsKuf9BtrCPfMZC7vDixFMB8GA1UdIwQYMBaAFE4L7xqkQFul +F2mHMMo0aEPQQa7yMGYGCCsGAQUFBwEBBFowWDAnBggrBgEFBQcwAYYbaHR0cDov +L29jc3Auc3RhcnRzc2wuY29tL2NhMC0GCCsGAQUFBzAChiFodHRwOi8vd3d3LnN0 +YXJ0c3NsLmNvbS9zZnNjYS5jcnQwWwYDVR0fBFQwUjAnoCWgI4YhaHR0cDovL3d3 +dy5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMCegJaAjhiFodHRwOi8vY3JsLnN0YXJ0 +c3NsLmNvbS9zZnNjYS5jcmwwgYAGA1UdIAR5MHcwdQYLKwYBBAGBtTcBAgEwZjAu +BggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9saWN5LnBkZjA0 +BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50ZXJtZWRpYXRl +LnBkZjANBgkqhkiG9w0BAQUFAAOCAgEAIQlJPqWIbuALi0jaMU2P91ZXouHTYlfp +tVbzhUV1O+VQHwSL5qBaPucAroXQ+/8gA2TLrQLhxpFy+KNN1t7ozD+hiqLjfDen +xk+PNdb01m4Ge90h2c9W/8swIkn+iQTzheWq8ecf6HWQTd35RvdCNPdFWAwRDYSw +xtpdPvkBnufh2lWVvnQce/xNFE+sflVHfXv0pQ1JHpXo9xLBzP92piVH0PN1Nb6X +t1gW66pceG/sUzCv6gRNzKkC4/C2BBL2MLERPZBOVmTX3DxDX3M570uvh+v2/miI +RHLq0gfGabDBoYvvF0nXYbFFSF87ICHpW7LM9NfpMfULFWE7epTj69m8f5SuauNi +YpaoZHy4h/OZMn6SolK+u/hlz8nyMPyLwcKmltdfieFcNID1j0cHL7SRv7Gifl9L +WtBbnySGBVFaaQNlQ0lxxeBvlDRr9hvYqbBMflPrj0jfyjO1SPo2ShpTpjMM0InN +SRXNiTE8kMBy12VLUjWKRhFEuT2OKGWmPnmeXAhEKa2wNREuIU640ucQPl2Eg7PD +wuTSxv0JS3QJ3fGz0xk+gA2iCxnwOOfFwq/iI9th4p1cbiCJSS4jarJiwUW0n6+L +p/EiO/h94pDQehn7Skzj0n1fSoMD7SfWI55rjbRZotnvbIIp3XUZPD9MEI3vu3Un +0q6Dp6jOW6c= +-----END CERTIFICATE-----` + +const startComRoot = `-----BEGIN CERTIFICATE----- +MIIHyTCCBbGgAwIBAgIBATANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM2WhcNMzYwOTE3MTk0NjM2WjB9 +MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi +U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk +pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf +OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C +Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT +Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi +HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM +Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w ++2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ +Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 +Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B +26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID +AQABo4ICUjCCAk4wDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAa4wHQYDVR0OBBYE +FE4L7xqkQFulF2mHMMo0aEPQQa7yMGQGA1UdHwRdMFswLKAqoCiGJmh0dHA6Ly9j +ZXJ0LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMCugKaAnhiVodHRwOi8vY3Js +LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMIIBXQYDVR0gBIIBVDCCAVAwggFM +BgsrBgEEAYG1NwEBATCCATswLwYIKwYBBQUHAgEWI2h0dHA6Ly9jZXJ0LnN0YXJ0 +Y29tLm9yZy9wb2xpY3kucGRmMDUGCCsGAQUFBwIBFilodHRwOi8vY2VydC5zdGFy +dGNvbS5vcmcvaW50ZXJtZWRpYXRlLnBkZjCB0AYIKwYBBQUHAgIwgcMwJxYgU3Rh +cnQgQ29tbWVyY2lhbCAoU3RhcnRDb20pIEx0ZC4wAwIBARqBl0xpbWl0ZWQgTGlh +YmlsaXR5LCByZWFkIHRoZSBzZWN0aW9uICpMZWdhbCBMaW1pdGF0aW9ucyogb2Yg +dGhlIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFBvbGljeSBhdmFp +bGFibGUgYXQgaHR0cDovL2NlcnQuc3RhcnRjb20ub3JnL3BvbGljeS5wZGYwEQYJ +YIZIAYb4QgEBBAQDAgAHMDgGCWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNT +TCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTANBgkqhkiG9w0BAQUFAAOCAgEAFmyZ +9GYMNPXQhV59CuzaEE44HF7fpiUFS5Eyweg78T3dRAlbB0mKKctmArexmvclmAk8 +jhvh3TaHK0u7aNM5Zj2gJsfyOZEdUauCe37Vzlrk4gNXcGmXCPleWKYK34wGmkUW +FjgKXlf2Ysd6AgXmvB618p70qSmD+LIU424oh0TDkBreOKk8rENNZEXO3SipXPJz +ewT4F+irsfMuXGRuczE6Eri8sxHkfY+BUZo7jYn0TZNmezwD7dOaHZrzZVD1oNB1 +ny+v8OqCQ5j4aZyJecRDjkZy42Q2Eq/3JR44iZB3fsNrarnDy0RLrHiQi+fHLB5L +EUTINFInzQpdn4XBidUaePKVEFMy3YCEZnXZtWgo+2EuvoSoOMCZEoalHmdkrQYu +L6lwhceWD3yJZfWOQ1QOq92lgDmUYMA0yZZwLKMS9R9Ie70cfmu3nZD0Ijuu+Pwq +yvqCUqDvr0tVk+vBtfAii6w0TiYiBKGHLHVKt+V9E9e4DGTANtLJL4YSjCMJwRuC +O3NJo2pXh5Tl1njFmUNj403gdy3hZZlyaQQaRwnmDwFWJPsfvw55qVguucQJAX6V +um0ABj6y6koQOdjQK/W/7HW/lwLFCRsI3FU34oH7N4RDYiDK51ZLZer+bMEkkySh +NOsF/5oirpt9P/FlUQqmMGqz9IgcgA38corog14= +-----END CERTIFICATE-----` diff --git a/src/pkg/crypto/x509/x509.go b/src/pkg/crypto/x509/x509.go new file mode 100644 index 000000000..8fda47159 --- /dev/null +++ b/src/pkg/crypto/x509/x509.go @@ -0,0 +1,1073 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package x509 parses X.509-encoded keys and certificates. +package x509 + +import ( + "asn1" + "big" + "bytes" + "crypto" + "crypto/dsa" + "crypto/rsa" + "crypto/sha1" + "crypto/x509/pkix" + "encoding/pem" + "io" + "os" + "time" +) + +// pkcs1PrivateKey is a structure which mirrors the PKCS#1 ASN.1 for an RSA private key. +type pkcs1PrivateKey struct { + Version int + N *big.Int + E int + D *big.Int + P *big.Int + Q *big.Int + // We ignore these values, if present, because rsa will calculate them. + Dp *big.Int `asn1:"optional"` + Dq *big.Int `asn1:"optional"` + Qinv *big.Int `asn1:"optional"` + + AdditionalPrimes []pkcs1AdditionalRSAPrime `asn1:"optional"` +} + +type pkcs1AdditionalRSAPrime struct { + Prime *big.Int + + // We ignore these values because rsa will calculate them. + Exp *big.Int + Coeff *big.Int +} + +// ParsePKCS1PrivateKey returns an RSA private key from its ASN.1 PKCS#1 DER encoded form. +func ParsePKCS1PrivateKey(der []byte) (key *rsa.PrivateKey, err os.Error) { + var priv pkcs1PrivateKey + rest, err := asn1.Unmarshal(der, &priv) + if len(rest) > 0 { + err = asn1.SyntaxError{"trailing data"} + return + } + if err != nil { + return + } + + if priv.Version > 1 { + return nil, os.NewError("x509: unsupported private key version") + } + + if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 { + return nil, os.NewError("private key contains zero or negative value") + } + + key = new(rsa.PrivateKey) + key.PublicKey = rsa.PublicKey{ + E: priv.E, + N: priv.N, + } + + key.D = priv.D + key.Primes = make([]*big.Int, 2+len(priv.AdditionalPrimes)) + key.Primes[0] = priv.P + key.Primes[1] = priv.Q + for i, a := range priv.AdditionalPrimes { + if a.Prime.Sign() <= 0 { + return nil, os.NewError("private key contains zero or negative prime") + } + key.Primes[i+2] = a.Prime + // We ignore the other two values because rsa will calculate + // them as needed. + } + + err = key.Validate() + if err != nil { + return nil, err + } + key.Precompute() + + return +} + +// MarshalPKCS1PrivateKey converts a private key to ASN.1 DER encoded form. +func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { + key.Precompute() + + version := 0 + if len(key.Primes) > 2 { + version = 1 + } + + priv := pkcs1PrivateKey{ + Version: version, + N: key.N, + E: key.PublicKey.E, + D: key.D, + P: key.Primes[0], + Q: key.Primes[1], + Dp: key.Precomputed.Dp, + Dq: key.Precomputed.Dq, + Qinv: key.Precomputed.Qinv, + } + + priv.AdditionalPrimes = make([]pkcs1AdditionalRSAPrime, len(key.Precomputed.CRTValues)) + for i, values := range key.Precomputed.CRTValues { + priv.AdditionalPrimes[i].Prime = key.Primes[2+i] + priv.AdditionalPrimes[i].Exp = values.Exp + priv.AdditionalPrimes[i].Coeff = values.Coeff + } + + b, _ := asn1.Marshal(priv) + return b +} + +// These structures reflect the ASN.1 structure of X.509 certificates.: + +type certificate struct { + Raw asn1.RawContent + TBSCertificate tbsCertificate + SignatureAlgorithm pkix.AlgorithmIdentifier + SignatureValue asn1.BitString +} + +type tbsCertificate struct { + Raw asn1.RawContent + Version int `asn1:"optional,explicit,default:1,tag:0"` + SerialNumber *big.Int + SignatureAlgorithm pkix.AlgorithmIdentifier + Issuer pkix.RDNSequence + Validity validity + Subject pkix.RDNSequence + PublicKey publicKeyInfo + UniqueId asn1.BitString `asn1:"optional,tag:1"` + SubjectUniqueId asn1.BitString `asn1:"optional,tag:2"` + Extensions []pkix.Extension `asn1:"optional,explicit,tag:3"` +} + +type dsaAlgorithmParameters struct { + P, Q, G *big.Int +} + +type dsaSignature struct { + R, S *big.Int +} + +type validity struct { + NotBefore, NotAfter *time.Time +} + +type publicKeyInfo struct { + Raw asn1.RawContent + Algorithm pkix.AlgorithmIdentifier + PublicKey asn1.BitString +} + +// RFC 5280, 4.2.1.1 +type authKeyId struct { + Id []byte `asn1:"optional,tag:0"` +} + +type SignatureAlgorithm int + +const ( + UnknownSignatureAlgorithm SignatureAlgorithm = iota + MD2WithRSA + MD5WithRSA + SHA1WithRSA + SHA256WithRSA + SHA384WithRSA + SHA512WithRSA + DSAWithSHA1 + DSAWithSHA256 +) + +type PublicKeyAlgorithm int + +const ( + UnknownPublicKeyAlgorithm PublicKeyAlgorithm = iota + RSA + DSA +) + +// OIDs for signature algorithms +// +// pkcs-1 OBJECT IDENTIFIER ::= { +// iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 1 } +// +// +// RFC 3279 2.2.1 RSA Signature Algorithms +// +// md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } +// +// md5WithRSAEncryption OBJECT IDENTIFER ::= { pkcs-1 4 } +// +// sha-1WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 5 } +// +// dsaWithSha1 OBJECT IDENTIFIER ::= { +// iso(1) member-body(2) us(840) x9-57(10040) x9cm(4) 3 } +// +// +// RFC 4055 5 PKCS #1 Version 1.5 +// +// sha256WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 11 } +// +// sha384WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 12 } +// +// sha512WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 13 } +// +// +// RFC 5758 3.1 DSA Signature Algorithms +// +// dsaWithSha356 OBJECT IDENTIFER ::= { +// joint-iso-ccitt(2) country(16) us(840) organization(1) gov(101) +// algorithms(4) id-dsa-with-sha2(3) 2} +// +var ( + oidSignatureMD2WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 2} + oidSignatureMD5WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 4} + oidSignatureSHA1WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5} + oidSignatureSHA256WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 11} + oidSignatureSHA384WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 12} + oidSignatureSHA512WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 13} + oidSignatureDSAWithSHA1 = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 3} + oidSignatureDSAWithSHA256 = asn1.ObjectIdentifier{2, 16, 840, 1, 101, 4, 3, 2} +) + +func getSignatureAlgorithmFromOID(oid asn1.ObjectIdentifier) SignatureAlgorithm { + switch { + case oid.Equal(oidSignatureMD2WithRSA): + return MD2WithRSA + case oid.Equal(oidSignatureMD5WithRSA): + return MD5WithRSA + case oid.Equal(oidSignatureSHA1WithRSA): + return SHA1WithRSA + case oid.Equal(oidSignatureSHA256WithRSA): + return SHA256WithRSA + case oid.Equal(oidSignatureSHA384WithRSA): + return SHA384WithRSA + case oid.Equal(oidSignatureSHA512WithRSA): + return SHA512WithRSA + case oid.Equal(oidSignatureDSAWithSHA1): + return DSAWithSHA1 + case oid.Equal(oidSignatureDSAWithSHA256): + return DSAWithSHA256 + } + return UnknownSignatureAlgorithm +} + +// RFC 3279, 2.3 Public Key Algorithms +// +// pkcs-1 OBJECT IDENTIFIER ::== { iso(1) member-body(2) us(840) +// rsadsi(113549) pkcs(1) 1 } +// +// rsaEncryption OBJECT IDENTIFIER ::== { pkcs1-1 1 } +// +// id-dsa OBJECT IDENTIFIER ::== { iso(1) member-body(2) us(840) +// x9-57(10040) x9cm(4) 1 } +var ( + oidPublicKeyRsa = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1} + oidPublicKeyDsa = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 1} +) + +func getPublicKeyAlgorithmFromOID(oid asn1.ObjectIdentifier) PublicKeyAlgorithm { + switch { + case oid.Equal(oidPublicKeyRsa): + return RSA + case oid.Equal(oidPublicKeyDsa): + return DSA + } + return UnknownPublicKeyAlgorithm +} + +// KeyUsage represents the set of actions that are valid for a given key. It's +// a bitmap of the KeyUsage* constants. +type KeyUsage int + +const ( + KeyUsageDigitalSignature KeyUsage = 1 << iota + KeyUsageContentCommitment + KeyUsageKeyEncipherment + KeyUsageDataEncipherment + KeyUsageKeyAgreement + KeyUsageCertSign + KeyUsageCRLSign + KeyUsageEncipherOnly + KeyUsageDecipherOnly +) + +// RFC 5280, 4.2.1.12 Extended Key Usage +// +// anyExtendedKeyUsage OBJECT IDENTIFIER ::= { id-ce-extKeyUsage 0 } +// +// id-kp OBJECT IDENTIFIER ::= { id-pkix 3 } +// +// id-kp-serverAuth OBJECT IDENTIFIER ::= { id-kp 1 } +// id-kp-clientAuth OBJECT IDENTIFIER ::= { id-kp 2 } +// id-kp-codeSigning OBJECT IDENTIFIER ::= { id-kp 3 } +// id-kp-emailProtection OBJECT IDENTIFIER ::= { id-kp 4 } +// id-kp-timeStamping OBJECT IDENTIFIER ::= { id-kp 8 } +// id-kp-OCSPSigning OBJECT IDENTIFIER ::= { id-kp 9 } +var ( + oidExtKeyUsageAny = asn1.ObjectIdentifier{2, 5, 29, 37, 0} + oidExtKeyUsageServerAuth = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 1} + oidExtKeyUsageClientAuth = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 2} + oidExtKeyUsageCodeSigning = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 3} + oidExtKeyUsageEmailProtection = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 4} + oidExtKeyUsageTimeStamping = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 8} + oidExtKeyUsageOCSPSigning = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 9} +) + +// ExtKeyUsage represents an extended set of actions that are valid for a given key. +// Each of the ExtKeyUsage* constants define a unique action. +type ExtKeyUsage int + +const ( + ExtKeyUsageAny ExtKeyUsage = iota + ExtKeyUsageServerAuth + ExtKeyUsageClientAuth + ExtKeyUsageCodeSigning + ExtKeyUsageEmailProtection + ExtKeyUsageTimeStamping + ExtKeyUsageOCSPSigning +) + +// A Certificate represents an X.509 certificate. +type Certificate struct { + Raw []byte // Complete ASN.1 DER content (certificate, signature algorithm and signature). + RawTBSCertificate []byte // Certificate part of raw ASN.1 DER content. + RawSubjectPublicKeyInfo []byte // DER encoded SubjectPublicKeyInfo. + + Signature []byte + SignatureAlgorithm SignatureAlgorithm + + PublicKeyAlgorithm PublicKeyAlgorithm + PublicKey interface{} + + Version int + SerialNumber *big.Int + Issuer pkix.Name + Subject pkix.Name + NotBefore, NotAfter *time.Time // Validity bounds. + KeyUsage KeyUsage + + ExtKeyUsage []ExtKeyUsage // Sequence of extended key usages. + UnknownExtKeyUsage []asn1.ObjectIdentifier // Encountered extended key usages unknown to this package. + + BasicConstraintsValid bool // if true then the next two fields are valid. + IsCA bool + MaxPathLen int + + SubjectKeyId []byte + AuthorityKeyId []byte + + // Subject Alternate Name values + DNSNames []string + EmailAddresses []string + + // Name constraints + PermittedDNSDomainsCritical bool // if true then the name constraints are marked critical. + PermittedDNSDomains []string + + PolicyIdentifiers []asn1.ObjectIdentifier +} + +// UnsupportedAlgorithmError results from attempting to perform an operation +// that involves algorithms that are not currently implemented. +type UnsupportedAlgorithmError struct{} + +func (UnsupportedAlgorithmError) String() string { + return "cannot verify signature: algorithm unimplemented" +} + +// ConstraintViolationError results when a requested usage is not permitted by +// a certificate. For example: checking a signature when the public key isn't a +// certificate signing key. +type ConstraintViolationError struct{} + +func (ConstraintViolationError) String() string { + return "invalid signature: parent certificate cannot sign this kind of certificate" +} + +func (c *Certificate) Equal(other *Certificate) bool { + return bytes.Equal(c.Raw, other.Raw) +} + +// CheckSignatureFrom verifies that the signature on c is a valid signature +// from parent. +func (c *Certificate) CheckSignatureFrom(parent *Certificate) (err os.Error) { + // RFC 5280, 4.2.1.9: + // "If the basic constraints extension is not present in a version 3 + // certificate, or the extension is present but the cA boolean is not + // asserted, then the certified public key MUST NOT be used to verify + // certificate signatures." + if parent.Version == 3 && !parent.BasicConstraintsValid || + parent.BasicConstraintsValid && !parent.IsCA { + return ConstraintViolationError{} + } + + if parent.KeyUsage != 0 && parent.KeyUsage&KeyUsageCertSign == 0 { + return ConstraintViolationError{} + } + + if parent.PublicKeyAlgorithm == UnknownPublicKeyAlgorithm { + return UnsupportedAlgorithmError{} + } + + // TODO(agl): don't ignore the path length constraint. + + return parent.CheckSignature(c.SignatureAlgorithm, c.RawTBSCertificate, c.Signature) +} + +// CheckSignature verifies that signature is a valid signature over signed from +// c's public key. +func (c *Certificate) CheckSignature(algo SignatureAlgorithm, signed, signature []byte) (err os.Error) { + var hashType crypto.Hash + + switch algo { + case SHA1WithRSA, DSAWithSHA1: + hashType = crypto.SHA1 + case SHA256WithRSA, DSAWithSHA256: + hashType = crypto.SHA256 + case SHA384WithRSA: + hashType = crypto.SHA384 + case SHA512WithRSA: + hashType = crypto.SHA512 + default: + return UnsupportedAlgorithmError{} + } + + h := hashType.New() + if h == nil { + return UnsupportedAlgorithmError{} + } + + h.Write(signed) + digest := h.Sum() + + switch pub := c.PublicKey.(type) { + case *rsa.PublicKey: + return rsa.VerifyPKCS1v15(pub, hashType, digest, signature) + case *dsa.PublicKey: + dsaSig := new(dsaSignature) + if _, err := asn1.Unmarshal(signature, dsaSig); err != nil { + return err + } + if dsaSig.R.Sign() <= 0 || dsaSig.S.Sign() <= 0 { + return os.NewError("DSA signature contained zero or negative values") + } + if !dsa.Verify(pub, digest, dsaSig.R, dsaSig.S) { + return os.NewError("DSA verification failure") + } + return + } + return UnsupportedAlgorithmError{} +} + +// CheckCRLSignature checks that the signature in crl is from c. +func (c *Certificate) CheckCRLSignature(crl *pkix.CertificateList) (err os.Error) { + algo := getSignatureAlgorithmFromOID(crl.SignatureAlgorithm.Algorithm) + return c.CheckSignature(algo, crl.TBSCertList.Raw, crl.SignatureValue.RightAlign()) +} + +type UnhandledCriticalExtension struct{} + +func (h UnhandledCriticalExtension) String() string { + return "unhandled critical extension" +} + +type basicConstraints struct { + IsCA bool `asn1:"optional"` + MaxPathLen int `asn1:"optional"` +} + +type rsaPublicKey struct { + N *big.Int + E int +} + +// RFC 5280 4.2.1.4 +type policyInformation struct { + Policy asn1.ObjectIdentifier + // policyQualifiers omitted +} + +// RFC 5280, 4.2.1.10 +type nameConstraints struct { + Permitted []generalSubtree `asn1:"optional,tag:0"` + Excluded []generalSubtree `asn1:"optional,tag:1"` +} + +type generalSubtree struct { + Name string `asn1:"tag:2,optional,ia5"` + Min int `asn1:"optional,tag:0"` + Max int `asn1:"optional,tag:1"` +} + +func parsePublicKey(algo PublicKeyAlgorithm, keyData *publicKeyInfo) (interface{}, os.Error) { + asn1Data := keyData.PublicKey.RightAlign() + switch algo { + case RSA: + p := new(rsaPublicKey) + _, err := asn1.Unmarshal(asn1Data, p) + if err != nil { + return nil, err + } + + pub := &rsa.PublicKey{ + E: p.E, + N: p.N, + } + return pub, nil + case DSA: + var p *big.Int + _, err := asn1.Unmarshal(asn1Data, &p) + if err != nil { + return nil, err + } + paramsData := keyData.Algorithm.Parameters.FullBytes + params := new(dsaAlgorithmParameters) + _, err = asn1.Unmarshal(paramsData, params) + if err != nil { + return nil, err + } + if p.Sign() <= 0 || params.P.Sign() <= 0 || params.Q.Sign() <= 0 || params.G.Sign() <= 0 { + return nil, os.NewError("zero or negative DSA parameter") + } + pub := &dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: params.P, + Q: params.Q, + G: params.G, + }, + Y: p, + } + return pub, nil + default: + return nil, nil + } + panic("unreachable") +} + +func parseCertificate(in *certificate) (*Certificate, os.Error) { + out := new(Certificate) + out.Raw = in.Raw + out.RawTBSCertificate = in.TBSCertificate.Raw + out.RawSubjectPublicKeyInfo = in.TBSCertificate.PublicKey.Raw + + out.Signature = in.SignatureValue.RightAlign() + out.SignatureAlgorithm = + getSignatureAlgorithmFromOID(in.TBSCertificate.SignatureAlgorithm.Algorithm) + + out.PublicKeyAlgorithm = + getPublicKeyAlgorithmFromOID(in.TBSCertificate.PublicKey.Algorithm.Algorithm) + var err os.Error + out.PublicKey, err = parsePublicKey(out.PublicKeyAlgorithm, &in.TBSCertificate.PublicKey) + if err != nil { + return nil, err + } + + if in.TBSCertificate.SerialNumber.Sign() < 0 { + return nil, os.NewError("negative serial number") + } + + out.Version = in.TBSCertificate.Version + 1 + out.SerialNumber = in.TBSCertificate.SerialNumber + out.Issuer.FillFromRDNSequence(&in.TBSCertificate.Issuer) + out.Subject.FillFromRDNSequence(&in.TBSCertificate.Subject) + out.NotBefore = in.TBSCertificate.Validity.NotBefore + out.NotAfter = in.TBSCertificate.Validity.NotAfter + + for _, e := range in.TBSCertificate.Extensions { + if len(e.Id) == 4 && e.Id[0] == 2 && e.Id[1] == 5 && e.Id[2] == 29 { + switch e.Id[3] { + case 15: + // RFC 5280, 4.2.1.3 + var usageBits asn1.BitString + _, err := asn1.Unmarshal(e.Value, &usageBits) + + if err == nil { + var usage int + for i := 0; i < 9; i++ { + if usageBits.At(i) != 0 { + usage |= 1 << uint(i) + } + } + out.KeyUsage = KeyUsage(usage) + continue + } + case 19: + // RFC 5280, 4.2.1.9 + var constraints basicConstraints + _, err := asn1.Unmarshal(e.Value, &constraints) + + if err == nil { + out.BasicConstraintsValid = true + out.IsCA = constraints.IsCA + out.MaxPathLen = constraints.MaxPathLen + continue + } + case 17: + // RFC 5280, 4.2.1.6 + + // SubjectAltName ::= GeneralNames + // + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + // + // GeneralName ::= CHOICE { + // otherName [0] OtherName, + // rfc822Name [1] IA5String, + // dNSName [2] IA5String, + // x400Address [3] ORAddress, + // directoryName [4] Name, + // ediPartyName [5] EDIPartyName, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // registeredID [8] OBJECT IDENTIFIER } + var seq asn1.RawValue + _, err := asn1.Unmarshal(e.Value, &seq) + if err != nil { + return nil, err + } + if !seq.IsCompound || seq.Tag != 16 || seq.Class != 0 { + return nil, asn1.StructuralError{"bad SAN sequence"} + } + + parsedName := false + + rest := seq.Bytes + for len(rest) > 0 { + var v asn1.RawValue + rest, err = asn1.Unmarshal(rest, &v) + if err != nil { + return nil, err + } + switch v.Tag { + case 1: + out.EmailAddresses = append(out.EmailAddresses, string(v.Bytes)) + parsedName = true + case 2: + out.DNSNames = append(out.DNSNames, string(v.Bytes)) + parsedName = true + } + } + + if parsedName { + continue + } + // If we didn't parse any of the names then we + // fall through to the critical check below. + + case 30: + // RFC 5280, 4.2.1.10 + + // NameConstraints ::= SEQUENCE { + // permittedSubtrees [0] GeneralSubtrees OPTIONAL, + // excludedSubtrees [1] GeneralSubtrees OPTIONAL } + // + // GeneralSubtrees ::= SEQUENCE SIZE (1..MAX) OF GeneralSubtree + // + // GeneralSubtree ::= SEQUENCE { + // base GeneralName, + // minimum [0] BaseDistance DEFAULT 0, + // maximum [1] BaseDistance OPTIONAL } + // + // BaseDistance ::= INTEGER (0..MAX) + + var constraints nameConstraints + _, err := asn1.Unmarshal(e.Value, &constraints) + if err != nil { + return nil, err + } + + if len(constraints.Excluded) > 0 && e.Critical { + return out, UnhandledCriticalExtension{} + } + + for _, subtree := range constraints.Permitted { + if subtree.Min > 0 || subtree.Max > 0 || len(subtree.Name) == 0 { + if e.Critical { + return out, UnhandledCriticalExtension{} + } + continue + } + out.PermittedDNSDomains = append(out.PermittedDNSDomains, subtree.Name) + } + continue + + case 35: + // RFC 5280, 4.2.1.1 + var a authKeyId + _, err = asn1.Unmarshal(e.Value, &a) + if err != nil { + return nil, err + } + out.AuthorityKeyId = a.Id + continue + + case 37: + // RFC 5280, 4.2.1.12. Extended Key Usage + + // id-ce-extKeyUsage OBJECT IDENTIFIER ::= { id-ce 37 } + // + // ExtKeyUsageSyntax ::= SEQUENCE SIZE (1..MAX) OF KeyPurposeId + // + // KeyPurposeId ::= OBJECT IDENTIFIER + + var keyUsage []asn1.ObjectIdentifier + _, err = asn1.Unmarshal(e.Value, &keyUsage) + if err != nil { + return nil, err + } + + for _, u := range keyUsage { + switch { + case u.Equal(oidExtKeyUsageAny): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageAny) + case u.Equal(oidExtKeyUsageServerAuth): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageServerAuth) + case u.Equal(oidExtKeyUsageClientAuth): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageClientAuth) + case u.Equal(oidExtKeyUsageCodeSigning): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageCodeSigning) + case u.Equal(oidExtKeyUsageEmailProtection): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageEmailProtection) + case u.Equal(oidExtKeyUsageTimeStamping): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageTimeStamping) + case u.Equal(oidExtKeyUsageOCSPSigning): + out.ExtKeyUsage = append(out.ExtKeyUsage, ExtKeyUsageOCSPSigning) + default: + out.UnknownExtKeyUsage = append(out.UnknownExtKeyUsage, u) + } + } + + continue + + case 14: + // RFC 5280, 4.2.1.2 + var keyid []byte + _, err = asn1.Unmarshal(e.Value, &keyid) + if err != nil { + return nil, err + } + out.SubjectKeyId = keyid + continue + + case 32: + // RFC 5280 4.2.1.4: Certificate Policies + var policies []policyInformation + if _, err = asn1.Unmarshal(e.Value, &policies); err != nil { + return nil, err + } + out.PolicyIdentifiers = make([]asn1.ObjectIdentifier, len(policies)) + for i, policy := range policies { + out.PolicyIdentifiers[i] = policy.Policy + } + } + } + + if e.Critical { + return out, UnhandledCriticalExtension{} + } + } + + return out, nil +} + +// ParseCertificate parses a single certificate from the given ASN.1 DER data. +func ParseCertificate(asn1Data []byte) (*Certificate, os.Error) { + var cert certificate + rest, err := asn1.Unmarshal(asn1Data, &cert) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, asn1.SyntaxError{"trailing data"} + } + + return parseCertificate(&cert) +} + +// ParseCertificates parses one or more certificates from the given ASN.1 DER +// data. The certificates must be concatenated with no intermediate padding. +func ParseCertificates(asn1Data []byte) ([]*Certificate, os.Error) { + var v []*certificate + + for len(asn1Data) > 0 { + cert := new(certificate) + var err os.Error + asn1Data, err = asn1.Unmarshal(asn1Data, cert) + if err != nil { + return nil, err + } + v = append(v, cert) + } + + ret := make([]*Certificate, len(v)) + for i, ci := range v { + cert, err := parseCertificate(ci) + if err != nil { + return nil, err + } + ret[i] = cert + } + + return ret, nil +} + +func reverseBitsInAByte(in byte) byte { + b1 := in>>4 | in<<4 + b2 := b1>>2&0x33 | b1<<2&0xcc + b3 := b2>>1&0x55 | b2<<1&0xaa + return b3 +} + +var ( + oidExtensionSubjectKeyId = []int{2, 5, 29, 14} + oidExtensionKeyUsage = []int{2, 5, 29, 15} + oidExtensionAuthorityKeyId = []int{2, 5, 29, 35} + oidExtensionBasicConstraints = []int{2, 5, 29, 19} + oidExtensionSubjectAltName = []int{2, 5, 29, 17} + oidExtensionCertificatePolicies = []int{2, 5, 29, 32} + oidExtensionNameConstraints = []int{2, 5, 29, 30} +) + +func buildExtensions(template *Certificate) (ret []pkix.Extension, err os.Error) { + ret = make([]pkix.Extension, 7 /* maximum number of elements. */ ) + n := 0 + + if template.KeyUsage != 0 { + ret[n].Id = oidExtensionKeyUsage + ret[n].Critical = true + + var a [2]byte + a[0] = reverseBitsInAByte(byte(template.KeyUsage)) + a[1] = reverseBitsInAByte(byte(template.KeyUsage >> 8)) + + l := 1 + if a[1] != 0 { + l = 2 + } + + ret[n].Value, err = asn1.Marshal(asn1.BitString{Bytes: a[0:l], BitLength: l * 8}) + if err != nil { + return + } + n++ + } + + if template.BasicConstraintsValid { + ret[n].Id = oidExtensionBasicConstraints + ret[n].Value, err = asn1.Marshal(basicConstraints{template.IsCA, template.MaxPathLen}) + ret[n].Critical = true + if err != nil { + return + } + n++ + } + + if len(template.SubjectKeyId) > 0 { + ret[n].Id = oidExtensionSubjectKeyId + ret[n].Value, err = asn1.Marshal(template.SubjectKeyId) + if err != nil { + return + } + n++ + } + + if len(template.AuthorityKeyId) > 0 { + ret[n].Id = oidExtensionAuthorityKeyId + ret[n].Value, err = asn1.Marshal(authKeyId{template.AuthorityKeyId}) + if err != nil { + return + } + n++ + } + + if len(template.DNSNames) > 0 { + ret[n].Id = oidExtensionSubjectAltName + rawValues := make([]asn1.RawValue, len(template.DNSNames)) + for i, name := range template.DNSNames { + rawValues[i] = asn1.RawValue{Tag: 2, Class: 2, Bytes: []byte(name)} + } + ret[n].Value, err = asn1.Marshal(rawValues) + if err != nil { + return + } + n++ + } + + if len(template.PolicyIdentifiers) > 0 { + ret[n].Id = oidExtensionCertificatePolicies + policies := make([]policyInformation, len(template.PolicyIdentifiers)) + for i, policy := range template.PolicyIdentifiers { + policies[i].Policy = policy + } + ret[n].Value, err = asn1.Marshal(policies) + if err != nil { + return + } + n++ + } + + if len(template.PermittedDNSDomains) > 0 { + ret[n].Id = oidExtensionNameConstraints + ret[n].Critical = template.PermittedDNSDomainsCritical + + var out nameConstraints + out.Permitted = make([]generalSubtree, len(template.PermittedDNSDomains)) + for i, permitted := range template.PermittedDNSDomains { + out.Permitted[i] = generalSubtree{Name: permitted} + } + ret[n].Value, err = asn1.Marshal(out) + if err != nil { + return + } + n++ + } + + // Adding another extension here? Remember to update the maximum number + // of elements in the make() at the top of the function. + + return ret[0:n], nil +} + +var ( + oidSHA1WithRSA = []int{1, 2, 840, 113549, 1, 1, 5} + oidRSA = []int{1, 2, 840, 113549, 1, 1, 1} +) + +// CreateSelfSignedCertificate creates a new certificate based on +// a template. The following members of template are used: SerialNumber, +// Subject, NotBefore, NotAfter, KeyUsage, BasicConstraintsValid, IsCA, +// MaxPathLen, SubjectKeyId, DNSNames, PermittedDNSDomainsCritical, +// PermittedDNSDomains. +// +// The certificate is signed by parent. If parent is equal to template then the +// certificate is self-signed. The parameter pub is the public key of the +// signee and priv is the private key of the signer. +// +// The returned slice is the certificate in DER encoding. +func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.PublicKey, priv *rsa.PrivateKey) (cert []byte, err os.Error) { + asn1PublicKey, err := asn1.Marshal(rsaPublicKey{ + N: pub.N, + E: pub.E, + }) + if err != nil { + return + } + + if len(parent.SubjectKeyId) > 0 { + template.AuthorityKeyId = parent.SubjectKeyId + } + + extensions, err := buildExtensions(template) + if err != nil { + return + } + + encodedPublicKey := asn1.BitString{BitLength: len(asn1PublicKey) * 8, Bytes: asn1PublicKey} + c := tbsCertificate{ + Version: 2, + SerialNumber: template.SerialNumber, + SignatureAlgorithm: pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, + Issuer: parent.Subject.ToRDNSequence(), + Validity: validity{template.NotBefore, template.NotAfter}, + Subject: template.Subject.ToRDNSequence(), + PublicKey: publicKeyInfo{nil, pkix.AlgorithmIdentifier{Algorithm: oidRSA}, encodedPublicKey}, + Extensions: extensions, + } + + tbsCertContents, err := asn1.Marshal(c) + if err != nil { + return + } + + c.Raw = tbsCertContents + + h := sha1.New() + h.Write(tbsCertContents) + digest := h.Sum() + + signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest) + if err != nil { + return + } + + cert, err = asn1.Marshal(certificate{ + nil, + c, + pkix.AlgorithmIdentifier{Algorithm: oidSHA1WithRSA}, + asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, + }) + return +} + +// pemCRLPrefix is the magic string that indicates that we have a PEM encoded +// CRL. +var pemCRLPrefix = []byte("-----BEGIN X509 CRL") +// pemType is the type of a PEM encoded CRL. +var pemType = "X509 CRL" + +// ParseCRL parses a CRL from the given bytes. It's often the case that PEM +// encoded CRLs will appear where they should be DER encoded, so this function +// will transparently handle PEM encoding as long as there isn't any leading +// garbage. +func ParseCRL(crlBytes []byte) (certList *pkix.CertificateList, err os.Error) { + if bytes.HasPrefix(crlBytes, pemCRLPrefix) { + block, _ := pem.Decode(crlBytes) + if block != nil && block.Type == pemType { + crlBytes = block.Bytes + } + } + return ParseDERCRL(crlBytes) +} + +// ParseDERCRL parses a DER encoded CRL from the given bytes. +func ParseDERCRL(derBytes []byte) (certList *pkix.CertificateList, err os.Error) { + certList = new(pkix.CertificateList) + _, err = asn1.Unmarshal(derBytes, certList) + if err != nil { + certList = nil + } + return +} + +// CreateCRL returns a DER encoded CRL, signed by this Certificate, that +// contains the given list of revoked certificates. +func (c *Certificate) CreateCRL(rand io.Reader, priv *rsa.PrivateKey, revokedCerts []pkix.RevokedCertificate, now, expiry *time.Time) (crlBytes []byte, err os.Error) { + tbsCertList := pkix.TBSCertificateList{ + Version: 2, + Signature: pkix.AlgorithmIdentifier{ + Algorithm: oidSignatureSHA1WithRSA, + }, + Issuer: c.Subject.ToRDNSequence(), + ThisUpdate: now, + NextUpdate: expiry, + RevokedCertificates: revokedCerts, + } + + tbsCertListContents, err := asn1.Marshal(tbsCertList) + if err != nil { + return + } + + h := sha1.New() + h.Write(tbsCertListContents) + digest := h.Sum() + + signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest) + if err != nil { + return + } + + return asn1.Marshal(pkix.CertificateList{ + TBSCertList: tbsCertList, + SignatureAlgorithm: pkix.AlgorithmIdentifier{ + Algorithm: oidSignatureSHA1WithRSA, + }, + SignatureValue: asn1.BitString{Bytes: signature, BitLength: len(signature) * 8}, + }) +} diff --git a/src/pkg/crypto/x509/x509_test.go b/src/pkg/crypto/x509/x509_test.go new file mode 100644 index 000000000..dc216505e --- /dev/null +++ b/src/pkg/crypto/x509/x509_test.go @@ -0,0 +1,431 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "asn1" + "big" + "crypto/dsa" + "crypto/rand" + "crypto/rsa" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "testing" + "time" +) + +func TestParsePKCS1PrivateKey(t *testing.T) { + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, err := ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse private key: %s", err) + return + } + if priv.PublicKey.N.Cmp(rsaPrivateKey.PublicKey.N) != 0 || + priv.PublicKey.E != rsaPrivateKey.PublicKey.E || + priv.D.Cmp(rsaPrivateKey.D) != 0 || + priv.Primes[0].Cmp(rsaPrivateKey.Primes[0]) != 0 || + priv.Primes[1].Cmp(rsaPrivateKey.Primes[1]) != 0 { + t.Errorf("got:%+v want:%+v", priv, rsaPrivateKey) + } +} + +var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +-----END RSA PRIVATE KEY----- +` + +func bigFromString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 10) + return ret +} + +func fromBase10(base10 string) *big.Int { + i := new(big.Int) + i.SetString(base10, 10) + return i +} + +func bigFromHexString(s string) *big.Int { + ret := new(big.Int) + ret.SetString(s, 16) + return ret +} + +var rsaPrivateKey = &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: bigFromString("9353930466774385905609975137998169297361893554149986716853295022578535724979677252958524466350471210367835187480748268864277464700638583474144061408845077"), + E: 65537, + }, + D: bigFromString("7266398431328116344057699379749222532279343923819063639497049039389899328538543087657733766554155839834519529439851673014800261285757759040931985506583861"), + Primes: []*big.Int{ + bigFromString("98920366548084643601728869055592650835572950932266967461790948584315647051443"), + bigFromString("94560208308847015747498523884063394671606671904944666360068158221458669711639"), + }, +} + +func TestMarshalRSAPrivateKey(t *testing.T) { + priv := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: fromBase10("16346378922382193400538269749936049106320265317511766357599732575277382844051791096569333808598921852351577762718529818072849191122419410612033592401403764925096136759934497687765453905884149505175426053037420486697072448609022753683683718057795566811401938833367954642951433473337066311978821180526439641496973296037000052546108507805269279414789035461158073156772151892452251106173507240488993608650881929629163465099476849643165682709047462010581308719577053905787496296934240246311806555924593059995202856826239801816771116902778517096212527979497399966526283516447337775509777558018145573127308919204297111496233"), + E: 3, + }, + D: fromBase10("10897585948254795600358846499957366070880176878341177571733155050184921896034527397712889205732614568234385175145686545381899460748279607074689061600935843283397424506622998458510302603922766336783617368686090042765718290914099334449154829375179958369993407724946186243249568928237086215759259909861748642124071874879861299389874230489928271621259294894142840428407196932444474088857746123104978617098858619445675532587787023228852383149557470077802718705420275739737958953794088728369933811184572620857678792001136676902250566845618813972833750098806496641114644760255910789397593428910198080271317419213080834885003"), + Primes: []*big.Int{ + fromBase10("1025363189502892836833747188838978207017355117492483312747347695538428729137306368764177201532277413433182799108299960196606011786562992097313508180436744488171474690412562218914213688661311117337381958560443"), + fromBase10("3467903426626310123395340254094941045497208049900750380025518552334536945536837294961497712862519984786362199788654739924501424784631315081391467293694361474867825728031147665777546570788493758372218019373"), + fromBase10("4597024781409332673052708605078359346966325141767460991205742124888960305710298765592730135879076084498363772408626791576005136245060321874472727132746643162385746062759369754202494417496879741537284589047"), + }, + } + + derBytes := MarshalPKCS1PrivateKey(priv) + + priv2, err := ParsePKCS1PrivateKey(derBytes) + if err != nil { + t.Errorf("error parsing serialized key: %s", err) + return + } + if priv.PublicKey.N.Cmp(priv2.PublicKey.N) != 0 || + priv.PublicKey.E != priv2.PublicKey.E || + priv.D.Cmp(priv2.D) != 0 || + len(priv2.Primes) != 3 || + priv.Primes[0].Cmp(priv2.Primes[0]) != 0 || + priv.Primes[1].Cmp(priv2.Primes[1]) != 0 || + priv.Primes[2].Cmp(priv2.Primes[2]) != 0 { + t.Errorf("got:%+v want:%+v", priv, priv2) + } +} + +type matchHostnamesTest struct { + pattern, host string + ok bool +} + +var matchHostnamesTests = []matchHostnamesTest{ + {"a.b.c", "a.b.c", true}, + {"a.b.c", "b.b.c", false}, + {"", "b.b.c", false}, + {"a.b.c", "", false}, + {"example.com", "example.com", true}, + {"example.com", "www.example.com", false}, + {"*.example.com", "www.example.com", true}, + {"*.example.com", "xyz.www.example.com", false}, + {"*.*.example.com", "xyz.www.example.com", true}, + {"*.www.*.com", "xyz.www.example.com", true}, +} + +func TestMatchHostnames(t *testing.T) { + for i, test := range matchHostnamesTests { + r := matchHostnames(test.pattern, test.host) + if r != test.ok { + t.Errorf("#%d mismatch got: %t want: %t", i, r, test.ok) + } + } +} + +func TestCertificateParse(t *testing.T) { + s, _ := hex.DecodeString(certBytes) + certs, err := ParseCertificates(s) + if err != nil { + t.Error(err) + } + if len(certs) != 2 { + t.Errorf("Wrong number of certs: got %d want 2", len(certs)) + return + } + + err = certs[0].CheckSignatureFrom(certs[1]) + if err != nil { + t.Error(err) + } + + if err := certs[0].VerifyHostname("mail.google.com"); err != nil { + t.Error(err) + } +} + +var certBytes = "308203223082028ba00302010202106edf0d9499fd4533dd1297fc42a93be1300d06092a864886" + + "f70d0101050500304c310b3009060355040613025a4131253023060355040a131c546861777465" + + "20436f6e73756c74696e67202850747929204c74642e311630140603550403130d546861777465" + + "20534743204341301e170d3039303332353136343932395a170d3130303332353136343932395a" + + "3069310b3009060355040613025553311330110603550408130a43616c69666f726e6961311630" + + "140603550407130d4d6f756e7461696e205669657731133011060355040a130a476f6f676c6520" + + "496e63311830160603550403130f6d61696c2e676f6f676c652e636f6d30819f300d06092a8648" + + "86f70d010101050003818d0030818902818100c5d6f892fccaf5614b064149e80a2c9581a218ef" + + "41ec35bd7a58125ae76f9ea54ddc893abbeb029f6b73616bf0ffd868791fba7af9c4aebf3706ba" + + "3eeaeed27435b4ddcfb157c05f351d66aa87fee0de072d66d773affbd36ab78bef090e0cc861a9" + + "03ac90dd98b51c9c41566c017f0beec3bff391051ffba0f5cc6850ad2a590203010001a381e730" + + "81e430280603551d250421301f06082b0601050507030106082b06010505070302060960864801" + + "86f842040130360603551d1f042f302d302ba029a0278625687474703a2f2f63726c2e74686177" + + "74652e636f6d2f54686177746553474343412e63726c307206082b060105050701010466306430" + + "2206082b060105050730018616687474703a2f2f6f6373702e7468617774652e636f6d303e0608" + + "2b060105050730028632687474703a2f2f7777772e7468617774652e636f6d2f7265706f736974" + + "6f72792f5468617774655f5347435f43412e637274300c0603551d130101ff04023000300d0609" + + "2a864886f70d01010505000381810062f1f3050ebc105e497c7aedf87e24d2f4a986bb3b837bd1" + + "9b91ebcad98b065992f6bd2b49b7d6d3cb2e427a99d606c7b1d46352527fac39e6a8b6726de5bf" + + "70212a52cba07634a5e332011bd1868e78eb5e3c93cf03072276786f207494feaa0ed9d53b2110" + + "a76571f90209cdae884385c882587030ee15f33d761e2e45a6bc308203233082028ca003020102" + + "020430000002300d06092a864886f70d0101050500305f310b3009060355040613025553311730" + + "15060355040a130e566572695369676e2c20496e632e31373035060355040b132e436c61737320" + + "33205075626c6963205072696d6172792043657274696669636174696f6e20417574686f726974" + + "79301e170d3034303531333030303030305a170d3134303531323233353935395a304c310b3009" + + "060355040613025a4131253023060355040a131c54686177746520436f6e73756c74696e672028" + + "50747929204c74642e311630140603550403130d5468617774652053474320434130819f300d06" + + "092a864886f70d010101050003818d0030818902818100d4d367d08d157faecd31fe7d1d91a13f" + + "0b713cacccc864fb63fc324b0794bd6f80ba2fe10493c033fc093323e90b742b71c403c6d2cde2" + + "2ff50963cdff48a500bfe0e7f388b72d32de9836e60aad007bc4644a3b847503f270927d0e62f5" + + "21ab693684317590f8bfc76c881b06957cc9e5a8de75a12c7a68dfd5ca1c875860190203010001" + + "a381fe3081fb30120603551d130101ff040830060101ff020100300b0603551d0f040403020106" + + "301106096086480186f842010104040302010630280603551d110421301fa41d301b3119301706" + + "035504031310507269766174654c6162656c332d313530310603551d1f042a30283026a024a022" + + "8620687474703a2f2f63726c2e766572697369676e2e636f6d2f706361332e63726c303206082b" + + "0601050507010104263024302206082b060105050730018616687474703a2f2f6f6373702e7468" + + "617774652e636f6d30340603551d25042d302b06082b0601050507030106082b06010505070302" + + "06096086480186f8420401060a6086480186f845010801300d06092a864886f70d010105050003" + + "81810055ac63eadea1ddd2905f9f0bce76be13518f93d9052bc81b774bad6950a1eededcfddb07" + + "e9e83994dcab72792f06bfab8170c4a8edea5334edef1e53d906c7562bd15cf4d18a8eb42bb137" + + "9048084225c53e8acb7feb6f04d16dc574a2f7a27c7b603c77cd0ece48027f012fb69b37e02a2a" + + "36dcd585d6ace53f546f961e05af" + +func TestCreateSelfSignedCertificate(t *testing.T) { + random := rand.Reader + + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, err := ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Errorf("Failed to parse private key: %s", err) + return + } + + template := Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "test.example.com", + Organization: []string{"Acme Co"}, + }, + NotBefore: time.SecondsToUTC(1000), + NotAfter: time.SecondsToUTC(100000), + + SubjectKeyId: []byte{1, 2, 3, 4}, + KeyUsage: KeyUsageCertSign, + + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{"test.example.com"}, + + PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, + PermittedDNSDomains: []string{".example.com", "example.com"}, + } + + derBytes, err := CreateCertificate(random, &template, &template, &priv.PublicKey, priv) + if err != nil { + t.Errorf("Failed to create certificate: %s", err) + return + } + + cert, err := ParseCertificate(derBytes) + if err != nil { + t.Errorf("Failed to parse certificate: %s", err) + return + } + + if len(cert.PolicyIdentifiers) != 1 || !cert.PolicyIdentifiers[0].Equal(template.PolicyIdentifiers[0]) { + t.Errorf("Failed to parse policy identifiers: got:%#v want:%#v", cert.PolicyIdentifiers, template.PolicyIdentifiers) + } + + if len(cert.PermittedDNSDomains) != 2 || cert.PermittedDNSDomains[0] != ".example.com" || cert.PermittedDNSDomains[1] != "example.com" { + t.Errorf("Failed to parse name constraints: %#v", cert.PermittedDNSDomains) + } + + err = cert.CheckSignatureFrom(cert) + if err != nil { + t.Errorf("Signature verification failed: %s", err) + return + } +} + +// Self-signed certificate using DSA with SHA1 +var dsaCertPem = `-----BEGIN CERTIFICATE----- +MIIEDTCCA82gAwIBAgIJALHPghaoxeDhMAkGByqGSM44BAMweTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAk5DMQ8wDQYDVQQHEwZOZXd0b24xFDASBgNVBAoTC0dvb2ds +ZSwgSW5jMRIwEAYDVQQDEwlKb24gQWxsaWUxIjAgBgkqhkiG9w0BCQEWE2pvbmFs +bGllQGdvb2dsZS5jb20wHhcNMTEwNTE0MDMwMTQ1WhcNMTEwNjEzMDMwMTQ1WjB5 +MQswCQYDVQQGEwJVUzELMAkGA1UECBMCTkMxDzANBgNVBAcTBk5ld3RvbjEUMBIG +A1UEChMLR29vZ2xlLCBJbmMxEjAQBgNVBAMTCUpvbiBBbGxpZTEiMCAGCSqGSIb3 +DQEJARYTam9uYWxsaWVAZ29vZ2xlLmNvbTCCAbcwggEsBgcqhkjOOAQBMIIBHwKB +gQC8hLUnQ7FpFYu4WXTj6DKvXvz8QrJkNJCVMTpKAT7uBpobk32S5RrPKXocd4gN +8lyGB9ggS03EVlEwXvSmO0DH2MQtke2jl9j1HLydClMf4sbx5V6TV9IFw505U1iW +jL7awRMgxge+FsudtJK254FjMFo03ZnOQ8ZJJ9E6AEDrlwIVAJpnBn9moyP11Ox5 +Asc/5dnjb6dPAoGBAJFHd4KVv1iTVCvEG6gGiYop5DJh28hUQcN9kul+2A0yPUSC +X93oN00P8Vh3eYgSaCWZsha7zDG53MrVJ0Zf6v/X/CoZNhLldeNOepivTRAzn+Rz +kKUYy5l1sxYLHQKF0UGNCXfFKZT0PCmgU+PWhYNBBMn6/cIh44vp85ideo5CA4GE +AAKBgFmifCafzeRaohYKXJgMGSEaggCVCRq5xdyDCat+wbOkjC4mfG01/um3G8u5 +LxasjlWRKTR/tcAL7t0QuokVyQaYdVypZXNaMtx1db7YBuHjj3aP+8JOQRI9xz8c +bp5NDJ5pISiFOv4p3GZfqZPcqckDt78AtkQrmnal2txhhjF6o4HeMIHbMB0GA1Ud +DgQWBBQVyyr7hO11ZFFpWX50298Sa3V+rzCBqwYDVR0jBIGjMIGggBQVyyr7hO11 +ZFFpWX50298Sa3V+r6F9pHsweTELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk5DMQ8w +DQYDVQQHEwZOZXd0b24xFDASBgNVBAoTC0dvb2dsZSwgSW5jMRIwEAYDVQQDEwlK +b24gQWxsaWUxIjAgBgkqhkiG9w0BCQEWE2pvbmFsbGllQGdvb2dsZS5jb22CCQCx +z4IWqMXg4TAMBgNVHRMEBTADAQH/MAkGByqGSM44BAMDLwAwLAIUPtn/5j8Q1jJI +7ggOIsgrhgUdjGQCFCsmDq1H11q9+9Wp9IMeGrTSKHIM +-----END CERTIFICATE----- +` + +func TestParseCertificateWithDsaPublicKey(t *testing.T) { + expectedKey := &dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: bigFromHexString("00BC84B52743B169158BB85974E3E832AF5EFCFC42B264349095313A4A013EEE069A1B937D92E51ACF297A1C77880DF25C8607D8204B4DC45651305EF4A63B40C7D8C42D91EDA397D8F51CBC9D0A531FE2C6F1E55E9357D205C39D395358968CBEDAC11320C607BE16CB9DB492B6E78163305A34DD99CE43C64927D13A0040EB97"), + Q: bigFromHexString("009A67067F66A323F5D4EC7902C73FE5D9E36FA74F"), + G: bigFromHexString("009147778295BF5893542BC41BA806898A29E43261DBC85441C37D92E97ED80D323D44825FDDE8374D0FF15877798812682599B216BBCC31B9DCCAD527465FEAFFD7FC2A193612E575E34E7A98AF4D10339FE47390A518CB9975B3160B1D0285D1418D0977C52994F43C29A053E3D685834104C9FAFDC221E38BE9F3989D7A8E42"), + }, + Y: bigFromHexString("59A27C269FCDE45AA2160A5C980C19211A820095091AB9C5DC8309AB7EC1B3A48C2E267C6D35FEE9B71BCBB92F16AC8E559129347FB5C00BEEDD10BA8915C90698755CA965735A32DC7575BED806E1E38F768FFBC24E41123DC73F1C6E9E4D0C9E692128853AFE29DC665FA993DCA9C903B7BF00B6442B9A76A5DADC6186317A"), + } + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + if cert.PublicKeyAlgorithm != DSA { + t.Errorf("Parsed key algorithm was not DSA") + } + parsedKey, ok := cert.PublicKey.(*dsa.PublicKey) + if !ok { + t.Fatalf("Parsed key was not a DSA key: %s", err) + } + if expectedKey.Y.Cmp(parsedKey.Y) != 0 || + expectedKey.P.Cmp(parsedKey.P) != 0 || + expectedKey.Q.Cmp(parsedKey.Q) != 0 || + expectedKey.G.Cmp(parsedKey.G) != 0 { + t.Fatal("Parsed key differs from expected key") + } +} + +func TestParseCertificateWithDSASignatureAlgorithm(t *testing.T) { + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + if cert.SignatureAlgorithm != DSAWithSHA1 { + t.Errorf("Parsed signature algorithm was not DSAWithSHA1") + } +} + +func TestVerifyCertificateWithDSASignature(t *testing.T) { + pemBlock, _ := pem.Decode([]byte(dsaCertPem)) + cert, err := ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("Failed to parse certificate: %s", err) + } + // test cert is self-signed + if err = cert.CheckSignatureFrom(cert); err != nil { + t.Fatalf("DSA Certificate verfication failed: %s", err) + } +} + +const pemCertificate = `-----BEGIN CERTIFICATE----- +MIIB5DCCAZCgAwIBAgIBATALBgkqhkiG9w0BAQUwLTEQMA4GA1UEChMHQWNtZSBDbzEZMBcGA1UE +AxMQdGVzdC5leGFtcGxlLmNvbTAeFw03MDAxMDEwMDE2NDBaFw03MDAxMDIwMzQ2NDBaMC0xEDAO +BgNVBAoTB0FjbWUgQ28xGTAXBgNVBAMTEHRlc3QuZXhhbXBsZS5jb20wWjALBgkqhkiG9w0BAQED +SwAwSAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0fd7Ai2KW5ToIwzFo +fvJcS/STa6HA5gQenRUCAwEAAaOBnjCBmzAOBgNVHQ8BAf8EBAMCAAQwDwYDVR0TAQH/BAUwAwEB +/zANBgNVHQ4EBgQEAQIDBDAPBgNVHSMECDAGgAQBAgMEMBsGA1UdEQQUMBKCEHRlc3QuZXhhbXBs +ZS5jb20wDwYDVR0gBAgwBjAEBgIqAzAqBgNVHR4EIzAhoB8wDoIMLmV4YW1wbGUuY29tMA2CC2V4 +YW1wbGUuY29tMAsGCSqGSIb3DQEBBQNBAHKZKoS1wEQOGhgklx4+/yFYQlnqwKXvar/ZecQvJwui +0seMQnwBhwdBkHfVIU2Fu5VUMRyxlf0ZNaDXcpU581k= +-----END CERTIFICATE-----` + +func TestCRLCreation(t *testing.T) { + block, _ := pem.Decode([]byte(pemPrivateKey)) + priv, _ := ParsePKCS1PrivateKey(block.Bytes) + block, _ = pem.Decode([]byte(pemCertificate)) + cert, _ := ParseCertificate(block.Bytes) + + now := time.SecondsToUTC(1000) + expiry := time.SecondsToUTC(10000) + + revokedCerts := []pkix.RevokedCertificate{ + { + SerialNumber: big.NewInt(1), + RevocationTime: now, + }, + { + SerialNumber: big.NewInt(42), + RevocationTime: now, + }, + } + + crlBytes, err := cert.CreateCRL(rand.Reader, priv, revokedCerts, now, expiry) + if err != nil { + t.Errorf("error creating CRL: %s", err) + } + + _, err = ParseDERCRL(crlBytes) + if err != nil { + t.Errorf("error reparsing CRL: %s", err) + } +} + +func fromBase64(in string) []byte { + out := make([]byte, base64.StdEncoding.DecodedLen(len(in))) + _, err := base64.StdEncoding.Decode(out, []byte(in)) + if err != nil { + panic("failed to base64 decode") + } + return out +} + +func TestParseDERCRL(t *testing.T) { + derBytes := fromBase64(derCRLBase64) + certList, err := ParseDERCRL(derBytes) + if err != nil { + t.Errorf("error parsing: %s", err) + return + } + numCerts := len(certList.TBSCertList.RevokedCertificates) + expected := 88 + if numCerts != expected { + t.Errorf("bad number of revoked certificates. got: %d want: %d", numCerts, expected) + } + + if certList.HasExpired(1302517272) { + t.Errorf("CRL has expired (but shouldn't have)") + } + + // Can't check the signature here without a package cycle. +} + +func TestParsePEMCRL(t *testing.T) { + pemBytes := fromBase64(pemCRLBase64) + certList, err := ParseCRL(pemBytes) + if err != nil { + t.Errorf("error parsing: %s", err) + return + } + numCerts := len(certList.TBSCertList.RevokedCertificates) + expected := 2 + if numCerts != expected { + t.Errorf("bad number of revoked certificates. got: %d want: %d", numCerts, expected) + } + + if certList.HasExpired(1302517272) { + t.Errorf("CRL has expired (but shouldn't have)") + } + + // Can't check the signature here without a package cycle. +} + +const derCRLBase64 = "MIINqzCCDJMCAQEwDQYJKoZIhvcNAQEFBQAwVjEZMBcGA1UEAxMQUEtJIEZJTk1FQ0NBTklDQTEVMBMGA1UEChMMRklOTUVDQ0FOSUNBMRUwEwYDVQQLEwxGSU5NRUNDQU5JQ0ExCzAJBgNVBAYTAklUFw0xMTA1MDQxNjU3NDJaFw0xMTA1MDQyMDU3NDJaMIIMBzAhAg4Ze1od49Lt1qIXBydAzhcNMDkwNzE2MDg0MzIyWjAAMCECDl0HSL9bcZ1Ci/UHJ0DPFw0wOTA3MTYwODQzMTNaMAAwIQIOESB9tVAmX3cY7QcnQNAXDTA5MDcxNjA4NDUyMlowADAhAg4S1tGAQ3mHt8uVBydA1RcNMDkwODA0MTUyNTIyWjAAMCECDlQ249Y7vtC25ScHJ0DWFw0wOTA4MDQxNTI1MzdaMAAwIQIOISMop3NkA4PfYwcnQNkXDTA5MDgwNDExMDAzNFowADAhAg56/BMoS29KEShTBydA2hcNMDkwODA0MTEwMTAzWjAAMCECDnBp/22HPH5CSWoHJ0DbFw0wOTA4MDQxMDU0NDlaMAAwIQIOV9IP+8CD8bK+XAcnQNwXDTA5MDgwNDEwNTcxN1owADAhAg4v5aRz0IxWqYiXBydA3RcNMDkwODA0MTA1NzQ1WjAAMCECDlOU34VzvZAybQwHJ0DeFw0wOTA4MDQxMDU4MjFaMAAwIAINO4CD9lluIxcwBydBAxcNMDkwNzIyMTUzMTU5WjAAMCECDgOllfO8Y1QA7/wHJ0ExFw0wOTA3MjQxMTQxNDNaMAAwIQIOJBX7jbiCdRdyjgcnQUQXDTA5MDkxNjA5MzAwOFowADAhAg5iYSAgmDrlH/RZBydBRRcNMDkwOTE2MDkzMDE3WjAAMCECDmu6k6srP3jcMaQHJ0FRFw0wOTA4MDQxMDU2NDBaMAAwIQIOX8aHlO0V+WVH4QcnQVMXDTA5MDgwNDEwNTcyOVowADAhAg5flK2rg3NnsRgDBydBzhcNMTEwMjAxMTUzMzQ2WjAAMCECDg35yJDL1jOPTgoHJ0HPFw0xMTAyMDExNTM0MjZaMAAwIQIOMyFJ6+e9iiGVBQcnQdAXDTA5MDkxODEzMjAwNVowADAhAg5Emb/Oykucmn8fBydB1xcNMDkwOTIxMTAxMDQ3WjAAMCECDjQKCncV+MnUavMHJ0HaFw0wOTA5MjIwODE1MjZaMAAwIQIOaxiFUt3dpd+tPwcnQfQXDTEwMDYxODA4NDI1MVowADAhAg5G7P8nO0tkrMt7BydB9RcNMTAwNjE4MDg0MjMwWjAAMCECDmTCC3SXhmDRst4HJ0H2Fw0wOTA5MjgxMjA3MjBaMAAwIQIOHoGhUr/pRwzTKgcnQfcXDTA5MDkyODEyMDcyNFowADAhAg50wrcrCiw8mQmPBydCBBcNMTAwMjE2MTMwMTA2WjAAMCECDifWmkvwyhEqwEcHJ0IFFw0xMDAyMTYxMzAxMjBaMAAwIQIOfgPmlW9fg+osNgcnQhwXDTEwMDQxMzA5NTIwMFowADAhAg4YHAGuA6LgCk7tBydCHRcNMTAwNDEzMDk1MTM4WjAAMCECDi1zH1bxkNJhokAHJ0IsFw0xMDA0MTMwOTU5MzBaMAAwIQIOMipNccsb/wo2fwcnQi0XDTEwMDQxMzA5NTkwMFowADAhAg46lCmvPl4GpP6ABydCShcNMTAwMTE5MDk1MjE3WjAAMCECDjaTcaj+wBpcGAsHJ0JLFw0xMDAxMTkwOTUyMzRaMAAwIQIOOMC13EOrBuxIOQcnQloXDTEwMDIwMTA5NDcwNVowADAhAg5KmZl+krz4RsmrBydCWxcNMTAwMjAxMDk0NjQwWjAAMCECDmLG3zQJ/fzdSsUHJ0JiFw0xMDAzMDEwOTUxNDBaMAAwIQIOP39ksgHdojf4owcnQmMXDTEwMDMwMTA5NTExN1owADAhAg4LDQzvWNRlD6v9BydCZBcNMTAwMzAxMDk0NjIyWjAAMCECDkmNfeclaFhIaaUHJ0JlFw0xMDAzMDEwOTQ2MDVaMAAwIQIOT/qWWfpH/m8NTwcnQpQXDTEwMDUxMTA5MTgyMVowADAhAg5m/ksYxvCEgJSvBydClRcNMTAwNTExMDkxODAxWjAAMCECDgvf3Ohq6JOPU9AHJ0KWFw0xMDA1MTEwOTIxMjNaMAAwIQIOKSPas10z4jNVIQcnQpcXDTEwMDUxMTA5MjEwMlowADAhAg4mCWmhoZ3lyKCDBydCohcNMTEwNDI4MTEwMjI1WjAAMCECDkeiyRsBMK0Gvr4HJ0KjFw0xMTA0MjgxMTAyMDdaMAAwIQIOa09b/nH2+55SSwcnQq4XDTExMDQwMTA4Mjk0NlowADAhAg5O7M7iq7gGplr1BydCrxcNMTEwNDAxMDgzMDE3WjAAMCECDjlT6mJxUjTvyogHJ0K1Fw0xMTAxMjcxNTQ4NTJaMAAwIQIODS/l4UUFLe21NAcnQrYXDTExMDEyNzE1NDgyOFowADAhAg5lPRA0XdOUF6lSBydDHhcNMTEwMTI4MTQzNTA1WjAAMCECDixKX4fFGGpENwgHJ0MfFw0xMTAxMjgxNDM1MzBaMAAwIQIORNBkqsPnpKTtbAcnQ08XDTEwMDkwOTA4NDg0MlowADAhAg5QL+EMM3lohedEBydDUBcNMTAwOTA5MDg0ODE5WjAAMCECDlhDnHK+HiTRAXcHJ0NUFw0xMDEwMTkxNjIxNDBaMAAwIQIOdBFqAzq/INz53gcnQ1UXDTEwMTAxOTE2MjA0NFowADAhAg4OjR7s8MgKles1BydDWhcNMTEwMTI3MTY1MzM2WjAAMCECDmfR/elHee+d0SoHJ0NbFw0xMTAxMjcxNjUzNTZaMAAwIQIOBTKv2ui+KFMI+wcnQ5YXDTEwMDkxNTEwMjE1N1owADAhAg49F3c/GSah+oRUBydDmxcNMTEwMTI3MTczMjMzWjAAMCECDggv4I61WwpKFMMHJ0OcFw0xMTAxMjcxNzMyNTVaMAAwIQIOXx/Y8sEvwS10LAcnQ6UXDTExMDEyODExMjkzN1owADAhAg5LSLbnVrSKaw/9BydDphcNMTEwMTI4MTEyOTIwWjAAMCECDmFFoCuhKUeACQQHJ0PfFw0xMTAxMTExMDE3MzdaMAAwIQIOQTDdFh2fSPF6AAcnQ+AXDTExMDExMTEwMTcxMFowADAhAg5B8AOXX61FpvbbBydD5RcNMTAxMDA2MTAxNDM2WjAAMCECDh41P2Gmi7PkwI4HJ0PmFw0xMDEwMDYxMDE2MjVaMAAwIQIOWUHGLQCd+Ale9gcnQ/0XDTExMDUwMjA3NTYxMFowADAhAg5Z2c9AYkikmgWOBydD/hcNMTEwNTAyMDc1NjM0WjAAMCECDmf/UD+/h8nf+74HJ0QVFw0xMTA0MTUwNzI4MzNaMAAwIQIOICvj4epy3MrqfwcnRBYXDTExMDQxNTA3Mjg1NlowADAhAg4bouRMfOYqgv4xBydEHxcNMTEwMzA4MTYyNDI1WjAAMCECDhebWHGoKiTp7pEHJ0QgFw0xMTAzMDgxNjI0NDhaMAAwIQIOX+qnxxAqJ8LtawcnRDcXDTExMDEzMTE1MTIyOFowADAhAg4j0fICqZ+wkOdqBydEOBcNMTEwMTMxMTUxMTQxWjAAMCECDhmXjsV4SUpWtAMHJ0RLFw0xMTAxMjgxMTI0MTJaMAAwIQIODno/w+zG43kkTwcnREwXDTExMDEyODExMjM1MlowADAhAg4b1gc88767Fr+LBydETxcNMTEwMTI4MTEwMjA4WjAAMCECDn+M3Pa1w2nyFeUHJ0RQFw0xMTAxMjgxMDU4NDVaMAAwIQIOaduoyIH61tqybAcnRJUXDTEwMTIxNTA5NDMyMlowADAhAg4nLqQPkyi3ESAKBydElhcNMTAxMjE1MDk0MzM2WjAAMCECDi504NIMH8578gQHJ0SbFw0xMTAyMTQxNDA1NDFaMAAwIQIOGuaM8PDaC5u1egcnRJwXDTExMDIxNDE0MDYwNFowADAhAg4ehYq/BXGnB5PWBydEnxcNMTEwMjA0MDgwOTUxWjAAMCECDkSD4eS4FxW5H20HJ0SgFw0xMTAyMDQwODA5MjVaMAAwIQIOOCcb6ilYObt1egcnRKEXDTExMDEyNjEwNDEyOVowADAhAg58tISWCCwFnKGnBydEohcNMTEwMjA0MDgxMzQyWjAAMCECDn5rjtabY/L/WL0HJ0TJFw0xMTAyMDQxMTAzNDFaMAAwDQYJKoZIhvcNAQEFBQADggEBAGnF2Gs0+LNiYCW1Ipm83OXQYP/bd5tFFRzyz3iepFqNfYs4D68/QihjFoRHQoXEB0OEe1tvaVnnPGnEOpi6krwekquMxo4H88B5SlyiFIqemCOIss0SxlCFs69LmfRYvPPvPEhoXtQ3ZThe0UvKG83GOklhvGl6OaiRf4Mt+m8zOT4Wox/j6aOBK6cw6qKCdmD+Yj1rrNqFGg1CnSWMoD6S6mwNgkzwdBUJZ22BwrzAAo4RHa2Uy3ef1FjwD0XtU5N3uDSxGGBEDvOe5z82rps3E22FpAA8eYl8kaXtmWqyvYU0epp4brGuTxCuBMCAsxt/OjIjeNNQbBGkwxgfYA0=" + +const pemCRLBase64 = "LS0tLS1CRUdJTiBYNTA5IENSTC0tLS0tDQpNSUlCOWpDQ0FWOENBUUV3RFFZSktvWklodmNOQVFFRkJRQXdiREVhTUJnR0ExVUVDaE1SVWxOQklGTmxZM1Z5DQphWFI1SUVsdVl5NHhIakFjQmdOVkJBTVRGVkpUUVNCUWRXSnNhV01nVW05dmRDQkRRU0IyTVRFdU1Dd0dDU3FHDQpTSWIzRFFFSkFSWWZjbk5oYTJWdmJuSnZiM1J6YVdkdVFISnpZWE5sWTNWeWFYUjVMbU52YlJjTk1URXdNakl6DQpNVGt5T0RNd1doY05NVEV3T0RJeU1Ua3lPRE13V2pDQmpEQktBaEVBckRxb2g5RkhKSFhUN09QZ3V1bjQrQmNODQpNRGt4TVRBeU1UUXlOekE1V2pBbU1Bb0dBMVVkRlFRRENnRUpNQmdHQTFVZEdBUVJHQTh5TURBNU1URXdNakUwDQpNalExTlZvd1BnSVJBTEd6blowOTVQQjVhQU9MUGc1N2ZNTVhEVEF5TVRBeU16RTBOVEF4TkZvd0dqQVlCZ05WDQpIUmdFRVJnUE1qQXdNakV3TWpNeE5EVXdNVFJhb0RBd0xqQWZCZ05WSFNNRUdEQVdnQlQxVERGNlVRTS9MTmVMDQpsNWx2cUhHUXEzZzltekFMQmdOVkhSUUVCQUlDQUlRd0RRWUpLb1pJaHZjTkFRRUZCUUFEZ1lFQUZVNUFzNk16DQpxNVBSc2lmYW9iUVBHaDFhSkx5QytNczVBZ2MwYld5QTNHQWR4dXI1U3BQWmVSV0NCamlQL01FSEJXSkNsQkhQDQpHUmNxNXlJZDNFakRrYUV5eFJhK2k2N0x6dmhJNmMyOUVlNks5cFNZd2ppLzdSVWhtbW5Qclh0VHhsTDBsckxyDQptUVFKNnhoRFJhNUczUUE0Q21VZHNITnZicnpnbUNZcHZWRT0NCi0tLS0tRU5EIFg1MDkgQ1JMLS0tLS0NCg0K" diff --git a/src/pkg/crypto/xtea/Makefile b/src/pkg/crypto/xtea/Makefile new file mode 100644 index 000000000..301621168 --- /dev/null +++ b/src/pkg/crypto/xtea/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/xtea +GOFILES=\ + cipher.go\ + block.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/xtea/block.go b/src/pkg/crypto/xtea/block.go new file mode 100644 index 000000000..bf5d24599 --- /dev/null +++ b/src/pkg/crypto/xtea/block.go @@ -0,0 +1,66 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Implementation adapted from Needham and Wheeler's paper: + http://www.cix.co.uk/~klockstone/xtea.pdf + + A precalculated look up table is used during encryption/decryption for values that are based purely on the key. +*/ + +package xtea + +// XTEA is based on 64 rounds. +const numRounds = 64 + +// blockToUint32 reads an 8 byte slice into two uint32s. +// The block is treated as big endian. +func blockToUint32(src []byte) (uint32, uint32) { + r0 := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) + r1 := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) + return r0, r1 +} + +// uint32ToBlock writes two uint32s into an 8 byte data block. +// Values are written as big endian. +func uint32ToBlock(v0, v1 uint32, dst []byte) { + dst[0] = byte(v0 >> 24) + dst[1] = byte(v0 >> 16) + dst[2] = byte(v0 >> 8) + dst[3] = byte(v0) + dst[4] = byte(v1 >> 24) + dst[5] = byte(v1 >> 16) + dst[6] = byte(v1 >> 8) + dst[7] = byte(v1 >> 0) +} + +// encryptBlock encrypts a single 8 byte block using XTEA. +func encryptBlock(c *Cipher, dst, src []byte) { + v0, v1 := blockToUint32(src) + + // Two rounds of XTEA applied per loop + for i := 0; i < numRounds; { + v0 += ((v1<<4 ^ v1>>5) + v1) ^ c.table[i] + i++ + v1 += ((v0<<4 ^ v0>>5) + v0) ^ c.table[i] + i++ + } + + uint32ToBlock(v0, v1, dst) +} + +// decryptBlock decrypt a single 8 byte block using XTEA. +func decryptBlock(c *Cipher, dst, src []byte) { + v0, v1 := blockToUint32(src) + + // Two rounds of XTEA applied per loop + for i := numRounds; i > 0; { + i-- + v1 -= ((v0<<4 ^ v0>>5) + v0) ^ c.table[i] + i-- + v0 -= ((v1<<4 ^ v1>>5) + v1) ^ c.table[i] + } + + uint32ToBlock(v0, v1, dst) +} diff --git a/src/pkg/crypto/xtea/cipher.go b/src/pkg/crypto/xtea/cipher.go new file mode 100644 index 000000000..b3fba3c84 --- /dev/null +++ b/src/pkg/crypto/xtea/cipher.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xtea implements XTEA encryption, as defined in Needham and Wheeler's +// 1997 technical report, "Tea extensions." +package xtea + +// For details, see http://www.cix.co.uk/~klockstone/xtea.pdf + +import ( + "os" + "strconv" +) + +// The XTEA block size in bytes. +const BlockSize = 8 + +// A Cipher is an instance of an XTEA cipher using a particular key. +// table contains a series of precalculated values that are used each round. +type Cipher struct { + table [64]uint32 +} + +type KeySizeError int + +func (k KeySizeError) String() string { + return "crypto/xtea: invalid key size " + strconv.Itoa(int(k)) +} + +// NewCipher creates and returns a new Cipher. +// The key argument should be the XTEA key. +// XTEA only supports 128 bit (16 byte) keys. +func NewCipher(key []byte) (*Cipher, os.Error) { + k := len(key) + switch k { + default: + return nil, KeySizeError(k) + case 16: + break + } + + c := new(Cipher) + initCipher(c, key) + + return c, nil +} + +// BlockSize returns the XTEA block size, 8 bytes. +// It is necessary to satisfy the Cipher interface in the +// package "crypto/cipher". +func (c *Cipher) BlockSize() int { return BlockSize } + +// Encrypt encrypts the 8 byte buffer src using the key and stores the result in dst. +// Note that for amounts of data larger than a block, +// it is not safe to just call Encrypt on successive blocks; +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). +func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c, dst, src) } + +// Decrypt decrypts the 8 byte buffer src using the key k and stores the result in dst. +func (c *Cipher) Decrypt(dst, src []byte) { decryptBlock(c, dst, src) } + +// Reset zeros the table, so that it will no longer appear in the process's memory. +func (c *Cipher) Reset() { + for i := 0; i < len(c.table); i++ { + c.table[i] = 0 + } +} + +// initCipher initializes the cipher context by creating a look up table +// of precalculated values that are based on the key. +func initCipher(c *Cipher, key []byte) { + // Load the key into four uint32s + var k [4]uint32 + for i := 0; i < len(k); i++ { + j := i << 2 // Multiply by 4 + k[i] = uint32(key[j+0])<<24 | uint32(key[j+1])<<16 | uint32(key[j+2])<<8 | uint32(key[j+3]) + } + + // Precalculate the table + const delta = 0x9E3779B9 + var sum uint32 = 0 + + // Two rounds of XTEA applied per loop + for i := 0; i < numRounds; { + c.table[i] = sum + k[sum&3] + i++ + sum += delta + c.table[i] = sum + k[(sum>>11)&3] + i++ + } +} diff --git a/src/pkg/crypto/xtea/xtea_test.go b/src/pkg/crypto/xtea/xtea_test.go new file mode 100644 index 000000000..217d96adc --- /dev/null +++ b/src/pkg/crypto/xtea/xtea_test.go @@ -0,0 +1,246 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xtea + +import ( + "testing" +) + +// A sample test key for when we just want to initialize a cipher +var testKey = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF} + +// Test that the block size for XTEA is correct +func TestBlocksize(t *testing.T) { + if BlockSize != 8 { + t.Errorf("BlockSize constant - expected 8, got %d", BlockSize) + return + } + + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + result := c.BlockSize() + if result != 8 { + t.Errorf("BlockSize function - expected 8, got %d", result) + return + } +} + +// A series of test values to confirm that the Cipher.table array was initialized correctly +var testTable = []uint32{ + 0x00112233, 0x6B1568B8, 0xE28CE030, 0xC5089E2D, 0xC5089E2D, 0x1EFBD3A2, 0xA7845C2A, 0x78EF0917, + 0x78EF0917, 0x172682D0, 0x5B6AC714, 0x822AC955, 0x3DE68511, 0xDC1DFECA, 0x2062430E, 0x3611343F, + 0xF1CCEFFB, 0x900469B4, 0xD448ADF8, 0x2E3BE36D, 0xB6C46BF5, 0x994029F2, 0x994029F2, 0xF3335F67, + 0x6AAAD6DF, 0x4D2694DC, 0x4D2694DC, 0xEB5E0E95, 0x2FA252D9, 0x4551440A, 0x121E10D6, 0xB0558A8F, + 0xE388BDC3, 0x0A48C004, 0xC6047BC0, 0x643BF579, 0xA88039BD, 0x02736F32, 0x8AFBF7BA, 0x5C66A4A7, + 0x5C66A4A7, 0xC76AEB2C, 0x3EE262A4, 0x215E20A1, 0x215E20A1, 0x7B515616, 0x03D9DE9E, 0x1988CFCF, + 0xD5448B8B, 0x737C0544, 0xB7C04988, 0xDE804BC9, 0x9A3C0785, 0x3873813E, 0x7CB7C582, 0xD6AAFAF7, + 0x4E22726F, 0x309E306C, 0x309E306C, 0x8A9165E1, 0x1319EE69, 0xF595AC66, 0xF595AC66, 0x4F88E1DB, +} + +// Test that the cipher context is initialized correctly +func TestCipherInit(t *testing.T) { + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + for i := 0; i < len(c.table); i++ { + if c.table[i] != testTable[i] { + t.Errorf("NewCipher() failed to initialize Cipher.table[%d] correctly. Expected %08X, got %08X", i, testTable[i], c.table[i]) + break + } + } +} + +// Test that invalid key sizes return an error +func TestInvalidKeySize(t *testing.T) { + // Test a long key + key := []byte{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + } + + _, err := NewCipher(key) + if err == nil { + t.Errorf("Invalid key size %d didn't result in an error.", len(key)) + } + + // Test a short key + key = []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77} + + _, err = NewCipher(key) + if err == nil { + t.Errorf("Invalid key size %d didn't result in an error.", len(key)) + } +} + +// Test that we can correctly decode some bytes we have encoded +func TestEncodeDecode(t *testing.T) { + original := []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF} + input := original + output := make([]byte, BlockSize) + + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + // Encrypt the input block + c.Encrypt(output, input) + + // Check that the output does not match the input + differs := false + for i := 0; i < len(input); i++ { + if output[i] != input[i] { + differs = true + break + } + } + if differs == false { + t.Error("Cipher.Encrypt: Failed to encrypt the input block.") + return + } + + // Decrypt the block we just encrypted + input = output + output = make([]byte, BlockSize) + c.Decrypt(output, input) + + // Check that the output from decrypt matches our initial input + for i := 0; i < len(input); i++ { + if output[i] != original[i] { + t.Errorf("Decrypted byte %d differed. Expected %02X, got %02X\n", i, original[i], output[i]) + return + } + } +} + +// Test Vectors +type CryptTest struct { + key []byte + plainText []byte + cipherText []byte +} + +var CryptTests = []CryptTest{ + // These were sourced from http://www.freemedialibrary.com/index.php/XTEA_test_vectors + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0x49, 0x7d, 0xf3, 0xd0, 0x72, 0x61, 0x2c, 0xb5}, + }, + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + []byte{0xe7, 0x8f, 0x2d, 0x13, 0x74, 0x43, 0x41, 0xd8}, + }, + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x5a, 0x5b, 0x6e, 0x27, 0x89, 0x48, 0xd7, 0x7f}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48}, + []byte{0xa0, 0x39, 0x05, 0x89, 0xf8, 0xb8, 0xef, 0xa5}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + []byte{0xed, 0x23, 0x37, 0x5a, 0x82, 0x1a, 0x8c, 0x2d}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x70, 0xe1, 0x22, 0x5d, 0x6e, 0x4e, 0x76, 0x55}, + []byte{0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41}, + }, + + // These vectors are from http://wiki.secondlife.com/wiki/XTEA_Strong_Encryption_Implementation#Bouncy_Castle_C.23_API + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0xDE, 0xE9, 0xD4, 0xD8, 0xF7, 0x13, 0x1E, 0xD9}, + }, + { + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + []byte{0x06, 0x5C, 0x1B, 0x89, 0x75, 0xC6, 0xA8, 0x16}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + []byte{0x1F, 0xF9, 0xA0, 0x26, 0x1A, 0xC6, 0x42, 0x64}, + }, + { + []byte{0x01, 0x23, 0x45, 0x67, 0x12, 0x34, 0x56, 0x78, 0x23, 0x45, 0x67, 0x89, 0x34, 0x56, 0x78, 0x9A}, + []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + []byte{0x8C, 0x67, 0x15, 0x5B, 0x2E, 0xF9, 0x1E, 0xAD}, + }, +} + +// Test encryption +func TestCipherEncrypt(t *testing.T) { + for i, tt := range CryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes), vector %d = %s", len(tt.key), i, err) + continue + } + + out := make([]byte, len(tt.plainText)) + c.Encrypt(out, tt.plainText) + + for j := 0; j < len(out); j++ { + if out[j] != tt.cipherText[j] { + t.Errorf("Cipher.Encrypt %d: out[%d] = %02X, expected %02X", i, j, out[j], tt.cipherText[j]) + break + } + } + } +} + +// Test decryption +func TestCipherDecrypt(t *testing.T) { + for i, tt := range CryptTests { + c, err := NewCipher(tt.key) + if err != nil { + t.Errorf("NewCipher(%d bytes), vector %d = %s", len(tt.key), i, err) + continue + } + + out := make([]byte, len(tt.cipherText)) + c.Decrypt(out, tt.cipherText) + + for j := 0; j < len(out); j++ { + if out[j] != tt.plainText[j] { + t.Errorf("Cipher.Decrypt %d: out[%d] = %02X, expected %02X", i, j, out[j], tt.plainText[j]) + break + } + } + } +} + +// Test resetting the cipher context +func TestReset(t *testing.T) { + c, err := NewCipher(testKey) + if err != nil { + t.Errorf("NewCipher(%d bytes) = %s", len(testKey), err) + return + } + + c.Reset() + for i := 0; i < len(c.table); i++ { + if c.table[i] != 0 { + t.Errorf("Cipher.Reset: Failed to clear Cipher.table[%d]. expected 0, got %08X", i, c.table[i]) + return + } + } +} diff --git a/src/pkg/csv/Makefile b/src/pkg/csv/Makefile new file mode 100644 index 000000000..e364d51d2 --- /dev/null +++ b/src/pkg/csv/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=csv +GOFILES=\ + reader.go\ + writer.go\ + +include ../../Make.pkg diff --git a/src/pkg/csv/reader.go b/src/pkg/csv/reader.go new file mode 100644 index 000000000..ea2c266a4 --- /dev/null +++ b/src/pkg/csv/reader.go @@ -0,0 +1,372 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package csv reads and writes comma-separated values (CSV) files. +// +// A csv file contains zero or more records of one or more fields per record. +// Each record is separated by the newline character. The final record may +// optionally be followed by a newline character. +// +// field1,field2,field3 +// +// White space is considered part of a field. +// +// Carriage returns before newline characters are silently removed. +// +// Blank lines are ignored. A line with only whitespace characters (excluding +// the ending newline character) is not considered a blank line. +// +// Fields which start and stop with the quote character " are called +// quoted-fields. The beginning and ending quote are not part of the +// field. +// +// The source: +// +// normal string,"quoted-field" +// +// results in the fields +// +// {`normal string`, `quoted-field`} +// +// Within a quoted-field a quote character followed by a second quote +// character is considered a single quote. +// +// "the ""word"" is true","a ""quoted-field""" +// +// results in +// +// {`the "word" is true`, `a "quoted-field"`} +// +// Newlines and commas may be included in a quoted-field +// +// "Multi-line +// field","comma is ," +// +// results in +// +// {`Multi-line +// field`, `comma is ,`} +package csv + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "unicode" +) + +// A ParseError is returned for parsing errors. +// The first line is 1. The first column is 0. +type ParseError struct { + Line int // Line where the error occurred + Column int // Column (rune index) where the error occurred + Error os.Error // The actual error +} + +func (e *ParseError) String() string { + return fmt.Sprintf("line %d, column %d: %s", e.Line, e.Column, e.Error) +} + +// These are the errors that can be returned in ParseError.Error +var ( + ErrTrailingComma = os.NewError("extra delimiter at end of line") + ErrBareQuote = os.NewError("bare \" in non-quoted-field") + ErrQuote = os.NewError("extraneous \" in field") + ErrFieldCount = os.NewError("wrong number of fields in line") +) + +// A Reader reads records from a CSV-encoded file. +// +// As returned by NewReader, a Reader expects input conforming to RFC 4180. +// The exported fields can be changed to customize the details before the +// first call to Read or ReadAll. +// +// Comma is the field delimiter. It defaults to ','. +// +// Comment, if not 0, is the comment character. Lines beginning with the +// Comment character are ignored. +// +// If FieldsPerRecord is positive, Read requires each record to +// have the given number of fields. If FieldsPerRecord is 0, Read sets it to +// the number of fields in the first record, so that future records must +// have the same field count. +// +// If LazyQuotes is true, a quote may appear in an unquoted field and a +// non-doubled quote may appear in a quoted field. +// +// If TrailingComma is true, the last field may be an unquoted empty field. +// +// If TrimLeadingSpace is true, leading white space in a field is ignored. +type Reader struct { + Comma int // Field delimiter (set to ',' by NewReader) + Comment int // Comment character for start of line + FieldsPerRecord int // Number of expected fields per record + LazyQuotes bool // Allow lazy quotes + TrailingComma bool // Allow trailing comma + TrimLeadingSpace bool // Trim leading space + line int + column int + r *bufio.Reader + field bytes.Buffer +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.Reader) *Reader { + return &Reader{ + Comma: ',', + r: bufio.NewReader(r), + } +} + +// error creates a new ParseError based on err. +func (r *Reader) error(err os.Error) os.Error { + return &ParseError{ + Line: r.line, + Column: r.column, + Error: err, + } +} + +// Read reads one record from r. The record is a slice of strings with each +// string representing one field. +func (r *Reader) Read() (record []string, err os.Error) { + for { + record, err = r.parseRecord() + if record != nil { + break + } + if err != nil { + return nil, err + } + } + + if r.FieldsPerRecord > 0 { + if len(record) != r.FieldsPerRecord { + r.column = 0 // report at start of record + return record, r.error(ErrFieldCount) + } + } else if r.FieldsPerRecord == 0 { + r.FieldsPerRecord = len(record) + } + return record, nil +} + +// ReadAll reads all the remaining records from r. +// Each record is a slice of fields. +func (r *Reader) ReadAll() (records [][]string, err os.Error) { + for { + record, err := r.Read() + if err == os.EOF { + return records, nil + } + if err != nil { + return nil, err + } + records = append(records, record) + } + panic("unreachable") +} + +// readRune reads one rune from r, folding \r\n to \n and keeping track +// of how far into the line we have read. r.column will point to the start +// of this rune, not the end of this rune. +func (r *Reader) readRune() (int, os.Error) { + rune, _, err := r.r.ReadRune() + + // Handle \r\n here. We make the simplifying assumption that + // anytime \r is followed by \n that it can be folded to \n. + // We will not detect files which contain both \r\n and bare \n. + if rune == '\r' { + rune, _, err = r.r.ReadRune() + if err == nil { + if rune != '\n' { + r.r.UnreadRune() + rune = '\r' + } + } + } + r.column++ + return rune, err +} + +// unreadRune puts the last rune read from r back. +func (r *Reader) unreadRune() { + r.r.UnreadRune() + r.column-- +} + +// skip reads runes up to and including the rune delim or until error. +func (r *Reader) skip(delim int) os.Error { + for { + rune, err := r.readRune() + if err != nil { + return err + } + if rune == delim { + return nil + } + } + panic("unreachable") +} + +// parseRecord reads and parses a single csv record from r. +func (r *Reader) parseRecord() (fields []string, err os.Error) { + // Each record starts on a new line. We increment our line + // number (lines start at 1, not 0) and set column to -1 + // so as we increment in readRune it points to the character we read. + r.line++ + r.column = -1 + + // Peek at the first rune. If it is an error we are done. + // If we are support comments and it is the comment character + // then skip to the end of line. + + rune, _, err := r.r.ReadRune() + if err != nil { + return nil, err + } + + if r.Comment != 0 && rune == r.Comment { + return nil, r.skip('\n') + } + r.r.UnreadRune() + + // At this point we have at least one field. + for { + haveField, delim, err := r.parseField() + if haveField { + fields = append(fields, r.field.String()) + } + if delim == '\n' || err == os.EOF { + return fields, err + } else if err != nil { + return nil, err + } + } + panic("unreachable") +} + +// parseField parses the next field in the record. The read field is +// located in r.field. Delim is the first character not part of the field +// (r.Comma or '\n'). +func (r *Reader) parseField() (haveField bool, delim int, err os.Error) { + r.field.Reset() + + rune, err := r.readRune() + if err != nil { + // If we have EOF and are not at the start of a line + // then we return the empty field. We have already + // checked for trailing commas if needed. + if err == os.EOF && r.column != 0 { + return true, 0, err + } + return false, 0, err + } + + if r.TrimLeadingSpace { + for unicode.IsSpace(rune) { + rune, err = r.readRune() + if err != nil { + return false, 0, err + } + } + } + + switch rune { + case r.Comma: + // will check below + + case '\n': + // We are a trailing empty field or a blank line + if r.column == 0 { + return false, rune, nil + } + return true, rune, nil + + case '"': + // quoted field + Quoted: + for { + rune, err = r.readRune() + if err != nil { + if err == os.EOF { + if r.LazyQuotes { + return true, 0, err + } + return false, 0, r.error(ErrQuote) + } + return false, 0, err + } + switch rune { + case '"': + rune, err = r.readRune() + if err != nil || rune == r.Comma { + break Quoted + } + if rune == '\n' { + return true, rune, nil + } + if rune != '"' { + if !r.LazyQuotes { + r.column-- + return false, 0, r.error(ErrQuote) + } + // accept the bare quote + r.field.WriteRune('"') + } + case '\n': + r.line++ + r.column = -1 + } + r.field.WriteRune(rune) + } + + default: + // unquoted field + for { + r.field.WriteRune(rune) + rune, err = r.readRune() + if err != nil || rune == r.Comma { + break + } + if rune == '\n' { + return true, rune, nil + } + if !r.LazyQuotes && rune == '"' { + return false, 0, r.error(ErrBareQuote) + } + } + } + + if err != nil { + if err == os.EOF { + return true, 0, err + } + return false, 0, err + } + + if !r.TrailingComma { + // We don't allow trailing commas. See if we + // are at the end of the line (being mindful + // of trimming spaces). + c := r.column + rune, err = r.readRune() + if r.TrimLeadingSpace { + for unicode.IsSpace(rune) { + rune, err = r.readRune() + if err != nil { + break + } + } + } + if err == os.EOF || rune == '\n' { + r.column = c // report the comma + return false, 0, r.error(ErrTrailingComma) + } + r.unreadRune() + } + return true, rune, nil +} diff --git a/src/pkg/csv/reader_test.go b/src/pkg/csv/reader_test.go new file mode 100644 index 000000000..0068bad1d --- /dev/null +++ b/src/pkg/csv/reader_test.go @@ -0,0 +1,265 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "reflect" + "strings" + "testing" +) + +var readTests = []struct { + Name string + Input string + Output [][]string + UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1 + + // These fields are copied into the Reader + Comma int + Comment int + FieldsPerRecord int + LazyQuotes bool + TrailingComma bool + TrimLeadingSpace bool + + Error string + Line int // Expected error line if != 0 + Column int // Expected error column if line != 0 +}{ + { + Name: "Simple", + Input: "a,b,c\n", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "CRLF", + Input: "a,b\r\nc,d\r\n", + Output: [][]string{{"a", "b"}, {"c", "d"}}, + }, + { + Name: "BareCR", + Input: "a,b\rc,d\r\n", + Output: [][]string{{"a", "b\rc", "d"}}, + }, + { + Name: "RFC4180test", + UseFieldsPerRecord: true, + Input: `#field1,field2,field3 +"aaa","bb +b","ccc" +"a,a","b""bb","ccc" +zzz,yyy,xxx +`, + Output: [][]string{ + {"#field1", "field2", "field3"}, + {"aaa", "bb\nb", "ccc"}, + {"a,a", `b"bb`, "ccc"}, + {"zzz", "yyy", "xxx"}, + }, + }, + { + Name: "NoEOLTest", + Input: "a,b,c", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "Semicolon", + Comma: ';', + Input: "a;b;c\n", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "MultiLine", + Input: `"two +line","one line","three +line +field"`, + Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}}, + }, + { + Name: "BlankLine", + Input: "a,b,c\n\nd,e,f\n\n", + Output: [][]string{ + {"a", "b", "c"}, + {"d", "e", "f"}, + }, + }, + { + Name: "TrimSpace", + Input: " a, b, c\n", + TrimLeadingSpace: true, + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "LeadingSpace", + Input: " a, b, c\n", + Output: [][]string{{" a", " b", " c"}}, + }, + { + Name: "Comment", + Comment: '#', + Input: "#1,2,3\na,b,c\n#comment", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "NoComment", + Input: "#1,2,3\na,b,c", + Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}}, + }, + { + Name: "LazyQuotes", + LazyQuotes: true, + Input: `a "word","1"2",a","b`, + Output: [][]string{{`a "word"`, `1"2`, `a"`, `b`}}, + }, + { + Name: "BareQuotes", + LazyQuotes: true, + Input: `a "word","1"2",a"`, + Output: [][]string{{`a "word"`, `1"2`, `a"`}}, + }, + { + Name: "BareDoubleQuotes", + LazyQuotes: true, + Input: `a""b,c`, + Output: [][]string{{`a""b`, `c`}}, + }, + { + Name: "BadDoubleQuotes", + Input: `a""b,c`, + Output: [][]string{{`a""b`, `c`}}, + Error: `bare " in non-quoted-field`, Line: 1, Column: 1, + }, + { + Name: "TrimQuote", + Input: ` "a"," b",c`, + TrimLeadingSpace: true, + Output: [][]string{{"a", " b", "c"}}, + }, + { + Name: "BadBareQuote", + Input: `a "word","b"`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 2, + }, + { + Name: "BadTrailingQuote", + Input: `"a word",b"`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 10, + }, + { + Name: "ExtraneousQuote", + Input: `"a "word","b"`, + Error: `extraneous " in field`, Line: 1, Column: 3, + }, + { + Name: "BadFieldCount", + UseFieldsPerRecord: true, + Input: "a,b,c\nd,e", + Error: "wrong number of fields", Line: 2, + }, + { + Name: "BadFieldCount1", + UseFieldsPerRecord: true, + FieldsPerRecord: 2, + Input: `a,b,c`, + Error: "wrong number of fields", Line: 1, + }, + { + Name: "FieldCount", + Input: "a,b,c\nd,e", + Output: [][]string{{"a", "b", "c"}, {"d", "e"}}, + }, + { + Name: "BadTrailingCommaEOF", + Input: "a,b,c,", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaEOL", + Input: "a,b,c,\n", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaSpaceEOF", + TrimLeadingSpace: true, + Input: "a,b,c, ", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaSpaceEOL", + TrimLeadingSpace: true, + Input: "a,b,c, \n", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaLine3", + TrimLeadingSpace: true, + Input: "a,b,c\nd,e,f\ng,hi,", + Error: "extra delimiter at end of line", Line: 3, Column: 4, + }, + { + Name: "NotTrailingComma3", + Input: "a,b,c, \n", + Output: [][]string{{"a", "b", "c", " "}}, + }, + { + Name: "CommaFieldTest", + TrailingComma: true, + Input: `x,y,z,w +x,y,z, +x,y,, +x,,, +,,, +"x","y","z","w" +"x","y","z","" +"x","y","","" +"x","","","" +"","","","" +`, + Output: [][]string{ + {"x", "y", "z", "w"}, + {"x", "y", "z", ""}, + {"x", "y", "", ""}, + {"x", "", "", ""}, + {"", "", "", ""}, + {"x", "y", "z", "w"}, + {"x", "y", "z", ""}, + {"x", "y", "", ""}, + {"x", "", "", ""}, + {"", "", "", ""}, + }, + }, +} + +func TestRead(t *testing.T) { + for _, tt := range readTests { + r := NewReader(strings.NewReader(tt.Input)) + r.Comment = tt.Comment + if tt.UseFieldsPerRecord { + r.FieldsPerRecord = tt.FieldsPerRecord + } else { + r.FieldsPerRecord = -1 + } + r.LazyQuotes = tt.LazyQuotes + r.TrailingComma = tt.TrailingComma + r.TrimLeadingSpace = tt.TrimLeadingSpace + if tt.Comma != 0 { + r.Comma = tt.Comma + } + out, err := r.ReadAll() + perr, _ := err.(*ParseError) + if tt.Error != "" { + if err == nil || !strings.Contains(err.String(), tt.Error) { + t.Errorf("%s: error %v, want error %q", tt.Name, err, tt.Error) + } else if tt.Line != 0 && (tt.Line != perr.Line || tt.Column != perr.Column) { + t.Errorf("%s: error at %d:%d expected %d:%d", tt.Name, perr.Line, perr.Column, tt.Line, tt.Column) + } + } else if err != nil { + t.Errorf("%s: unexpected error %v", tt.Name, err) + } else if !reflect.DeepEqual(out, tt.Output) { + t.Errorf("%s: out=%q want %q", tt.Name, out, tt.Output) + } + } +} diff --git a/src/pkg/csv/writer.go b/src/pkg/csv/writer.go new file mode 100644 index 000000000..ccf703f0f --- /dev/null +++ b/src/pkg/csv/writer.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "bufio" + "io" + "os" + "strings" + "unicode" + "utf8" +) + +// A Writer writes records to a CSV encoded file. +// +// As returned by NewWriter, a Writer writes records terminated by a +// newline and uses ',' as the field delimiter. The exported fields can be +// changed to customize the details before the first call to Write or WriteAll. +// +// Comma is the field delimiter. +// +// If UseCRLF is true, the Writer ends each record with \r\n instead of \n. +type Writer struct { + Comma int // Field delimiter (set to to ',' by NewWriter) + UseCRLF bool // True to use \r\n as the line terminator + w *bufio.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{ + Comma: ',', + w: bufio.NewWriter(w), + } +} + +// Writer writes a single CSV record to w along with any necessary quoting. +// A record is a slice of strings with each string being one field. +func (w *Writer) Write(record []string) (err os.Error) { + for n, field := range record { + if n > 0 { + if _, err = w.w.WriteRune(w.Comma); err != nil { + return + } + } + + // If we don't have to have a quoted field then just + // write out the field and continue to the next field. + if !w.fieldNeedsQuotes(field) { + if _, err = w.w.WriteString(field); err != nil { + return + } + continue + } + if err = w.w.WriteByte('"'); err != nil { + return + } + + for _, rune := range field { + switch rune { + case '"': + _, err = w.w.WriteString(`""`) + case '\r': + if !w.UseCRLF { + err = w.w.WriteByte('\r') + } + case '\n': + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } + default: + _, err = w.w.WriteRune(rune) + } + if err != nil { + return + } + } + + if err = w.w.WriteByte('"'); err != nil { + return + } + } + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } + return +} + +// Flush writes any buffered data to the underlying io.Writer. +func (w *Writer) Flush() { + w.w.Flush() +} + +// WriteAll writes multiple CSV records to w using Write and then calls Flush. +func (w *Writer) WriteAll(records [][]string) (err os.Error) { + for _, record := range records { + err = w.Write(record) + if err != nil { + break + } + } + w.Flush() + return nil +} + +// fieldNeedsQuotes returns true if our field must be enclosed in quotes. +// Empty fields, files with a Comma, fields with a quote or newline, and +// fields which start with a space must be enclosed in quotes. +func (w *Writer) fieldNeedsQuotes(field string) bool { + if len(field) == 0 || strings.IndexRune(field, w.Comma) >= 0 || strings.IndexAny(field, "\"\r\n") >= 0 { + return true + } + + rune, _ := utf8.DecodeRuneInString(field) + return unicode.IsSpace(rune) +} diff --git a/src/pkg/csv/writer_test.go b/src/pkg/csv/writer_test.go new file mode 100644 index 000000000..578959007 --- /dev/null +++ b/src/pkg/csv/writer_test.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "bytes" + "testing" +) + +var writeTests = []struct { + Input [][]string + Output string + UseCRLF bool +}{ + {Input: [][]string{{"abc"}}, Output: "abc\n"}, + {Input: [][]string{{"abc"}}, Output: "abc\r\n", UseCRLF: true}, + {Input: [][]string{{`"abc"`}}, Output: `"""abc"""` + "\n"}, + {Input: [][]string{{`a"b`}}, Output: `"a""b"` + "\n"}, + {Input: [][]string{{`"a"b"`}}, Output: `"""a""b"""` + "\n"}, + {Input: [][]string{{" abc"}}, Output: `" abc"` + "\n"}, + {Input: [][]string{{"abc,def"}}, Output: `"abc,def"` + "\n"}, + {Input: [][]string{{"abc", "def"}}, Output: "abc,def\n"}, + {Input: [][]string{{"abc"}, {"def"}}, Output: "abc\ndef\n"}, + {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\ndef\"\n"}, + {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\r\ndef\"\r\n", UseCRLF: true}, +} + +func TestWrite(t *testing.T) { + for n, tt := range writeTests { + b := &bytes.Buffer{} + f := NewWriter(b) + f.UseCRLF = tt.UseCRLF + err := f.WriteAll(tt.Input) + if err != nil { + t.Errorf("Unexpected error: %s\n", err) + } + out := b.String() + if out != tt.Output { + t.Errorf("#%d: out=%q want %q", n, out, tt.Output) + } + } +} diff --git a/src/pkg/debug/dwarf/Makefile b/src/pkg/debug/dwarf/Makefile new file mode 100644 index 000000000..c4203188e --- /dev/null +++ b/src/pkg/debug/dwarf/Makefile @@ -0,0 +1,16 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/dwarf +GOFILES=\ + buf.go\ + const.go\ + entry.go\ + open.go\ + type.go\ + unit.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/dwarf/buf.go b/src/pkg/debug/dwarf/buf.go new file mode 100644 index 000000000..2d29cebdd --- /dev/null +++ b/src/pkg/debug/dwarf/buf.go @@ -0,0 +1,154 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Buffered reading and decoding of DWARF data streams. + +package dwarf + +import ( + "encoding/binary" + "os" + "strconv" +) + +// Data buffer being decoded. +type buf struct { + dwarf *Data + order binary.ByteOrder + name string + off Offset + data []byte + addrsize int + err os.Error +} + +func makeBuf(d *Data, name string, off Offset, data []byte, addrsize int) buf { + return buf{d, d.order, name, off, data, addrsize, nil} +} + +func (b *buf) uint8() uint8 { + if len(b.data) < 1 { + b.error("underflow") + return 0 + } + val := b.data[0] + b.data = b.data[1:] + b.off++ + return val +} + +func (b *buf) bytes(n int) []byte { + if len(b.data) < n { + b.error("underflow") + return nil + } + data := b.data[0:n] + b.data = b.data[n:] + b.off += Offset(n) + return data +} + +func (b *buf) skip(n int) { b.bytes(n) } + +func (b *buf) string() string { + for i := 0; i < len(b.data); i++ { + if b.data[i] == 0 { + s := string(b.data[0:i]) + b.data = b.data[i+1:] + b.off += Offset(i + 1) + return s + } + } + b.error("underflow") + return "" +} + +func (b *buf) uint16() uint16 { + a := b.bytes(2) + if a == nil { + return 0 + } + return b.order.Uint16(a) +} + +func (b *buf) uint32() uint32 { + a := b.bytes(4) + if a == nil { + return 0 + } + return b.order.Uint32(a) +} + +func (b *buf) uint64() uint64 { + a := b.bytes(8) + if a == nil { + return 0 + } + return b.order.Uint64(a) +} + +// Read a varint, which is 7 bits per byte, little endian. +// the 0x80 bit means read another byte. +func (b *buf) varint() (c uint64, bits uint) { + for i := 0; i < len(b.data); i++ { + byte := b.data[i] + c |= uint64(byte&0x7F) << bits + bits += 7 + if byte&0x80 == 0 { + b.off += Offset(i + 1) + b.data = b.data[i+1:] + return c, bits + } + } + return 0, 0 +} + +// Unsigned int is just a varint. +func (b *buf) uint() uint64 { + x, _ := b.varint() + return x +} + +// Signed int is a sign-extended varint. +func (b *buf) int() int64 { + ux, bits := b.varint() + x := int64(ux) + if x&(1<<(bits-1)) != 0 { + x |= -1 << bits + } + return x +} + +// Address-sized uint. +func (b *buf) addr() uint64 { + switch b.addrsize { + case 1: + return uint64(b.uint8()) + case 2: + return uint64(b.uint16()) + case 4: + return uint64(b.uint32()) + case 8: + return uint64(b.uint64()) + } + b.error("unknown address size") + return 0 +} + +func (b *buf) error(s string) { + if b.err == nil { + b.data = nil + b.err = DecodeError{b.name, b.off, s} + } +} + +type DecodeError struct { + Name string + Offset Offset + Error string +} + +func (e DecodeError) String() string { + return "decoding dwarf section " + e.Name + " at offset 0x" + strconv.Itob64(int64(e.Offset), 16) + ": " + e.Error +} diff --git a/src/pkg/debug/dwarf/const.go b/src/pkg/debug/dwarf/const.go new file mode 100644 index 000000000..1a3fec155 --- /dev/null +++ b/src/pkg/debug/dwarf/const.go @@ -0,0 +1,433 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Constants + +package dwarf + +import "strconv" + +// An Attr identifies the attribute type in a DWARF Entry's Field. +type Attr uint32 + +const ( + AttrSibling Attr = 0x01 + AttrLocation Attr = 0x02 + AttrName Attr = 0x03 + AttrOrdering Attr = 0x09 + AttrByteSize Attr = 0x0B + AttrBitOffset Attr = 0x0C + AttrBitSize Attr = 0x0D + AttrStmtList Attr = 0x10 + AttrLowpc Attr = 0x11 + AttrHighpc Attr = 0x12 + AttrLanguage Attr = 0x13 + AttrDiscr Attr = 0x15 + AttrDiscrValue Attr = 0x16 + AttrVisibility Attr = 0x17 + AttrImport Attr = 0x18 + AttrStringLength Attr = 0x19 + AttrCommonRef Attr = 0x1A + AttrCompDir Attr = 0x1B + AttrConstValue Attr = 0x1C + AttrContainingType Attr = 0x1D + AttrDefaultValue Attr = 0x1E + AttrInline Attr = 0x20 + AttrIsOptional Attr = 0x21 + AttrLowerBound Attr = 0x22 + AttrProducer Attr = 0x25 + AttrPrototyped Attr = 0x27 + AttrReturnAddr Attr = 0x2A + AttrStartScope Attr = 0x2C + AttrStrideSize Attr = 0x2E + AttrUpperBound Attr = 0x2F + AttrAbstractOrigin Attr = 0x31 + AttrAccessibility Attr = 0x32 + AttrAddrClass Attr = 0x33 + AttrArtificial Attr = 0x34 + AttrBaseTypes Attr = 0x35 + AttrCalling Attr = 0x36 + AttrCount Attr = 0x37 + AttrDataMemberLoc Attr = 0x38 + AttrDeclColumn Attr = 0x39 + AttrDeclFile Attr = 0x3A + AttrDeclLine Attr = 0x3B + AttrDeclaration Attr = 0x3C + AttrDiscrList Attr = 0x3D + AttrEncoding Attr = 0x3E + AttrExternal Attr = 0x3F + AttrFrameBase Attr = 0x40 + AttrFriend Attr = 0x41 + AttrIdentifierCase Attr = 0x42 + AttrMacroInfo Attr = 0x43 + AttrNamelistItem Attr = 0x44 + AttrPriority Attr = 0x45 + AttrSegment Attr = 0x46 + AttrSpecification Attr = 0x47 + AttrStaticLink Attr = 0x48 + AttrType Attr = 0x49 + AttrUseLocation Attr = 0x4A + AttrVarParam Attr = 0x4B + AttrVirtuality Attr = 0x4C + AttrVtableElemLoc Attr = 0x4D + AttrAllocated Attr = 0x4E + AttrAssociated Attr = 0x4F + AttrDataLocation Attr = 0x50 + AttrStride Attr = 0x51 + AttrEntrypc Attr = 0x52 + AttrUseUTF8 Attr = 0x53 + AttrExtension Attr = 0x54 + AttrRanges Attr = 0x55 + AttrTrampoline Attr = 0x56 + AttrCallColumn Attr = 0x57 + AttrCallFile Attr = 0x58 + AttrCallLine Attr = 0x59 + AttrDescription Attr = 0x5A +) + +var attrNames = [...]string{ + AttrSibling: "Sibling", + AttrLocation: "Location", + AttrName: "Name", + AttrOrdering: "Ordering", + AttrByteSize: "ByteSize", + AttrBitOffset: "BitOffset", + AttrBitSize: "BitSize", + AttrStmtList: "StmtList", + AttrLowpc: "Lowpc", + AttrHighpc: "Highpc", + AttrLanguage: "Language", + AttrDiscr: "Discr", + AttrDiscrValue: "DiscrValue", + AttrVisibility: "Visibility", + AttrImport: "Import", + AttrStringLength: "StringLength", + AttrCommonRef: "CommonRef", + AttrCompDir: "CompDir", + AttrConstValue: "ConstValue", + AttrContainingType: "ContainingType", + AttrDefaultValue: "DefaultValue", + AttrInline: "Inline", + AttrIsOptional: "IsOptional", + AttrLowerBound: "LowerBound", + AttrProducer: "Producer", + AttrPrototyped: "Prototyped", + AttrReturnAddr: "ReturnAddr", + AttrStartScope: "StartScope", + AttrStrideSize: "StrideSize", + AttrUpperBound: "UpperBound", + AttrAbstractOrigin: "AbstractOrigin", + AttrAccessibility: "Accessibility", + AttrAddrClass: "AddrClass", + AttrArtificial: "Artificial", + AttrBaseTypes: "BaseTypes", + AttrCalling: "Calling", + AttrCount: "Count", + AttrDataMemberLoc: "DataMemberLoc", + AttrDeclColumn: "DeclColumn", + AttrDeclFile: "DeclFile", + AttrDeclLine: "DeclLine", + AttrDeclaration: "Declaration", + AttrDiscrList: "DiscrList", + AttrEncoding: "Encoding", + AttrExternal: "External", + AttrFrameBase: "FrameBase", + AttrFriend: "Friend", + AttrIdentifierCase: "IdentifierCase", + AttrMacroInfo: "MacroInfo", + AttrNamelistItem: "NamelistItem", + AttrPriority: "Priority", + AttrSegment: "Segment", + AttrSpecification: "Specification", + AttrStaticLink: "StaticLink", + AttrType: "Type", + AttrUseLocation: "UseLocation", + AttrVarParam: "VarParam", + AttrVirtuality: "Virtuality", + AttrVtableElemLoc: "VtableElemLoc", + AttrAllocated: "Allocated", + AttrAssociated: "Associated", + AttrDataLocation: "DataLocation", + AttrStride: "Stride", + AttrEntrypc: "Entrypc", + AttrUseUTF8: "UseUTF8", + AttrExtension: "Extension", + AttrRanges: "Ranges", + AttrTrampoline: "Trampoline", + AttrCallColumn: "CallColumn", + AttrCallFile: "CallFile", + AttrCallLine: "CallLine", + AttrDescription: "Description", +} + +func (a Attr) String() string { + if int(a) < len(attrNames) { + s := attrNames[a] + if s != "" { + return s + } + } + return strconv.Itoa(int(a)) +} + +func (a Attr) GoString() string { + if int(a) < len(attrNames) { + s := attrNames[a] + if s != "" { + return "dwarf.Attr" + s + } + } + return "dwarf.Attr(" + strconv.Itoa64(int64(a)) + ")" +} + +// A format is a DWARF data encoding format. +type format uint32 + +const ( + // value formats + formAddr format = 0x01 + formDwarfBlock2 format = 0x03 + formDwarfBlock4 format = 0x04 + formData2 format = 0x05 + formData4 format = 0x06 + formData8 format = 0x07 + formString format = 0x08 + formDwarfBlock format = 0x09 + formDwarfBlock1 format = 0x0A + formData1 format = 0x0B + formFlag format = 0x0C + formSdata format = 0x0D + formStrp format = 0x0E + formUdata format = 0x0F + formRefAddr format = 0x10 + formRef1 format = 0x11 + formRef2 format = 0x12 + formRef4 format = 0x13 + formRef8 format = 0x14 + formRefUdata format = 0x15 + formIndirect format = 0x16 +) + +// A Tag is the classification (the type) of an Entry. +type Tag uint32 + +const ( + TagArrayType Tag = 0x01 + TagClassType Tag = 0x02 + TagEntryPoint Tag = 0x03 + TagEnumerationType Tag = 0x04 + TagFormalParameter Tag = 0x05 + TagImportedDeclaration Tag = 0x08 + TagLabel Tag = 0x0A + TagLexDwarfBlock Tag = 0x0B + TagMember Tag = 0x0D + TagPointerType Tag = 0x0F + TagReferenceType Tag = 0x10 + TagCompileUnit Tag = 0x11 + TagStringType Tag = 0x12 + TagStructType Tag = 0x13 + TagSubroutineType Tag = 0x15 + TagTypedef Tag = 0x16 + TagUnionType Tag = 0x17 + TagUnspecifiedParameters Tag = 0x18 + TagVariant Tag = 0x19 + TagCommonDwarfBlock Tag = 0x1A + TagCommonInclusion Tag = 0x1B + TagInheritance Tag = 0x1C + TagInlinedSubroutine Tag = 0x1D + TagModule Tag = 0x1E + TagPtrToMemberType Tag = 0x1F + TagSetType Tag = 0x20 + TagSubrangeType Tag = 0x21 + TagWithStmt Tag = 0x22 + TagAccessDeclaration Tag = 0x23 + TagBaseType Tag = 0x24 + TagCatchDwarfBlock Tag = 0x25 + TagConstType Tag = 0x26 + TagConstant Tag = 0x27 + TagEnumerator Tag = 0x28 + TagFileType Tag = 0x29 + TagFriend Tag = 0x2A + TagNamelist Tag = 0x2B + TagNamelistItem Tag = 0x2C + TagPackedType Tag = 0x2D + TagSubprogram Tag = 0x2E + TagTemplateTypeParameter Tag = 0x2F + TagTemplateValueParameter Tag = 0x30 + TagThrownType Tag = 0x31 + TagTryDwarfBlock Tag = 0x32 + TagVariantPart Tag = 0x33 + TagVariable Tag = 0x34 + TagVolatileType Tag = 0x35 + TagDwarfProcedure Tag = 0x36 + TagRestrictType Tag = 0x37 + TagInterfaceType Tag = 0x38 + TagNamespace Tag = 0x39 + TagImportedModule Tag = 0x3A + TagUnspecifiedType Tag = 0x3B + TagPartialUnit Tag = 0x3C + TagImportedUnit Tag = 0x3D + TagMutableType Tag = 0x3E +) + +var tagNames = [...]string{ + TagArrayType: "ArrayType", + TagClassType: "ClassType", + TagEntryPoint: "EntryPoint", + TagEnumerationType: "EnumerationType", + TagFormalParameter: "FormalParameter", + TagImportedDeclaration: "ImportedDeclaration", + TagLabel: "Label", + TagLexDwarfBlock: "LexDwarfBlock", + TagMember: "Member", + TagPointerType: "PointerType", + TagReferenceType: "ReferenceType", + TagCompileUnit: "CompileUnit", + TagStringType: "StringType", + TagStructType: "StructType", + TagSubroutineType: "SubroutineType", + TagTypedef: "Typedef", + TagUnionType: "UnionType", + TagUnspecifiedParameters: "UnspecifiedParameters", + TagVariant: "Variant", + TagCommonDwarfBlock: "CommonDwarfBlock", + TagCommonInclusion: "CommonInclusion", + TagInheritance: "Inheritance", + TagInlinedSubroutine: "InlinedSubroutine", + TagModule: "Module", + TagPtrToMemberType: "PtrToMemberType", + TagSetType: "SetType", + TagSubrangeType: "SubrangeType", + TagWithStmt: "WithStmt", + TagAccessDeclaration: "AccessDeclaration", + TagBaseType: "BaseType", + TagCatchDwarfBlock: "CatchDwarfBlock", + TagConstType: "ConstType", + TagConstant: "Constant", + TagEnumerator: "Enumerator", + TagFileType: "FileType", + TagFriend: "Friend", + TagNamelist: "Namelist", + TagNamelistItem: "NamelistItem", + TagPackedType: "PackedType", + TagSubprogram: "Subprogram", + TagTemplateTypeParameter: "TemplateTypeParameter", + TagTemplateValueParameter: "TemplateValueParameter", + TagThrownType: "ThrownType", + TagTryDwarfBlock: "TryDwarfBlock", + TagVariantPart: "VariantPart", + TagVariable: "Variable", + TagVolatileType: "VolatileType", + TagDwarfProcedure: "DwarfProcedure", + TagRestrictType: "RestrictType", + TagInterfaceType: "InterfaceType", + TagNamespace: "Namespace", + TagImportedModule: "ImportedModule", + TagUnspecifiedType: "UnspecifiedType", + TagPartialUnit: "PartialUnit", + TagImportedUnit: "ImportedUnit", + TagMutableType: "MutableType", +} + +func (t Tag) String() string { + if int(t) < len(tagNames) { + s := tagNames[t] + if s != "" { + return s + } + } + return strconv.Itoa(int(t)) +} + +func (t Tag) GoString() string { + if int(t) < len(tagNames) { + s := tagNames[t] + if s != "" { + return "dwarf.Tag" + s + } + } + return "dwarf.Tag(" + strconv.Itoa64(int64(t)) + ")" +} + +// Location expression operators. +// The debug info encodes value locations like 8(R3) +// as a sequence of these op codes. +// This package does not implement full expressions; +// the opPlusUconst operator is expected by the type parser. +const ( + opAddr = 0x03 /* 1 op, const addr */ + opDeref = 0x06 + opConst1u = 0x08 /* 1 op, 1 byte const */ + opConst1s = 0x09 /* " signed */ + opConst2u = 0x0A /* 1 op, 2 byte const */ + opConst2s = 0x0B /* " signed */ + opConst4u = 0x0C /* 1 op, 4 byte const */ + opConst4s = 0x0D /* " signed */ + opConst8u = 0x0E /* 1 op, 8 byte const */ + opConst8s = 0x0F /* " signed */ + opConstu = 0x10 /* 1 op, LEB128 const */ + opConsts = 0x11 /* " signed */ + opDup = 0x12 + opDrop = 0x13 + opOver = 0x14 + opPick = 0x15 /* 1 op, 1 byte stack index */ + opSwap = 0x16 + opRot = 0x17 + opXderef = 0x18 + opAbs = 0x19 + opAnd = 0x1A + opDiv = 0x1B + opMinus = 0x1C + opMod = 0x1D + opMul = 0x1E + opNeg = 0x1F + opNot = 0x20 + opOr = 0x21 + opPlus = 0x22 + opPlusUconst = 0x23 /* 1 op, ULEB128 addend */ + opShl = 0x24 + opShr = 0x25 + opShra = 0x26 + opXor = 0x27 + opSkip = 0x2F /* 1 op, signed 2-byte constant */ + opBra = 0x28 /* 1 op, signed 2-byte constant */ + opEq = 0x29 + opGe = 0x2A + opGt = 0x2B + opLe = 0x2C + opLt = 0x2D + opNe = 0x2E + opLit0 = 0x30 + /* OpLitN = OpLit0 + N for N = 0..31 */ + opReg0 = 0x50 + /* OpRegN = OpReg0 + N for N = 0..31 */ + opBreg0 = 0x70 /* 1 op, signed LEB128 constant */ + /* OpBregN = OpBreg0 + N for N = 0..31 */ + opRegx = 0x90 /* 1 op, ULEB128 register */ + opFbreg = 0x91 /* 1 op, SLEB128 offset */ + opBregx = 0x92 /* 2 op, ULEB128 reg; SLEB128 off */ + opPiece = 0x93 /* 1 op, ULEB128 size of piece */ + opDerefSize = 0x94 /* 1-byte size of data retrieved */ + opXderefSize = 0x95 /* 1-byte size of data retrieved */ + opNop = 0x96 + /* next four new in Dwarf v3 */ + opPushObjAddr = 0x97 + opCall2 = 0x98 /* 2-byte offset of DIE */ + opCall4 = 0x99 /* 4-byte offset of DIE */ + opCallRef = 0x9A /* 4- or 8- byte offset of DIE */ + /* 0xE0-0xFF reserved for user-specific */ +) + +// Basic type encodings -- the value for AttrEncoding in a TagBaseType Entry. +const ( + encAddress = 0x01 + encBoolean = 0x02 + encComplexFloat = 0x03 + encFloat = 0x04 + encSigned = 0x05 + encSignedChar = 0x06 + encUnsigned = 0x07 + encUnsignedChar = 0x08 + encImaginaryFloat = 0x09 +) diff --git a/src/pkg/debug/dwarf/entry.go b/src/pkg/debug/dwarf/entry.go new file mode 100644 index 000000000..549e5c2cc --- /dev/null +++ b/src/pkg/debug/dwarf/entry.go @@ -0,0 +1,343 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DWARF debug information entry parser. +// An entry is a sequence of data items of a given format. +// The first word in the entry is an index into what DWARF +// calls the ``abbreviation table.'' An abbreviation is really +// just a type descriptor: it's an array of attribute tag/value format pairs. + +package dwarf + +import "os" + +// a single entry's description: a sequence of attributes +type abbrev struct { + tag Tag + children bool + field []afield +} + +type afield struct { + attr Attr + fmt format +} + +// a map from entry format ids to their descriptions +type abbrevTable map[uint32]abbrev + +// ParseAbbrev returns the abbreviation table that starts at byte off +// in the .debug_abbrev section. +func (d *Data) parseAbbrev(off uint32) (abbrevTable, os.Error) { + if m, ok := d.abbrevCache[off]; ok { + return m, nil + } + + data := d.abbrev + if off > uint32(len(data)) { + data = nil + } else { + data = data[off:] + } + b := makeBuf(d, "abbrev", 0, data, 0) + + // Error handling is simplified by the buf getters + // returning an endless stream of 0s after an error. + m := make(abbrevTable) + for { + // Table ends with id == 0. + id := uint32(b.uint()) + if id == 0 { + break + } + + // Walk over attributes, counting. + n := 0 + b1 := b // Read from copy of b. + b1.uint() + b1.uint8() + for { + tag := b1.uint() + fmt := b1.uint() + if tag == 0 && fmt == 0 { + break + } + n++ + } + if b1.err != nil { + return nil, b1.err + } + + // Walk over attributes again, this time writing them down. + var a abbrev + a.tag = Tag(b.uint()) + a.children = b.uint8() != 0 + a.field = make([]afield, n) + for i := range a.field { + a.field[i].attr = Attr(b.uint()) + a.field[i].fmt = format(b.uint()) + } + b.uint() + b.uint() + + m[id] = a + } + if b.err != nil { + return nil, b.err + } + d.abbrevCache[off] = m + return m, nil +} + +// An entry is a sequence of attribute/value pairs. +type Entry struct { + Offset Offset // offset of Entry in DWARF info + Tag Tag // tag (kind of Entry) + Children bool // whether Entry is followed by children + Field []Field +} + +// A Field is a single attribute/value pair in an Entry. +type Field struct { + Attr Attr + Val interface{} +} + +// Val returns the value associated with attribute Attr in Entry, +// or nil if there is no such attribute. +// +// A common idiom is to merge the check for nil return with +// the check that the value has the expected dynamic type, as in: +// v, ok := e.Val(AttrSibling).(int64); +// +func (e *Entry) Val(a Attr) interface{} { + for _, f := range e.Field { + if f.Attr == a { + return f.Val + } + } + return nil +} + +// An Offset represents the location of an Entry within the DWARF info. +// (See Reader.Seek.) +type Offset uint32 + +// Entry reads a single entry from buf, decoding +// according to the given abbreviation table. +func (b *buf) entry(atab abbrevTable, ubase Offset) *Entry { + off := b.off + id := uint32(b.uint()) + if id == 0 { + return &Entry{} + } + a, ok := atab[id] + if !ok { + b.error("unknown abbreviation table index") + return nil + } + e := &Entry{ + Offset: off, + Tag: a.tag, + Children: a.children, + Field: make([]Field, len(a.field)), + } + for i := range e.Field { + e.Field[i].Attr = a.field[i].attr + fmt := a.field[i].fmt + if fmt == formIndirect { + fmt = format(b.uint()) + } + var val interface{} + switch fmt { + default: + b.error("unknown entry attr format") + + // address + case formAddr: + val = b.addr() + + // block + case formDwarfBlock1: + val = b.bytes(int(b.uint8())) + case formDwarfBlock2: + val = b.bytes(int(b.uint16())) + case formDwarfBlock4: + val = b.bytes(int(b.uint32())) + case formDwarfBlock: + val = b.bytes(int(b.uint())) + + // constant + case formData1: + val = int64(b.uint8()) + case formData2: + val = int64(b.uint16()) + case formData4: + val = int64(b.uint32()) + case formData8: + val = int64(b.uint64()) + case formSdata: + val = int64(b.int()) + case formUdata: + val = int64(b.uint()) + + // flag + case formFlag: + val = b.uint8() == 1 + + // reference to other entry + case formRefAddr: + val = Offset(b.addr()) + case formRef1: + val = Offset(b.uint8()) + ubase + case formRef2: + val = Offset(b.uint16()) + ubase + case formRef4: + val = Offset(b.uint32()) + ubase + case formRef8: + val = Offset(b.uint64()) + ubase + case formRefUdata: + val = Offset(b.uint()) + ubase + + // string + case formString: + val = b.string() + case formStrp: + off := b.uint32() // offset into .debug_str + if b.err != nil { + return nil + } + b1 := makeBuf(b.dwarf, "str", 0, b.dwarf.str, 0) + b1.skip(int(off)) + val = b1.string() + if b1.err != nil { + b.err = b1.err + return nil + } + } + e.Field[i].Val = val + } + if b.err != nil { + return nil + } + return e +} + +// A Reader allows reading Entry structures from a DWARF ``info'' section. +// The Entry structures are arranged in a tree. The Reader's Next function +// return successive entries from a pre-order traversal of the tree. +// If an entry has children, its Children field will be true, and the children +// follow, terminated by an Entry with Tag 0. +type Reader struct { + b buf + d *Data + err os.Error + unit int + lastChildren bool // .Children of last entry returned by Next + lastSibling Offset // .Val(AttrSibling) of last entry returned by Next +} + +// Reader returns a new Reader for Data. +// The reader is positioned at byte offset 0 in the DWARF ``info'' section. +func (d *Data) Reader() *Reader { + r := &Reader{d: d} + r.Seek(0) + return r +} + +// Seek positions the Reader at offset off in the encoded entry stream. +// Offset 0 can be used to denote the first entry. +func (r *Reader) Seek(off Offset) { + d := r.d + r.err = nil + r.lastChildren = false + if off == 0 { + if len(d.unit) == 0 { + return + } + u := &d.unit[0] + r.unit = 0 + r.b = makeBuf(r.d, "info", u.off, u.data, u.addrsize) + return + } + + // TODO(rsc): binary search (maybe a new package) + var i int + var u *unit + for i = range d.unit { + u = &d.unit[i] + if u.off <= off && off < u.off+Offset(len(u.data)) { + r.unit = i + r.b = makeBuf(r.d, "info", off, u.data[off-u.off:], u.addrsize) + return + } + } + r.err = os.NewError("offset out of range") +} + +// maybeNextUnit advances to the next unit if this one is finished. +func (r *Reader) maybeNextUnit() { + for len(r.b.data) == 0 && r.unit+1 < len(r.d.unit) { + r.unit++ + u := &r.d.unit[r.unit] + r.b = makeBuf(r.d, "info", u.off, u.data, u.addrsize) + } +} + +// Next reads the next entry from the encoded entry stream. +// It returns nil, nil when it reaches the end of the section. +// It returns an error if the current offset is invalid or the data at the +// offset cannot be decoded as a valid Entry. +func (r *Reader) Next() (*Entry, os.Error) { + if r.err != nil { + return nil, r.err + } + r.maybeNextUnit() + if len(r.b.data) == 0 { + return nil, nil + } + u := &r.d.unit[r.unit] + e := r.b.entry(u.atable, u.base) + if r.b.err != nil { + r.err = r.b.err + return nil, r.err + } + if e != nil { + r.lastChildren = e.Children + if r.lastChildren { + r.lastSibling, _ = e.Val(AttrSibling).(Offset) + } + } else { + r.lastChildren = false + } + return e, nil +} + +// SkipChildren skips over the child entries associated with +// the last Entry returned by Next. If that Entry did not have +// children or Next has not been called, SkipChildren is a no-op. +func (r *Reader) SkipChildren() { + if r.err != nil || !r.lastChildren { + return + } + + // If the last entry had a sibling attribute, + // that attribute gives the offset of the next + // sibling, so we can avoid decoding the + // child subtrees. + if r.lastSibling >= r.b.off { + r.Seek(r.lastSibling) + return + } + + for { + e, err := r.Next() + if err != nil || e == nil || e.Tag == 0 { + break + } + if e.Children { + r.SkipChildren() + } + } +} diff --git a/src/pkg/debug/dwarf/open.go b/src/pkg/debug/dwarf/open.go new file mode 100644 index 000000000..d9525f788 --- /dev/null +++ b/src/pkg/debug/dwarf/open.go @@ -0,0 +1,80 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dwarf provides access to DWARF debugging information loaded from +// executable files, as defined in the DWARF 2.0 Standard at +// http://dwarfstd.org/doc/dwarf-2.0.0.pdf +package dwarf + +import ( + "encoding/binary" + "os" +) + +// Data represents the DWARF debugging information +// loaded from an executable file (for example, an ELF or Mach-O executable). +type Data struct { + // raw data + abbrev []byte + aranges []byte + frame []byte + info []byte + line []byte + pubnames []byte + ranges []byte + str []byte + + // parsed data + abbrevCache map[uint32]abbrevTable + addrsize int + order binary.ByteOrder + typeCache map[Offset]Type + unit []unit +} + +// New returns a new Data object initialized from the given parameters. +// Clients should typically use [TODO(rsc): method to be named later] instead of calling +// New directly. +// +// The []byte arguments are the data from the corresponding debug section +// in the object file; for example, for an ELF object, abbrev is the contents of +// the ".debug_abbrev" section. +func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, os.Error) { + d := &Data{ + abbrev: abbrev, + aranges: aranges, + frame: frame, + info: info, + line: line, + pubnames: pubnames, + ranges: ranges, + str: str, + abbrevCache: make(map[uint32]abbrevTable), + typeCache: make(map[Offset]Type), + } + + // Sniff .debug_info to figure out byte order. + // bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3). + if len(d.info) < 6 { + return nil, DecodeError{"info", Offset(len(d.info)), "too short"} + } + x, y := d.info[4], d.info[5] + switch { + case x == 0 && y == 0: + return nil, DecodeError{"info", 4, "unsupported version 0"} + case x == 0: + d.order = binary.BigEndian + case y == 0: + d.order = binary.LittleEndian + default: + return nil, DecodeError{"info", 4, "cannot determine byte order"} + } + + u, err := d.parseUnits() + if err != nil { + return nil, err + } + d.unit = u + return d, nil +} diff --git a/src/pkg/debug/dwarf/testdata/typedef.c b/src/pkg/debug/dwarf/testdata/typedef.c new file mode 100644 index 000000000..664d021ce --- /dev/null +++ b/src/pkg/debug/dwarf/testdata/typedef.c @@ -0,0 +1,79 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Linux ELF: +gcc -gdwarf-2 -m64 -c typedef.c && gcc -gdwarf-2 -m64 -o typedef.elf typedef.o + +OS X Mach-O: +gcc -gdwarf-2 -m64 -c typedef.c -o typedef.macho +*/ +#include + +typedef volatile int* t_ptr_volatile_int; +typedef const char *t_ptr_const_char; +typedef long t_long; +typedef unsigned short t_ushort; +typedef int t_func_int_of_float_double(float, double); +typedef int (*t_ptr_func_int_of_float_double)(float, double); +typedef int (*t_ptr_func_int_of_float_complex)(float complex); +typedef int (*t_ptr_func_int_of_double_complex)(double complex); +typedef int (*t_ptr_func_int_of_long_double_complex)(long double complex); +typedef int *t_func_ptr_int_of_char_schar_uchar(char, signed char, unsigned char); +typedef void t_func_void_of_char(char); +typedef void t_func_void_of_void(void); +typedef void t_func_void_of_ptr_char_dots(char*, ...); +typedef struct my_struct { + volatile int vi; + char x : 1; + int y : 4; + long long array[40]; +} t_my_struct; +typedef union my_union { + volatile int vi; + char x : 1; + int y : 4; + long long array[40]; +} t_my_union; +typedef enum my_enum { + e1 = 1, + e2 = 2, + e3 = -5, + e4 = 1000000000000000LL, +} t_my_enum; + +typedef struct list t_my_list; +struct list { + short val; + t_my_list *next; +}; + +typedef struct tree { + struct tree *left, *right; + unsigned long long val; +} t_my_tree; + +t_ptr_volatile_int *a2; +t_ptr_const_char **a3a; +t_long *a4; +t_ushort *a5; +t_func_int_of_float_double *a6; +t_ptr_func_int_of_float_double *a7; +t_func_ptr_int_of_char_schar_uchar *a8; +t_func_void_of_char *a9; +t_func_void_of_void *a10; +t_func_void_of_ptr_char_dots *a11; +t_my_struct *a12; +t_my_union *a12a; +t_my_enum *a13; +t_my_list *a14; +t_my_tree *a15; +t_ptr_func_int_of_float_complex *a16; +t_ptr_func_int_of_double_complex *a17; +t_ptr_func_int_of_long_double_complex *a18; + +int main() +{ + return 0; +} diff --git a/src/pkg/debug/dwarf/testdata/typedef.elf b/src/pkg/debug/dwarf/testdata/typedef.elf new file mode 100755 index 000000000..44df8da9b Binary files /dev/null and b/src/pkg/debug/dwarf/testdata/typedef.elf differ diff --git a/src/pkg/debug/dwarf/testdata/typedef.macho b/src/pkg/debug/dwarf/testdata/typedef.macho new file mode 100644 index 000000000..41019c1e1 Binary files /dev/null and b/src/pkg/debug/dwarf/testdata/typedef.macho differ diff --git a/src/pkg/debug/dwarf/type.go b/src/pkg/debug/dwarf/type.go new file mode 100644 index 000000000..f35365ebe --- /dev/null +++ b/src/pkg/debug/dwarf/type.go @@ -0,0 +1,584 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DWARF type information structures. +// The format is heavily biased toward C, but for simplicity +// the String methods use a pseudo-Go syntax. + +package dwarf + +import ( + "os" + "strconv" +) + +// A Type conventionally represents a pointer to any of the +// specific Type structures (CharType, StructType, etc.). +type Type interface { + Common() *CommonType + String() string + Size() int64 +} + +// A CommonType holds fields common to multiple types. +// If a field is not known or not applicable for a given type, +// the zero value is used. +type CommonType struct { + ByteSize int64 // size of value of this type, in bytes + Name string // name that can be used to refer to type +} + +func (c *CommonType) Common() *CommonType { return c } + +func (c *CommonType) Size() int64 { return c.ByteSize } + +// Basic types + +// A BasicType holds fields common to all basic types. +type BasicType struct { + CommonType + BitSize int64 + BitOffset int64 +} + +func (b *BasicType) Basic() *BasicType { return b } + +func (t *BasicType) String() string { + if t.Name != "" { + return t.Name + } + return "?" +} + +// A CharType represents a signed character type. +type CharType struct { + BasicType +} + +// A UcharType represents an unsigned character type. +type UcharType struct { + BasicType +} + +// An IntType represents a signed integer type. +type IntType struct { + BasicType +} + +// A UintType represents an unsigned integer type. +type UintType struct { + BasicType +} + +// A FloatType represents a floating point type. +type FloatType struct { + BasicType +} + +// A ComplexType represents a complex floating point type. +type ComplexType struct { + BasicType +} + +// A BoolType represents a boolean type. +type BoolType struct { + BasicType +} + +// An AddrType represents a machine address type. +type AddrType struct { + BasicType +} + +// qualifiers + +// A QualType represents a type that has the C/C++ "const", "restrict", or "volatile" qualifier. +type QualType struct { + CommonType + Qual string + Type Type +} + +func (t *QualType) String() string { return t.Qual + " " + t.Type.String() } + +func (t *QualType) Size() int64 { return t.Type.Size() } + +// An ArrayType represents a fixed size array type. +type ArrayType struct { + CommonType + Type Type + StrideBitSize int64 // if > 0, number of bits to hold each element + Count int64 // if == -1, an incomplete array, like char x[]. +} + +func (t *ArrayType) String() string { + return "[" + strconv.Itoa64(t.Count) + "]" + t.Type.String() +} + +func (t *ArrayType) Size() int64 { return t.Count * t.Type.Size() } + +// A VoidType represents the C void type. +type VoidType struct { + CommonType +} + +func (t *VoidType) String() string { return "void" } + +// A PtrType represents a pointer type. +type PtrType struct { + CommonType + Type Type +} + +func (t *PtrType) String() string { return "*" + t.Type.String() } + +// A StructType represents a struct, union, or C++ class type. +type StructType struct { + CommonType + StructName string + Kind string // "struct", "union", or "class". + Field []*StructField + Incomplete bool // if true, struct, union, class is declared but not defined +} + +// A StructField represents a field in a struct, union, or C++ class type. +type StructField struct { + Name string + Type Type + ByteOffset int64 + ByteSize int64 + BitOffset int64 // within the ByteSize bytes at ByteOffset + BitSize int64 // zero if not a bit field +} + +func (t *StructType) String() string { + if t.StructName != "" { + return t.Kind + " " + t.StructName + } + return t.Defn() +} + +func (t *StructType) Defn() string { + s := t.Kind + if t.StructName != "" { + s += " " + t.StructName + } + if t.Incomplete { + s += " /*incomplete*/" + return s + } + s += " {" + for i, f := range t.Field { + if i > 0 { + s += "; " + } + s += f.Name + " " + f.Type.String() + s += "@" + strconv.Itoa64(f.ByteOffset) + if f.BitSize > 0 { + s += " : " + strconv.Itoa64(f.BitSize) + s += "@" + strconv.Itoa64(f.BitOffset) + } + } + s += "}" + return s +} + +// An EnumType represents an enumerated type. +// The only indication of its native integer type is its ByteSize +// (inside CommonType). +type EnumType struct { + CommonType + EnumName string + Val []*EnumValue +} + +// An EnumValue represents a single enumeration value. +type EnumValue struct { + Name string + Val int64 +} + +func (t *EnumType) String() string { + s := "enum" + if t.EnumName != "" { + s += " " + t.EnumName + } + s += " {" + for i, v := range t.Val { + if i > 0 { + s += "; " + } + s += v.Name + "=" + strconv.Itoa64(v.Val) + } + s += "}" + return s +} + +// A FuncType represents a function type. +type FuncType struct { + CommonType + ReturnType Type + ParamType []Type +} + +func (t *FuncType) String() string { + s := "func(" + for i, t := range t.ParamType { + if i > 0 { + s += ", " + } + s += t.String() + } + s += ")" + if t.ReturnType != nil { + s += " " + t.ReturnType.String() + } + return s +} + +// A DotDotDotType represents the variadic ... function parameter. +type DotDotDotType struct { + CommonType +} + +func (t *DotDotDotType) String() string { return "..." } + +// A TypedefType represents a named type. +type TypedefType struct { + CommonType + Type Type +} + +func (t *TypedefType) String() string { return t.Name } + +func (t *TypedefType) Size() int64 { return t.Type.Size() } + +func (d *Data) Type(off Offset) (Type, os.Error) { + if t, ok := d.typeCache[off]; ok { + return t, nil + } + + r := d.Reader() + r.Seek(off) + e, err := r.Next() + if err != nil { + return nil, err + } + if e == nil || e.Offset != off { + return nil, DecodeError{"info", off, "no type at offset"} + } + + // Parse type from Entry. + // Must always set d.typeCache[off] before calling + // d.Type recursively, to handle circular types correctly. + var typ Type + + // Get next child; set err if error happens. + next := func() *Entry { + if !e.Children { + return nil + } + kid, err1 := r.Next() + if err1 != nil { + err = err1 + return nil + } + if kid == nil { + err = DecodeError{"info", r.b.off, "unexpected end of DWARF entries"} + return nil + } + if kid.Tag == 0 { + return nil + } + return kid + } + + // Get Type referred to by Entry's AttrType field. + // Set err if error happens. Not having a type is an error. + typeOf := func(e *Entry) Type { + toff, ok := e.Val(AttrType).(Offset) + if !ok { + // It appears that no Type means "void". + return new(VoidType) + } + var t Type + if t, err = d.Type(toff); err != nil { + return nil + } + return t + } + + switch e.Tag { + case TagArrayType: + // Multi-dimensional array. (DWARF v2 §5.4) + // Attributes: + // AttrType:subtype [required] + // AttrStrideSize: size in bits of each element of the array + // AttrByteSize: size of entire array + // Children: + // TagSubrangeType or TagEnumerationType giving one dimension. + // dimensions are in left to right order. + t := new(ArrayType) + typ = t + d.typeCache[off] = t + if t.Type = typeOf(e); err != nil { + goto Error + } + t.StrideBitSize, _ = e.Val(AttrStrideSize).(int64) + + // Accumulate dimensions, + ndim := 0 + for kid := next(); kid != nil; kid = next() { + // TODO(rsc): Can also be TagEnumerationType + // but haven't seen that in the wild yet. + switch kid.Tag { + case TagSubrangeType: + max, ok := kid.Val(AttrUpperBound).(int64) + if !ok { + max = -2 // Count == -1, as in x[]. + } + if ndim == 0 { + t.Count = max + 1 + } else { + // Multidimensional array. + // Create new array type underneath this one. + t.Type = &ArrayType{Type: t.Type, Count: max + 1} + } + ndim++ + case TagEnumerationType: + err = DecodeError{"info", kid.Offset, "cannot handle enumeration type as array bound"} + goto Error + } + } + if ndim == 0 { + // LLVM generates this for x[]. + t.Count = -1 + } + + case TagBaseType: + // Basic type. (DWARF v2 §5.1) + // Attributes: + // AttrName: name of base type in programming language of the compilation unit [required] + // AttrEncoding: encoding value for type (encFloat etc) [required] + // AttrByteSize: size of type in bytes [required] + // AttrBitOffset: for sub-byte types, size in bits + // AttrBitSize: for sub-byte types, bit offset of high order bit in the AttrByteSize bytes + name, _ := e.Val(AttrName).(string) + enc, ok := e.Val(AttrEncoding).(int64) + if !ok { + err = DecodeError{"info", e.Offset, "missing encoding attribute for " + name} + goto Error + } + switch enc { + default: + err = DecodeError{"info", e.Offset, "unrecognized encoding attribute value"} + goto Error + + case encAddress: + typ = new(AddrType) + case encBoolean: + typ = new(BoolType) + case encComplexFloat: + typ = new(ComplexType) + case encFloat: + typ = new(FloatType) + case encSigned: + typ = new(IntType) + case encUnsigned: + typ = new(UintType) + case encSignedChar: + typ = new(CharType) + case encUnsignedChar: + typ = new(UcharType) + } + d.typeCache[off] = typ + t := typ.(interface { + Basic() *BasicType + }).Basic() + t.Name = name + t.BitSize, _ = e.Val(AttrBitSize).(int64) + t.BitOffset, _ = e.Val(AttrBitOffset).(int64) + + case TagClassType, TagStructType, TagUnionType: + // Structure, union, or class type. (DWARF v2 §5.5) + // Attributes: + // AttrName: name of struct, union, or class + // AttrByteSize: byte size [required] + // AttrDeclaration: if true, struct/union/class is incomplete + // Children: + // TagMember to describe one member. + // AttrName: name of member [required] + // AttrType: type of member [required] + // AttrByteSize: size in bytes + // AttrBitOffset: bit offset within bytes for bit fields + // AttrBitSize: bit size for bit fields + // AttrDataMemberLoc: location within struct [required for struct, class] + // There is much more to handle C++, all ignored for now. + t := new(StructType) + typ = t + d.typeCache[off] = t + switch e.Tag { + case TagClassType: + t.Kind = "class" + case TagStructType: + t.Kind = "struct" + case TagUnionType: + t.Kind = "union" + } + t.StructName, _ = e.Val(AttrName).(string) + t.Incomplete = e.Val(AttrDeclaration) != nil + t.Field = make([]*StructField, 0, 8) + for kid := next(); kid != nil; kid = next() { + if kid.Tag == TagMember { + f := new(StructField) + if f.Type = typeOf(kid); err != nil { + goto Error + } + if loc, ok := kid.Val(AttrDataMemberLoc).([]byte); ok { + b := makeBuf(d, "location", 0, loc, d.addrsize) + if b.uint8() != opPlusUconst { + err = DecodeError{"info", kid.Offset, "unexpected opcode"} + goto Error + } + f.ByteOffset = int64(b.uint()) + if b.err != nil { + err = b.err + goto Error + } + } + f.Name, _ = kid.Val(AttrName).(string) + f.ByteSize, _ = kid.Val(AttrByteSize).(int64) + f.BitOffset, _ = kid.Val(AttrBitOffset).(int64) + f.BitSize, _ = kid.Val(AttrBitSize).(int64) + t.Field = append(t.Field, f) + } + } + + case TagConstType, TagVolatileType, TagRestrictType: + // Type modifier (DWARF v2 §5.2) + // Attributes: + // AttrType: subtype + t := new(QualType) + typ = t + d.typeCache[off] = t + if t.Type = typeOf(e); err != nil { + goto Error + } + switch e.Tag { + case TagConstType: + t.Qual = "const" + case TagRestrictType: + t.Qual = "restrict" + case TagVolatileType: + t.Qual = "volatile" + } + + case TagEnumerationType: + // Enumeration type (DWARF v2 §5.6) + // Attributes: + // AttrName: enum name if any + // AttrByteSize: bytes required to represent largest value + // Children: + // TagEnumerator: + // AttrName: name of constant + // AttrConstValue: value of constant + t := new(EnumType) + typ = t + d.typeCache[off] = t + t.EnumName, _ = e.Val(AttrName).(string) + t.Val = make([]*EnumValue, 0, 8) + for kid := next(); kid != nil; kid = next() { + if kid.Tag == TagEnumerator { + f := new(EnumValue) + f.Name, _ = kid.Val(AttrName).(string) + f.Val, _ = kid.Val(AttrConstValue).(int64) + n := len(t.Val) + if n >= cap(t.Val) { + val := make([]*EnumValue, n, n*2) + copy(val, t.Val) + t.Val = val + } + t.Val = t.Val[0 : n+1] + t.Val[n] = f + } + } + + case TagPointerType: + // Type modifier (DWARF v2 §5.2) + // Attributes: + // AttrType: subtype [not required! void* has no AttrType] + // AttrAddrClass: address class [ignored] + t := new(PtrType) + typ = t + d.typeCache[off] = t + if e.Val(AttrType) == nil { + t.Type = &VoidType{} + break + } + t.Type = typeOf(e) + + case TagSubroutineType: + // Subroutine type. (DWARF v2 §5.7) + // Attributes: + // AttrType: type of return value if any + // AttrName: possible name of type [ignored] + // AttrPrototyped: whether used ANSI C prototype [ignored] + // Children: + // TagFormalParameter: typed parameter + // AttrType: type of parameter + // TagUnspecifiedParameter: final ... + t := new(FuncType) + typ = t + d.typeCache[off] = t + if t.ReturnType = typeOf(e); err != nil { + goto Error + } + t.ParamType = make([]Type, 0, 8) + for kid := next(); kid != nil; kid = next() { + var tkid Type + switch kid.Tag { + default: + continue + case TagFormalParameter: + if tkid = typeOf(kid); err != nil { + goto Error + } + case TagUnspecifiedParameters: + tkid = &DotDotDotType{} + } + t.ParamType = append(t.ParamType, tkid) + } + + case TagTypedef: + // Typedef (DWARF v2 §5.3) + // Attributes: + // AttrName: name [required] + // AttrType: type definition [required] + t := new(TypedefType) + typ = t + d.typeCache[off] = t + t.Name, _ = e.Val(AttrName).(string) + t.Type = typeOf(e) + } + + if err != nil { + goto Error + } + + { + b, ok := e.Val(AttrByteSize).(int64) + if !ok { + b = -1 + } + typ.Common().ByteSize = b + } + return typ, nil + +Error: + // If the parse fails, take the type out of the cache + // so that the next call with this offset doesn't hit + // the cache and return success. + d.typeCache[off] = nil, false + return nil, err +} diff --git a/src/pkg/debug/dwarf/type_test.go b/src/pkg/debug/dwarf/type_test.go new file mode 100644 index 000000000..b9470a4fc --- /dev/null +++ b/src/pkg/debug/dwarf/type_test.go @@ -0,0 +1,111 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf_test + +import ( + . "debug/dwarf" + "debug/elf" + "debug/macho" + "testing" +) + +var typedefTests = map[string]string{ + "t_ptr_volatile_int": "*volatile int", + "t_ptr_const_char": "*const char", + "t_long": "long int", + "t_ushort": "short unsigned int", + "t_func_int_of_float_double": "func(float, double) int", + "t_ptr_func_int_of_float_double": "*func(float, double) int", + "t_ptr_func_int_of_float_complex": "*func(complex float) int", + "t_ptr_func_int_of_double_complex": "*func(complex double) int", + "t_ptr_func_int_of_long_double_complex": "*func(complex long double) int", + "t_func_ptr_int_of_char_schar_uchar": "func(char, signed char, unsigned char) *int", + "t_func_void_of_char": "func(char) void", + "t_func_void_of_void": "func() void", + "t_func_void_of_ptr_char_dots": "func(*char, ...) void", + "t_my_struct": "struct my_struct {vi volatile int@0; x char@4 : 1@7; y int@4 : 4@27; array [40]long long int@8}", + "t_my_union": "union my_union {vi volatile int@0; x char@0 : 1@7; y int@0 : 4@28; array [40]long long int@0}", + "t_my_enum": "enum my_enum {e1=1; e2=2; e3=-5; e4=1000000000000000}", + "t_my_list": "struct list {val short int@0; next *t_my_list@8}", + "t_my_tree": "struct tree {left *struct tree@0; right *struct tree@8; val long long unsigned int@16}", +} + +func elfData(t *testing.T, name string) *Data { + f, err := elf.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + +func machoData(t *testing.T, name string) *Data { + f, err := macho.Open(name) + if err != nil { + t.Fatal(err) + } + + d, err := f.DWARF() + if err != nil { + t.Fatal(err) + } + return d +} + +func TestTypedefsELF(t *testing.T) { testTypedefs(t, elfData(t, "testdata/typedef.elf")) } + +func TestTypedefsMachO(t *testing.T) { + testTypedefs(t, machoData(t, "testdata/typedef.macho")) +} + +func testTypedefs(t *testing.T, d *Data) { + r := d.Reader() + seen := make(map[string]bool) + for { + e, err := r.Next() + if err != nil { + t.Fatal("r.Next:", err) + } + if e == nil { + break + } + if e.Tag == TagTypedef { + typ, err := d.Type(e.Offset) + if err != nil { + t.Fatal("d.Type:", err) + } + t1 := typ.(*TypedefType) + var typstr string + if ts, ok := t1.Type.(*StructType); ok { + typstr = ts.Defn() + } else { + typstr = t1.Type.String() + } + + if want, ok := typedefTests[t1.Name]; ok { + if seen[t1.Name] { + t.Errorf("multiple definitions for %s", t1.Name) + } + seen[t1.Name] = true + if typstr != want { + t.Errorf("%s:\n\thave %s\n\twant %s", t1.Name, typstr, want) + } + } + } + if e.Tag != TagCompileUnit { + r.SkipChildren() + } + } + + for k := range typedefTests { + if !seen[k] { + t.Errorf("missing %s", k) + } + } +} diff --git a/src/pkg/debug/dwarf/unit.go b/src/pkg/debug/dwarf/unit.go new file mode 100644 index 000000000..02cb363b4 --- /dev/null +++ b/src/pkg/debug/dwarf/unit.go @@ -0,0 +1,62 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dwarf + +import ( + "os" + "strconv" +) + +// DWARF debug info is split into a sequence of compilation units. +// Each unit has its own abbreviation table and address size. + +type unit struct { + base Offset // byte offset of header within the aggregate info + off Offset // byte offset of data within the aggregate info + data []byte + atable abbrevTable + addrsize int +} + +func (d *Data) parseUnits() ([]unit, os.Error) { + // Count units. + nunit := 0 + b := makeBuf(d, "info", 0, d.info, 0) + for len(b.data) > 0 { + b.skip(int(b.uint32())) + nunit++ + } + if b.err != nil { + return nil, b.err + } + + // Again, this time writing them down. + b = makeBuf(d, "info", 0, d.info, 0) + units := make([]unit, nunit) + for i := range units { + u := &units[i] + u.base = b.off + n := b.uint32() + if vers := b.uint16(); vers != 2 { + b.error("unsupported DWARF version " + strconv.Itoa(int(vers))) + break + } + atable, err := d.parseAbbrev(b.uint32()) + if err != nil { + if b.err == nil { + b.err = err + } + break + } + u.atable = atable + u.addrsize = int(b.uint8()) + u.off = b.off + u.data = b.bytes(int(n - (2 + 4 + 1))) + } + if b.err != nil { + return nil, b.err + } + return units, nil +} diff --git a/src/pkg/debug/elf/Makefile b/src/pkg/debug/elf/Makefile new file mode 100644 index 000000000..dd431f653 --- /dev/null +++ b/src/pkg/debug/elf/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/elf +GOFILES=\ + elf.go\ + file.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/elf/elf.go b/src/pkg/debug/elf/elf.go new file mode 100644 index 000000000..c71b230bd --- /dev/null +++ b/src/pkg/debug/elf/elf.go @@ -0,0 +1,1521 @@ +/* + * ELF constants and data structures + * + * Derived from: + * $FreeBSD: src/sys/sys/elf32.h,v 1.8.14.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/sys/elf64.h,v 1.10.14.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/sys/elf_common.h,v 1.15.8.1 2005/12/30 22:13:58 marcel Exp $ + * $FreeBSD: src/sys/alpha/include/elf.h,v 1.14 2003/09/25 01:10:22 peter Exp $ + * $FreeBSD: src/sys/amd64/include/elf.h,v 1.18 2004/08/03 08:21:48 dfr Exp $ + * $FreeBSD: src/sys/arm/include/elf.h,v 1.5.2.1 2006/06/30 21:42:52 cognet Exp $ + * $FreeBSD: src/sys/i386/include/elf.h,v 1.16 2004/08/02 19:12:17 dfr Exp $ + * $FreeBSD: src/sys/powerpc/include/elf.h,v 1.7 2004/11/02 09:47:01 ssouhlal Exp $ + * $FreeBSD: src/sys/sparc64/include/elf.h,v 1.12 2003/09/25 01:10:26 peter Exp $ + * + * Copyright (c) 1996-1998 John D. Polstra. All rights reserved. + * Copyright (c) 2001 David E. O'Brien + * Portions Copyright 2009 The Go Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +package elf + +import "strconv" + +/* + * Constants + */ + +// Indexes into the Header.Ident array. +const ( + EI_CLASS = 4 /* Class of machine. */ + EI_DATA = 5 /* Data format. */ + EI_VERSION = 6 /* ELF format version. */ + EI_OSABI = 7 /* Operating system / ABI identification */ + EI_ABIVERSION = 8 /* ABI version */ + EI_PAD = 9 /* Start of padding (per SVR4 ABI). */ + EI_NIDENT = 16 /* Size of e_ident array. */ +) + +// Initial magic number for ELF files. +const ELFMAG = "\177ELF" + +// Version is found in Header.Ident[EI_VERSION] and Header.Version. +type Version byte + +const ( + EV_NONE Version = 0 + EV_CURRENT Version = 1 +) + +var versionStrings = []intName{ + {0, "EV_NONE"}, + {1, "EV_CURRENT"}, +} + +func (i Version) String() string { return stringName(uint32(i), versionStrings, false) } +func (i Version) GoString() string { return stringName(uint32(i), versionStrings, true) } + +// Class is found in Header.Ident[EI_CLASS] and Header.Class. +type Class byte + +const ( + ELFCLASSNONE Class = 0 /* Unknown class. */ + ELFCLASS32 Class = 1 /* 32-bit architecture. */ + ELFCLASS64 Class = 2 /* 64-bit architecture. */ +) + +var classStrings = []intName{ + {0, "ELFCLASSNONE"}, + {1, "ELFCLASS32"}, + {2, "ELFCLASS64"}, +} + +func (i Class) String() string { return stringName(uint32(i), classStrings, false) } +func (i Class) GoString() string { return stringName(uint32(i), classStrings, true) } + +// Data is found in Header.Ident[EI_DATA] and Header.Data. +type Data byte + +const ( + ELFDATANONE Data = 0 /* Unknown data format. */ + ELFDATA2LSB Data = 1 /* 2's complement little-endian. */ + ELFDATA2MSB Data = 2 /* 2's complement big-endian. */ +) + +var dataStrings = []intName{ + {0, "ELFDATANONE"}, + {1, "ELFDATA2LSB"}, + {2, "ELFDATA2MSB"}, +} + +func (i Data) String() string { return stringName(uint32(i), dataStrings, false) } +func (i Data) GoString() string { return stringName(uint32(i), dataStrings, true) } + +// OSABI is found in Header.Ident[EI_OSABI] and Header.OSABI. +type OSABI byte + +const ( + ELFOSABI_NONE OSABI = 0 /* UNIX System V ABI */ + ELFOSABI_HPUX OSABI = 1 /* HP-UX operating system */ + ELFOSABI_NETBSD OSABI = 2 /* NetBSD */ + ELFOSABI_LINUX OSABI = 3 /* GNU/Linux */ + ELFOSABI_HURD OSABI = 4 /* GNU/Hurd */ + ELFOSABI_86OPEN OSABI = 5 /* 86Open common IA32 ABI */ + ELFOSABI_SOLARIS OSABI = 6 /* Solaris */ + ELFOSABI_AIX OSABI = 7 /* AIX */ + ELFOSABI_IRIX OSABI = 8 /* IRIX */ + ELFOSABI_FREEBSD OSABI = 9 /* FreeBSD */ + ELFOSABI_TRU64 OSABI = 10 /* TRU64 UNIX */ + ELFOSABI_MODESTO OSABI = 11 /* Novell Modesto */ + ELFOSABI_OPENBSD OSABI = 12 /* OpenBSD */ + ELFOSABI_OPENVMS OSABI = 13 /* Open VMS */ + ELFOSABI_NSK OSABI = 14 /* HP Non-Stop Kernel */ + ELFOSABI_ARM OSABI = 97 /* ARM */ + ELFOSABI_STANDALONE OSABI = 255 /* Standalone (embedded) application */ +) + +var osabiStrings = []intName{ + {0, "ELFOSABI_NONE"}, + {1, "ELFOSABI_HPUX"}, + {2, "ELFOSABI_NETBSD"}, + {3, "ELFOSABI_LINUX"}, + {4, "ELFOSABI_HURD"}, + {5, "ELFOSABI_86OPEN"}, + {6, "ELFOSABI_SOLARIS"}, + {7, "ELFOSABI_AIX"}, + {8, "ELFOSABI_IRIX"}, + {9, "ELFOSABI_FREEBSD"}, + {10, "ELFOSABI_TRU64"}, + {11, "ELFOSABI_MODESTO"}, + {12, "ELFOSABI_OPENBSD"}, + {13, "ELFOSABI_OPENVMS"}, + {14, "ELFOSABI_NSK"}, + {97, "ELFOSABI_ARM"}, + {255, "ELFOSABI_STANDALONE"}, +} + +func (i OSABI) String() string { return stringName(uint32(i), osabiStrings, false) } +func (i OSABI) GoString() string { return stringName(uint32(i), osabiStrings, true) } + +// Type is found in Header.Type. +type Type uint16 + +const ( + ET_NONE Type = 0 /* Unknown type. */ + ET_REL Type = 1 /* Relocatable. */ + ET_EXEC Type = 2 /* Executable. */ + ET_DYN Type = 3 /* Shared object. */ + ET_CORE Type = 4 /* Core file. */ + ET_LOOS Type = 0xfe00 /* First operating system specific. */ + ET_HIOS Type = 0xfeff /* Last operating system-specific. */ + ET_LOPROC Type = 0xff00 /* First processor-specific. */ + ET_HIPROC Type = 0xffff /* Last processor-specific. */ +) + +var typeStrings = []intName{ + {0, "ET_NONE"}, + {1, "ET_REL"}, + {2, "ET_EXEC"}, + {3, "ET_DYN"}, + {4, "ET_CORE"}, + {0xfe00, "ET_LOOS"}, + {0xfeff, "ET_HIOS"}, + {0xff00, "ET_LOPROC"}, + {0xffff, "ET_HIPROC"}, +} + +func (i Type) String() string { return stringName(uint32(i), typeStrings, false) } +func (i Type) GoString() string { return stringName(uint32(i), typeStrings, true) } + +// Machine is found in Header.Machine. +type Machine uint16 + +const ( + EM_NONE Machine = 0 /* Unknown machine. */ + EM_M32 Machine = 1 /* AT&T WE32100. */ + EM_SPARC Machine = 2 /* Sun SPARC. */ + EM_386 Machine = 3 /* Intel i386. */ + EM_68K Machine = 4 /* Motorola 68000. */ + EM_88K Machine = 5 /* Motorola 88000. */ + EM_860 Machine = 7 /* Intel i860. */ + EM_MIPS Machine = 8 /* MIPS R3000 Big-Endian only. */ + EM_S370 Machine = 9 /* IBM System/370. */ + EM_MIPS_RS3_LE Machine = 10 /* MIPS R3000 Little-Endian. */ + EM_PARISC Machine = 15 /* HP PA-RISC. */ + EM_VPP500 Machine = 17 /* Fujitsu VPP500. */ + EM_SPARC32PLUS Machine = 18 /* SPARC v8plus. */ + EM_960 Machine = 19 /* Intel 80960. */ + EM_PPC Machine = 20 /* PowerPC 32-bit. */ + EM_PPC64 Machine = 21 /* PowerPC 64-bit. */ + EM_S390 Machine = 22 /* IBM System/390. */ + EM_V800 Machine = 36 /* NEC V800. */ + EM_FR20 Machine = 37 /* Fujitsu FR20. */ + EM_RH32 Machine = 38 /* TRW RH-32. */ + EM_RCE Machine = 39 /* Motorola RCE. */ + EM_ARM Machine = 40 /* ARM. */ + EM_SH Machine = 42 /* Hitachi SH. */ + EM_SPARCV9 Machine = 43 /* SPARC v9 64-bit. */ + EM_TRICORE Machine = 44 /* Siemens TriCore embedded processor. */ + EM_ARC Machine = 45 /* Argonaut RISC Core. */ + EM_H8_300 Machine = 46 /* Hitachi H8/300. */ + EM_H8_300H Machine = 47 /* Hitachi H8/300H. */ + EM_H8S Machine = 48 /* Hitachi H8S. */ + EM_H8_500 Machine = 49 /* Hitachi H8/500. */ + EM_IA_64 Machine = 50 /* Intel IA-64 Processor. */ + EM_MIPS_X Machine = 51 /* Stanford MIPS-X. */ + EM_COLDFIRE Machine = 52 /* Motorola ColdFire. */ + EM_68HC12 Machine = 53 /* Motorola M68HC12. */ + EM_MMA Machine = 54 /* Fujitsu MMA. */ + EM_PCP Machine = 55 /* Siemens PCP. */ + EM_NCPU Machine = 56 /* Sony nCPU. */ + EM_NDR1 Machine = 57 /* Denso NDR1 microprocessor. */ + EM_STARCORE Machine = 58 /* Motorola Star*Core processor. */ + EM_ME16 Machine = 59 /* Toyota ME16 processor. */ + EM_ST100 Machine = 60 /* STMicroelectronics ST100 processor. */ + EM_TINYJ Machine = 61 /* Advanced Logic Corp. TinyJ processor. */ + EM_X86_64 Machine = 62 /* Advanced Micro Devices x86-64 */ + + /* Non-standard or deprecated. */ + EM_486 Machine = 6 /* Intel i486. */ + EM_MIPS_RS4_BE Machine = 10 /* MIPS R4000 Big-Endian */ + EM_ALPHA_STD Machine = 41 /* Digital Alpha (standard value). */ + EM_ALPHA Machine = 0x9026 /* Alpha (written in the absence of an ABI) */ +) + +var machineStrings = []intName{ + {0, "EM_NONE"}, + {1, "EM_M32"}, + {2, "EM_SPARC"}, + {3, "EM_386"}, + {4, "EM_68K"}, + {5, "EM_88K"}, + {7, "EM_860"}, + {8, "EM_MIPS"}, + {9, "EM_S370"}, + {10, "EM_MIPS_RS3_LE"}, + {15, "EM_PARISC"}, + {17, "EM_VPP500"}, + {18, "EM_SPARC32PLUS"}, + {19, "EM_960"}, + {20, "EM_PPC"}, + {21, "EM_PPC64"}, + {22, "EM_S390"}, + {36, "EM_V800"}, + {37, "EM_FR20"}, + {38, "EM_RH32"}, + {39, "EM_RCE"}, + {40, "EM_ARM"}, + {42, "EM_SH"}, + {43, "EM_SPARCV9"}, + {44, "EM_TRICORE"}, + {45, "EM_ARC"}, + {46, "EM_H8_300"}, + {47, "EM_H8_300H"}, + {48, "EM_H8S"}, + {49, "EM_H8_500"}, + {50, "EM_IA_64"}, + {51, "EM_MIPS_X"}, + {52, "EM_COLDFIRE"}, + {53, "EM_68HC12"}, + {54, "EM_MMA"}, + {55, "EM_PCP"}, + {56, "EM_NCPU"}, + {57, "EM_NDR1"}, + {58, "EM_STARCORE"}, + {59, "EM_ME16"}, + {60, "EM_ST100"}, + {61, "EM_TINYJ"}, + {62, "EM_X86_64"}, + + /* Non-standard or deprecated. */ + {6, "EM_486"}, + {10, "EM_MIPS_RS4_BE"}, + {41, "EM_ALPHA_STD"}, + {0x9026, "EM_ALPHA"}, +} + +func (i Machine) String() string { return stringName(uint32(i), machineStrings, false) } +func (i Machine) GoString() string { return stringName(uint32(i), machineStrings, true) } + +// Special section indices. +type SectionIndex int + +const ( + SHN_UNDEF SectionIndex = 0 /* Undefined, missing, irrelevant. */ + SHN_LORESERVE SectionIndex = 0xff00 /* First of reserved range. */ + SHN_LOPROC SectionIndex = 0xff00 /* First processor-specific. */ + SHN_HIPROC SectionIndex = 0xff1f /* Last processor-specific. */ + SHN_LOOS SectionIndex = 0xff20 /* First operating system-specific. */ + SHN_HIOS SectionIndex = 0xff3f /* Last operating system-specific. */ + SHN_ABS SectionIndex = 0xfff1 /* Absolute values. */ + SHN_COMMON SectionIndex = 0xfff2 /* Common data. */ + SHN_XINDEX SectionIndex = 0xffff /* Escape -- index stored elsewhere. */ + SHN_HIRESERVE SectionIndex = 0xffff /* Last of reserved range. */ +) + +var shnStrings = []intName{ + {0, "SHN_UNDEF"}, + {0xff00, "SHN_LOPROC"}, + {0xff20, "SHN_LOOS"}, + {0xfff1, "SHN_ABS"}, + {0xfff2, "SHN_COMMON"}, + {0xffff, "SHN_XINDEX"}, +} + +func (i SectionIndex) String() string { return stringName(uint32(i), shnStrings, false) } +func (i SectionIndex) GoString() string { return stringName(uint32(i), shnStrings, true) } + +// Section type. +type SectionType uint32 + +const ( + SHT_NULL SectionType = 0 /* inactive */ + SHT_PROGBITS SectionType = 1 /* program defined information */ + SHT_SYMTAB SectionType = 2 /* symbol table section */ + SHT_STRTAB SectionType = 3 /* string table section */ + SHT_RELA SectionType = 4 /* relocation section with addends */ + SHT_HASH SectionType = 5 /* symbol hash table section */ + SHT_DYNAMIC SectionType = 6 /* dynamic section */ + SHT_NOTE SectionType = 7 /* note section */ + SHT_NOBITS SectionType = 8 /* no space section */ + SHT_REL SectionType = 9 /* relocation section - no addends */ + SHT_SHLIB SectionType = 10 /* reserved - purpose unknown */ + SHT_DYNSYM SectionType = 11 /* dynamic symbol table section */ + SHT_INIT_ARRAY SectionType = 14 /* Initialization function pointers. */ + SHT_FINI_ARRAY SectionType = 15 /* Termination function pointers. */ + SHT_PREINIT_ARRAY SectionType = 16 /* Pre-initialization function ptrs. */ + SHT_GROUP SectionType = 17 /* Section group. */ + SHT_SYMTAB_SHNDX SectionType = 18 /* Section indexes (see SHN_XINDEX). */ + SHT_LOOS SectionType = 0x60000000 /* First of OS specific semantics */ + SHT_GNU_ATTRIBUTES SectionType = 0x6ffffff5 /* GNU object attributes */ + SHT_GNU_HASH SectionType = 0x6ffffff6 /* GNU hash table */ + SHT_GNU_LIBLIST SectionType = 0x6ffffff7 /* GNU prelink library list */ + SHT_GNU_VERDEF SectionType = 0x6ffffffd /* GNU version definition section */ + SHT_GNU_VERNEED SectionType = 0x6ffffffe /* GNU version needs section */ + SHT_GNU_VERSYM SectionType = 0x6fffffff /* GNU version symbol table */ + SHT_HIOS SectionType = 0x6fffffff /* Last of OS specific semantics */ + SHT_LOPROC SectionType = 0x70000000 /* reserved range for processor */ + SHT_HIPROC SectionType = 0x7fffffff /* specific section header types */ + SHT_LOUSER SectionType = 0x80000000 /* reserved range for application */ + SHT_HIUSER SectionType = 0xffffffff /* specific indexes */ +) + +var shtStrings = []intName{ + {0, "SHT_NULL"}, + {1, "SHT_PROGBITS"}, + {2, "SHT_SYMTAB"}, + {3, "SHT_STRTAB"}, + {4, "SHT_RELA"}, + {5, "SHT_HASH"}, + {6, "SHT_DYNAMIC"}, + {7, "SHT_NOTE"}, + {8, "SHT_NOBITS"}, + {9, "SHT_REL"}, + {10, "SHT_SHLIB"}, + {11, "SHT_DYNSYM"}, + {14, "SHT_INIT_ARRAY"}, + {15, "SHT_FINI_ARRAY"}, + {16, "SHT_PREINIT_ARRAY"}, + {17, "SHT_GROUP"}, + {18, "SHT_SYMTAB_SHNDX"}, + {0x60000000, "SHT_LOOS"}, + {0x6ffffff5, "SHT_GNU_ATTRIBUTES"}, + {0x6ffffff6, "SHT_GNU_HASH"}, + {0x6ffffff7, "SHT_GNU_LIBLIST"}, + {0x6ffffffd, "SHT_GNU_VERDEF"}, + {0x6ffffffe, "SHT_GNU_VERNEED"}, + {0x6fffffff, "SHT_GNU_VERSYM"}, + {0x70000000, "SHT_LOPROC"}, + {0x7fffffff, "SHT_HIPROC"}, + {0x80000000, "SHT_LOUSER"}, + {0xffffffff, "SHT_HIUSER"}, +} + +func (i SectionType) String() string { return stringName(uint32(i), shtStrings, false) } +func (i SectionType) GoString() string { return stringName(uint32(i), shtStrings, true) } + +// Section flags. +type SectionFlag uint32 + +const ( + SHF_WRITE SectionFlag = 0x1 /* Section contains writable data. */ + SHF_ALLOC SectionFlag = 0x2 /* Section occupies memory. */ + SHF_EXECINSTR SectionFlag = 0x4 /* Section contains instructions. */ + SHF_MERGE SectionFlag = 0x10 /* Section may be merged. */ + SHF_STRINGS SectionFlag = 0x20 /* Section contains strings. */ + SHF_INFO_LINK SectionFlag = 0x40 /* sh_info holds section index. */ + SHF_LINK_ORDER SectionFlag = 0x80 /* Special ordering requirements. */ + SHF_OS_NONCONFORMING SectionFlag = 0x100 /* OS-specific processing required. */ + SHF_GROUP SectionFlag = 0x200 /* Member of section group. */ + SHF_TLS SectionFlag = 0x400 /* Section contains TLS data. */ + SHF_MASKOS SectionFlag = 0x0ff00000 /* OS-specific semantics. */ + SHF_MASKPROC SectionFlag = 0xf0000000 /* Processor-specific semantics. */ +) + +var shfStrings = []intName{ + {0x1, "SHF_WRITE"}, + {0x2, "SHF_ALLOC"}, + {0x4, "SHF_EXECINSTR"}, + {0x10, "SHF_MERGE"}, + {0x20, "SHF_STRINGS"}, + {0x40, "SHF_INFO_LINK"}, + {0x80, "SHF_LINK_ORDER"}, + {0x100, "SHF_OS_NONCONFORMING"}, + {0x200, "SHF_GROUP"}, + {0x400, "SHF_TLS"}, +} + +func (i SectionFlag) String() string { return flagName(uint32(i), shfStrings, false) } +func (i SectionFlag) GoString() string { return flagName(uint32(i), shfStrings, true) } + +// Prog.Type +type ProgType int + +const ( + PT_NULL ProgType = 0 /* Unused entry. */ + PT_LOAD ProgType = 1 /* Loadable segment. */ + PT_DYNAMIC ProgType = 2 /* Dynamic linking information segment. */ + PT_INTERP ProgType = 3 /* Pathname of interpreter. */ + PT_NOTE ProgType = 4 /* Auxiliary information. */ + PT_SHLIB ProgType = 5 /* Reserved (not used). */ + PT_PHDR ProgType = 6 /* Location of program header itself. */ + PT_TLS ProgType = 7 /* Thread local storage segment */ + PT_LOOS ProgType = 0x60000000 /* First OS-specific. */ + PT_HIOS ProgType = 0x6fffffff /* Last OS-specific. */ + PT_LOPROC ProgType = 0x70000000 /* First processor-specific type. */ + PT_HIPROC ProgType = 0x7fffffff /* Last processor-specific type. */ +) + +var ptStrings = []intName{ + {0, "PT_NULL"}, + {1, "PT_LOAD"}, + {2, "PT_DYNAMIC"}, + {3, "PT_INTERP"}, + {4, "PT_NOTE"}, + {5, "PT_SHLIB"}, + {6, "PT_PHDR"}, + {7, "PT_TLS"}, + {0x60000000, "PT_LOOS"}, + {0x6fffffff, "PT_HIOS"}, + {0x70000000, "PT_LOPROC"}, + {0x7fffffff, "PT_HIPROC"}, +} + +func (i ProgType) String() string { return stringName(uint32(i), ptStrings, false) } +func (i ProgType) GoString() string { return stringName(uint32(i), ptStrings, true) } + +// Prog.Flag +type ProgFlag uint32 + +const ( + PF_X ProgFlag = 0x1 /* Executable. */ + PF_W ProgFlag = 0x2 /* Writable. */ + PF_R ProgFlag = 0x4 /* Readable. */ + PF_MASKOS ProgFlag = 0x0ff00000 /* Operating system-specific. */ + PF_MASKPROC ProgFlag = 0xf0000000 /* Processor-specific. */ +) + +var pfStrings = []intName{ + {0x1, "PF_X"}, + {0x2, "PF_W"}, + {0x4, "PF_R"}, +} + +func (i ProgFlag) String() string { return flagName(uint32(i), pfStrings, false) } +func (i ProgFlag) GoString() string { return flagName(uint32(i), pfStrings, true) } + +// Dyn.Tag +type DynTag int + +const ( + DT_NULL DynTag = 0 /* Terminating entry. */ + DT_NEEDED DynTag = 1 /* String table offset of a needed shared library. */ + DT_PLTRELSZ DynTag = 2 /* Total size in bytes of PLT relocations. */ + DT_PLTGOT DynTag = 3 /* Processor-dependent address. */ + DT_HASH DynTag = 4 /* Address of symbol hash table. */ + DT_STRTAB DynTag = 5 /* Address of string table. */ + DT_SYMTAB DynTag = 6 /* Address of symbol table. */ + DT_RELA DynTag = 7 /* Address of ElfNN_Rela relocations. */ + DT_RELASZ DynTag = 8 /* Total size of ElfNN_Rela relocations. */ + DT_RELAENT DynTag = 9 /* Size of each ElfNN_Rela relocation entry. */ + DT_STRSZ DynTag = 10 /* Size of string table. */ + DT_SYMENT DynTag = 11 /* Size of each symbol table entry. */ + DT_INIT DynTag = 12 /* Address of initialization function. */ + DT_FINI DynTag = 13 /* Address of finalization function. */ + DT_SONAME DynTag = 14 /* String table offset of shared object name. */ + DT_RPATH DynTag = 15 /* String table offset of library path. [sup] */ + DT_SYMBOLIC DynTag = 16 /* Indicates "symbolic" linking. [sup] */ + DT_REL DynTag = 17 /* Address of ElfNN_Rel relocations. */ + DT_RELSZ DynTag = 18 /* Total size of ElfNN_Rel relocations. */ + DT_RELENT DynTag = 19 /* Size of each ElfNN_Rel relocation. */ + DT_PLTREL DynTag = 20 /* Type of relocation used for PLT. */ + DT_DEBUG DynTag = 21 /* Reserved (not used). */ + DT_TEXTREL DynTag = 22 /* Indicates there may be relocations in non-writable segments. [sup] */ + DT_JMPREL DynTag = 23 /* Address of PLT relocations. */ + DT_BIND_NOW DynTag = 24 /* [sup] */ + DT_INIT_ARRAY DynTag = 25 /* Address of the array of pointers to initialization functions */ + DT_FINI_ARRAY DynTag = 26 /* Address of the array of pointers to termination functions */ + DT_INIT_ARRAYSZ DynTag = 27 /* Size in bytes of the array of initialization functions. */ + DT_FINI_ARRAYSZ DynTag = 28 /* Size in bytes of the array of terminationfunctions. */ + DT_RUNPATH DynTag = 29 /* String table offset of a null-terminated library search path string. */ + DT_FLAGS DynTag = 30 /* Object specific flag values. */ + DT_ENCODING DynTag = 32 /* Values greater than or equal to DT_ENCODING + and less than DT_LOOS follow the rules for + the interpretation of the d_un union + as follows: even == 'd_ptr', even == 'd_val' + or none */ + DT_PREINIT_ARRAY DynTag = 32 /* Address of the array of pointers to pre-initialization functions. */ + DT_PREINIT_ARRAYSZ DynTag = 33 /* Size in bytes of the array of pre-initialization functions. */ + DT_LOOS DynTag = 0x6000000d /* First OS-specific */ + DT_HIOS DynTag = 0x6ffff000 /* Last OS-specific */ + DT_VERSYM DynTag = 0x6ffffff0 + DT_VERNEED DynTag = 0x6ffffffe + DT_VERNEEDNUM DynTag = 0x6fffffff + DT_LOPROC DynTag = 0x70000000 /* First processor-specific type. */ + DT_HIPROC DynTag = 0x7fffffff /* Last processor-specific type. */ +) + +var dtStrings = []intName{ + {0, "DT_NULL"}, + {1, "DT_NEEDED"}, + {2, "DT_PLTRELSZ"}, + {3, "DT_PLTGOT"}, + {4, "DT_HASH"}, + {5, "DT_STRTAB"}, + {6, "DT_SYMTAB"}, + {7, "DT_RELA"}, + {8, "DT_RELASZ"}, + {9, "DT_RELAENT"}, + {10, "DT_STRSZ"}, + {11, "DT_SYMENT"}, + {12, "DT_INIT"}, + {13, "DT_FINI"}, + {14, "DT_SONAME"}, + {15, "DT_RPATH"}, + {16, "DT_SYMBOLIC"}, + {17, "DT_REL"}, + {18, "DT_RELSZ"}, + {19, "DT_RELENT"}, + {20, "DT_PLTREL"}, + {21, "DT_DEBUG"}, + {22, "DT_TEXTREL"}, + {23, "DT_JMPREL"}, + {24, "DT_BIND_NOW"}, + {25, "DT_INIT_ARRAY"}, + {26, "DT_FINI_ARRAY"}, + {27, "DT_INIT_ARRAYSZ"}, + {28, "DT_FINI_ARRAYSZ"}, + {29, "DT_RUNPATH"}, + {30, "DT_FLAGS"}, + {32, "DT_ENCODING"}, + {32, "DT_PREINIT_ARRAY"}, + {33, "DT_PREINIT_ARRAYSZ"}, + {0x6000000d, "DT_LOOS"}, + {0x6ffff000, "DT_HIOS"}, + {0x6ffffff0, "DT_VERSYM"}, + {0x6ffffffe, "DT_VERNEED"}, + {0x6fffffff, "DT_VERNEEDNUM"}, + {0x70000000, "DT_LOPROC"}, + {0x7fffffff, "DT_HIPROC"}, +} + +func (i DynTag) String() string { return stringName(uint32(i), dtStrings, false) } +func (i DynTag) GoString() string { return stringName(uint32(i), dtStrings, true) } + +// DT_FLAGS values. +type DynFlag int + +const ( + DF_ORIGIN DynFlag = 0x0001 /* Indicates that the object being loaded may + make reference to the + $ORIGIN substitution string */ + DF_SYMBOLIC DynFlag = 0x0002 /* Indicates "symbolic" linking. */ + DF_TEXTREL DynFlag = 0x0004 /* Indicates there may be relocations in non-writable segments. */ + DF_BIND_NOW DynFlag = 0x0008 /* Indicates that the dynamic linker should + process all relocations for the object + containing this entry before transferring + control to the program. */ + DF_STATIC_TLS DynFlag = 0x0010 /* Indicates that the shared object or + executable contains code using a static + thread-local storage scheme. */ +) + +var dflagStrings = []intName{ + {0x0001, "DF_ORIGIN"}, + {0x0002, "DF_SYMBOLIC"}, + {0x0004, "DF_TEXTREL"}, + {0x0008, "DF_BIND_NOW"}, + {0x0010, "DF_STATIC_TLS"}, +} + +func (i DynFlag) String() string { return flagName(uint32(i), dflagStrings, false) } +func (i DynFlag) GoString() string { return flagName(uint32(i), dflagStrings, true) } + +// NType values; used in core files. +type NType int + +const ( + NT_PRSTATUS NType = 1 /* Process status. */ + NT_FPREGSET NType = 2 /* Floating point registers. */ + NT_PRPSINFO NType = 3 /* Process state info. */ +) + +var ntypeStrings = []intName{ + {1, "NT_PRSTATUS"}, + {2, "NT_FPREGSET"}, + {3, "NT_PRPSINFO"}, +} + +func (i NType) String() string { return stringName(uint32(i), ntypeStrings, false) } +func (i NType) GoString() string { return stringName(uint32(i), ntypeStrings, true) } + +/* Symbol Binding - ELFNN_ST_BIND - st_info */ +type SymBind int + +const ( + STB_LOCAL SymBind = 0 /* Local symbol */ + STB_GLOBAL SymBind = 1 /* Global symbol */ + STB_WEAK SymBind = 2 /* like global - lower precedence */ + STB_LOOS SymBind = 10 /* Reserved range for operating system */ + STB_HIOS SymBind = 12 /* specific semantics. */ + STB_LOPROC SymBind = 13 /* reserved range for processor */ + STB_HIPROC SymBind = 15 /* specific semantics. */ +) + +var stbStrings = []intName{ + {0, "STB_LOCAL"}, + {1, "STB_GLOBAL"}, + {2, "STB_WEAK"}, + {10, "STB_LOOS"}, + {12, "STB_HIOS"}, + {13, "STB_LOPROC"}, + {15, "STB_HIPROC"}, +} + +func (i SymBind) String() string { return stringName(uint32(i), stbStrings, false) } +func (i SymBind) GoString() string { return stringName(uint32(i), stbStrings, true) } + +/* Symbol type - ELFNN_ST_TYPE - st_info */ +type SymType int + +const ( + STT_NOTYPE SymType = 0 /* Unspecified type. */ + STT_OBJECT SymType = 1 /* Data object. */ + STT_FUNC SymType = 2 /* Function. */ + STT_SECTION SymType = 3 /* Section. */ + STT_FILE SymType = 4 /* Source file. */ + STT_COMMON SymType = 5 /* Uninitialized common block. */ + STT_TLS SymType = 6 /* TLS object. */ + STT_LOOS SymType = 10 /* Reserved range for operating system */ + STT_HIOS SymType = 12 /* specific semantics. */ + STT_LOPROC SymType = 13 /* reserved range for processor */ + STT_HIPROC SymType = 15 /* specific semantics. */ +) + +var sttStrings = []intName{ + {0, "STT_NOTYPE"}, + {1, "STT_OBJECT"}, + {2, "STT_FUNC"}, + {3, "STT_SECTION"}, + {4, "STT_FILE"}, + {5, "STT_COMMON"}, + {6, "STT_TLS"}, + {10, "STT_LOOS"}, + {12, "STT_HIOS"}, + {13, "STT_LOPROC"}, + {15, "STT_HIPROC"}, +} + +func (i SymType) String() string { return stringName(uint32(i), sttStrings, false) } +func (i SymType) GoString() string { return stringName(uint32(i), sttStrings, true) } + +/* Symbol visibility - ELFNN_ST_VISIBILITY - st_other */ +type SymVis int + +const ( + STV_DEFAULT SymVis = 0x0 /* Default visibility (see binding). */ + STV_INTERNAL SymVis = 0x1 /* Special meaning in relocatable objects. */ + STV_HIDDEN SymVis = 0x2 /* Not visible. */ + STV_PROTECTED SymVis = 0x3 /* Visible but not preemptible. */ +) + +var stvStrings = []intName{ + {0x0, "STV_DEFAULT"}, + {0x1, "STV_INTERNAL"}, + {0x2, "STV_HIDDEN"}, + {0x3, "STV_PROTECTED"}, +} + +func (i SymVis) String() string { return stringName(uint32(i), stvStrings, false) } +func (i SymVis) GoString() string { return stringName(uint32(i), stvStrings, true) } + +/* + * Relocation types. + */ + +// Relocation types for x86-64. +type R_X86_64 int + +const ( + R_X86_64_NONE R_X86_64 = 0 /* No relocation. */ + R_X86_64_64 R_X86_64 = 1 /* Add 64 bit symbol value. */ + R_X86_64_PC32 R_X86_64 = 2 /* PC-relative 32 bit signed sym value. */ + R_X86_64_GOT32 R_X86_64 = 3 /* PC-relative 32 bit GOT offset. */ + R_X86_64_PLT32 R_X86_64 = 4 /* PC-relative 32 bit PLT offset. */ + R_X86_64_COPY R_X86_64 = 5 /* Copy data from shared object. */ + R_X86_64_GLOB_DAT R_X86_64 = 6 /* Set GOT entry to data address. */ + R_X86_64_JMP_SLOT R_X86_64 = 7 /* Set GOT entry to code address. */ + R_X86_64_RELATIVE R_X86_64 = 8 /* Add load address of shared object. */ + R_X86_64_GOTPCREL R_X86_64 = 9 /* Add 32 bit signed pcrel offset to GOT. */ + R_X86_64_32 R_X86_64 = 10 /* Add 32 bit zero extended symbol value */ + R_X86_64_32S R_X86_64 = 11 /* Add 32 bit sign extended symbol value */ + R_X86_64_16 R_X86_64 = 12 /* Add 16 bit zero extended symbol value */ + R_X86_64_PC16 R_X86_64 = 13 /* Add 16 bit signed extended pc relative symbol value */ + R_X86_64_8 R_X86_64 = 14 /* Add 8 bit zero extended symbol value */ + R_X86_64_PC8 R_X86_64 = 15 /* Add 8 bit signed extended pc relative symbol value */ + R_X86_64_DTPMOD64 R_X86_64 = 16 /* ID of module containing symbol */ + R_X86_64_DTPOFF64 R_X86_64 = 17 /* Offset in TLS block */ + R_X86_64_TPOFF64 R_X86_64 = 18 /* Offset in static TLS block */ + R_X86_64_TLSGD R_X86_64 = 19 /* PC relative offset to GD GOT entry */ + R_X86_64_TLSLD R_X86_64 = 20 /* PC relative offset to LD GOT entry */ + R_X86_64_DTPOFF32 R_X86_64 = 21 /* Offset in TLS block */ + R_X86_64_GOTTPOFF R_X86_64 = 22 /* PC relative offset to IE GOT entry */ + R_X86_64_TPOFF32 R_X86_64 = 23 /* Offset in static TLS block */ +) + +var rx86_64Strings = []intName{ + {0, "R_X86_64_NONE"}, + {1, "R_X86_64_64"}, + {2, "R_X86_64_PC32"}, + {3, "R_X86_64_GOT32"}, + {4, "R_X86_64_PLT32"}, + {5, "R_X86_64_COPY"}, + {6, "R_X86_64_GLOB_DAT"}, + {7, "R_X86_64_JMP_SLOT"}, + {8, "R_X86_64_RELATIVE"}, + {9, "R_X86_64_GOTPCREL"}, + {10, "R_X86_64_32"}, + {11, "R_X86_64_32S"}, + {12, "R_X86_64_16"}, + {13, "R_X86_64_PC16"}, + {14, "R_X86_64_8"}, + {15, "R_X86_64_PC8"}, + {16, "R_X86_64_DTPMOD64"}, + {17, "R_X86_64_DTPOFF64"}, + {18, "R_X86_64_TPOFF64"}, + {19, "R_X86_64_TLSGD"}, + {20, "R_X86_64_TLSLD"}, + {21, "R_X86_64_DTPOFF32"}, + {22, "R_X86_64_GOTTPOFF"}, + {23, "R_X86_64_TPOFF32"}, +} + +func (i R_X86_64) String() string { return stringName(uint32(i), rx86_64Strings, false) } +func (i R_X86_64) GoString() string { return stringName(uint32(i), rx86_64Strings, true) } + +// Relocation types for Alpha. +type R_ALPHA int + +const ( + R_ALPHA_NONE R_ALPHA = 0 /* No reloc */ + R_ALPHA_REFLONG R_ALPHA = 1 /* Direct 32 bit */ + R_ALPHA_REFQUAD R_ALPHA = 2 /* Direct 64 bit */ + R_ALPHA_GPREL32 R_ALPHA = 3 /* GP relative 32 bit */ + R_ALPHA_LITERAL R_ALPHA = 4 /* GP relative 16 bit w/optimization */ + R_ALPHA_LITUSE R_ALPHA = 5 /* Optimization hint for LITERAL */ + R_ALPHA_GPDISP R_ALPHA = 6 /* Add displacement to GP */ + R_ALPHA_BRADDR R_ALPHA = 7 /* PC+4 relative 23 bit shifted */ + R_ALPHA_HINT R_ALPHA = 8 /* PC+4 relative 16 bit shifted */ + R_ALPHA_SREL16 R_ALPHA = 9 /* PC relative 16 bit */ + R_ALPHA_SREL32 R_ALPHA = 10 /* PC relative 32 bit */ + R_ALPHA_SREL64 R_ALPHA = 11 /* PC relative 64 bit */ + R_ALPHA_OP_PUSH R_ALPHA = 12 /* OP stack push */ + R_ALPHA_OP_STORE R_ALPHA = 13 /* OP stack pop and store */ + R_ALPHA_OP_PSUB R_ALPHA = 14 /* OP stack subtract */ + R_ALPHA_OP_PRSHIFT R_ALPHA = 15 /* OP stack right shift */ + R_ALPHA_GPVALUE R_ALPHA = 16 + R_ALPHA_GPRELHIGH R_ALPHA = 17 + R_ALPHA_GPRELLOW R_ALPHA = 18 + R_ALPHA_IMMED_GP_16 R_ALPHA = 19 + R_ALPHA_IMMED_GP_HI32 R_ALPHA = 20 + R_ALPHA_IMMED_SCN_HI32 R_ALPHA = 21 + R_ALPHA_IMMED_BR_HI32 R_ALPHA = 22 + R_ALPHA_IMMED_LO32 R_ALPHA = 23 + R_ALPHA_COPY R_ALPHA = 24 /* Copy symbol at runtime */ + R_ALPHA_GLOB_DAT R_ALPHA = 25 /* Create GOT entry */ + R_ALPHA_JMP_SLOT R_ALPHA = 26 /* Create PLT entry */ + R_ALPHA_RELATIVE R_ALPHA = 27 /* Adjust by program base */ +) + +var ralphaStrings = []intName{ + {0, "R_ALPHA_NONE"}, + {1, "R_ALPHA_REFLONG"}, + {2, "R_ALPHA_REFQUAD"}, + {3, "R_ALPHA_GPREL32"}, + {4, "R_ALPHA_LITERAL"}, + {5, "R_ALPHA_LITUSE"}, + {6, "R_ALPHA_GPDISP"}, + {7, "R_ALPHA_BRADDR"}, + {8, "R_ALPHA_HINT"}, + {9, "R_ALPHA_SREL16"}, + {10, "R_ALPHA_SREL32"}, + {11, "R_ALPHA_SREL64"}, + {12, "R_ALPHA_OP_PUSH"}, + {13, "R_ALPHA_OP_STORE"}, + {14, "R_ALPHA_OP_PSUB"}, + {15, "R_ALPHA_OP_PRSHIFT"}, + {16, "R_ALPHA_GPVALUE"}, + {17, "R_ALPHA_GPRELHIGH"}, + {18, "R_ALPHA_GPRELLOW"}, + {19, "R_ALPHA_IMMED_GP_16"}, + {20, "R_ALPHA_IMMED_GP_HI32"}, + {21, "R_ALPHA_IMMED_SCN_HI32"}, + {22, "R_ALPHA_IMMED_BR_HI32"}, + {23, "R_ALPHA_IMMED_LO32"}, + {24, "R_ALPHA_COPY"}, + {25, "R_ALPHA_GLOB_DAT"}, + {26, "R_ALPHA_JMP_SLOT"}, + {27, "R_ALPHA_RELATIVE"}, +} + +func (i R_ALPHA) String() string { return stringName(uint32(i), ralphaStrings, false) } +func (i R_ALPHA) GoString() string { return stringName(uint32(i), ralphaStrings, true) } + +// Relocation types for ARM. +type R_ARM int + +const ( + R_ARM_NONE R_ARM = 0 /* No relocation. */ + R_ARM_PC24 R_ARM = 1 + R_ARM_ABS32 R_ARM = 2 + R_ARM_REL32 R_ARM = 3 + R_ARM_PC13 R_ARM = 4 + R_ARM_ABS16 R_ARM = 5 + R_ARM_ABS12 R_ARM = 6 + R_ARM_THM_ABS5 R_ARM = 7 + R_ARM_ABS8 R_ARM = 8 + R_ARM_SBREL32 R_ARM = 9 + R_ARM_THM_PC22 R_ARM = 10 + R_ARM_THM_PC8 R_ARM = 11 + R_ARM_AMP_VCALL9 R_ARM = 12 + R_ARM_SWI24 R_ARM = 13 + R_ARM_THM_SWI8 R_ARM = 14 + R_ARM_XPC25 R_ARM = 15 + R_ARM_THM_XPC22 R_ARM = 16 + R_ARM_COPY R_ARM = 20 /* Copy data from shared object. */ + R_ARM_GLOB_DAT R_ARM = 21 /* Set GOT entry to data address. */ + R_ARM_JUMP_SLOT R_ARM = 22 /* Set GOT entry to code address. */ + R_ARM_RELATIVE R_ARM = 23 /* Add load address of shared object. */ + R_ARM_GOTOFF R_ARM = 24 /* Add GOT-relative symbol address. */ + R_ARM_GOTPC R_ARM = 25 /* Add PC-relative GOT table address. */ + R_ARM_GOT32 R_ARM = 26 /* Add PC-relative GOT offset. */ + R_ARM_PLT32 R_ARM = 27 /* Add PC-relative PLT offset. */ + R_ARM_GNU_VTENTRY R_ARM = 100 + R_ARM_GNU_VTINHERIT R_ARM = 101 + R_ARM_RSBREL32 R_ARM = 250 + R_ARM_THM_RPC22 R_ARM = 251 + R_ARM_RREL32 R_ARM = 252 + R_ARM_RABS32 R_ARM = 253 + R_ARM_RPC24 R_ARM = 254 + R_ARM_RBASE R_ARM = 255 +) + +var rarmStrings = []intName{ + {0, "R_ARM_NONE"}, + {1, "R_ARM_PC24"}, + {2, "R_ARM_ABS32"}, + {3, "R_ARM_REL32"}, + {4, "R_ARM_PC13"}, + {5, "R_ARM_ABS16"}, + {6, "R_ARM_ABS12"}, + {7, "R_ARM_THM_ABS5"}, + {8, "R_ARM_ABS8"}, + {9, "R_ARM_SBREL32"}, + {10, "R_ARM_THM_PC22"}, + {11, "R_ARM_THM_PC8"}, + {12, "R_ARM_AMP_VCALL9"}, + {13, "R_ARM_SWI24"}, + {14, "R_ARM_THM_SWI8"}, + {15, "R_ARM_XPC25"}, + {16, "R_ARM_THM_XPC22"}, + {20, "R_ARM_COPY"}, + {21, "R_ARM_GLOB_DAT"}, + {22, "R_ARM_JUMP_SLOT"}, + {23, "R_ARM_RELATIVE"}, + {24, "R_ARM_GOTOFF"}, + {25, "R_ARM_GOTPC"}, + {26, "R_ARM_GOT32"}, + {27, "R_ARM_PLT32"}, + {100, "R_ARM_GNU_VTENTRY"}, + {101, "R_ARM_GNU_VTINHERIT"}, + {250, "R_ARM_RSBREL32"}, + {251, "R_ARM_THM_RPC22"}, + {252, "R_ARM_RREL32"}, + {253, "R_ARM_RABS32"}, + {254, "R_ARM_RPC24"}, + {255, "R_ARM_RBASE"}, +} + +func (i R_ARM) String() string { return stringName(uint32(i), rarmStrings, false) } +func (i R_ARM) GoString() string { return stringName(uint32(i), rarmStrings, true) } + +// Relocation types for 386. +type R_386 int + +const ( + R_386_NONE R_386 = 0 /* No relocation. */ + R_386_32 R_386 = 1 /* Add symbol value. */ + R_386_PC32 R_386 = 2 /* Add PC-relative symbol value. */ + R_386_GOT32 R_386 = 3 /* Add PC-relative GOT offset. */ + R_386_PLT32 R_386 = 4 /* Add PC-relative PLT offset. */ + R_386_COPY R_386 = 5 /* Copy data from shared object. */ + R_386_GLOB_DAT R_386 = 6 /* Set GOT entry to data address. */ + R_386_JMP_SLOT R_386 = 7 /* Set GOT entry to code address. */ + R_386_RELATIVE R_386 = 8 /* Add load address of shared object. */ + R_386_GOTOFF R_386 = 9 /* Add GOT-relative symbol address. */ + R_386_GOTPC R_386 = 10 /* Add PC-relative GOT table address. */ + R_386_TLS_TPOFF R_386 = 14 /* Negative offset in static TLS block */ + R_386_TLS_IE R_386 = 15 /* Absolute address of GOT for -ve static TLS */ + R_386_TLS_GOTIE R_386 = 16 /* GOT entry for negative static TLS block */ + R_386_TLS_LE R_386 = 17 /* Negative offset relative to static TLS */ + R_386_TLS_GD R_386 = 18 /* 32 bit offset to GOT (index,off) pair */ + R_386_TLS_LDM R_386 = 19 /* 32 bit offset to GOT (index,zero) pair */ + R_386_TLS_GD_32 R_386 = 24 /* 32 bit offset to GOT (index,off) pair */ + R_386_TLS_GD_PUSH R_386 = 25 /* pushl instruction for Sun ABI GD sequence */ + R_386_TLS_GD_CALL R_386 = 26 /* call instruction for Sun ABI GD sequence */ + R_386_TLS_GD_POP R_386 = 27 /* popl instruction for Sun ABI GD sequence */ + R_386_TLS_LDM_32 R_386 = 28 /* 32 bit offset to GOT (index,zero) pair */ + R_386_TLS_LDM_PUSH R_386 = 29 /* pushl instruction for Sun ABI LD sequence */ + R_386_TLS_LDM_CALL R_386 = 30 /* call instruction for Sun ABI LD sequence */ + R_386_TLS_LDM_POP R_386 = 31 /* popl instruction for Sun ABI LD sequence */ + R_386_TLS_LDO_32 R_386 = 32 /* 32 bit offset from start of TLS block */ + R_386_TLS_IE_32 R_386 = 33 /* 32 bit offset to GOT static TLS offset entry */ + R_386_TLS_LE_32 R_386 = 34 /* 32 bit offset within static TLS block */ + R_386_TLS_DTPMOD32 R_386 = 35 /* GOT entry containing TLS index */ + R_386_TLS_DTPOFF32 R_386 = 36 /* GOT entry containing TLS offset */ + R_386_TLS_TPOFF32 R_386 = 37 /* GOT entry of -ve static TLS offset */ +) + +var r386Strings = []intName{ + {0, "R_386_NONE"}, + {1, "R_386_32"}, + {2, "R_386_PC32"}, + {3, "R_386_GOT32"}, + {4, "R_386_PLT32"}, + {5, "R_386_COPY"}, + {6, "R_386_GLOB_DAT"}, + {7, "R_386_JMP_SLOT"}, + {8, "R_386_RELATIVE"}, + {9, "R_386_GOTOFF"}, + {10, "R_386_GOTPC"}, + {14, "R_386_TLS_TPOFF"}, + {15, "R_386_TLS_IE"}, + {16, "R_386_TLS_GOTIE"}, + {17, "R_386_TLS_LE"}, + {18, "R_386_TLS_GD"}, + {19, "R_386_TLS_LDM"}, + {24, "R_386_TLS_GD_32"}, + {25, "R_386_TLS_GD_PUSH"}, + {26, "R_386_TLS_GD_CALL"}, + {27, "R_386_TLS_GD_POP"}, + {28, "R_386_TLS_LDM_32"}, + {29, "R_386_TLS_LDM_PUSH"}, + {30, "R_386_TLS_LDM_CALL"}, + {31, "R_386_TLS_LDM_POP"}, + {32, "R_386_TLS_LDO_32"}, + {33, "R_386_TLS_IE_32"}, + {34, "R_386_TLS_LE_32"}, + {35, "R_386_TLS_DTPMOD32"}, + {36, "R_386_TLS_DTPOFF32"}, + {37, "R_386_TLS_TPOFF32"}, +} + +func (i R_386) String() string { return stringName(uint32(i), r386Strings, false) } +func (i R_386) GoString() string { return stringName(uint32(i), r386Strings, true) } + +// Relocation types for PowerPC. +type R_PPC int + +const ( + R_PPC_NONE R_PPC = 0 /* No relocation. */ + R_PPC_ADDR32 R_PPC = 1 + R_PPC_ADDR24 R_PPC = 2 + R_PPC_ADDR16 R_PPC = 3 + R_PPC_ADDR16_LO R_PPC = 4 + R_PPC_ADDR16_HI R_PPC = 5 + R_PPC_ADDR16_HA R_PPC = 6 + R_PPC_ADDR14 R_PPC = 7 + R_PPC_ADDR14_BRTAKEN R_PPC = 8 + R_PPC_ADDR14_BRNTAKEN R_PPC = 9 + R_PPC_REL24 R_PPC = 10 + R_PPC_REL14 R_PPC = 11 + R_PPC_REL14_BRTAKEN R_PPC = 12 + R_PPC_REL14_BRNTAKEN R_PPC = 13 + R_PPC_GOT16 R_PPC = 14 + R_PPC_GOT16_LO R_PPC = 15 + R_PPC_GOT16_HI R_PPC = 16 + R_PPC_GOT16_HA R_PPC = 17 + R_PPC_PLTREL24 R_PPC = 18 + R_PPC_COPY R_PPC = 19 + R_PPC_GLOB_DAT R_PPC = 20 + R_PPC_JMP_SLOT R_PPC = 21 + R_PPC_RELATIVE R_PPC = 22 + R_PPC_LOCAL24PC R_PPC = 23 + R_PPC_UADDR32 R_PPC = 24 + R_PPC_UADDR16 R_PPC = 25 + R_PPC_REL32 R_PPC = 26 + R_PPC_PLT32 R_PPC = 27 + R_PPC_PLTREL32 R_PPC = 28 + R_PPC_PLT16_LO R_PPC = 29 + R_PPC_PLT16_HI R_PPC = 30 + R_PPC_PLT16_HA R_PPC = 31 + R_PPC_SDAREL16 R_PPC = 32 + R_PPC_SECTOFF R_PPC = 33 + R_PPC_SECTOFF_LO R_PPC = 34 + R_PPC_SECTOFF_HI R_PPC = 35 + R_PPC_SECTOFF_HA R_PPC = 36 + R_PPC_TLS R_PPC = 67 + R_PPC_DTPMOD32 R_PPC = 68 + R_PPC_TPREL16 R_PPC = 69 + R_PPC_TPREL16_LO R_PPC = 70 + R_PPC_TPREL16_HI R_PPC = 71 + R_PPC_TPREL16_HA R_PPC = 72 + R_PPC_TPREL32 R_PPC = 73 + R_PPC_DTPREL16 R_PPC = 74 + R_PPC_DTPREL16_LO R_PPC = 75 + R_PPC_DTPREL16_HI R_PPC = 76 + R_PPC_DTPREL16_HA R_PPC = 77 + R_PPC_DTPREL32 R_PPC = 78 + R_PPC_GOT_TLSGD16 R_PPC = 79 + R_PPC_GOT_TLSGD16_LO R_PPC = 80 + R_PPC_GOT_TLSGD16_HI R_PPC = 81 + R_PPC_GOT_TLSGD16_HA R_PPC = 82 + R_PPC_GOT_TLSLD16 R_PPC = 83 + R_PPC_GOT_TLSLD16_LO R_PPC = 84 + R_PPC_GOT_TLSLD16_HI R_PPC = 85 + R_PPC_GOT_TLSLD16_HA R_PPC = 86 + R_PPC_GOT_TPREL16 R_PPC = 87 + R_PPC_GOT_TPREL16_LO R_PPC = 88 + R_PPC_GOT_TPREL16_HI R_PPC = 89 + R_PPC_GOT_TPREL16_HA R_PPC = 90 + R_PPC_EMB_NADDR32 R_PPC = 101 + R_PPC_EMB_NADDR16 R_PPC = 102 + R_PPC_EMB_NADDR16_LO R_PPC = 103 + R_PPC_EMB_NADDR16_HI R_PPC = 104 + R_PPC_EMB_NADDR16_HA R_PPC = 105 + R_PPC_EMB_SDAI16 R_PPC = 106 + R_PPC_EMB_SDA2I16 R_PPC = 107 + R_PPC_EMB_SDA2REL R_PPC = 108 + R_PPC_EMB_SDA21 R_PPC = 109 + R_PPC_EMB_MRKREF R_PPC = 110 + R_PPC_EMB_RELSEC16 R_PPC = 111 + R_PPC_EMB_RELST_LO R_PPC = 112 + R_PPC_EMB_RELST_HI R_PPC = 113 + R_PPC_EMB_RELST_HA R_PPC = 114 + R_PPC_EMB_BIT_FLD R_PPC = 115 + R_PPC_EMB_RELSDA R_PPC = 116 +) + +var rppcStrings = []intName{ + {0, "R_PPC_NONE"}, + {1, "R_PPC_ADDR32"}, + {2, "R_PPC_ADDR24"}, + {3, "R_PPC_ADDR16"}, + {4, "R_PPC_ADDR16_LO"}, + {5, "R_PPC_ADDR16_HI"}, + {6, "R_PPC_ADDR16_HA"}, + {7, "R_PPC_ADDR14"}, + {8, "R_PPC_ADDR14_BRTAKEN"}, + {9, "R_PPC_ADDR14_BRNTAKEN"}, + {10, "R_PPC_REL24"}, + {11, "R_PPC_REL14"}, + {12, "R_PPC_REL14_BRTAKEN"}, + {13, "R_PPC_REL14_BRNTAKEN"}, + {14, "R_PPC_GOT16"}, + {15, "R_PPC_GOT16_LO"}, + {16, "R_PPC_GOT16_HI"}, + {17, "R_PPC_GOT16_HA"}, + {18, "R_PPC_PLTREL24"}, + {19, "R_PPC_COPY"}, + {20, "R_PPC_GLOB_DAT"}, + {21, "R_PPC_JMP_SLOT"}, + {22, "R_PPC_RELATIVE"}, + {23, "R_PPC_LOCAL24PC"}, + {24, "R_PPC_UADDR32"}, + {25, "R_PPC_UADDR16"}, + {26, "R_PPC_REL32"}, + {27, "R_PPC_PLT32"}, + {28, "R_PPC_PLTREL32"}, + {29, "R_PPC_PLT16_LO"}, + {30, "R_PPC_PLT16_HI"}, + {31, "R_PPC_PLT16_HA"}, + {32, "R_PPC_SDAREL16"}, + {33, "R_PPC_SECTOFF"}, + {34, "R_PPC_SECTOFF_LO"}, + {35, "R_PPC_SECTOFF_HI"}, + {36, "R_PPC_SECTOFF_HA"}, + + {67, "R_PPC_TLS"}, + {68, "R_PPC_DTPMOD32"}, + {69, "R_PPC_TPREL16"}, + {70, "R_PPC_TPREL16_LO"}, + {71, "R_PPC_TPREL16_HI"}, + {72, "R_PPC_TPREL16_HA"}, + {73, "R_PPC_TPREL32"}, + {74, "R_PPC_DTPREL16"}, + {75, "R_PPC_DTPREL16_LO"}, + {76, "R_PPC_DTPREL16_HI"}, + {77, "R_PPC_DTPREL16_HA"}, + {78, "R_PPC_DTPREL32"}, + {79, "R_PPC_GOT_TLSGD16"}, + {80, "R_PPC_GOT_TLSGD16_LO"}, + {81, "R_PPC_GOT_TLSGD16_HI"}, + {82, "R_PPC_GOT_TLSGD16_HA"}, + {83, "R_PPC_GOT_TLSLD16"}, + {84, "R_PPC_GOT_TLSLD16_LO"}, + {85, "R_PPC_GOT_TLSLD16_HI"}, + {86, "R_PPC_GOT_TLSLD16_HA"}, + {87, "R_PPC_GOT_TPREL16"}, + {88, "R_PPC_GOT_TPREL16_LO"}, + {89, "R_PPC_GOT_TPREL16_HI"}, + {90, "R_PPC_GOT_TPREL16_HA"}, + + {101, "R_PPC_EMB_NADDR32"}, + {102, "R_PPC_EMB_NADDR16"}, + {103, "R_PPC_EMB_NADDR16_LO"}, + {104, "R_PPC_EMB_NADDR16_HI"}, + {105, "R_PPC_EMB_NADDR16_HA"}, + {106, "R_PPC_EMB_SDAI16"}, + {107, "R_PPC_EMB_SDA2I16"}, + {108, "R_PPC_EMB_SDA2REL"}, + {109, "R_PPC_EMB_SDA21"}, + {110, "R_PPC_EMB_MRKREF"}, + {111, "R_PPC_EMB_RELSEC16"}, + {112, "R_PPC_EMB_RELST_LO"}, + {113, "R_PPC_EMB_RELST_HI"}, + {114, "R_PPC_EMB_RELST_HA"}, + {115, "R_PPC_EMB_BIT_FLD"}, + {116, "R_PPC_EMB_RELSDA"}, +} + +func (i R_PPC) String() string { return stringName(uint32(i), rppcStrings, false) } +func (i R_PPC) GoString() string { return stringName(uint32(i), rppcStrings, true) } + +// Relocation types for SPARC. +type R_SPARC int + +const ( + R_SPARC_NONE R_SPARC = 0 + R_SPARC_8 R_SPARC = 1 + R_SPARC_16 R_SPARC = 2 + R_SPARC_32 R_SPARC = 3 + R_SPARC_DISP8 R_SPARC = 4 + R_SPARC_DISP16 R_SPARC = 5 + R_SPARC_DISP32 R_SPARC = 6 + R_SPARC_WDISP30 R_SPARC = 7 + R_SPARC_WDISP22 R_SPARC = 8 + R_SPARC_HI22 R_SPARC = 9 + R_SPARC_22 R_SPARC = 10 + R_SPARC_13 R_SPARC = 11 + R_SPARC_LO10 R_SPARC = 12 + R_SPARC_GOT10 R_SPARC = 13 + R_SPARC_GOT13 R_SPARC = 14 + R_SPARC_GOT22 R_SPARC = 15 + R_SPARC_PC10 R_SPARC = 16 + R_SPARC_PC22 R_SPARC = 17 + R_SPARC_WPLT30 R_SPARC = 18 + R_SPARC_COPY R_SPARC = 19 + R_SPARC_GLOB_DAT R_SPARC = 20 + R_SPARC_JMP_SLOT R_SPARC = 21 + R_SPARC_RELATIVE R_SPARC = 22 + R_SPARC_UA32 R_SPARC = 23 + R_SPARC_PLT32 R_SPARC = 24 + R_SPARC_HIPLT22 R_SPARC = 25 + R_SPARC_LOPLT10 R_SPARC = 26 + R_SPARC_PCPLT32 R_SPARC = 27 + R_SPARC_PCPLT22 R_SPARC = 28 + R_SPARC_PCPLT10 R_SPARC = 29 + R_SPARC_10 R_SPARC = 30 + R_SPARC_11 R_SPARC = 31 + R_SPARC_64 R_SPARC = 32 + R_SPARC_OLO10 R_SPARC = 33 + R_SPARC_HH22 R_SPARC = 34 + R_SPARC_HM10 R_SPARC = 35 + R_SPARC_LM22 R_SPARC = 36 + R_SPARC_PC_HH22 R_SPARC = 37 + R_SPARC_PC_HM10 R_SPARC = 38 + R_SPARC_PC_LM22 R_SPARC = 39 + R_SPARC_WDISP16 R_SPARC = 40 + R_SPARC_WDISP19 R_SPARC = 41 + R_SPARC_GLOB_JMP R_SPARC = 42 + R_SPARC_7 R_SPARC = 43 + R_SPARC_5 R_SPARC = 44 + R_SPARC_6 R_SPARC = 45 + R_SPARC_DISP64 R_SPARC = 46 + R_SPARC_PLT64 R_SPARC = 47 + R_SPARC_HIX22 R_SPARC = 48 + R_SPARC_LOX10 R_SPARC = 49 + R_SPARC_H44 R_SPARC = 50 + R_SPARC_M44 R_SPARC = 51 + R_SPARC_L44 R_SPARC = 52 + R_SPARC_REGISTER R_SPARC = 53 + R_SPARC_UA64 R_SPARC = 54 + R_SPARC_UA16 R_SPARC = 55 +) + +var rsparcStrings = []intName{ + {0, "R_SPARC_NONE"}, + {1, "R_SPARC_8"}, + {2, "R_SPARC_16"}, + {3, "R_SPARC_32"}, + {4, "R_SPARC_DISP8"}, + {5, "R_SPARC_DISP16"}, + {6, "R_SPARC_DISP32"}, + {7, "R_SPARC_WDISP30"}, + {8, "R_SPARC_WDISP22"}, + {9, "R_SPARC_HI22"}, + {10, "R_SPARC_22"}, + {11, "R_SPARC_13"}, + {12, "R_SPARC_LO10"}, + {13, "R_SPARC_GOT10"}, + {14, "R_SPARC_GOT13"}, + {15, "R_SPARC_GOT22"}, + {16, "R_SPARC_PC10"}, + {17, "R_SPARC_PC22"}, + {18, "R_SPARC_WPLT30"}, + {19, "R_SPARC_COPY"}, + {20, "R_SPARC_GLOB_DAT"}, + {21, "R_SPARC_JMP_SLOT"}, + {22, "R_SPARC_RELATIVE"}, + {23, "R_SPARC_UA32"}, + {24, "R_SPARC_PLT32"}, + {25, "R_SPARC_HIPLT22"}, + {26, "R_SPARC_LOPLT10"}, + {27, "R_SPARC_PCPLT32"}, + {28, "R_SPARC_PCPLT22"}, + {29, "R_SPARC_PCPLT10"}, + {30, "R_SPARC_10"}, + {31, "R_SPARC_11"}, + {32, "R_SPARC_64"}, + {33, "R_SPARC_OLO10"}, + {34, "R_SPARC_HH22"}, + {35, "R_SPARC_HM10"}, + {36, "R_SPARC_LM22"}, + {37, "R_SPARC_PC_HH22"}, + {38, "R_SPARC_PC_HM10"}, + {39, "R_SPARC_PC_LM22"}, + {40, "R_SPARC_WDISP16"}, + {41, "R_SPARC_WDISP19"}, + {42, "R_SPARC_GLOB_JMP"}, + {43, "R_SPARC_7"}, + {44, "R_SPARC_5"}, + {45, "R_SPARC_6"}, + {46, "R_SPARC_DISP64"}, + {47, "R_SPARC_PLT64"}, + {48, "R_SPARC_HIX22"}, + {49, "R_SPARC_LOX10"}, + {50, "R_SPARC_H44"}, + {51, "R_SPARC_M44"}, + {52, "R_SPARC_L44"}, + {53, "R_SPARC_REGISTER"}, + {54, "R_SPARC_UA64"}, + {55, "R_SPARC_UA16"}, +} + +func (i R_SPARC) String() string { return stringName(uint32(i), rsparcStrings, false) } +func (i R_SPARC) GoString() string { return stringName(uint32(i), rsparcStrings, true) } + +// Magic number for the elf trampoline, chosen wisely to be an immediate value. +const ARM_MAGIC_TRAMP_NUMBER = 0x5c000003 + +// ELF32 File header. +type Header32 struct { + Ident [EI_NIDENT]byte /* File identification. */ + Type uint16 /* File type. */ + Machine uint16 /* Machine architecture. */ + Version uint32 /* ELF format version. */ + Entry uint32 /* Entry point. */ + Phoff uint32 /* Program header file offset. */ + Shoff uint32 /* Section header file offset. */ + Flags uint32 /* Architecture-specific flags. */ + Ehsize uint16 /* Size of ELF header in bytes. */ + Phentsize uint16 /* Size of program header entry. */ + Phnum uint16 /* Number of program header entries. */ + Shentsize uint16 /* Size of section header entry. */ + Shnum uint16 /* Number of section header entries. */ + Shstrndx uint16 /* Section name strings section. */ +} + +// ELF32 Section header. +type Section32 struct { + Name uint32 /* Section name (index into the section header string table). */ + Type uint32 /* Section type. */ + Flags uint32 /* Section flags. */ + Addr uint32 /* Address in memory image. */ + Off uint32 /* Offset in file. */ + Size uint32 /* Size in bytes. */ + Link uint32 /* Index of a related section. */ + Info uint32 /* Depends on section type. */ + Addralign uint32 /* Alignment in bytes. */ + Entsize uint32 /* Size of each entry in section. */ +} + +// ELF32 Program header. +type Prog32 struct { + Type uint32 /* Entry type. */ + Off uint32 /* File offset of contents. */ + Vaddr uint32 /* Virtual address in memory image. */ + Paddr uint32 /* Physical address (not used). */ + Filesz uint32 /* Size of contents in file. */ + Memsz uint32 /* Size of contents in memory. */ + Flags uint32 /* Access permission flags. */ + Align uint32 /* Alignment in memory and file. */ +} + +// ELF32 Dynamic structure. The ".dynamic" section contains an array of them. +type Dyn32 struct { + Tag int32 /* Entry type. */ + Val uint32 /* Integer/Address value. */ +} + +/* + * Relocation entries. + */ + +// ELF32 Relocations that don't need an addend field. +type Rel32 struct { + Off uint32 /* Location to be relocated. */ + Info uint32 /* Relocation type and symbol index. */ +} + +// ELF32 Relocations that need an addend field. +type Rela32 struct { + Off uint32 /* Location to be relocated. */ + Info uint32 /* Relocation type and symbol index. */ + Addend int32 /* Addend. */ +} + +func R_SYM32(info uint32) uint32 { return uint32(info >> 8) } +func R_TYPE32(info uint32) uint32 { return uint32(info & 0xff) } +func R_INFO32(sym, typ uint32) uint32 { return sym<<8 | typ } + +// ELF32 Symbol. +type Sym32 struct { + Name uint32 + Value uint32 + Size uint32 + Info uint8 + Other uint8 + Shndx uint16 +} + +const Sym32Size = 16 + +func ST_BIND(info uint8) SymBind { return SymBind(info >> 4) } +func ST_TYPE(info uint8) SymType { return SymType(info & 0xF) } +func ST_INFO(bind SymBind, typ SymType) uint8 { + return uint8(bind)<<4 | uint8(typ)&0xf +} +func ST_VISIBILITY(other uint8) SymVis { return SymVis(other & 3) } + +/* + * ELF64 + */ + +// ELF64 file header. +type Header64 struct { + Ident [EI_NIDENT]byte /* File identification. */ + Type uint16 /* File type. */ + Machine uint16 /* Machine architecture. */ + Version uint32 /* ELF format version. */ + Entry uint64 /* Entry point. */ + Phoff uint64 /* Program header file offset. */ + Shoff uint64 /* Section header file offset. */ + Flags uint32 /* Architecture-specific flags. */ + Ehsize uint16 /* Size of ELF header in bytes. */ + Phentsize uint16 /* Size of program header entry. */ + Phnum uint16 /* Number of program header entries. */ + Shentsize uint16 /* Size of section header entry. */ + Shnum uint16 /* Number of section header entries. */ + Shstrndx uint16 /* Section name strings section. */ +} + +// ELF64 Section header. +type Section64 struct { + Name uint32 /* Section name (index into the section header string table). */ + Type uint32 /* Section type. */ + Flags uint64 /* Section flags. */ + Addr uint64 /* Address in memory image. */ + Off uint64 /* Offset in file. */ + Size uint64 /* Size in bytes. */ + Link uint32 /* Index of a related section. */ + Info uint32 /* Depends on section type. */ + Addralign uint64 /* Alignment in bytes. */ + Entsize uint64 /* Size of each entry in section. */ +} + +// ELF64 Program header. +type Prog64 struct { + Type uint32 /* Entry type. */ + Flags uint32 /* Access permission flags. */ + Off uint64 /* File offset of contents. */ + Vaddr uint64 /* Virtual address in memory image. */ + Paddr uint64 /* Physical address (not used). */ + Filesz uint64 /* Size of contents in file. */ + Memsz uint64 /* Size of contents in memory. */ + Align uint64 /* Alignment in memory and file. */ +} + +// ELF64 Dynamic structure. The ".dynamic" section contains an array of them. +type Dyn64 struct { + Tag int64 /* Entry type. */ + Val uint64 /* Integer/address value */ +} + +/* + * Relocation entries. + */ + +/* ELF64 relocations that don't need an addend field. */ +type Rel64 struct { + Off uint64 /* Location to be relocated. */ + Info uint64 /* Relocation type and symbol index. */ +} + +/* ELF64 relocations that need an addend field. */ +type Rela64 struct { + Off uint64 /* Location to be relocated. */ + Info uint64 /* Relocation type and symbol index. */ + Addend int64 /* Addend. */ +} + +func R_SYM64(info uint64) uint32 { return uint32(info >> 32) } +func R_TYPE64(info uint64) uint32 { return uint32(info) } +func R_INFO(sym, typ uint32) uint64 { return uint64(sym)<<32 | uint64(typ) } + +// ELF64 symbol table entries. +type Sym64 struct { + Name uint32 /* String table index of name. */ + Info uint8 /* Type and binding information. */ + Other uint8 /* Reserved (not used). */ + Shndx uint16 /* Section index of symbol. */ + Value uint64 /* Symbol value. */ + Size uint64 /* Size of associated object. */ +} + +const Sym64Size = 24 + +type intName struct { + i uint32 + s string +} + +func stringName(i uint32, names []intName, goSyntax bool) string { + for _, n := range names { + if n.i == i { + if goSyntax { + return "elf." + n.s + } + return n.s + } + } + + // second pass - look for smaller to add with. + // assume sorted already + for j := len(names) - 1; j >= 0; j-- { + n := names[j] + if n.i < i { + s := n.s + if goSyntax { + s = "elf." + s + } + return s + "+" + strconv.Uitoa64(uint64(i-n.i)) + } + } + + return strconv.Uitoa64(uint64(i)) +} + +func flagName(i uint32, names []intName, goSyntax bool) string { + s := "" + for _, n := range names { + if n.i&i == n.i { + if len(s) > 0 { + s += "+" + } + if goSyntax { + s += "elf." + } + s += n.s + i -= n.i + } + } + if len(s) == 0 { + return "0x" + strconv.Uitob64(uint64(i), 16) + } + if i != 0 { + s += "+0x" + strconv.Uitob64(uint64(i), 16) + } + return s +} diff --git a/src/pkg/debug/elf/elf_test.go b/src/pkg/debug/elf/elf_test.go new file mode 100644 index 000000000..67b961b5c --- /dev/null +++ b/src/pkg/debug/elf/elf_test.go @@ -0,0 +1,49 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elf + +import ( + "fmt" + "testing" +) + +type nameTest struct { + val interface{} + str string +} + +var nameTests = []nameTest{ + {ELFOSABI_LINUX, "ELFOSABI_LINUX"}, + {ET_EXEC, "ET_EXEC"}, + {EM_860, "EM_860"}, + {SHN_LOPROC, "SHN_LOPROC"}, + {SHT_PROGBITS, "SHT_PROGBITS"}, + {SHF_MERGE + SHF_TLS, "SHF_MERGE+SHF_TLS"}, + {PT_LOAD, "PT_LOAD"}, + {PF_W + PF_R + 0x50, "PF_W+PF_R+0x50"}, + {DT_SYMBOLIC, "DT_SYMBOLIC"}, + {DF_BIND_NOW, "DF_BIND_NOW"}, + {NT_FPREGSET, "NT_FPREGSET"}, + {STB_GLOBAL, "STB_GLOBAL"}, + {STT_COMMON, "STT_COMMON"}, + {STV_HIDDEN, "STV_HIDDEN"}, + {R_X86_64_PC32, "R_X86_64_PC32"}, + {R_ALPHA_OP_PUSH, "R_ALPHA_OP_PUSH"}, + {R_ARM_THM_ABS5, "R_ARM_THM_ABS5"}, + {R_386_GOT32, "R_386_GOT32"}, + {R_PPC_GOT16_HI, "R_PPC_GOT16_HI"}, + {R_SPARC_GOT22, "R_SPARC_GOT22"}, + {ET_LOOS + 5, "ET_LOOS+5"}, + {ProgFlag(0x50), "0x50"}, +} + +func TestNames(t *testing.T) { + for i, tt := range nameTests { + s := fmt.Sprint(tt.val) + if s != tt.str { + t.Errorf("#%d: want %q have %q", i, s, tt.str) + } + } +} diff --git a/src/pkg/debug/elf/file.go b/src/pkg/debug/elf/file.go new file mode 100644 index 000000000..a0ddb1fc7 --- /dev/null +++ b/src/pkg/debug/elf/file.go @@ -0,0 +1,761 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package elf implements access to ELF object files. +package elf + +import ( + "bytes" + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" +) + +// TODO: error reporting detail + +/* + * Internal ELF representation + */ + +// A FileHeader represents an ELF file header. +type FileHeader struct { + Class Class + Data Data + Version Version + OSABI OSABI + ABIVersion uint8 + ByteOrder binary.ByteOrder + Type Type + Machine Machine +} + +// A File represents an open ELF file. +type File struct { + FileHeader + Sections []*Section + Progs []*Prog + closer io.Closer + gnuNeed []verneed + gnuVersym []byte +} + +// A SectionHeader represents a single ELF section header. +type SectionHeader struct { + Name string + Type SectionType + Flags SectionFlag + Addr uint64 + Offset uint64 + Size uint64 + Link uint32 + Info uint32 + Addralign uint64 + Entsize uint64 +} + +// A Section represents a single section in an ELF file. +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the ELF section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// stringTable reads and returns the string table given by the +// specified link value. +func (f *File) stringTable(link uint32) ([]byte, os.Error) { + if link <= 0 || link >= uint32(len(f.Sections)) { + return nil, os.NewError("section has invalid string table link") + } + return f.Sections[link].Data() +} + +// Open returns a new ReadSeeker reading the ELF section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +// A ProgHeader represents a single ELF program header. +type ProgHeader struct { + Type ProgType + Flags ProgFlag + Off uint64 + Vaddr uint64 + Paddr uint64 + Filesz uint64 + Memsz uint64 + Align uint64 +} + +// A Prog represents a single ELF program header in an ELF binary. +type Prog struct { + ProgHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Open returns a new ReadSeeker reading the ELF program body. +func (p *Prog) Open() io.ReadSeeker { return io.NewSectionReader(p.sr, 0, 1<<63-1) } + +// A Symbol represents an entry in an ELF symbol table section. +type Symbol struct { + Name string + Info, Other byte + Section SectionIndex + Value, Size uint64 +} + +/* + * ELF reader + */ + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v' ", e.val) + } + msg += fmt.Sprintf("in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as an ELF binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// SectionByType returns the first section in f with the +// given type, or nil if there is no such section. +func (f *File) SectionByType(typ SectionType) *Section { + for _, s := range f.Sections { + if s.Type == typ { + return s + } + } + return nil +} + +// NewFile creates a new File for accessing an ELF binary in an underlying reader. +// The ELF binary is expected to start at position 0 in the ReaderAt. +func NewFile(r io.ReaderAt) (*File, os.Error) { + sr := io.NewSectionReader(r, 0, 1<<63-1) + // Read and decode ELF identifier + var ident [16]uint8 + if _, err := r.ReadAt(ident[0:], 0); err != nil { + return nil, err + } + if ident[0] != '\x7f' || ident[1] != 'E' || ident[2] != 'L' || ident[3] != 'F' { + return nil, &FormatError{0, "bad magic number", ident[0:4]} + } + + f := new(File) + f.Class = Class(ident[EI_CLASS]) + switch f.Class { + case ELFCLASS32: + case ELFCLASS64: + // ok + default: + return nil, &FormatError{0, "unknown ELF class", f.Class} + } + + f.Data = Data(ident[EI_DATA]) + switch f.Data { + case ELFDATA2LSB: + f.ByteOrder = binary.LittleEndian + case ELFDATA2MSB: + f.ByteOrder = binary.BigEndian + default: + return nil, &FormatError{0, "unknown ELF data encoding", f.Data} + } + + f.Version = Version(ident[EI_VERSION]) + if f.Version != EV_CURRENT { + return nil, &FormatError{0, "unknown ELF version", f.Version} + } + + f.OSABI = OSABI(ident[EI_OSABI]) + f.ABIVersion = ident[EI_ABIVERSION] + + // Read ELF file header + var phoff int64 + var phentsize, phnum int + var shoff int64 + var shentsize, shnum, shstrndx int + shstrndx = -1 + switch f.Class { + case ELFCLASS32: + hdr := new(Header32) + sr.Seek(0, os.SEEK_SET) + if err := binary.Read(sr, f.ByteOrder, hdr); err != nil { + return nil, err + } + f.Type = Type(hdr.Type) + f.Machine = Machine(hdr.Machine) + if v := Version(hdr.Version); v != f.Version { + return nil, &FormatError{0, "mismatched ELF version", v} + } + phoff = int64(hdr.Phoff) + phentsize = int(hdr.Phentsize) + phnum = int(hdr.Phnum) + shoff = int64(hdr.Shoff) + shentsize = int(hdr.Shentsize) + shnum = int(hdr.Shnum) + shstrndx = int(hdr.Shstrndx) + case ELFCLASS64: + hdr := new(Header64) + sr.Seek(0, os.SEEK_SET) + if err := binary.Read(sr, f.ByteOrder, hdr); err != nil { + return nil, err + } + f.Type = Type(hdr.Type) + f.Machine = Machine(hdr.Machine) + if v := Version(hdr.Version); v != f.Version { + return nil, &FormatError{0, "mismatched ELF version", v} + } + phoff = int64(hdr.Phoff) + phentsize = int(hdr.Phentsize) + phnum = int(hdr.Phnum) + shoff = int64(hdr.Shoff) + shentsize = int(hdr.Shentsize) + shnum = int(hdr.Shnum) + shstrndx = int(hdr.Shstrndx) + } + if shstrndx < 0 || shstrndx >= shnum { + return nil, &FormatError{0, "invalid ELF shstrndx", shstrndx} + } + + // Read program headers + f.Progs = make([]*Prog, phnum) + for i := 0; i < phnum; i++ { + off := phoff + int64(i)*int64(phentsize) + sr.Seek(off, os.SEEK_SET) + p := new(Prog) + switch f.Class { + case ELFCLASS32: + ph := new(Prog32) + if err := binary.Read(sr, f.ByteOrder, ph); err != nil { + return nil, err + } + p.ProgHeader = ProgHeader{ + Type: ProgType(ph.Type), + Flags: ProgFlag(ph.Flags), + Off: uint64(ph.Off), + Vaddr: uint64(ph.Vaddr), + Paddr: uint64(ph.Paddr), + Filesz: uint64(ph.Filesz), + Memsz: uint64(ph.Memsz), + Align: uint64(ph.Align), + } + case ELFCLASS64: + ph := new(Prog64) + if err := binary.Read(sr, f.ByteOrder, ph); err != nil { + return nil, err + } + p.ProgHeader = ProgHeader{ + Type: ProgType(ph.Type), + Flags: ProgFlag(ph.Flags), + Off: uint64(ph.Off), + Vaddr: uint64(ph.Vaddr), + Paddr: uint64(ph.Paddr), + Filesz: uint64(ph.Filesz), + Memsz: uint64(ph.Memsz), + Align: uint64(ph.Align), + } + } + p.sr = io.NewSectionReader(r, int64(p.Off), int64(p.Filesz)) + p.ReaderAt = p.sr + f.Progs[i] = p + } + + // Read section headers + f.Sections = make([]*Section, shnum) + names := make([]uint32, shnum) + for i := 0; i < shnum; i++ { + off := shoff + int64(i)*int64(shentsize) + sr.Seek(off, os.SEEK_SET) + s := new(Section) + switch f.Class { + case ELFCLASS32: + sh := new(Section32) + if err := binary.Read(sr, f.ByteOrder, sh); err != nil { + return nil, err + } + names[i] = sh.Name + s.SectionHeader = SectionHeader{ + Type: SectionType(sh.Type), + Flags: SectionFlag(sh.Flags), + Addr: uint64(sh.Addr), + Offset: uint64(sh.Off), + Size: uint64(sh.Size), + Link: uint32(sh.Link), + Info: uint32(sh.Info), + Addralign: uint64(sh.Addralign), + Entsize: uint64(sh.Entsize), + } + case ELFCLASS64: + sh := new(Section64) + if err := binary.Read(sr, f.ByteOrder, sh); err != nil { + return nil, err + } + names[i] = sh.Name + s.SectionHeader = SectionHeader{ + Type: SectionType(sh.Type), + Flags: SectionFlag(sh.Flags), + Offset: uint64(sh.Off), + Size: uint64(sh.Size), + Addr: uint64(sh.Addr), + Link: uint32(sh.Link), + Info: uint32(sh.Info), + Addralign: uint64(sh.Addralign), + Entsize: uint64(sh.Entsize), + } + } + s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.Size)) + s.ReaderAt = s.sr + f.Sections[i] = s + } + + // Load section header string table. + shstrtab, err := f.Sections[shstrndx].Data() + if err != nil { + return nil, err + } + for i, s := range f.Sections { + var ok bool + s.Name, ok = getString(shstrtab, int(names[i])) + if !ok { + return nil, &FormatError{shoff + int64(i*shentsize), "bad section name index", names[i]} + } + } + + return f, nil +} + +// getSymbols returns a slice of Symbols from parsing the symbol table +// with the given type, along with the associated string table. +func (f *File) getSymbols(typ SectionType) ([]Symbol, []byte, os.Error) { + switch f.Class { + case ELFCLASS64: + return f.getSymbols64(typ) + + case ELFCLASS32: + return f.getSymbols32(typ) + } + + return nil, nil, os.NewError("not implemented") +} + +func (f *File) getSymbols32(typ SectionType) ([]Symbol, []byte, os.Error) { + symtabSection := f.SectionByType(typ) + if symtabSection == nil { + return nil, nil, os.NewError("no symbol section") + } + + data, err := symtabSection.Data() + if err != nil { + return nil, nil, os.NewError("cannot load symbol section") + } + symtab := bytes.NewBuffer(data) + if symtab.Len()%Sym32Size != 0 { + return nil, nil, os.NewError("length of symbol section is not a multiple of SymSize") + } + + strdata, err := f.stringTable(symtabSection.Link) + if err != nil { + return nil, nil, os.NewError("cannot load string table section") + } + + // The first entry is all zeros. + var skip [Sym32Size]byte + symtab.Read(skip[0:]) + + symbols := make([]Symbol, symtab.Len()/Sym32Size) + + i := 0 + var sym Sym32 + for symtab.Len() > 0 { + binary.Read(symtab, f.ByteOrder, &sym) + str, _ := getString(strdata, int(sym.Name)) + symbols[i].Name = str + symbols[i].Info = sym.Info + symbols[i].Other = sym.Other + symbols[i].Section = SectionIndex(sym.Shndx) + symbols[i].Value = uint64(sym.Value) + symbols[i].Size = uint64(sym.Size) + i++ + } + + return symbols, strdata, nil +} + +func (f *File) getSymbols64(typ SectionType) ([]Symbol, []byte, os.Error) { + symtabSection := f.SectionByType(typ) + if symtabSection == nil { + return nil, nil, os.NewError("no symbol section") + } + + data, err := symtabSection.Data() + if err != nil { + return nil, nil, os.NewError("cannot load symbol section") + } + symtab := bytes.NewBuffer(data) + if symtab.Len()%Sym64Size != 0 { + return nil, nil, os.NewError("length of symbol section is not a multiple of Sym64Size") + } + + strdata, err := f.stringTable(symtabSection.Link) + if err != nil { + return nil, nil, os.NewError("cannot load string table section") + } + + // The first entry is all zeros. + var skip [Sym64Size]byte + symtab.Read(skip[0:]) + + symbols := make([]Symbol, symtab.Len()/Sym64Size) + + i := 0 + var sym Sym64 + for symtab.Len() > 0 { + binary.Read(symtab, f.ByteOrder, &sym) + str, _ := getString(strdata, int(sym.Name)) + symbols[i].Name = str + symbols[i].Info = sym.Info + symbols[i].Other = sym.Other + symbols[i].Section = SectionIndex(sym.Shndx) + symbols[i].Value = sym.Value + symbols[i].Size = sym.Size + i++ + } + + return symbols, strdata, nil +} + +// getString extracts a string from an ELF string table. +func getString(section []byte, start int) (string, bool) { + if start < 0 || start >= len(section) { + return "", false + } + + for end := start; end < len(section); end++ { + if section[end] == 0 { + return string(section[start:end]), true + } + } + return "", false +} + +// Section returns a section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +// applyRelocations applies relocations to dst. rels is a relocations section +// in RELA format. +func (f *File) applyRelocations(dst []byte, rels []byte) os.Error { + if f.Class == ELFCLASS64 && f.Machine == EM_X86_64 { + return f.applyRelocationsAMD64(dst, rels) + } + + return os.NewError("not implemented") +} + +func (f *File) applyRelocationsAMD64(dst []byte, rels []byte) os.Error { + if len(rels)%Sym64Size != 0 { + return os.NewError("length of relocation section is not a multiple of Sym64Size") + } + + symbols, _, err := f.getSymbols(SHT_SYMTAB) + if err != nil { + return err + } + + b := bytes.NewBuffer(rels) + var rela Rela64 + + for b.Len() > 0 { + binary.Read(b, f.ByteOrder, &rela) + symNo := rela.Info >> 32 + t := R_X86_64(rela.Info & 0xffff) + + if symNo >= uint64(len(symbols)) { + continue + } + sym := &symbols[symNo] + if SymType(sym.Info&0xf) != STT_SECTION { + // We don't handle non-section relocations for now. + continue + } + + switch t { + case R_X86_64_64: + if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { + continue + } + f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], uint64(rela.Addend)) + case R_X86_64_32: + if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { + continue + } + f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], uint32(rela.Addend)) + } + } + + return nil +} + +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = ".debug_" + name + s := f.Section(name) + if s == nil { + continue + } + b, err := s.Data() + if err != nil && uint64(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + // If there's a relocation table for .debug_info, we have to process it + // now otherwise the data in .debug_info is invalid for x86-64 objects. + rela := f.Section(".rela.debug_info") + if rela != nil && rela.Type == SHT_RELA && f.Machine == EM_X86_64 { + data, err := rela.Data() + if err != nil { + return nil, err + } + err = f.applyRelocations(dat[1], data) + if err != nil { + return nil, err + } + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// Symbols returns the symbol table for f. +func (f *File) Symbols() ([]Symbol, os.Error) { + sym, _, err := f.getSymbols(SHT_SYMTAB) + return sym, err +} + +type ImportedSymbol struct { + Name string + Version string + Library string +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +// It does not return weak symbols. +func (f *File) ImportedSymbols() ([]ImportedSymbol, os.Error) { + sym, str, err := f.getSymbols(SHT_DYNSYM) + if err != nil { + return nil, err + } + f.gnuVersionInit(str) + var all []ImportedSymbol + for i, s := range sym { + if ST_BIND(s.Info) == STB_GLOBAL && s.Section == SHN_UNDEF { + all = append(all, ImportedSymbol{Name: s.Name}) + f.gnuVersion(i, &all[len(all)-1]) + } + } + return all, nil +} + +type verneed struct { + File string + Name string +} + +// gnuVersionInit parses the GNU version tables +// for use by calls to gnuVersion. +func (f *File) gnuVersionInit(str []byte) { + // Accumulate verneed information. + vn := f.SectionByType(SHT_GNU_VERNEED) + if vn == nil { + return + } + d, _ := vn.Data() + + var need []verneed + i := 0 + for { + if i+16 > len(d) { + break + } + vers := f.ByteOrder.Uint16(d[i : i+2]) + if vers != 1 { + break + } + cnt := f.ByteOrder.Uint16(d[i+2 : i+4]) + fileoff := f.ByteOrder.Uint32(d[i+4 : i+8]) + aux := f.ByteOrder.Uint32(d[i+8 : i+12]) + next := f.ByteOrder.Uint32(d[i+12 : i+16]) + file, _ := getString(str, int(fileoff)) + + var name string + j := i + int(aux) + for c := 0; c < int(cnt); c++ { + if j+16 > len(d) { + break + } + // hash := f.ByteOrder.Uint32(d[j:j+4]) + // flags := f.ByteOrder.Uint16(d[j+4:j+6]) + other := f.ByteOrder.Uint16(d[j+6 : j+8]) + nameoff := f.ByteOrder.Uint32(d[j+8 : j+12]) + next := f.ByteOrder.Uint32(d[j+12 : j+16]) + name, _ = getString(str, int(nameoff)) + ndx := int(other) + if ndx >= len(need) { + a := make([]verneed, 2*(ndx+1)) + copy(a, need) + need = a + } + + need[ndx] = verneed{file, name} + if next == 0 { + break + } + j += int(next) + } + + if next == 0 { + break + } + i += int(next) + } + + // Versym parallels symbol table, indexing into verneed. + vs := f.SectionByType(SHT_GNU_VERSYM) + if vs == nil { + return + } + d, _ = vs.Data() + + f.gnuNeed = need + f.gnuVersym = d +} + +// gnuVersion adds Library and Version information to sym, +// which came from offset i of the symbol table. +func (f *File) gnuVersion(i int, sym *ImportedSymbol) { + // Each entry is two bytes; skip undef entry at beginning. + i = (i + 1) * 2 + if i >= len(f.gnuVersym) { + return + } + j := int(f.ByteOrder.Uint16(f.gnuVersym[i:])) + if j < 2 || j >= len(f.gnuNeed) { + return + } + n := &f.gnuNeed[j] + sym.Library = n.File + sym.Version = n.Name +} + +// ImportedLibraries returns the names of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + ds := f.SectionByType(SHT_DYNAMIC) + if ds == nil { + // not dynamic, so no libraries + return nil, nil + } + d, err := ds.Data() + if err != nil { + return nil, err + } + str, err := f.stringTable(ds.Link) + if err != nil { + return nil, err + } + var all []string + for len(d) > 0 { + var tag DynTag + var value uint64 + switch f.Class { + case ELFCLASS32: + tag = DynTag(f.ByteOrder.Uint32(d[0:4])) + value = uint64(f.ByteOrder.Uint32(d[4:8])) + d = d[8:] + case ELFCLASS64: + tag = DynTag(f.ByteOrder.Uint64(d[0:8])) + value = f.ByteOrder.Uint64(d[8:16]) + d = d[16:] + } + if tag == DT_NEEDED { + s, ok := getString(str, int(value)) + if ok { + all = append(all, s) + } + } + } + + return all, nil +} diff --git a/src/pkg/debug/elf/file_test.go b/src/pkg/debug/elf/file_test.go new file mode 100644 index 000000000..451d3d514 --- /dev/null +++ b/src/pkg/debug/elf/file_test.go @@ -0,0 +1,241 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package elf + +import ( + "debug/dwarf" + "encoding/binary" + "net" + "os" + "reflect" + "runtime" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + sections []SectionHeader + progs []ProgHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-freebsd-exec", + FileHeader{ELFCLASS32, ELFDATA2LSB, EV_CURRENT, ELFOSABI_FREEBSD, 0, binary.LittleEndian, ET_EXEC, EM_386}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x80480d4, 0xd4, 0x15, 0x0, 0x0, 0x1, 0x0}, + {".hash", SHT_HASH, SHF_ALLOC, 0x80480ec, 0xec, 0x90, 0x3, 0x0, 0x4, 0x4}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x804817c, 0x17c, 0x110, 0x4, 0x1, 0x4, 0x10}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x804828c, 0x28c, 0xbb, 0x0, 0x0, 0x1, 0x0}, + {".rel.plt", SHT_REL, SHF_ALLOC, 0x8048348, 0x348, 0x20, 0x3, 0x7, 0x4, 0x8}, + {".init", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x8048368, 0x368, 0x11, 0x0, 0x0, 0x4, 0x0}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x804837c, 0x37c, 0x50, 0x0, 0x0, 0x4, 0x4}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x80483cc, 0x3cc, 0x180, 0x0, 0x0, 0x4, 0x0}, + {".fini", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x804854c, 0x54c, 0xc, 0x0, 0x0, 0x4, 0x0}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x8048558, 0x558, 0xa3, 0x0, 0x0, 0x1, 0x0}, + {".data", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80495fc, 0x5fc, 0xc, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x8049608, 0x608, 0x4, 0x0, 0x0, 0x4, 0x0}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x804960c, 0x60c, 0x98, 0x4, 0x0, 0x4, 0x8}, + {".ctors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496a4, 0x6a4, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".dtors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496ac, 0x6ac, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".jcr", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496b4, 0x6b4, 0x4, 0x0, 0x0, 0x4, 0x0}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x80496b8, 0x6b8, 0x1c, 0x0, 0x0, 0x4, 0x4}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x80496d4, 0x6d4, 0x20, 0x0, 0x0, 0x4, 0x0}, + {".comment", SHT_PROGBITS, 0x0, 0x0, 0x6d4, 0x12d, 0x0, 0x0, 0x1, 0x0}, + {".debug_aranges", SHT_PROGBITS, 0x0, 0x0, 0x801, 0x20, 0x0, 0x0, 0x1, 0x0}, + {".debug_pubnames", SHT_PROGBITS, 0x0, 0x0, 0x821, 0x1b, 0x0, 0x0, 0x1, 0x0}, + {".debug_info", SHT_PROGBITS, 0x0, 0x0, 0x83c, 0x11d, 0x0, 0x0, 0x1, 0x0}, + {".debug_abbrev", SHT_PROGBITS, 0x0, 0x0, 0x959, 0x41, 0x0, 0x0, 0x1, 0x0}, + {".debug_line", SHT_PROGBITS, 0x0, 0x0, 0x99a, 0x35, 0x0, 0x0, 0x1, 0x0}, + {".debug_frame", SHT_PROGBITS, 0x0, 0x0, 0x9d0, 0x30, 0x0, 0x0, 0x4, 0x0}, + {".debug_str", SHT_PROGBITS, 0x0, 0x0, 0xa00, 0xd, 0x0, 0x0, 0x1, 0x0}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0xa0d, 0xf8, 0x0, 0x0, 0x1, 0x0}, + {".symtab", SHT_SYMTAB, 0x0, 0x0, 0xfb8, 0x4b0, 0x1d, 0x38, 0x4, 0x10}, + {".strtab", SHT_STRTAB, 0x0, 0x0, 0x1468, 0x206, 0x0, 0x0, 0x1, 0x0}, + }, + []ProgHeader{ + {PT_PHDR, PF_R + PF_X, 0x34, 0x8048034, 0x8048034, 0xa0, 0xa0, 0x4}, + {PT_INTERP, PF_R, 0xd4, 0x80480d4, 0x80480d4, 0x15, 0x15, 0x1}, + {PT_LOAD, PF_R + PF_X, 0x0, 0x8048000, 0x8048000, 0x5fb, 0x5fb, 0x1000}, + {PT_LOAD, PF_R + PF_W, 0x5fc, 0x80495fc, 0x80495fc, 0xd8, 0xf8, 0x1000}, + {PT_DYNAMIC, PF_R + PF_W, 0x60c, 0x804960c, 0x804960c, 0x98, 0x98, 0x4}, + }, + }, + { + "testdata/gcc-amd64-linux-exec", + FileHeader{ELFCLASS64, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE, 0, binary.LittleEndian, ET_EXEC, EM_X86_64}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x400200, 0x200, 0x1c, 0x0, 0x0, 0x1, 0x0}, + {".note.ABI-tag", SHT_NOTE, SHF_ALLOC, 0x40021c, 0x21c, 0x20, 0x0, 0x0, 0x4, 0x0}, + {".hash", SHT_HASH, SHF_ALLOC, 0x400240, 0x240, 0x24, 0x5, 0x0, 0x8, 0x4}, + {".gnu.hash", SHT_LOOS + 268435446, SHF_ALLOC, 0x400268, 0x268, 0x1c, 0x5, 0x0, 0x8, 0x0}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x400288, 0x288, 0x60, 0x6, 0x1, 0x8, 0x18}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x4002e8, 0x2e8, 0x3d, 0x0, 0x0, 0x1, 0x0}, + {".gnu.version", SHT_HIOS, SHF_ALLOC, 0x400326, 0x326, 0x8, 0x5, 0x0, 0x2, 0x2}, + {".gnu.version_r", SHT_LOOS + 268435454, SHF_ALLOC, 0x400330, 0x330, 0x20, 0x6, 0x1, 0x8, 0x0}, + {".rela.dyn", SHT_RELA, SHF_ALLOC, 0x400350, 0x350, 0x18, 0x5, 0x0, 0x8, 0x18}, + {".rela.plt", SHT_RELA, SHF_ALLOC, 0x400368, 0x368, 0x30, 0x5, 0xc, 0x8, 0x18}, + {".init", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x400398, 0x398, 0x18, 0x0, 0x0, 0x4, 0x0}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x4003b0, 0x3b0, 0x30, 0x0, 0x0, 0x4, 0x10}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x4003e0, 0x3e0, 0x1b4, 0x0, 0x0, 0x10, 0x0}, + {".fini", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x400594, 0x594, 0xe, 0x0, 0x0, 0x4, 0x0}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x4005a4, 0x5a4, 0x11, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame_hdr", SHT_PROGBITS, SHF_ALLOC, 0x4005b8, 0x5b8, 0x24, 0x0, 0x0, 0x4, 0x0}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x4005e0, 0x5e0, 0xa4, 0x0, 0x0, 0x8, 0x0}, + {".ctors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600688, 0x688, 0x10, 0x0, 0x0, 0x8, 0x0}, + {".dtors", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600698, 0x698, 0x10, 0x0, 0x0, 0x8, 0x0}, + {".jcr", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x6006a8, 0x6a8, 0x8, 0x0, 0x0, 0x8, 0x0}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x6006b0, 0x6b0, 0x1a0, 0x6, 0x0, 0x8, 0x10}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600850, 0x850, 0x8, 0x0, 0x0, 0x8, 0x8}, + {".got.plt", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600858, 0x858, 0x28, 0x0, 0x0, 0x8, 0x8}, + {".data", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x600880, 0x880, 0x18, 0x0, 0x0, 0x8, 0x0}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x600898, 0x898, 0x8, 0x0, 0x0, 0x4, 0x0}, + {".comment", SHT_PROGBITS, 0x0, 0x0, 0x898, 0x126, 0x0, 0x0, 0x1, 0x0}, + {".debug_aranges", SHT_PROGBITS, 0x0, 0x0, 0x9c0, 0x90, 0x0, 0x0, 0x10, 0x0}, + {".debug_pubnames", SHT_PROGBITS, 0x0, 0x0, 0xa50, 0x25, 0x0, 0x0, 0x1, 0x0}, + {".debug_info", SHT_PROGBITS, 0x0, 0x0, 0xa75, 0x1a7, 0x0, 0x0, 0x1, 0x0}, + {".debug_abbrev", SHT_PROGBITS, 0x0, 0x0, 0xc1c, 0x6f, 0x0, 0x0, 0x1, 0x0}, + {".debug_line", SHT_PROGBITS, 0x0, 0x0, 0xc8b, 0x13f, 0x0, 0x0, 0x1, 0x0}, + {".debug_str", SHT_PROGBITS, SHF_MERGE + SHF_STRINGS, 0x0, 0xdca, 0xb1, 0x0, 0x0, 0x1, 0x1}, + {".debug_ranges", SHT_PROGBITS, 0x0, 0x0, 0xe80, 0x90, 0x0, 0x0, 0x10, 0x0}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0xf10, 0x149, 0x0, 0x0, 0x1, 0x0}, + {".symtab", SHT_SYMTAB, 0x0, 0x0, 0x19a0, 0x6f0, 0x24, 0x39, 0x8, 0x18}, + {".strtab", SHT_STRTAB, 0x0, 0x0, 0x2090, 0x1fc, 0x0, 0x0, 0x1, 0x0}, + }, + []ProgHeader{ + {PT_PHDR, PF_R + PF_X, 0x40, 0x400040, 0x400040, 0x1c0, 0x1c0, 0x8}, + {PT_INTERP, PF_R, 0x200, 0x400200, 0x400200, 0x1c, 0x1c, 1}, + {PT_LOAD, PF_R + PF_X, 0x0, 0x400000, 0x400000, 0x684, 0x684, 0x200000}, + {PT_LOAD, PF_R + PF_W, 0x688, 0x600688, 0x600688, 0x210, 0x218, 0x200000}, + {PT_DYNAMIC, PF_R + PF_W, 0x6b0, 0x6006b0, 0x6006b0, 0x1a0, 0x1a0, 0x8}, + {PT_NOTE, PF_R, 0x21c, 0x40021c, 0x40021c, 0x20, 0x20, 0x4}, + {PT_LOOS + 0x474E550, PF_R, 0x5b8, 0x4005b8, 0x4005b8, 0x24, 0x24, 0x4}, + {PT_LOOS + 0x474E551, PF_R + PF_W, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + for i, s := range f.Sections { + if i >= len(tt.sections) { + break + } + sh := &tt.sections[i] + if !reflect.DeepEqual(&s.SectionHeader, sh) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, &s.SectionHeader, sh) + } + } + for i, p := range f.Progs { + if i >= len(tt.progs) { + break + } + ph := &tt.progs[i] + if !reflect.DeepEqual(&p.ProgHeader, ph) { + t.Errorf("open %s, program %d:\n\thave %#v\n\twant %#v\n", tt.file, i, &p.ProgHeader, ph) + } + } + tn := len(tt.sections) + fn := len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + tn = len(tt.progs) + fn = len(f.Progs) + if tn != fn { + t.Errorf("open %s: len(Progs) = %d, want %d", tt.file, fn, tn) + } + } +} + +type relocationTest struct { + file string + firstEntry *dwarf.Entry +} + +var relocationTests = []relocationTest{ + { + "testdata/go-relocation-test-gcc441-x86-64.obj", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, + { + "testdata/go-relocation-test-gcc441-x86.obj", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "t.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x5)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, + { + "testdata/go-relocation-test-gcc424-x86-64.obj", + &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.2.4 (Ubuntu 4.2.4-1ubuntu4)"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test-gcc424.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}, + }, +} + +func TestDWARFRelocations(t *testing.T) { + for i, test := range relocationTests { + f, err := Open(test.file) + if err != nil { + t.Error(err) + continue + } + dwarf, err := f.DWARF() + if err != nil { + t.Error(err) + continue + } + reader := dwarf.Reader() + // Checking only the first entry is sufficient since it has + // many different strings. If the relocation had failed, all + // the string offsets would be zero and all the strings would + // end up being the same. + firstEntry, err := reader.Next() + if err != nil { + t.Error(err) + continue + } + + if !reflect.DeepEqual(test.firstEntry, firstEntry) { + t.Errorf("#%d: mismatch: got:%#v want:%#v", i, firstEntry, test.firstEntry) + continue + } + } +} + +func TestNoSectionOverlaps(t *testing.T) { + // Ensure 6l outputs sections without overlaps. + if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { + return // not ELF + } + _ = net.ResolveIPAddr // force dynamic linkage + f, err := Open(os.Args[0]) + if err != nil { + t.Error(err) + return + } + for i, si := range f.Sections { + sih := si.SectionHeader + for j, sj := range f.Sections { + sjh := sj.SectionHeader + if i == j || sjh.Type == SHT_NOBITS || sih.Offset == sjh.Offset && sih.Size == 0 { + continue + } + if sih.Offset >= sjh.Offset && sih.Offset < sjh.Offset+sjh.Size { + t.Errorf("ld produced ELF with section %s within %s: 0x%x <= 0x%x..0x%x < 0x%x", + sih.Name, sjh.Name, sjh.Offset, sih.Offset, sih.Offset+sih.Size, sjh.Offset+sjh.Size) + } + } + } +} diff --git a/src/pkg/debug/elf/testdata/gcc-386-freebsd-exec b/src/pkg/debug/elf/testdata/gcc-386-freebsd-exec new file mode 100755 index 000000000..7af9c58ca Binary files /dev/null and b/src/pkg/debug/elf/testdata/gcc-386-freebsd-exec differ diff --git a/src/pkg/debug/elf/testdata/gcc-amd64-linux-exec b/src/pkg/debug/elf/testdata/gcc-amd64-linux-exec new file mode 100755 index 000000000..c6cb1de28 Binary files /dev/null and b/src/pkg/debug/elf/testdata/gcc-amd64-linux-exec differ diff --git a/src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj b/src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj new file mode 100644 index 000000000..a7c6d6e56 Binary files /dev/null and b/src/pkg/debug/elf/testdata/go-relocation-test-gcc424-x86-64.obj differ diff --git a/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj new file mode 100644 index 000000000..2d37ab6e6 Binary files /dev/null and b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86-64.obj differ diff --git a/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj new file mode 100644 index 000000000..0d59fe303 Binary files /dev/null and b/src/pkg/debug/elf/testdata/go-relocation-test-gcc441-x86.obj differ diff --git a/src/pkg/debug/gosym/Makefile b/src/pkg/debug/gosym/Makefile new file mode 100644 index 000000000..4f420e729 --- /dev/null +++ b/src/pkg/debug/gosym/Makefile @@ -0,0 +1,19 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/gosym +GOFILES=\ + pclntab.go\ + symtab.go\ + +include ../../../Make.pkg + +test: make-pclinetest + +testshort: make-pclinetest + +make-pclinetest: + @if [ "`uname`-`uname -m`" = Linux-x86_64 -a $(GOARCH) = amd64 ]; then mkdir -p _test && $(AS) pclinetest.s && $(LD) -E main -o _test/pclinetest pclinetest.$O; fi diff --git a/src/pkg/debug/gosym/pclinetest.h b/src/pkg/debug/gosym/pclinetest.h new file mode 100644 index 000000000..a6c40e76c --- /dev/null +++ b/src/pkg/debug/gosym/pclinetest.h @@ -0,0 +1,7 @@ +// Empty include file to generate z symbols + + + + + +// EOF diff --git a/src/pkg/debug/gosym/pclinetest.s b/src/pkg/debug/gosym/pclinetest.s new file mode 100644 index 000000000..6305435b0 --- /dev/null +++ b/src/pkg/debug/gosym/pclinetest.s @@ -0,0 +1,58 @@ +TEXT linefrompc(SB),7,$0 // Each byte stores its line delta +BYTE $2; +BYTE $1; +BYTE $1; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; +BYTE $1; +BYTE $1; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +BYTE $1; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; BYTE $0; +#include "pclinetest.h" +BYTE $2; +#include "pclinetest.h" +BYTE $2; + +TEXT pcfromline(SB),7,$0 // Each record stores its line delta, then n, then n more bytes +BYTE $31; BYTE $0; +BYTE $1; BYTE $1; BYTE $0; +BYTE $1; BYTE $0; + +BYTE $2; BYTE $4; BYTE $0; BYTE $0; BYTE $0; BYTE $0; + + +#include "pclinetest.h" +BYTE $4; BYTE $0; + + +BYTE $3; BYTE $3; BYTE $0; BYTE $0; BYTE $0; +#include "pclinetest.h" + + +BYTE $4; BYTE $3; BYTE $0; BYTE $0; BYTE $0; + +TEXT main(SB),7,$0 + // Prevent GC of our test symbols + CALL linefrompc(SB) + CALL pcfromline(SB) + +// Keep the linker happy +TEXT main·main(SB),7,$0 + RET + +TEXT main·init(SB),7,$0 + RET diff --git a/src/pkg/debug/gosym/pclntab.go b/src/pkg/debug/gosym/pclntab.go new file mode 100644 index 000000000..9d7b0d15f --- /dev/null +++ b/src/pkg/debug/gosym/pclntab.go @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Line tables + */ + +package gosym + +import "encoding/binary" + +type LineTable struct { + Data []byte + PC uint64 + Line int +} + +// TODO(rsc): Need to pull in quantum from architecture definition. +const quantum = 1 + +func (t *LineTable) parse(targetPC uint64, targetLine int) (b []byte, pc uint64, line int) { + // The PC/line table can be thought of as a sequence of + // * + // batches. Each update batch results in a (pc, line) pair, + // where line applies to every PC from pc up to but not + // including the pc of the next pair. + // + // Here we process each update individually, which simplifies + // the code, but makes the corner cases more confusing. + b, pc, line = t.Data, t.PC, t.Line + for pc <= targetPC && line != targetLine && len(b) > 0 { + code := b[0] + b = b[1:] + switch { + case code == 0: + if len(b) < 4 { + b = b[0:0] + break + } + val := binary.BigEndian.Uint32(b) + b = b[4:] + line += int(val) + case code <= 64: + line += int(code) + case code <= 128: + line -= int(code - 64) + default: + pc += quantum * uint64(code-128) + continue + } + pc += quantum + } + return b, pc, line +} + +func (t *LineTable) slice(pc uint64) *LineTable { + data, pc, line := t.parse(pc, -1) + return &LineTable{data, pc, line} +} + +func (t *LineTable) PCToLine(pc uint64) int { + _, _, line := t.parse(pc, -1) + return line +} + +func (t *LineTable) LineToPC(line int, maxpc uint64) uint64 { + _, pc, line1 := t.parse(maxpc, line) + if line1 != line { + return 0 + } + // Subtract quantum from PC to account for post-line increment + return pc - quantum +} + +// NewLineTable returns a new PC/line table +// corresponding to the encoded data. +// Text must be the start address of the +// corresponding text segment. +func NewLineTable(data []byte, text uint64) *LineTable { + return &LineTable{data, text, 0} +} diff --git a/src/pkg/debug/gosym/pclntab_test.go b/src/pkg/debug/gosym/pclntab_test.go new file mode 100644 index 000000000..c83e64eab --- /dev/null +++ b/src/pkg/debug/gosym/pclntab_test.go @@ -0,0 +1,204 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gosym + +import ( + "debug/elf" + "os" + "testing" + "syscall" +) + +func dotest() bool { + // For now, only works on ELF platforms. + return syscall.OS == "linux" && os.Getenv("GOARCH") == "amd64" +} + +func getTable(t *testing.T) *Table { + f, tab := crack(os.Args[0], t) + f.Close() + return tab +} + +func crack(file string, t *testing.T) (*elf.File, *Table) { + // Open self + f, err := elf.Open(file) + if err != nil { + t.Fatal(err) + } + return parse(file, f, t) +} + +func parse(file string, f *elf.File, t *testing.T) (*elf.File, *Table) { + symdat, err := f.Section(".gosymtab").Data() + if err != nil { + f.Close() + t.Fatalf("reading %s gosymtab: %v", file, err) + } + pclndat, err := f.Section(".gopclntab").Data() + if err != nil { + f.Close() + t.Fatalf("reading %s gopclntab: %v", file, err) + } + + pcln := NewLineTable(pclndat, f.Section(".text").Addr) + tab, err := NewTable(symdat, pcln) + if err != nil { + f.Close() + t.Fatalf("parsing %s gosymtab: %v", file, err) + } + + return f, tab +} + +var goarch = os.Getenv("O") + +func TestLineFromAline(t *testing.T) { + if !dotest() { + return + } + + tab := getTable(t) + + // Find the sym package + pkg := tab.LookupFunc("debug/gosym.TestLineFromAline").Obj + if pkg == nil { + t.Fatalf("nil pkg") + } + + // Walk every absolute line and ensure that we hit every + // source line monotonically + lastline := make(map[string]int) + final := -1 + for i := 0; i < 10000; i++ { + path, line := pkg.lineFromAline(i) + // Check for end of object + if path == "" { + if final == -1 { + final = i - 1 + } + continue + } else if final != -1 { + t.Fatalf("reached end of package at absolute line %d, but absolute line %d mapped to %s:%d", final, i, path, line) + } + // It's okay to see files multiple times (e.g., sys.a) + if line == 1 { + lastline[path] = 1 + continue + } + // Check that the is the next line in path + ll, ok := lastline[path] + if !ok { + t.Errorf("file %s starts on line %d", path, line) + } else if line != ll+1 { + t.Errorf("expected next line of file %s to be %d, got %d", path, ll+1, line) + } + lastline[path] = line + } + if final == -1 { + t.Errorf("never reached end of object") + } +} + +func TestLineAline(t *testing.T) { + if !dotest() { + return + } + + tab := getTable(t) + + for _, o := range tab.Files { + // A source file can appear multiple times in a + // object. alineFromLine will always return alines in + // the first file, so track which lines we've seen. + found := make(map[string]int) + for i := 0; i < 1000; i++ { + path, line := o.lineFromAline(i) + if path == "" { + break + } + + // cgo files are full of 'Z' symbols, which we don't handle + if len(path) > 4 && path[len(path)-4:] == ".cgo" { + continue + } + + if minline, ok := found[path]; path != "" && ok { + if minline >= line { + // We've already covered this file + continue + } + } + found[path] = line + + a, err := o.alineFromLine(path, line) + if err != nil { + t.Errorf("absolute line %d in object %s maps to %s:%d, but mapping that back gives error %s", i, o.Paths[0].Name, path, line, err) + } else if a != i { + t.Errorf("absolute line %d in object %s maps to %s:%d, which maps back to absolute line %d\n", i, o.Paths[0].Name, path, line, a) + } + } + } +} + +func TestPCLine(t *testing.T) { + if !dotest() { + return + } + + f, tab := crack("_test/pclinetest", t) + text := f.Section(".text") + textdat, err := text.Data() + if err != nil { + t.Fatalf("reading .text: %v", err) + } + + // Test PCToLine + sym := tab.LookupFunc("linefrompc") + wantLine := 0 + for pc := sym.Entry; pc < sym.End; pc++ { + file, line, fn := tab.PCToLine(pc) + off := pc - text.Addr // TODO(rsc): should not need off; bug in 8g + wantLine += int(textdat[off]) + if fn == nil { + t.Errorf("failed to get line of PC %#x", pc) + } else if len(file) < 12 || file[len(file)-12:] != "pclinetest.s" || line != wantLine || fn != sym { + t.Errorf("expected %s:%d (%s) at PC %#x, got %s:%d (%s)", "pclinetest.s", wantLine, sym.Name, pc, file, line, fn.Name) + } + } + + // Test LineToPC + sym = tab.LookupFunc("pcfromline") + lookupline := -1 + wantLine = 0 + off := uint64(0) // TODO(rsc): should not need off; bug in 8g + for pc := sym.Value; pc < sym.End; pc += 2 + uint64(textdat[off]) { + file, line, fn := tab.PCToLine(pc) + off = pc - text.Addr + wantLine += int(textdat[off]) + if line != wantLine { + t.Errorf("expected line %d at PC %#x in pcfromline, got %d", wantLine, pc, line) + off = pc + 1 - text.Addr + continue + } + if lookupline == -1 { + lookupline = line + } + for ; lookupline <= line; lookupline++ { + pc2, fn2, err := tab.LineToPC(file, lookupline) + if lookupline != line { + // Should be nothing on this line + if err == nil { + t.Errorf("expected no PC at line %d, got %#x (%s)", lookupline, pc2, fn2.Name) + } + } else if err != nil { + t.Errorf("failed to get PC of line %d: %s", lookupline, err) + } else if pc != pc2 { + t.Errorf("expected PC %#x (%s) at line %d, got PC %#x (%s)", pc, fn.Name, line, pc2, fn2.Name) + } + } + off = pc + 1 - text.Addr + } +} diff --git a/src/pkg/debug/gosym/symtab.go b/src/pkg/debug/gosym/symtab.go new file mode 100644 index 000000000..dea460d71 --- /dev/null +++ b/src/pkg/debug/gosym/symtab.go @@ -0,0 +1,548 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gosym implements access to the Go symbol +// and line number tables embedded in Go binaries generated +// by the gc compilers. +package gosym + +// The table format is a variant of the format used in Plan 9's a.out +// format, documented at http://plan9.bell-labs.com/magic/man2html/6/a.out. +// The best reference for the differences between the Plan 9 format +// and the Go format is the runtime source, specifically ../../runtime/symtab.c. + +import ( + "encoding/binary" + "fmt" + "os" + "strconv" + "strings" +) + +/* + * Symbols + */ + +// A Sym represents a single symbol table entry. +type Sym struct { + Value uint64 + Type byte + Name string + GoType uint64 + // If this symbol if a function symbol, the corresponding Func + Func *Func +} + +// Static returns whether this symbol is static (not visible outside its file). +func (s *Sym) Static() bool { return s.Type >= 'a' } + +// PackageName returns the package part of the symbol name, +// or the empty string if there is none. +func (s *Sym) PackageName() string { + if i := strings.Index(s.Name, "."); i != -1 { + return s.Name[0:i] + } + return "" +} + +// ReceiverName returns the receiver type name of this symbol, +// or the empty string if there is none. +func (s *Sym) ReceiverName() string { + l := strings.Index(s.Name, ".") + r := strings.LastIndex(s.Name, ".") + if l == -1 || r == -1 || l == r { + return "" + } + return s.Name[l+1 : r] +} + +// BaseName returns the symbol name without the package or receiver name. +func (s *Sym) BaseName() string { + if i := strings.LastIndex(s.Name, "."); i != -1 { + return s.Name[i+1:] + } + return s.Name +} + +// A Func collects information about a single function. +type Func struct { + Entry uint64 + *Sym + End uint64 + Params []*Sym + Locals []*Sym + FrameSize int + LineTable *LineTable + Obj *Obj +} + +// An Obj represents a single object file. +type Obj struct { + Funcs []Func + Paths []Sym +} + +/* + * Symbol tables + */ + +// Table represents a Go symbol table. It stores all of the +// symbols decoded from the program and provides methods to translate +// between symbols, names, and addresses. +type Table struct { + Syms []Sym + Funcs []Func + Files map[string]*Obj + Objs []Obj + // textEnd uint64; +} + +type sym struct { + value uint32 + gotype uint32 + typ byte + name []byte +} + +func walksymtab(data []byte, fn func(sym) os.Error) os.Error { + var s sym + p := data + for len(p) >= 6 { + s.value = binary.BigEndian.Uint32(p[0:4]) + typ := p[4] + if typ&0x80 == 0 { + return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ} + } + typ &^= 0x80 + s.typ = typ + p = p[5:] + var i int + var nnul int + for i = 0; i < len(p); i++ { + if p[i] == 0 { + nnul = 1 + break + } + } + switch typ { + case 'z', 'Z': + p = p[i+nnul:] + for i = 0; i+2 <= len(p); i += 2 { + if p[i] == 0 && p[i+1] == 0 { + nnul = 2 + break + } + } + } + if i+nnul+4 > len(p) { + return &DecodingError{len(data), "unexpected EOF", nil} + } + s.name = p[0:i] + i += nnul + s.gotype = binary.BigEndian.Uint32(p[i : i+4]) + p = p[i+4:] + fn(s) + } + return nil +} + +// NewTable decodes the Go symbol table in data, +// returning an in-memory representation. +func NewTable(symtab []byte, pcln *LineTable) (*Table, os.Error) { + var n int + err := walksymtab(symtab, func(s sym) os.Error { + n++ + return nil + }) + if err != nil { + return nil, err + } + + var t Table + fname := make(map[uint16]string) + t.Syms = make([]Sym, 0, n) + nf := 0 + nz := 0 + lasttyp := uint8(0) + err = walksymtab(symtab, func(s sym) os.Error { + n := len(t.Syms) + t.Syms = t.Syms[0 : n+1] + ts := &t.Syms[n] + ts.Type = s.typ + ts.Value = uint64(s.value) + ts.GoType = uint64(s.gotype) + switch s.typ { + default: + // rewrite name to use . instead of · (c2 b7) + w := 0 + b := s.name + for i := 0; i < len(b); i++ { + if b[i] == 0xc2 && i+1 < len(b) && b[i+1] == 0xb7 { + i++ + b[i] = '.' + } + b[w] = b[i] + w++ + } + ts.Name = string(s.name[0:w]) + case 'z', 'Z': + if lasttyp != 'z' && lasttyp != 'Z' { + nz++ + } + for i := 0; i < len(s.name); i += 2 { + eltIdx := binary.BigEndian.Uint16(s.name[i : i+2]) + elt, ok := fname[eltIdx] + if !ok { + return &DecodingError{-1, "bad filename code", eltIdx} + } + if n := len(ts.Name); n > 0 && ts.Name[n-1] != '/' { + ts.Name += "/" + } + ts.Name += elt + } + } + switch s.typ { + case 'T', 't', 'L', 'l': + nf++ + case 'f': + fname[uint16(s.value)] = ts.Name + } + lasttyp = s.typ + return nil + }) + if err != nil { + return nil, err + } + + t.Funcs = make([]Func, 0, nf) + t.Objs = make([]Obj, 0, nz) + t.Files = make(map[string]*Obj) + + // Count text symbols and attach frame sizes, parameters, and + // locals to them. Also, find object file boundaries. + var obj *Obj + lastf := 0 + for i := 0; i < len(t.Syms); i++ { + sym := &t.Syms[i] + switch sym.Type { + case 'Z', 'z': // path symbol + // Finish the current object + if obj != nil { + obj.Funcs = t.Funcs[lastf:] + } + lastf = len(t.Funcs) + + // Start new object + n := len(t.Objs) + t.Objs = t.Objs[0 : n+1] + obj = &t.Objs[n] + + // Count & copy path symbols + var end int + for end = i + 1; end < len(t.Syms); end++ { + if c := t.Syms[end].Type; c != 'Z' && c != 'z' { + break + } + } + obj.Paths = t.Syms[i:end] + i = end - 1 // loop will i++ + + // Record file names + depth := 0 + for j := range obj.Paths { + s := &obj.Paths[j] + if s.Name == "" { + depth-- + } else { + if depth == 0 { + t.Files[s.Name] = obj + } + depth++ + } + } + + case 'T', 't', 'L', 'l': // text symbol + if n := len(t.Funcs); n > 0 { + t.Funcs[n-1].End = sym.Value + } + if sym.Name == "etext" { + continue + } + + // Count parameter and local (auto) syms + var np, na int + var end int + countloop: + for end = i + 1; end < len(t.Syms); end++ { + switch t.Syms[end].Type { + case 'T', 't', 'L', 'l', 'Z', 'z': + break countloop + case 'p': + np++ + case 'a': + na++ + } + } + + // Fill in the function symbol + n := len(t.Funcs) + t.Funcs = t.Funcs[0 : n+1] + fn := &t.Funcs[n] + sym.Func = fn + fn.Params = make([]*Sym, 0, np) + fn.Locals = make([]*Sym, 0, na) + fn.Sym = sym + fn.Entry = sym.Value + fn.Obj = obj + if pcln != nil { + fn.LineTable = pcln.slice(fn.Entry) + pcln = fn.LineTable + } + for j := i; j < end; j++ { + s := &t.Syms[j] + switch s.Type { + case 'm': + fn.FrameSize = int(s.Value) + case 'p': + n := len(fn.Params) + fn.Params = fn.Params[0 : n+1] + fn.Params[n] = s + case 'a': + n := len(fn.Locals) + fn.Locals = fn.Locals[0 : n+1] + fn.Locals[n] = s + } + } + i = end - 1 // loop will i++ + } + } + if obj != nil { + obj.Funcs = t.Funcs[lastf:] + } + return &t, nil +} + +// PCToFunc returns the function containing the program counter pc, +// or nil if there is no such function. +func (t *Table) PCToFunc(pc uint64) *Func { + funcs := t.Funcs + for len(funcs) > 0 { + m := len(funcs) / 2 + fn := &funcs[m] + switch { + case pc < fn.Entry: + funcs = funcs[0:m] + case fn.Entry <= pc && pc < fn.End: + return fn + default: + funcs = funcs[m+1:] + } + } + return nil +} + +// PCToLine looks up line number information for a program counter. +// If there is no information, it returns fn == nil. +func (t *Table) PCToLine(pc uint64) (file string, line int, fn *Func) { + if fn = t.PCToFunc(pc); fn == nil { + return + } + file, line = fn.Obj.lineFromAline(fn.LineTable.PCToLine(pc)) + return +} + +// LineToPC looks up the first program counter on the given line in +// the named file. Returns UnknownPathError or UnknownLineError if +// there is an error looking up this line. +func (t *Table) LineToPC(file string, line int) (pc uint64, fn *Func, err os.Error) { + obj, ok := t.Files[file] + if !ok { + return 0, nil, UnknownFileError(file) + } + abs, err := obj.alineFromLine(file, line) + if err != nil { + return + } + for i := range obj.Funcs { + f := &obj.Funcs[i] + pc := f.LineTable.LineToPC(abs, f.End) + if pc != 0 { + return pc, f, nil + } + } + return 0, nil, &UnknownLineError{file, line} +} + +// LookupSym returns the text, data, or bss symbol with the given name, +// or nil if no such symbol is found. +func (t *Table) LookupSym(name string) *Sym { + // TODO(austin) Maybe make a map + for i := range t.Syms { + s := &t.Syms[i] + switch s.Type { + case 'T', 't', 'L', 'l', 'D', 'd', 'B', 'b': + if s.Name == name { + return s + } + } + } + return nil +} + +// LookupFunc returns the text, data, or bss symbol with the given name, +// or nil if no such symbol is found. +func (t *Table) LookupFunc(name string) *Func { + for i := range t.Funcs { + f := &t.Funcs[i] + if f.Sym.Name == name { + return f + } + } + return nil +} + +// SymByAddr returns the text, data, or bss symbol starting at the given address. +// TODO(rsc): Allow lookup by any address within the symbol. +func (t *Table) SymByAddr(addr uint64) *Sym { + // TODO(austin) Maybe make a map + for i := range t.Syms { + s := &t.Syms[i] + switch s.Type { + case 'T', 't', 'L', 'l', 'D', 'd', 'B', 'b': + if s.Value == addr { + return s + } + } + } + return nil +} + +/* + * Object files + */ + +func (o *Obj) lineFromAline(aline int) (string, int) { + type stackEnt struct { + path string + start int + offset int + prev *stackEnt + } + + noPath := &stackEnt{"", 0, 0, nil} + tos := noPath + + // TODO(austin) I have no idea how 'Z' symbols work, except + // that they pop the stack. +pathloop: + for _, s := range o.Paths { + val := int(s.Value) + switch { + case val > aline: + break pathloop + + case val == 1: + // Start a new stack + tos = &stackEnt{s.Name, val, 0, noPath} + + case s.Name == "": + // Pop + if tos == noPath { + return "", 0 + } + tos.prev.offset += val - tos.start + tos = tos.prev + + default: + // Push + tos = &stackEnt{s.Name, val, 0, tos} + } + } + + if tos == noPath { + return "", 0 + } + return tos.path, aline - tos.start - tos.offset + 1 +} + +func (o *Obj) alineFromLine(path string, line int) (int, os.Error) { + if line < 1 { + return 0, &UnknownLineError{path, line} + } + + for i, s := range o.Paths { + // Find this path + if s.Name != path { + continue + } + + // Find this line at this stack level + depth := 0 + var incstart int + line += int(s.Value) + pathloop: + for _, s := range o.Paths[i:] { + val := int(s.Value) + switch { + case depth == 1 && val >= line: + return line - 1, nil + + case s.Name == "": + depth-- + if depth == 0 { + break pathloop + } else if depth == 1 { + line += val - incstart + } + + default: + if depth == 1 { + incstart = val + } + depth++ + } + } + return 0, &UnknownLineError{path, line} + } + return 0, UnknownFileError(path) +} + +/* + * Errors + */ + +// UnknownFileError represents a failure to find the specific file in +// the symbol table. +type UnknownFileError string + +func (e UnknownFileError) String() string { return "unknown file: " + string(e) } + +// UnknownLineError represents a failure to map a line to a program +// counter, either because the line is beyond the bounds of the file +// or because there is no code on the given line. +type UnknownLineError struct { + File string + Line int +} + +func (e *UnknownLineError) String() string { + return "no code at " + e.File + ":" + strconv.Itoa(e.Line) +} + +// DecodingError represents an error during the decoding of +// the symbol table. +type DecodingError struct { + off int + msg string + val interface{} +} + +func (e *DecodingError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" at byte %#x", e.off) + return msg +} diff --git a/src/pkg/debug/macho/Makefile b/src/pkg/debug/macho/Makefile new file mode 100644 index 000000000..5fbbf1efe --- /dev/null +++ b/src/pkg/debug/macho/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/macho +GOFILES=\ + macho.go\ + file.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/macho/file.go b/src/pkg/debug/macho/file.go new file mode 100644 index 000000000..721a4c416 --- /dev/null +++ b/src/pkg/debug/macho/file.go @@ -0,0 +1,517 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package macho implements access to Mach-O object files, as defined by +// http://developer.apple.com/mac/library/documentation/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html. +package macho + +// High level access to low level data structures. + +import ( + "bytes" + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" +) + +// A File represents an open Mach-O file. +type File struct { + FileHeader + ByteOrder binary.ByteOrder + Loads []Load + Sections []*Section + + Symtab *Symtab + Dysymtab *Dysymtab + + closer io.Closer +} + +// A Load represents any Mach-O load command. +type Load interface { + Raw() []byte +} + +// A LoadBytes is the uninterpreted bytes of a Mach-O load command. +type LoadBytes []byte + +func (b LoadBytes) Raw() []byte { return b } + +// A SegmentHeader is the header for a Mach-O 32-bit or 64-bit load segment command. +type SegmentHeader struct { + Cmd LoadCmd + Len uint32 + Name string + Addr uint64 + Memsz uint64 + Offset uint64 + Filesz uint64 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A Segment represents a Mach-O 32-bit or 64-bit load segment command. +type Segment struct { + LoadBytes + SegmentHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the segment. +func (s *Segment) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the segment. +func (s *Segment) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +type SectionHeader struct { + Name string + Seg string + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 +} + +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +// Data reads and returns the contents of the Mach-O section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the Mach-O section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +// A Dylib represents a Mach-O load dynamic library command. +type Dylib struct { + LoadBytes + Name string + Time uint32 + CurrentVersion uint32 + CompatVersion uint32 +} + +// A Symtab represents a Mach-O symbol table command. +type Symtab struct { + LoadBytes + SymtabCmd + Syms []Symbol +} + +// A Dysymtab represents a Mach-O dynamic symbol table command. +type Dysymtab struct { + LoadBytes + DysymtabCmd + IndirectSyms []uint32 // indices into Symtab.Syms +} + +/* + * Mach-O reader + */ + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as a Mach-O binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// NewFile creates a new File for accessing a Mach-O binary in an underlying reader. +// The Mach-O binary is expected to start at position 0 in the ReaderAt. +func NewFile(r io.ReaderAt) (*File, os.Error) { + f := new(File) + sr := io.NewSectionReader(r, 0, 1<<63-1) + + // Read and decode Mach magic to determine byte order, size. + // Magic32 and Magic64 differ only in the bottom bit. + var ident [4]byte + if _, err := r.ReadAt(ident[0:], 0); err != nil { + return nil, err + } + be := binary.BigEndian.Uint32(ident[0:]) + le := binary.LittleEndian.Uint32(ident[0:]) + switch Magic32 &^ 1 { + case be &^ 1: + f.ByteOrder = binary.BigEndian + f.Magic = be + case le &^ 1: + f.ByteOrder = binary.LittleEndian + f.Magic = le + default: + return nil, &FormatError{0, "invalid magic number", nil} + } + + // Read entire file header. + if err := binary.Read(sr, f.ByteOrder, &f.FileHeader); err != nil { + return nil, err + } + + // Then load commands. + offset := int64(fileHeaderSize32) + if f.Magic == Magic64 { + offset = fileHeaderSize64 + } + dat := make([]byte, f.Cmdsz) + if _, err := r.ReadAt(dat, offset); err != nil { + return nil, err + } + f.Loads = make([]Load, f.Ncmd) + bo := f.ByteOrder + for i := range f.Loads { + // Each load command begins with uint32 command and length. + if len(dat) < 8 { + return nil, &FormatError{offset, "command block too small", nil} + } + cmd, siz := LoadCmd(bo.Uint32(dat[0:4])), bo.Uint32(dat[4:8]) + if siz < 8 || siz > uint32(len(dat)) { + return nil, &FormatError{offset, "invalid command block size", nil} + } + var cmddat []byte + cmddat, dat = dat[0:siz], dat[siz:] + offset += int64(siz) + var s *Segment + switch cmd { + default: + f.Loads[i] = LoadBytes(cmddat) + + case LoadCmdDylib: + var hdr DylibCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + l := new(Dylib) + if hdr.Name >= uint32(len(cmddat)) { + return nil, &FormatError{offset, "invalid name in dynamic library command", hdr.Name} + } + l.Name = cstring(cmddat[hdr.Name:]) + l.Time = hdr.Time + l.CurrentVersion = hdr.CurrentVersion + l.CompatVersion = hdr.CompatVersion + l.LoadBytes = LoadBytes(cmddat) + f.Loads[i] = l + + case LoadCmdSymtab: + var hdr SymtabCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + strtab := make([]byte, hdr.Strsize) + if _, err := r.ReadAt(strtab, int64(hdr.Stroff)); err != nil { + return nil, err + } + var symsz int + if f.Magic == Magic64 { + symsz = 16 + } else { + symsz = 12 + } + symdat := make([]byte, int(hdr.Nsyms)*symsz) + if _, err := r.ReadAt(symdat, int64(hdr.Symoff)); err != nil { + return nil, err + } + st, err := f.parseSymtab(symdat, strtab, cmddat, &hdr, offset) + if err != nil { + return nil, err + } + f.Loads[i] = st + f.Symtab = st + + case LoadCmdDysymtab: + var hdr DysymtabCmd + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &hdr); err != nil { + return nil, err + } + dat := make([]byte, hdr.Nindirectsyms*4) + if _, err := r.ReadAt(dat, int64(hdr.Indirectsymoff)); err != nil { + return nil, err + } + x := make([]uint32, hdr.Nindirectsyms) + if err := binary.Read(bytes.NewBuffer(dat), bo, x); err != nil { + return nil, err + } + st := new(Dysymtab) + st.LoadBytes = LoadBytes(cmddat) + st.DysymtabCmd = hdr + st.IndirectSyms = x + f.Loads[i] = st + f.Dysymtab = st + + case LoadCmdSegment: + var seg32 Segment32 + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &seg32); err != nil { + return nil, err + } + s = new(Segment) + s.LoadBytes = cmddat + s.Cmd = cmd + s.Len = siz + s.Name = cstring(seg32.Name[0:]) + s.Addr = uint64(seg32.Addr) + s.Memsz = uint64(seg32.Memsz) + s.Offset = uint64(seg32.Offset) + s.Filesz = uint64(seg32.Filesz) + s.Maxprot = seg32.Maxprot + s.Prot = seg32.Prot + s.Nsect = seg32.Nsect + s.Flag = seg32.Flag + f.Loads[i] = s + for i := 0; i < int(s.Nsect); i++ { + var sh32 Section32 + if err := binary.Read(b, bo, &sh32); err != nil { + return nil, err + } + sh := new(Section) + sh.Name = cstring(sh32.Name[0:]) + sh.Seg = cstring(sh32.Seg[0:]) + sh.Addr = uint64(sh32.Addr) + sh.Size = uint64(sh32.Size) + sh.Offset = sh32.Offset + sh.Align = sh32.Align + sh.Reloff = sh32.Reloff + sh.Nreloc = sh32.Nreloc + sh.Flags = sh32.Flags + f.pushSection(sh, r) + } + + case LoadCmdSegment64: + var seg64 Segment64 + b := bytes.NewBuffer(cmddat) + if err := binary.Read(b, bo, &seg64); err != nil { + return nil, err + } + s = new(Segment) + s.LoadBytes = cmddat + s.Cmd = cmd + s.Len = siz + s.Name = cstring(seg64.Name[0:]) + s.Addr = seg64.Addr + s.Memsz = seg64.Memsz + s.Offset = seg64.Offset + s.Filesz = seg64.Filesz + s.Maxprot = seg64.Maxprot + s.Prot = seg64.Prot + s.Nsect = seg64.Nsect + s.Flag = seg64.Flag + f.Loads[i] = s + for i := 0; i < int(s.Nsect); i++ { + var sh64 Section64 + if err := binary.Read(b, bo, &sh64); err != nil { + return nil, err + } + sh := new(Section) + sh.Name = cstring(sh64.Name[0:]) + sh.Seg = cstring(sh64.Seg[0:]) + sh.Addr = sh64.Addr + sh.Size = sh64.Size + sh.Offset = sh64.Offset + sh.Align = sh64.Align + sh.Reloff = sh64.Reloff + sh.Nreloc = sh64.Nreloc + sh.Flags = sh64.Flags + f.pushSection(sh, r) + } + } + if s != nil { + s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.Filesz)) + s.ReaderAt = s.sr + } + } + return f, nil +} + +func (f *File) parseSymtab(symdat, strtab, cmddat []byte, hdr *SymtabCmd, offset int64) (*Symtab, os.Error) { + bo := f.ByteOrder + symtab := make([]Symbol, hdr.Nsyms) + b := bytes.NewBuffer(symdat) + for i := range symtab { + var n Nlist64 + if f.Magic == Magic64 { + if err := binary.Read(b, bo, &n); err != nil { + return nil, err + } + } else { + var n32 Nlist32 + if err := binary.Read(b, bo, &n32); err != nil { + return nil, err + } + n.Name = n32.Name + n.Type = n32.Type + n.Sect = n32.Sect + n.Desc = n32.Desc + n.Value = uint64(n32.Value) + } + sym := &symtab[i] + if n.Name >= uint32(len(strtab)) { + return nil, &FormatError{offset, "invalid name in symbol table", n.Name} + } + sym.Name = cstring(strtab[n.Name:]) + sym.Type = n.Type + sym.Sect = n.Sect + sym.Desc = n.Desc + sym.Value = n.Value + } + st := new(Symtab) + st.LoadBytes = LoadBytes(cmddat) + st.Syms = symtab + return st, nil +} + +func (f *File) pushSection(sh *Section, r io.ReaderAt) { + f.Sections = append(f.Sections, sh) + sh.sr = io.NewSectionReader(r, int64(sh.Offset), int64(sh.Size)) + sh.ReaderAt = sh.sr +} + +func cstring(b []byte) string { + var i int + for i = 0; i < len(b) && b[i] != 0; i++ { + } + return string(b[0:i]) +} + +// Segment returns the first Segment with the given name, or nil if no such segment exists. +func (f *File) Segment(name string) *Segment { + for _, l := range f.Loads { + if s, ok := l.(*Segment); ok && s.Name == name { + return s + } + } + return nil +} + +// Section returns the first section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +// DWARF returns the DWARF debug information for the Mach-O file. +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = "__debug_" + name + s := f.Section(name) + if s == nil { + return nil, os.NewError("missing Mach-O section " + name) + } + b, err := s.Data() + if err != nil && uint64(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +func (f *File) ImportedSymbols() ([]string, os.Error) { + if f.Dysymtab == nil || f.Symtab == nil { + return nil, &FormatError{0, "missing symbol table", nil} + } + + st := f.Symtab + dt := f.Dysymtab + var all []string + for _, s := range st.Syms[dt.Iundefsym : dt.Iundefsym+dt.Nundefsym] { + all = append(all, s.Name) + } + return all, nil +} + +// ImportedLibraries returns the paths of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + var all []string + for _, l := range f.Loads { + if lib, ok := l.(*Dylib); ok { + all = append(all, lib.Name) + } + } + return all, nil +} diff --git a/src/pkg/debug/macho/file_test.go b/src/pkg/debug/macho/file_test.go new file mode 100644 index 000000000..56d8a20be --- /dev/null +++ b/src/pkg/debug/macho/file_test.go @@ -0,0 +1,167 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package macho + +import ( + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + segments []*SegmentHeader + sections []*SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-darwin-exec", + FileHeader{0xfeedface, Cpu386, 0x3, 0x2, 0xc, 0x3c0, 0x85}, + []*SegmentHeader{ + &SegmentHeader{LoadCmdSegment, 0x38, "__PAGEZERO", 0x0, 0x1000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + &SegmentHeader{LoadCmdSegment, 0xc0, "__TEXT", 0x1000, 0x1000, 0x0, 0x1000, 0x7, 0x5, 0x2, 0x0}, + &SegmentHeader{LoadCmdSegment, 0xc0, "__DATA", 0x2000, 0x1000, 0x1000, 0x1000, 0x7, 0x3, 0x2, 0x0}, + &SegmentHeader{LoadCmdSegment, 0x7c, "__IMPORT", 0x3000, 0x1000, 0x2000, 0x1000, 0x7, 0x7, 0x1, 0x0}, + &SegmentHeader{LoadCmdSegment, 0x38, "__LINKEDIT", 0x4000, 0x1000, 0x3000, 0x12c, 0x7, 0x1, 0x0, 0x0}, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x1f68, 0x88, 0xf68, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__cstring", "__TEXT", 0x1ff0, 0xd, 0xff0, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__data", "__DATA", 0x2000, 0x14, 0x1000, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x2014, 0x1c, 0x1014, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__jump_table", "__IMPORT", 0x3000, 0xa, 0x2000, 0x6, 0x0, 0x0, 0x4000008}, + }, + }, + { + "testdata/gcc-amd64-darwin-exec", + FileHeader{0xfeedfacf, CpuAmd64, 0x80000003, 0x2, 0xb, 0x568, 0x85}, + []*SegmentHeader{ + &SegmentHeader{LoadCmdSegment64, 0x48, "__PAGEZERO", 0x0, 0x100000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x1d8, "__TEXT", 0x100000000, 0x1000, 0x0, 0x1000, 0x7, 0x5, 0x5, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x138, "__DATA", 0x100001000, 0x1000, 0x1000, 0x1000, 0x7, 0x3, 0x3, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x48, "__LINKEDIT", 0x100002000, 0x1000, 0x2000, 0x140, 0x7, 0x1, 0x0, 0x0}, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x100000f14, 0x6d, 0xf14, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__symbol_stub1", "__TEXT", 0x100000f81, 0xc, 0xf81, 0x0, 0x0, 0x0, 0x80000408}, + &SectionHeader{"__stub_helper", "__TEXT", 0x100000f90, 0x18, 0xf90, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__cstring", "__TEXT", 0x100000fa8, 0xd, 0xfa8, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__eh_frame", "__TEXT", 0x100000fb8, 0x48, 0xfb8, 0x3, 0x0, 0x0, 0x6000000b}, + &SectionHeader{"__data", "__DATA", 0x100001000, 0x1c, 0x1000, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x100001020, 0x38, 0x1020, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__la_symbol_ptr", "__DATA", 0x100001058, 0x10, 0x1058, 0x2, 0x0, 0x0, 0x7}, + }, + }, + { + "testdata/gcc-amd64-darwin-exec-debug", + FileHeader{0xfeedfacf, CpuAmd64, 0x80000003, 0xa, 0x4, 0x5a0, 0}, + []*SegmentHeader{ + nil, + &SegmentHeader{LoadCmdSegment64, 0x1d8, "__TEXT", 0x100000000, 0x1000, 0x0, 0x0, 0x7, 0x5, 0x5, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x138, "__DATA", 0x100001000, 0x1000, 0x0, 0x0, 0x7, 0x3, 0x3, 0x0}, + &SegmentHeader{LoadCmdSegment64, 0x278, "__DWARF", 0x100002000, 0x1000, 0x1000, 0x1bc, 0x7, 0x3, 0x7, 0x0}, + }, + []*SectionHeader{ + &SectionHeader{"__text", "__TEXT", 0x100000f14, 0x0, 0x0, 0x2, 0x0, 0x0, 0x80000400}, + &SectionHeader{"__symbol_stub1", "__TEXT", 0x100000f81, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80000408}, + &SectionHeader{"__stub_helper", "__TEXT", 0x100000f90, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0}, + &SectionHeader{"__cstring", "__TEXT", 0x100000fa8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, + &SectionHeader{"__eh_frame", "__TEXT", 0x100000fb8, 0x0, 0x0, 0x3, 0x0, 0x0, 0x6000000b}, + &SectionHeader{"__data", "__DATA", 0x100001000, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__dyld", "__DATA", 0x100001020, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0}, + &SectionHeader{"__la_symbol_ptr", "__DATA", 0x100001058, 0x0, 0x0, 0x2, 0x0, 0x0, 0x7}, + &SectionHeader{"__debug_abbrev", "__DWARF", 0x100002000, 0x36, 0x1000, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_aranges", "__DWARF", 0x100002036, 0x30, 0x1036, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_frame", "__DWARF", 0x100002066, 0x40, 0x1066, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_info", "__DWARF", 0x1000020a6, 0x54, 0x10a6, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_line", "__DWARF", 0x1000020fa, 0x47, 0x10fa, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_pubnames", "__DWARF", 0x100002141, 0x1b, 0x1141, 0x0, 0x0, 0x0, 0x0}, + &SectionHeader{"__debug_str", "__DWARF", 0x10000215c, 0x60, 0x115c, 0x0, 0x0, 0x0, 0x0}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + for i, l := range f.Loads { + if i >= len(tt.segments) { + break + } + sh := tt.segments[i] + s, ok := l.(*Segment) + if sh == nil { + if ok { + t.Errorf("open %s, section %d: skipping %#v\n", tt.file, i, &s.SegmentHeader) + } + continue + } + if !ok { + t.Errorf("open %s, section %d: not *Segment\n", tt.file, i) + continue + } + have := &s.SegmentHeader + want := sh + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, segment %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn := len(tt.segments) + fn := len(f.Loads) + if tn != fn { + t.Errorf("open %s: len(Loads) = %d, want %d", tt.file, fn, tn) + } + + for i, sh := range f.Sections { + if i >= len(tt.sections) { + break + } + have := &sh.SectionHeader + want := tt.sections[i] + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn = len(tt.sections) + fn = len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + + } +} + +func TestOpenFailure(t *testing.T) { + filename := "file.go" // not a Mach-O file + _, err := Open(filename) // don't crash + if err == nil { + t.Errorf("open %s: succeeded unexpectedly", filename) + } +} diff --git a/src/pkg/debug/macho/macho.go b/src/pkg/debug/macho/macho.go new file mode 100644 index 000000000..1386f5acf --- /dev/null +++ b/src/pkg/debug/macho/macho.go @@ -0,0 +1,305 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Mach-O header data structures +// http://developer.apple.com/mac/library/documentation/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html + +package macho + +import "strconv" + +// A FileHeader represents a Mach-O file header. +type FileHeader struct { + Magic uint32 + Cpu Cpu + SubCpu uint32 + Type Type + Ncmd uint32 + Cmdsz uint32 + Flags uint32 +} + +const ( + fileHeaderSize32 = 7 * 4 + fileHeaderSize64 = 8 * 4 +) + +const ( + Magic32 uint32 = 0xfeedface + Magic64 uint32 = 0xfeedfacf +) + +// A Type is a Mach-O file type, either an object or an executable. +type Type uint32 + +const ( + TypeObj Type = 1 + TypeExec Type = 2 +) + +// A Cpu is a Mach-O cpu type. +type Cpu uint32 + +const ( + Cpu386 Cpu = 7 + CpuAmd64 Cpu = Cpu386 + 1<<24 +) + +var cpuStrings = []intName{ + {uint32(Cpu386), "Cpu386"}, + {uint32(CpuAmd64), "CpuAmd64"}, +} + +func (i Cpu) String() string { return stringName(uint32(i), cpuStrings, false) } +func (i Cpu) GoString() string { return stringName(uint32(i), cpuStrings, true) } + +// A LoadCmd is a Mach-O load command. +type LoadCmd uint32 + +const ( + LoadCmdSegment LoadCmd = 1 + LoadCmdSymtab LoadCmd = 2 + LoadCmdThread LoadCmd = 4 + LoadCmdUnixThread LoadCmd = 5 // thread+stack + LoadCmdDysymtab LoadCmd = 11 + LoadCmdDylib LoadCmd = 12 + LoadCmdDylinker LoadCmd = 15 + LoadCmdSegment64 LoadCmd = 25 +) + +var cmdStrings = []intName{ + {uint32(LoadCmdSegment), "LoadCmdSegment"}, + {uint32(LoadCmdThread), "LoadCmdThread"}, + {uint32(LoadCmdUnixThread), "LoadCmdUnixThread"}, + {uint32(LoadCmdDylib), "LoadCmdDylib"}, + {uint32(LoadCmdSegment64), "LoadCmdSegment64"}, +} + +func (i LoadCmd) String() string { return stringName(uint32(i), cmdStrings, false) } +func (i LoadCmd) GoString() string { return stringName(uint32(i), cmdStrings, true) } + +// A Segment64 is a 64-bit Mach-O segment load command. +type Segment64 struct { + Cmd LoadCmd + Len uint32 + Name [16]byte + Addr uint64 + Memsz uint64 + Offset uint64 + Filesz uint64 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A Segment32 is a 32-bit Mach-O segment load command. +type Segment32 struct { + Cmd LoadCmd + Len uint32 + Name [16]byte + Addr uint32 + Memsz uint32 + Offset uint32 + Filesz uint32 + Maxprot uint32 + Prot uint32 + Nsect uint32 + Flag uint32 +} + +// A DylibCmd is a Mach-O load dynamic library command. +type DylibCmd struct { + Cmd LoadCmd + Len uint32 + Name uint32 + Time uint32 + CurrentVersion uint32 + CompatVersion uint32 +} + +// A Section32 is a 32-bit Mach-O section header. +type Section32 struct { + Name [16]byte + Seg [16]byte + Addr uint32 + Size uint32 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 + Reserve1 uint32 + Reserve2 uint32 +} + +// A Section32 is a 64-bit Mach-O section header. +type Section64 struct { + Name [16]byte + Seg [16]byte + Addr uint64 + Size uint64 + Offset uint32 + Align uint32 + Reloff uint32 + Nreloc uint32 + Flags uint32 + Reserve1 uint32 + Reserve2 uint32 + Reserve3 uint32 +} + +// A SymtabCmd is a Mach-O symbol table command. +type SymtabCmd struct { + Cmd LoadCmd + Len uint32 + Symoff uint32 + Nsyms uint32 + Stroff uint32 + Strsize uint32 +} + +// A DysymtabCmd is a Mach-O dynamic symbol table command. +type DysymtabCmd struct { + Cmd LoadCmd + Len uint32 + Ilocalsym uint32 + Nlocalsym uint32 + Iextdefsym uint32 + Nextdefsym uint32 + Iundefsym uint32 + Nundefsym uint32 + Tocoffset uint32 + Ntoc uint32 + Modtaboff uint32 + Nmodtab uint32 + Extrefsymoff uint32 + Nextrefsyms uint32 + Indirectsymoff uint32 + Nindirectsyms uint32 + Extreloff uint32 + Nextrel uint32 + Locreloff uint32 + Nlocrel uint32 +} + +// An Nlist32 is a Mach-O 32-bit symbol table entry. +type Nlist32 struct { + Name uint32 + Type uint8 + Sect uint8 + Desc uint16 + Value uint32 +} + +// An Nlist64 is a Mach-O 64-bit symbol table entry. +type Nlist64 struct { + Name uint32 + Type uint8 + Sect uint8 + Desc uint16 + Value uint64 +} + +// A Symbol is a Mach-O 32-bit or 64-bit symbol table entry. +type Symbol struct { + Name string + Type uint8 + Sect uint8 + Desc uint16 + Value uint64 +} + +// A Thread is a Mach-O thread state command. +type Thread struct { + Cmd LoadCmd + Len uint32 + Type uint32 + Data []uint32 +} + +// Regs386 is the Mach-O 386 register structure. +type Regs386 struct { + AX uint32 + BX uint32 + CX uint32 + DX uint32 + DI uint32 + SI uint32 + BP uint32 + SP uint32 + SS uint32 + FLAGS uint32 + IP uint32 + CS uint32 + DS uint32 + ES uint32 + FS uint32 + GS uint32 +} + +// RegsAMD64 is the Mach-O AMD64 register structure. +type RegsAMD64 struct { + AX uint64 + BX uint64 + CX uint64 + DX uint64 + DI uint64 + SI uint64 + BP uint64 + SP uint64 + R8 uint64 + R9 uint64 + R10 uint64 + R11 uint64 + R12 uint64 + R13 uint64 + R14 uint64 + R15 uint64 + IP uint64 + FLAGS uint64 + CS uint64 + FS uint64 + GS uint64 +} + +type intName struct { + i uint32 + s string +} + +func stringName(i uint32, names []intName, goSyntax bool) string { + for _, n := range names { + if n.i == i { + if goSyntax { + return "macho." + n.s + } + return n.s + } + } + return strconv.Uitoa64(uint64(i)) +} + +func flagName(i uint32, names []intName, goSyntax bool) string { + s := "" + for _, n := range names { + if n.i&i == n.i { + if len(s) > 0 { + s += "+" + } + if goSyntax { + s += "macho." + } + s += n.s + i -= n.i + } + } + if len(s) == 0 { + return "0x" + strconv.Uitob64(uint64(i), 16) + } + if i != 0 { + s += "+0x" + strconv.Uitob64(uint64(i), 16) + } + return s +} diff --git a/src/pkg/debug/macho/testdata/gcc-386-darwin-exec b/src/pkg/debug/macho/testdata/gcc-386-darwin-exec new file mode 100755 index 000000000..03ba1bafa Binary files /dev/null and b/src/pkg/debug/macho/testdata/gcc-386-darwin-exec differ diff --git a/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec new file mode 100755 index 000000000..5155a5a26 Binary files /dev/null and b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec differ diff --git a/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug new file mode 100644 index 000000000..a47d3aef7 Binary files /dev/null and b/src/pkg/debug/macho/testdata/gcc-amd64-darwin-exec-debug differ diff --git a/src/pkg/debug/macho/testdata/hello.c b/src/pkg/debug/macho/testdata/hello.c new file mode 100644 index 000000000..a689d3644 --- /dev/null +++ b/src/pkg/debug/macho/testdata/hello.c @@ -0,0 +1,8 @@ +#include + +int +main(void) +{ + printf("hello, world\n"); + return 0; +} diff --git a/src/pkg/debug/pe/Makefile b/src/pkg/debug/pe/Makefile new file mode 100644 index 000000000..998e6a418 --- /dev/null +++ b/src/pkg/debug/pe/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=debug/pe +GOFILES=\ + pe.go\ + file.go\ + +include ../../../Make.pkg diff --git a/src/pkg/debug/pe/file.go b/src/pkg/debug/pe/file.go new file mode 100644 index 000000000..d86d916f5 --- /dev/null +++ b/src/pkg/debug/pe/file.go @@ -0,0 +1,315 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pe implements access to PE (Microsoft Windows Portable Executable) files. +package pe + +import ( + "debug/dwarf" + "encoding/binary" + "fmt" + "io" + "os" + "strconv" +) + +// A File represents an open PE file. +type File struct { + FileHeader + Sections []*Section + + closer io.Closer +} + +type SectionHeader struct { + Name string + VirtualSize uint32 + VirtualAddress uint32 + Size uint32 + Offset uint32 + PointerToRelocations uint32 + PointerToLineNumbers uint32 + NumberOfRelocations uint16 + NumberOfLineNumbers uint16 + Characteristics uint32 +} + +type Section struct { + SectionHeader + + // Embed ReaderAt for ReadAt method. + // Do not embed SectionReader directly + // to avoid having Read and Seek. + // If a client wants Read and Seek it must use + // Open() to avoid fighting over the seek offset + // with other clients. + io.ReaderAt + sr *io.SectionReader +} + +type ImportDirectory struct { + OriginalFirstThunk uint32 + TimeDateStamp uint32 + ForwarderChain uint32 + Name uint32 + FirstThunk uint32 + + dll string +} + +// Data reads and returns the contents of the PE section. +func (s *Section) Data() ([]byte, os.Error) { + dat := make([]byte, s.sr.Size()) + n, err := s.sr.ReadAt(dat, 0) + return dat[0:n], err +} + +// Open returns a new ReadSeeker reading the PE section. +func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } + +type FormatError struct { + off int64 + msg string + val interface{} +} + +func (e *FormatError) String() string { + msg := e.msg + if e.val != nil { + msg += fmt.Sprintf(" '%v'", e.val) + } + msg += fmt.Sprintf(" in record at byte %#x", e.off) + return msg +} + +// Open opens the named file using os.Open and prepares it for use as a PE binary. +func Open(name string) (*File, os.Error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + ff, err := NewFile(f) + if err != nil { + f.Close() + return nil, err + } + ff.closer = f + return ff, nil +} + +// Close closes the File. +// If the File was created using NewFile directly instead of Open, +// Close has no effect. +func (f *File) Close() os.Error { + var err os.Error + if f.closer != nil { + err = f.closer.Close() + f.closer = nil + } + return err +} + +// NewFile creates a new File for accessing a PE binary in an underlying reader. +func NewFile(r io.ReaderAt) (*File, os.Error) { + f := new(File) + sr := io.NewSectionReader(r, 0, 1<<63-1) + + var dosheader [96]byte + if _, err := r.ReadAt(dosheader[0:], 0); err != nil { + return nil, err + } + var base int64 + if dosheader[0] == 'M' && dosheader[1] == 'Z' { + var sign [4]byte + r.ReadAt(sign[0:], int64(dosheader[0x3c])) + if !(sign[0] == 'P' && sign[1] == 'E' && sign[2] == 0 && sign[3] == 0) { + return nil, os.NewError("Invalid PE File Format.") + } + base = int64(dosheader[0x3c]) + 4 + } else { + base = int64(0) + } + sr.Seek(base, os.SEEK_SET) + if err := binary.Read(sr, binary.LittleEndian, &f.FileHeader); err != nil { + return nil, err + } + if f.FileHeader.Machine != IMAGE_FILE_MACHINE_UNKNOWN && f.FileHeader.Machine != IMAGE_FILE_MACHINE_AMD64 && f.FileHeader.Machine != IMAGE_FILE_MACHINE_I386 { + return nil, os.NewError("Invalid PE File Format.") + } + // get symbol string table + sr.Seek(int64(f.FileHeader.PointerToSymbolTable+18*f.FileHeader.NumberOfSymbols), os.SEEK_SET) + var l uint32 + if err := binary.Read(sr, binary.LittleEndian, &l); err != nil { + return nil, err + } + ss := make([]byte, l) + if _, err := r.ReadAt(ss, int64(f.FileHeader.PointerToSymbolTable+18*f.FileHeader.NumberOfSymbols)); err != nil { + return nil, err + } + sr.Seek(base, os.SEEK_SET) + binary.Read(sr, binary.LittleEndian, &f.FileHeader) + sr.Seek(int64(f.FileHeader.SizeOfOptionalHeader), os.SEEK_CUR) //Skip OptionalHeader + f.Sections = make([]*Section, f.FileHeader.NumberOfSections) + for i := 0; i < int(f.FileHeader.NumberOfSections); i++ { + sh := new(SectionHeader32) + if err := binary.Read(sr, binary.LittleEndian, sh); err != nil { + return nil, err + } + var name string + if sh.Name[0] == '\x2F' { + si, _ := strconv.Atoi(cstring(sh.Name[1:])) + name, _ = getString(ss, si) + } else { + name = cstring(sh.Name[0:]) + } + s := new(Section) + s.SectionHeader = SectionHeader{ + Name: name, + VirtualSize: uint32(sh.VirtualSize), + VirtualAddress: uint32(sh.VirtualAddress), + Size: uint32(sh.SizeOfRawData), + Offset: uint32(sh.PointerToRawData), + PointerToRelocations: uint32(sh.PointerToRelocations), + PointerToLineNumbers: uint32(sh.PointerToLineNumbers), + NumberOfRelocations: uint16(sh.NumberOfRelocations), + NumberOfLineNumbers: uint16(sh.NumberOfLineNumbers), + Characteristics: uint32(sh.Characteristics), + } + s.sr = io.NewSectionReader(r, int64(s.SectionHeader.Offset), int64(s.SectionHeader.Size)) + s.ReaderAt = s.sr + f.Sections[i] = s + } + return f, nil +} + +func cstring(b []byte) string { + var i int + for i = 0; i < len(b) && b[i] != 0; i++ { + } + return string(b[0:i]) +} + +// getString extracts a string from symbol string table. +func getString(section []byte, start int) (string, bool) { + if start < 0 || start >= len(section) { + return "", false + } + + for end := start; end < len(section); end++ { + if section[end] == 0 { + return string(section[start:end]), true + } + } + return "", false +} + +// Section returns the first section with the given name, or nil if no such +// section exists. +func (f *File) Section(name string) *Section { + for _, s := range f.Sections { + if s.Name == name { + return s + } + } + return nil +} + +func (f *File) DWARF() (*dwarf.Data, os.Error) { + // There are many other DWARF sections, but these + // are the required ones, and the debug/dwarf package + // does not use the others, so don't bother loading them. + var names = [...]string{"abbrev", "info", "str"} + var dat [len(names)][]byte + for i, name := range names { + name = ".debug_" + name + s := f.Section(name) + if s == nil { + continue + } + b, err := s.Data() + if err != nil && uint32(len(b)) < s.Size { + return nil, err + } + dat[i] = b + } + + abbrev, info, str := dat[0], dat[1], dat[2] + return dwarf.New(abbrev, nil, nil, info, nil, nil, nil, str) +} + +// ImportedSymbols returns the names of all symbols +// referred to by the binary f that are expected to be +// satisfied by other libraries at dynamic load time. +// It does not return weak symbols. +func (f *File) ImportedSymbols() ([]string, os.Error) { + pe64 := f.Machine == IMAGE_FILE_MACHINE_AMD64 + ds := f.Section(".idata") + if ds == nil { + // not dynamic, so no libraries + return nil, nil + } + d, err := ds.Data() + if err != nil { + return nil, err + } + var ida []ImportDirectory + for len(d) > 0 { + var dt ImportDirectory + dt.OriginalFirstThunk = binary.LittleEndian.Uint32(d[0:4]) + dt.Name = binary.LittleEndian.Uint32(d[12:16]) + dt.FirstThunk = binary.LittleEndian.Uint32(d[16:20]) + d = d[20:] + if dt.OriginalFirstThunk == 0 { + break + } + ida = append(ida, dt) + } + names, _ := ds.Data() + var all []string + for _, dt := range ida { + dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress)) + d, _ = ds.Data() + // seek to OriginalFirstThunk + d = d[dt.OriginalFirstThunk-ds.VirtualAddress:] + for len(d) > 0 { + if pe64 { // 64bit + va := binary.LittleEndian.Uint64(d[0:8]) + d = d[8:] + if va == 0 { + break + } + if va&0x8000000000000000 > 0 { // is Ordinal + // TODO add dynimport ordinal support. + } else { + fn, _ := getString(names, int(uint32(va)-ds.VirtualAddress+2)) + all = append(all, fn+":"+dt.dll) + } + } else { // 32bit + va := binary.LittleEndian.Uint32(d[0:4]) + d = d[4:] + if va == 0 { + break + } + if va&0x80000000 > 0 { // is Ordinal + // TODO add dynimport ordinal support. + //ord := va&0x0000FFFF + } else { + fn, _ := getString(names, int(va-ds.VirtualAddress+2)) + all = append(all, fn+":"+dt.dll) + } + } + } + } + + return all, nil +} + +// ImportedLibraries returns the names of all libraries +// referred to by the binary f that are expected to be +// linked with the binary at dynamic link time. +func (f *File) ImportedLibraries() ([]string, os.Error) { + // TODO + // cgo -dynimport don't use this for windows PE, so just return. + return nil, nil +} diff --git a/src/pkg/debug/pe/file_test.go b/src/pkg/debug/pe/file_test.go new file mode 100644 index 000000000..2c5c25b8c --- /dev/null +++ b/src/pkg/debug/pe/file_test.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pe + +import ( + "reflect" + "testing" +) + +type fileTest struct { + file string + hdr FileHeader + sections []*SectionHeader +} + +var fileTests = []fileTest{ + { + "testdata/gcc-386-mingw-obj", + FileHeader{0x014c, 0x000c, 0x0, 0x64a, 0x1e, 0x0, 0x104}, + []*SectionHeader{ + &SectionHeader{".text", 0, 0, 36, 500, 1440, 0, 3, 0, 0x60300020}, + &SectionHeader{".data", 0, 0, 0, 0, 0, 0, 0, 0, 3224371264}, + &SectionHeader{".bss", 0, 0, 0, 0, 0, 0, 0, 0, 3224371328}, + &SectionHeader{".debug_abbrev", 0, 0, 137, 536, 0, 0, 0, 0, 0x42100000}, + &SectionHeader{".debug_info", 0, 0, 418, 673, 1470, 0, 7, 0, 1108344832}, + &SectionHeader{".debug_line", 0, 0, 128, 1091, 1540, 0, 1, 0, 1108344832}, + &SectionHeader{".rdata", 0, 0, 16, 1219, 0, 0, 0, 0, 1076887616}, + &SectionHeader{".debug_frame", 0, 0, 52, 1235, 1550, 0, 2, 0, 1110441984}, + &SectionHeader{".debug_loc", 0, 0, 56, 1287, 0, 0, 0, 0, 1108344832}, + &SectionHeader{".debug_pubnames", 0, 0, 27, 1343, 1570, 0, 1, 0, 1108344832}, + &SectionHeader{".debug_pubtypes", 0, 0, 38, 1370, 1580, 0, 1, 0, 1108344832}, + &SectionHeader{".debug_aranges", 0, 0, 32, 1408, 1590, 0, 2, 0, 1108344832}, + }, + }, + { + "testdata/gcc-386-mingw-exec", + FileHeader{0x014c, 0x000f, 0x4c6a1b60, 0x3c00, 0x282, 0xe0, 0x107}, + []*SectionHeader{ + &SectionHeader{Name: ".text", VirtualSize: 0xcd8, VirtualAddress: 0x1000, Size: 0xe00, Offset: 0x400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x60500060}, + &SectionHeader{Name: ".data", VirtualSize: 0x10, VirtualAddress: 0x2000, Size: 0x200, Offset: 0x1200, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".rdata", VirtualSize: 0x120, VirtualAddress: 0x3000, Size: 0x200, Offset: 0x1400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x40300040}, + &SectionHeader{Name: ".bss", VirtualSize: 0xdc, VirtualAddress: 0x4000, Size: 0x0, Offset: 0x0, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0400080}, + &SectionHeader{Name: ".idata", VirtualSize: 0x3c8, VirtualAddress: 0x5000, Size: 0x400, Offset: 0x1600, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".CRT", VirtualSize: 0x18, VirtualAddress: 0x6000, Size: 0x200, Offset: 0x1a00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".tls", VirtualSize: 0x20, VirtualAddress: 0x7000, Size: 0x200, Offset: 0x1c00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0xc0300040}, + &SectionHeader{Name: ".debug_aranges", VirtualSize: 0x20, VirtualAddress: 0x8000, Size: 0x200, Offset: 0x1e00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_pubnames", VirtualSize: 0x51, VirtualAddress: 0x9000, Size: 0x200, Offset: 0x2000, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_pubtypes", VirtualSize: 0x91, VirtualAddress: 0xa000, Size: 0x200, Offset: 0x2200, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_info", VirtualSize: 0xe22, VirtualAddress: 0xb000, Size: 0x1000, Offset: 0x2400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_abbrev", VirtualSize: 0x157, VirtualAddress: 0xc000, Size: 0x200, Offset: 0x3400, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_line", VirtualSize: 0x144, VirtualAddress: 0xd000, Size: 0x200, Offset: 0x3600, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + &SectionHeader{Name: ".debug_frame", VirtualSize: 0x34, VirtualAddress: 0xe000, Size: 0x200, Offset: 0x3800, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42300000}, + &SectionHeader{Name: ".debug_loc", VirtualSize: 0x38, VirtualAddress: 0xf000, Size: 0x200, Offset: 0x3a00, PointerToRelocations: 0x0, PointerToLineNumbers: 0x0, NumberOfRelocations: 0x0, NumberOfLineNumbers: 0x0, Characteristics: 0x42100000}, + }, + }, +} + +func TestOpen(t *testing.T) { + for i := range fileTests { + tt := &fileTests[i] + + f, err := Open(tt.file) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(f.FileHeader, tt.hdr) { + t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) + continue + } + + for i, sh := range f.Sections { + if i >= len(tt.sections) { + break + } + have := &sh.SectionHeader + want := tt.sections[i] + if !reflect.DeepEqual(have, want) { + t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) + } + } + tn := len(tt.sections) + fn := len(f.Sections) + if tn != fn { + t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) + } + + } +} + +func TestOpenFailure(t *testing.T) { + filename := "file.go" // not a PE file + _, err := Open(filename) // don't crash + if err == nil { + t.Errorf("open %s: succeeded unexpectedly", filename) + } +} diff --git a/src/pkg/debug/pe/pe.go b/src/pkg/debug/pe/pe.go new file mode 100644 index 000000000..b3dab739a --- /dev/null +++ b/src/pkg/debug/pe/pe.go @@ -0,0 +1,51 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pe + +type FileHeader struct { + Machine uint16 + NumberOfSections uint16 + TimeDateStamp uint32 + PointerToSymbolTable uint32 + NumberOfSymbols uint32 + SizeOfOptionalHeader uint16 + Characteristics uint16 +} + +type SectionHeader32 struct { + Name [8]uint8 + VirtualSize uint32 + VirtualAddress uint32 + SizeOfRawData uint32 + PointerToRawData uint32 + PointerToRelocations uint32 + PointerToLineNumbers uint32 + NumberOfRelocations uint16 + NumberOfLineNumbers uint16 + Characteristics uint32 +} + +const ( + IMAGE_FILE_MACHINE_UNKNOWN = 0x0 + IMAGE_FILE_MACHINE_AM33 = 0x1d3 + IMAGE_FILE_MACHINE_AMD64 = 0x8664 + IMAGE_FILE_MACHINE_ARM = 0x1c0 + IMAGE_FILE_MACHINE_EBC = 0xebc + IMAGE_FILE_MACHINE_I386 = 0x14c + IMAGE_FILE_MACHINE_IA64 = 0x200 + IMAGE_FILE_MACHINE_M32R = 0x9041 + IMAGE_FILE_MACHINE_MIPS16 = 0x266 + IMAGE_FILE_MACHINE_MIPSFPU = 0x366 + IMAGE_FILE_MACHINE_MIPSFPU16 = 0x466 + IMAGE_FILE_MACHINE_POWERPC = 0x1f0 + IMAGE_FILE_MACHINE_POWERPCFP = 0x1f1 + IMAGE_FILE_MACHINE_R4000 = 0x166 + IMAGE_FILE_MACHINE_SH3 = 0x1a2 + IMAGE_FILE_MACHINE_SH3DSP = 0x1a3 + IMAGE_FILE_MACHINE_SH4 = 0x1a6 + IMAGE_FILE_MACHINE_SH5 = 0x1a8 + IMAGE_FILE_MACHINE_THUMB = 0x1c2 + IMAGE_FILE_MACHINE_WCEMIPSV2 = 0x169 +) diff --git a/src/pkg/debug/pe/testdata/gcc-386-mingw-exec b/src/pkg/debug/pe/testdata/gcc-386-mingw-exec new file mode 100644 index 000000000..4b808d043 Binary files /dev/null and b/src/pkg/debug/pe/testdata/gcc-386-mingw-exec differ diff --git a/src/pkg/debug/pe/testdata/gcc-386-mingw-obj b/src/pkg/debug/pe/testdata/gcc-386-mingw-obj new file mode 100644 index 000000000..0c84d898d Binary files /dev/null and b/src/pkg/debug/pe/testdata/gcc-386-mingw-obj differ diff --git a/src/pkg/debug/pe/testdata/hello.c b/src/pkg/debug/pe/testdata/hello.c new file mode 100644 index 000000000..a689d3644 --- /dev/null +++ b/src/pkg/debug/pe/testdata/hello.c @@ -0,0 +1,8 @@ +#include + +int +main(void) +{ + printf("hello, world\n"); + return 0; +} diff --git a/src/pkg/deps.bash b/src/pkg/deps.bash new file mode 100755 index 000000000..2095ec1d8 --- /dev/null +++ b/src/pkg/deps.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +eval $(gomake --no-print-directory -f ../Make.inc go-env) + +OUT="Make.deps" +TMP="Make.deps.tmp" + +if [ -f $OUT ] && ! [ -w $OUT ]; then + echo "$0: $OUT is read-only; aborting." 1>&2 + exit 1 +fi + +# Get list of directories from Makefile +dirs=$(gomake --no-print-directory echo-dirs) +dirpat=$(echo $dirs C | awk '{ + for(i=1;i<=NF;i++){ + x=$i + gsub("/", "\\/", x) + printf("/^(%s)$/\n", x) + } +}') + +for dir in $dirs; do ( + cd $dir || exit 1 + + sources=$(sed -n 's/^[ ]*\([^ ]*\.go\)[ ]*\\*[ ]*$/\1/p' Makefile) + sources=$(echo $sources | sed 's/\$(GOOS)/'$GOOS'/g') + sources=$(echo $sources | sed 's/\$(GOARCH)/'$GOARCH'/g') + # /dev/null here means we get an empty dependency list if $sources is empty + # instead of listing every file in the directory. + sources=$(ls $sources /dev/null 2> /dev/null) # remove .s, .c, etc. + + deps=$( + sed -n '/^import.*"/p; /^import[ \t]*(/,/^)/p' $sources /dev/null | + cut -d '"' -f2 | + awk "$dirpat" | + grep -v "^$dir\$" | + sed 's/$/.install/' | + sed 's;^C\.install;runtime/cgo.install;' | + sort -u + ) + + echo $dir.install: $deps +) done > $TMP + +mv $TMP $OUT diff --git a/src/pkg/ebnf/Makefile b/src/pkg/ebnf/Makefile new file mode 100644 index 000000000..f5555d272 --- /dev/null +++ b/src/pkg/ebnf/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=ebnf +GOFILES=\ + ebnf.go\ + parser.go\ + +include ../../Make.pkg diff --git a/src/pkg/ebnf/ebnf.go b/src/pkg/ebnf/ebnf.go new file mode 100644 index 000000000..69da11767 --- /dev/null +++ b/src/pkg/ebnf/ebnf.go @@ -0,0 +1,245 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ebnf is a library for EBNF grammars. The input is text ([]byte) +// satisfying the following grammar (represented itself in EBNF): +// +// Production = name "=" [ Expression ] "." . +// Expression = Alternative { "|" Alternative } . +// Alternative = Term { Term } . +// Term = name | token [ "…" token ] | Group | Option | Repetition . +// Group = "(" Expression ")" . +// Option = "[" Expression "]" . +// Repetition = "{" Expression "}" . +// +// A name is a Go identifier, a token is a Go string, and comments +// and white space follow the same rules as for the Go language. +// Production names starting with an uppercase Unicode letter denote +// non-terminal productions (i.e., productions which allow white-space +// and comments between tokens); all other production names denote +// lexical productions. +// +package ebnf + +import ( + "go/scanner" + "go/token" + "os" + "unicode" + "utf8" +) + +// ---------------------------------------------------------------------------- +// Internal representation + +type ( + // An Expression node represents a production expression. + Expression interface { + // Pos is the position of the first character of the syntactic construct + Pos() token.Pos + } + + // An Alternative node represents a non-empty list of alternative expressions. + Alternative []Expression // x | y | z + + // A Sequence node represents a non-empty list of sequential expressions. + Sequence []Expression // x y z + + // A Name node represents a production name. + Name struct { + StringPos token.Pos + String string + } + + // A Token node represents a literal. + Token struct { + StringPos token.Pos + String string + } + + // A List node represents a range of characters. + Range struct { + Begin, End *Token // begin ... end + } + + // A Group node represents a grouped expression. + Group struct { + Lparen token.Pos + Body Expression // (body) + } + + // An Option node represents an optional expression. + Option struct { + Lbrack token.Pos + Body Expression // [body] + } + + // A Repetition node represents a repeated expression. + Repetition struct { + Lbrace token.Pos + Body Expression // {body} + } + + // A Bad node stands for pieces of source code that lead to a parse error. + Bad struct { + TokPos token.Pos + Error string // parser error message + } + + // A Production node represents an EBNF production. + Production struct { + Name *Name + Expr Expression + } + + // A Grammar is a set of EBNF productions. The map + // is indexed by production name. + // + Grammar map[string]*Production +) + +func (x Alternative) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Alternative +func (x Sequence) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Sequences +func (x *Name) Pos() token.Pos { return x.StringPos } +func (x *Token) Pos() token.Pos { return x.StringPos } +func (x *Range) Pos() token.Pos { return x.Begin.Pos() } +func (x *Group) Pos() token.Pos { return x.Lparen } +func (x *Option) Pos() token.Pos { return x.Lbrack } +func (x *Repetition) Pos() token.Pos { return x.Lbrace } +func (x *Bad) Pos() token.Pos { return x.TokPos } +func (x *Production) Pos() token.Pos { return x.Name.Pos() } + +// ---------------------------------------------------------------------------- +// Grammar verification + +func isLexical(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return !unicode.IsUpper(ch) +} + +type verifier struct { + fset *token.FileSet + scanner.ErrorVector + worklist []*Production + reached Grammar // set of productions reached from (and including) the root production + grammar Grammar +} + +func (v *verifier) error(pos token.Pos, msg string) { + v.Error(v.fset.Position(pos), msg) +} + +func (v *verifier) push(prod *Production) { + name := prod.Name.String + if _, found := v.reached[name]; !found { + v.worklist = append(v.worklist, prod) + v.reached[name] = prod + } +} + +func (v *verifier) verifyChar(x *Token) int { + s := x.String + if utf8.RuneCountInString(s) != 1 { + v.error(x.Pos(), "single char expected, found "+s) + return 0 + } + ch, _ := utf8.DecodeRuneInString(s) + return ch +} + +func (v *verifier) verifyExpr(expr Expression, lexical bool) { + switch x := expr.(type) { + case nil: + // empty expression + case Alternative: + for _, e := range x { + v.verifyExpr(e, lexical) + } + case Sequence: + for _, e := range x { + v.verifyExpr(e, lexical) + } + case *Name: + // a production with this name must exist; + // add it to the worklist if not yet processed + if prod, found := v.grammar[x.String]; found { + v.push(prod) + } else { + v.error(x.Pos(), "missing production "+x.String) + } + // within a lexical production references + // to non-lexical productions are invalid + if lexical && !isLexical(x.String) { + v.error(x.Pos(), "reference to non-lexical production "+x.String) + } + case *Token: + // nothing to do for now + case *Range: + i := v.verifyChar(x.Begin) + j := v.verifyChar(x.End) + if i >= j { + v.error(x.Pos(), "decreasing character range") + } + case *Group: + v.verifyExpr(x.Body, lexical) + case *Option: + v.verifyExpr(x.Body, lexical) + case *Repetition: + v.verifyExpr(x.Body, lexical) + default: + panic("unreachable") + } +} + +func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) { + // find root production + root, found := grammar[start] + if !found { + // token.NoPos doesn't require a file set; + // ok to set v.fset only afterwards + v.error(token.NoPos, "no start production "+start) + return + } + + // initialize verifier + v.fset = fset + v.ErrorVector.Reset() + v.worklist = v.worklist[0:0] + v.reached = make(Grammar) + v.grammar = grammar + + // work through the worklist + v.push(root) + for { + n := len(v.worklist) - 1 + if n < 0 { + break + } + prod := v.worklist[n] + v.worklist = v.worklist[0:n] + v.verifyExpr(prod.Expr, isLexical(prod.Name.String)) + } + + // check if all productions were reached + if len(v.reached) < len(v.grammar) { + for name, prod := range v.grammar { + if _, found := v.reached[name]; !found { + v.error(prod.Pos(), name+" is unreachable") + } + } + } +} + +// Verify checks that: +// - all productions used are defined +// - all productions defined are used when beginning at start +// - lexical productions refer only to other lexical productions +// +// Position information is interpreted relative to the file set fset. +// +func Verify(fset *token.FileSet, grammar Grammar, start string) os.Error { + var v verifier + v.verify(fset, grammar, start) + return v.GetError(scanner.Sorted) +} diff --git a/src/pkg/ebnf/ebnf_test.go b/src/pkg/ebnf/ebnf_test.go new file mode 100644 index 000000000..b086facc3 --- /dev/null +++ b/src/pkg/ebnf/ebnf_test.go @@ -0,0 +1,87 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ebnf + +import ( + "go/token" + "io/ioutil" + "testing" +) + +var fset = token.NewFileSet() + +var goodGrammars = []string{ + `Program = .`, + + `Program = foo . + foo = "foo" .`, + + `Program = "a" | "b" "c" .`, + + `Program = "a" … "z" .`, + + `Program = Song . + Song = { Note } . + Note = Do | (Re | Mi | Fa | So | La) | Ti . + Do = "c" . + Re = "d" . + Mi = "e" . + Fa = "f" . + So = "g" . + La = "a" . + Ti = ti . + ti = "b" .`, +} + +var badGrammars = []string{ + `Program = | .`, + `Program = | b .`, + `Program = a … b .`, + `Program = "a" … .`, + `Program = … "b" .`, + `Program = () .`, + `Program = [] .`, + `Program = {} .`, +} + +func checkGood(t *testing.T, filename string, src []byte) { + grammar, err := Parse(fset, filename, src) + if err != nil { + t.Errorf("Parse(%s) failed: %v", src, err) + } + if err = Verify(fset, grammar, "Program"); err != nil { + t.Errorf("Verify(%s) failed: %v", src, err) + } +} + +func checkBad(t *testing.T, filename string, src []byte) { + _, err := Parse(fset, filename, src) + if err == nil { + t.Errorf("Parse(%s) should have failed", src) + } +} + +func TestGrammars(t *testing.T) { + for _, src := range goodGrammars { + checkGood(t, "", []byte(src)) + } + for _, src := range badGrammars { + checkBad(t, "", []byte(src)) + } +} + +var files = []string{ +// TODO(gri) add some test files +} + +func TestFiles(t *testing.T) { + for _, filename := range files { + src, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + checkGood(t, filename, src) + } +} diff --git a/src/pkg/ebnf/parser.go b/src/pkg/ebnf/parser.go new file mode 100644 index 000000000..ef2fac000 --- /dev/null +++ b/src/pkg/ebnf/parser.go @@ -0,0 +1,197 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ebnf + +import ( + "go/scanner" + "go/token" + "os" + "strconv" +) + +type parser struct { + fset *token.FileSet + scanner.ErrorVector + scanner scanner.Scanner + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal +} + +func (p *parser) next() { + p.pos, p.tok, p.lit = p.scanner.Scan() + if p.tok.IsKeyword() { + // TODO Should keyword mapping always happen outside scanner? + // Or should there be a flag to scanner to enable keyword mapping? + p.tok = token.IDENT + } +} + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.fset.Position(pos), msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + +func (p *parser) parseIdentifier() *Name { + pos := p.pos + name := p.lit + p.expect(token.IDENT) + return &Name{pos, name} +} + +func (p *parser) parseToken() *Token { + pos := p.pos + value := "" + if p.tok == token.STRING { + value, _ = strconv.Unquote(p.lit) + // Unquote may fail with an error, but only if the scanner found + // an illegal string in the first place. In this case the error + // has already been reported. + p.next() + } else { + p.expect(token.STRING) + } + return &Token{pos, value} +} + +// ParseTerm returns nil if no term was found. +func (p *parser) parseTerm() (x Expression) { + pos := p.pos + + switch p.tok { + case token.IDENT: + x = p.parseIdentifier() + + case token.STRING: + tok := p.parseToken() + x = tok + const ellipsis = "…" // U+2026, the horizontal ellipsis character + if p.tok == token.ILLEGAL && p.lit == ellipsis { + p.next() + x = &Range{tok, p.parseToken()} + } + + case token.LPAREN: + p.next() + x = &Group{pos, p.parseExpression()} + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + x = &Option{pos, p.parseExpression()} + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + x = &Repetition{pos, p.parseExpression()} + p.expect(token.RBRACE) + } + + return x +} + +func (p *parser) parseSequence() Expression { + var list Sequence + + for x := p.parseTerm(); x != nil; x = p.parseTerm() { + list = append(list, x) + } + + // no need for a sequence if list.Len() < 2 + switch len(list) { + case 0: + p.errorExpected(p.pos, "term") + return &Bad{p.pos, "term expected"} + case 1: + return list[0] + } + + return list +} + +func (p *parser) parseExpression() Expression { + var list Alternative + + for { + list = append(list, p.parseSequence()) + if p.tok != token.OR { + break + } + p.next() + } + // len(list) > 0 + + // no need for an Alternative node if list.Len() < 2 + if len(list) == 1 { + return list[0] + } + + return list +} + +func (p *parser) parseProduction() *Production { + name := p.parseIdentifier() + p.expect(token.ASSIGN) + var expr Expression + if p.tok != token.PERIOD { + expr = p.parseExpression() + } + p.expect(token.PERIOD) + return &Production{name, expr} +} + +func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar { + // initialize parser + p.fset = fset + p.ErrorVector.Reset() + p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, scanner.AllowIllegalChars) + p.next() // initializes pos, tok, lit + + grammar := make(Grammar) + for p.tok != token.EOF { + prod := p.parseProduction() + name := prod.Name.String + if _, found := grammar[name]; !found { + grammar[name] = prod + } else { + p.error(prod.Pos(), name+" declared already") + } + } + + return grammar +} + +// Parse parses a set of EBNF productions from source src. +// It returns a set of productions. Errors are reported +// for incorrect syntax and if a production is declared +// more than once. Position information is recorded relative +// to the file set fset. +// +func Parse(fset *token.FileSet, filename string, src []byte) (Grammar, os.Error) { + var p parser + grammar := p.parse(fset, filename, src) + return grammar, p.GetError(scanner.Sorted) +} diff --git a/src/pkg/encoding/ascii85/Makefile b/src/pkg/encoding/ascii85/Makefile new file mode 100644 index 000000000..412383efd --- /dev/null +++ b/src/pkg/encoding/ascii85/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/ascii85 +GOFILES=\ + ascii85.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/ascii85/ascii85.go b/src/pkg/encoding/ascii85/ascii85.go new file mode 100644 index 000000000..ead0c2475 --- /dev/null +++ b/src/pkg/encoding/ascii85/ascii85.go @@ -0,0 +1,300 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ascii85 implements the ascii85 data encoding +// as used in the btoa tool and Adobe's PostScript and PDF document formats. +package ascii85 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encoder + */ + +// Encode encodes src into at most MaxEncodedLen(len(src)) +// bytes of dst, returning the actual number of bytes written. +// +// The encoding handles 4-byte chunks, using a special encoding +// for the last fragment, so Encode is not appropriate for use on +// individual blocks of a large data stream. Use NewEncoder() instead. +// +// Often, ascii85-encoded data is wrapped in <~ and ~> symbols. +// Encode does not add these. +func Encode(dst, src []byte) int { + if len(src) == 0 { + return 0 + } + + n := 0 + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + dst[4] = 0 + + // Unpack 4 bytes into uint32 to repack into base 85 5-byte. + var v uint32 + switch len(src) { + default: + v |= uint32(src[3]) + fallthrough + case 3: + v |= uint32(src[2]) << 8 + fallthrough + case 2: + v |= uint32(src[1]) << 16 + fallthrough + case 1: + v |= uint32(src[0]) << 24 + } + + // Special case: zero (!!!!!) shortens to z. + if v == 0 && len(src) >= 4 { + dst[0] = 'z' + dst = dst[1:] + n++ + continue + } + + // Otherwise, 5 base 85 digits starting at !. + for i := 4; i >= 0; i-- { + dst[i] = '!' + byte(v%85) + v /= 85 + } + + // If src was short, discard the low destination bytes. + m := 5 + if len(src) < 4 { + m -= 4 - len(src) + src = nil + } else { + src = src[4:] + } + dst = dst[m:] + n += m + } + return n +} + +// MaxEncodedLen returns the maximum length of an encoding of n source bytes. +func MaxEncodedLen(n int) int { return (n + 3) / 4 * 5 } + +// NewEncoder returns a new ascii85 stream encoder. Data written to +// the returned writer will be encoded and then written to w. +// Ascii85 encodings operate in 32-bit blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// trailing partial block. +func NewEncoder(w io.Writer) io.WriteCloser { return &encoder{w: w} } + +type encoder struct { + err os.Error + w io.Writer + buf [4]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 4; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 4 { + return + } + nout := Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 4 { + nn := len(e.out) / 5 * 4 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 4 + if nn > 0 { + nout := Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + nout := Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:nout]) + } + return e.err +} + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal ascii85 data at input byte " + strconv.Itoa64(int64(e)) +} + +// Decode decodes src into dst, returning both the number +// of bytes written to dst and the number consumed from src. +// If src contains invalid ascii85 data, Decode will return the +// number of bytes successfully written and a CorruptInputError. +// Decode ignores space and control characters in src. +// Often, ascii85-encoded data is wrapped in <~ and ~> symbols. +// Decode expects these to have been stripped by the caller. +// +// If flush is true, Decode assumes that src represents the +// end of the input stream and processes it completely rather +// than wait for the completion of another 32-bit block. +// +// NewDecoder wraps an io.Reader interface around Decode. +// +func Decode(dst, src []byte, flush bool) (ndst, nsrc int, err os.Error) { + var v uint32 + var nb int + for i, b := range src { + if len(dst)-ndst < 4 { + return + } + switch { + case b <= ' ': + continue + case b == 'z' && nb == 0: + nb = 5 + v = 0 + case '!' <= b && b <= 'u': + v = v*85 + uint32(b-'!') + nb++ + default: + return 0, 0, CorruptInputError(i) + } + if nb == 5 { + nsrc = i + 1 + dst[ndst] = byte(v >> 24) + dst[ndst+1] = byte(v >> 16) + dst[ndst+2] = byte(v >> 8) + dst[ndst+3] = byte(v) + ndst += 4 + nb = 0 + v = 0 + } + } + if flush { + nsrc = len(src) + if nb > 0 { + // The number of output bytes in the last fragment + // is the number of leftover input bytes - 1: + // the extra byte provides enough bits to cover + // the inefficiency of the encoding for the block. + if nb == 1 { + return 0, 0, CorruptInputError(len(src)) + } + for i := nb; i < 5; i++ { + // The short encoding truncated the output value. + // We have to assume the worst case values (digit 84) + // in order to ensure that the top bits are correct. + v = v*85 + 84 + } + for i := 0; i < nb-1; i++ { + dst[ndst] = byte(v >> 24) + v <<= 8 + ndst++ + } + } + } + return +} + +// NewDecoder constructs a new ascii85 stream decoder. +func NewDecoder(r io.Reader) io.Reader { return &decoder{r: r} } + +type decoder struct { + err os.Error + readErr os.Error + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return 0, nil + } + if d.err != nil { + return 0, d.err + } + + for { + // Copy leftover output from last decode. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return + } + + // Decode leftover input from last read. + var nn, nsrc, ndst int + if d.nbuf > 0 { + ndst, nsrc, d.err = Decode(d.outbuf[0:], d.buf[0:d.nbuf], d.readErr != nil) + if ndst > 0 { + d.out = d.outbuf[0:ndst] + d.nbuf = copy(d.buf[0:], d.buf[nsrc:d.nbuf]) + continue // copy out and return + } + } + + // Out of input, out of decoded output. Check errors. + if d.err != nil { + return 0, d.err + } + if d.readErr != nil { + d.err = d.readErr + return 0, d.err + } + + // Read more data. + nn, d.readErr = d.r.Read(d.buf[d.nbuf:]) + d.nbuf += nn + } + panic("unreachable") +} diff --git a/src/pkg/encoding/ascii85/ascii85_test.go b/src/pkg/encoding/ascii85/ascii85_test.go new file mode 100644 index 000000000..fdfeb889f --- /dev/null +++ b/src/pkg/encoding/ascii85/ascii85_test.go @@ -0,0 +1,188 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii85 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // Wikipedia example + { + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKFCj@.4Gp$d7F!,L7@<6@)/0JDEF@3BB/F*&OCAfu2/AKY\n" + + "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF-FD5W8ARlolDIa\n" + + "l(DIduD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", + }, +} + +var bigtest = pairs[len(pairs)-1] + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func strip85(s string) string { + t := make([]byte, len(s)) + w := 0 + for r := 0; r < len(s); r++ { + c := s[r] + if c > ' ' { + t[w] = c + w++ + } + } + return string(t[0:w]) +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, MaxEncodedLen(len(p.decoded))) + n := Encode(buf, []byte(p.decoded)) + buf = buf[0:n] + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, strip85(string(buf)), strip85(p.encoded)) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, strip85(bb.String()), strip85(p.encoded)) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, strip85(bb.String()), strip85(bigtest.encoded)) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, 4*len(p.encoded)) + ndst, nsrc, err := Decode(dbuf, []byte(p.encoded), true) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = nsrc %v, want %v", p.encoded, nsrc, len(p.encoded)) + testEqual(t, "Decode(%q) = ndst %v, want %v", p.encoded, ndst, len(p.decoded)) + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:ndst]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + dbuf, err := ioutil.ReadAll(decoder) + if err != nil { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, len(dbuf), len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf), p.decoded) + if err != nil { + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"v", 0}, + {"!z!!!!!!!!!", 1}, + } + + for _, e := range examples { + dbuf := make([]byte, 4*len(e.e)) + _, _, err := Decode(dbuf, []byte(e.e), true) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(encoded)) + if err != nil { + t.Fatalf("io.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/base32/Makefile b/src/pkg/encoding/base32/Makefile new file mode 100644 index 000000000..c0e85b644 --- /dev/null +++ b/src/pkg/encoding/base32/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include $(GOROOT)/src/Make.inc + +TARG=encoding/base32 +GOFILES=\ + base32.go\ + +include $(GOROOT)/src/Make.pkg diff --git a/src/pkg/encoding/base32/base32.go b/src/pkg/encoding/base32/base32.go new file mode 100644 index 000000000..acace30d6 --- /dev/null +++ b/src/pkg/encoding/base32/base32.go @@ -0,0 +1,368 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base32 implements base32 encoding as specified by RFC 4648. +package base32 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 32 encoding/decoding scheme, defined by a +// 32-character alphabet. The most common is the "base32" encoding +// introduced for SASL GSSAPI and standardized in RFC 4648. +// The alternate "base32hex" encoding is used in DNSSEC. +type Encoding struct { + encode string + decodeMap [256]byte +} + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 32-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// StdEncoding is the standard base32 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncoding(encodeStd) + +// HexEncoding is the ``Extended Hex Alphabet'' defined in RFC 4648. +// It is typically used in DNS. +var HexEncoding = NewEncoding(encodeHex) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 8 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + dst[4] = 0 + dst[5] = 0 + dst[6] = 0 + dst[7] = 0 + + // Unpack 8x 5-bit source blocks into a 5 byte + // destination quantum + switch len(src) { + default: + dst[7] |= src[4] & 0x1F + dst[6] |= src[4] >> 5 + fallthrough + case 4: + dst[6] |= (src[3] << 3) & 0x1F + dst[5] |= (src[3] >> 2) & 0x1F + dst[4] |= src[3] >> 7 + fallthrough + case 3: + dst[4] |= (src[2] << 1) & 0x1F + dst[3] |= (src[2] >> 4) & 0x1F + fallthrough + case 2: + dst[3] |= (src[1] << 4) & 0x1F + dst[2] |= (src[1] >> 1) & 0x1F + dst[1] |= (src[1] >> 6) & 0x1F + fallthrough + case 1: + dst[1] |= (src[0] << 2) & 0x1F + dst[0] |= src[0] >> 3 + } + + // Encode 5-bit blocks using the base32 alphabet + for j := 0; j < 8; j++ { + dst[j] = enc.encode[dst[j]] + } + + // Pad the final quantum + if len(src) < 5 { + dst[7] = '=' + if len(src) < 4 { + dst[6] = '=' + dst[5] = '=' + if len(src) < 3 { + dst[4] = '=' + if len(src) < 2 { + dst[3] = '=' + dst[2] = '=' + } + } + } + break + } + src = src[5:] + dst = dst[8:] + } +} + +type encoder struct { + err os.Error + enc *Encoding + w io.Writer + buf [5]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 5; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 5 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:8]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 5 { + nn := len(e.out) / 8 * 5 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 5 + if nn > 0 { + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/5*8]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:8]) + } + return e.err +} + +// NewEncoder returns a new base32 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base32 encodings operate in 5-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base32 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { return (n + 4) / 5 * 8 } + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal base32 data at input byte " + strconv.Itoa64(int64(e)) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. decode also assumes len(src)%8==0, +// since it is meant for internal use. +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err os.Error) { + for i := 0; i < len(src)/8 && !end; i++ { + // Decode quantum using the base32 alphabet + var dbuf [8]byte + dlen := 8 + + // do the top bytes contain any data? + dbufloop: + for j := 0; j < 8; j++ { + in := src[i*8+j] + if in == '=' && j >= 2 && i == len(src)/8-1 { + // We've reached the end and there's + // padding, the rest should be padded + for k := j; k < 8; k++ { + if src[i*8+k] != '=' { + return n, false, CorruptInputError(i*8 + j) + } + } + dlen = j + end = true + break dbufloop + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(i*8 + j) + } + } + + // Pack 8x 5-bit source blocks into 5 byte destination + // quantum + switch dlen { + case 7, 8: + dst[i*5+4] = dbuf[6]<<5 | dbuf[7] + fallthrough + case 6, 5: + dst[i*5+3] = dbuf[4]<<7 | dbuf[5]<<2 | dbuf[6]>>3 + fallthrough + case 4: + dst[i*5+2] = dbuf[3]<<4 | dbuf[4]>>1 + fallthrough + case 3: + dst[i*5+1] = dbuf[1]<<6 | dbuf[2]<<1 | dbuf[3]>>4 + fallthrough + case 2: + dst[i*5+0] = dbuf[0]<<3 | dbuf[1]>>2 + } + switch dlen { + case 2: + n += 1 + case 3, 4: + n += 2 + case 5: + n += 3 + case 6, 7: + n += 4 + case 8: + n += 5 + } + } + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base32 data, it will return the +// number of bytes successfully written and CorruptInputError. +func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { + if len(src)%8 != 0 { + return 0, CorruptInputError(len(src) / 8 * 8) + } + + n, _, err = enc.decode(dst, src) + return +} + +type decoder struct { + err os.Error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 8 * 5]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 5 * 8 + if nn < 8 { + nn = 8 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + d.nbuf += nn + if d.nbuf < 8 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 8 * 8 + nw := d.nbuf / 8 * 5 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +// NewDecoder constructs a new base32 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: r} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base32-encoded data. +func (enc *Encoding) DecodedLen(n int) int { return n / 8 * 5 } diff --git a/src/pkg/encoding/base32/base32_test.go b/src/pkg/encoding/base32/base32_test.go new file mode 100644 index 000000000..3fa1c2b26 --- /dev/null +++ b/src/pkg/encoding/base32/base32_test.go @@ -0,0 +1,193 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base32 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // RFC 4648 examples + {"", ""}, + {"f", "MY======"}, + {"fo", "MZXQ===="}, + {"foo", "MZXW6==="}, + {"foob", "MZXW6YQ="}, + {"fooba", "MZXW6YTB"}, + {"foobar", "MZXW6YTBOI======"}, + + // Wikipedia examples, converted to base32 + {"sure.", "ON2XEZJO"}, + {"sure", "ON2XEZI="}, + {"sur", "ON2XE==="}, + {"su", "ON2Q===="}, + {"leasure.", "NRSWC43VOJSS4==="}, + {"easure.", "MVQXG5LSMUXA===="}, + {"asure.", "MFZXK4TFFY======"}, + {"sure.", "ON2XEZJO"}, +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "KR3WC4ZAMJZGS3DMNFTSYIDBNZSCA5DIMUQHG3DJORUHSIDUN53GK4Y=", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + buf := make([]byte, StdEncoding.EncodedLen(len(p.decoded))) + StdEncoding.Encode(buf, []byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, + string(dbuf[0:count]), + p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != os.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != os.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"!!!!", 0}, + {"x===", 0}, + {"AA=A====", 2}, + {"AAA=AAAA", 3}, + {"MMMMMMMMM", 8}, + {"MMMMMM", 0}, + } + + for _, e := range examples { + dbuf := make([]byte, StdEncoding.DecodedLen(len(e.e))) + _, err := StdEncoding.Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/base64/Makefile b/src/pkg/encoding/base64/Makefile new file mode 100644 index 000000000..2f54ed839 --- /dev/null +++ b/src/pkg/encoding/base64/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/base64 +GOFILES=\ + base64.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/base64/base64.go b/src/pkg/encoding/base64/base64.go new file mode 100644 index 000000000..c6b2a13e4 --- /dev/null +++ b/src/pkg/encoding/base64/base64.go @@ -0,0 +1,343 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package base64 implements base64 encoding as specified by RFC 4648. +package base64 + +import ( + "io" + "os" + "strconv" +) + +/* + * Encodings + */ + +// An Encoding is a radix 64 encoding/decoding scheme, defined by a +// 64-character alphabet. The most common encoding is the "base64" +// encoding defined in RFC 4648 and used in MIME (RFC 2045) and PEM +// (RFC 1421). RFC 4648 also defines an alternate encoding, which is +// the standard encoding with - and _ substituted for + and /. +type Encoding struct { + encode string + decodeMap [256]byte +} + +const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +const encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + +// NewEncoding returns a new Encoding defined by the given alphabet, +// which must be a 64-byte string. +func NewEncoding(encoder string) *Encoding { + e := new(Encoding) + e.encode = encoder + for i := 0; i < len(e.decodeMap); i++ { + e.decodeMap[i] = 0xFF + } + for i := 0; i < len(encoder); i++ { + e.decodeMap[encoder[i]] = byte(i) + } + return e +} + +// StdEncoding is the standard base64 encoding, as defined in +// RFC 4648. +var StdEncoding = NewEncoding(encodeStd) + +// URLEncoding is the alternate base64 encoding defined in RFC 4648. +// It is typically used in URLs and file names. +var URLEncoding = NewEncoding(encodeURL) + +/* + * Encoder + */ + +// Encode encodes src using the encoding enc, writing +// EncodedLen(len(src)) bytes to dst. +// +// The encoding pads the output to a multiple of 4 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. Use NewEncoder() instead. +func (enc *Encoding) Encode(dst, src []byte) { + if len(src) == 0 { + return + } + + for len(src) > 0 { + dst[0] = 0 + dst[1] = 0 + dst[2] = 0 + dst[3] = 0 + + // Unpack 4x 6-bit source blocks into a 4 byte + // destination quantum + switch len(src) { + default: + dst[3] |= src[2] & 0x3F + dst[2] |= src[2] >> 6 + fallthrough + case 2: + dst[2] |= (src[1] << 2) & 0x3F + dst[1] |= src[1] >> 4 + fallthrough + case 1: + dst[1] |= (src[0] << 4) & 0x3F + dst[0] |= src[0] >> 2 + } + + // Encode 6-bit blocks using the base64 alphabet + for j := 0; j < 4; j++ { + dst[j] = enc.encode[dst[j]] + } + + // Pad the final quantum + if len(src) < 3 { + dst[3] = '=' + if len(src) < 2 { + dst[2] = '=' + } + break + } + + src = src[3:] + dst = dst[4:] + } +} + +// EncodeToString returns the base64 encoding of src. +func (enc *Encoding) EncodeToString(src []byte) string { + buf := make([]byte, enc.EncodedLen(len(src))) + enc.Encode(buf, src) + return string(buf) +} + +type encoder struct { + err os.Error + enc *Encoding + w io.Writer + buf [3]byte // buffered data waiting to be encoded + nbuf int // number of bytes in buf + out [1024]byte // output buffer +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 3; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 3 { + return + } + e.enc.Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:4]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 3 { + nn := len(e.out) / 4 * 3 + if nn > len(p) { + nn = len(p) + } + nn -= nn % 3 + if nn > 0 { + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/3*4]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +// Close flushes any pending output from the encoder. +// It is an error to call Write after calling Close. +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:4]) + } + return e.err +} + +// NewEncoder returns a new base64 stream encoder. Data written to +// the returned writer will be encoded using enc and then written to w. +// Base64 encodings operate in 4-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { + return &encoder{enc: enc, w: w} +} + +// EncodedLen returns the length in bytes of the base64 encoding +// of an input buffer of length n. +func (enc *Encoding) EncodedLen(n int) int { return (n + 2) / 3 * 4 } + +/* + * Decoder + */ + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal base64 data at input byte " + strconv.Itoa64(int64(e)) +} + +// decode is like Decode but returns an additional 'end' value, which +// indicates if end-of-message padding was encountered and thus any +// additional data is an error. decode also assumes len(src)%4==0, +// since it is meant for internal use. +func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err os.Error) { + for i := 0; i < len(src)/4 && !end; i++ { + // Decode quantum using the base64 alphabet + var dbuf [4]byte + dlen := 4 + + dbufloop: + for j := 0; j < 4; j++ { + in := src[i*4+j] + if in == '=' && j >= 2 && i == len(src)/4-1 { + // We've reached the end and there's + // padding + if src[i*4+3] != '=' { + return n, false, CorruptInputError(i*4 + 2) + } + dlen = j + end = true + break dbufloop + } + dbuf[j] = enc.decodeMap[in] + if dbuf[j] == 0xFF { + return n, false, CorruptInputError(i*4 + j) + } + } + + // Pack 4x 6-bit source blocks into 3 byte destination + // quantum + switch dlen { + case 4: + dst[i*3+2] = dbuf[2]<<6 | dbuf[3] + fallthrough + case 3: + dst[i*3+1] = dbuf[1]<<4 | dbuf[2]>>2 + fallthrough + case 2: + dst[i*3+0] = dbuf[0]<<2 | dbuf[1]>>4 + } + n += dlen - 1 + } + + return n, end, nil +} + +// Decode decodes src using the encoding enc. It writes at most +// DecodedLen(len(src)) bytes to dst and returns the number of bytes +// written. If src contains invalid base64 data, it will return the +// number of bytes successfully written and CorruptInputError. +func (enc *Encoding) Decode(dst, src []byte) (n int, err os.Error) { + if len(src)%4 != 0 { + return 0, CorruptInputError(len(src) / 4 * 4) + } + + n, _, err = enc.decode(dst, src) + return +} + +// DecodeString returns the bytes represented by the base64 string s. +func (enc *Encoding) DecodeString(s string) ([]byte, os.Error) { + dbuf := make([]byte, enc.DecodedLen(len(s))) + n, err := enc.Decode(dbuf, []byte(s)) + return dbuf[:n], err +} + +type decoder struct { + err os.Error + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 4 * 3]byte +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if d.err != nil { + return 0, d.err + } + + // Use leftover decoded output from last read. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return n, nil + } + + // Read a chunk. + nn := len(p) / 3 * 4 + if nn < 4 { + nn = 4 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 4-d.nbuf) + d.nbuf += nn + if d.nbuf < 4 { + return 0, d.err + } + + // Decode chunk into p, or d.out and then p if p is too small. + nr := d.nbuf / 4 * 4 + nw := d.nbuf / 4 * 3 + if nw > len(p) { + nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + d.out = d.outbuf[0:nw] + n = copy(p, d.out) + d.out = d.out[n:] + } else { + n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + } + d.nbuf -= nr + for i := 0; i < d.nbuf; i++ { + d.buf[i] = d.buf[i+nr] + } + + if d.err == nil { + d.err = err + } + return n, d.err +} + +// NewDecoder constructs a new base64 stream decoder. +func NewDecoder(enc *Encoding, r io.Reader) io.Reader { + return &decoder{enc: enc, r: r} +} + +// DecodedLen returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base64-encoded data. +func (enc *Encoding) DecodedLen(n int) int { return n / 4 * 3 } diff --git a/src/pkg/encoding/base64/base64_test.go b/src/pkg/encoding/base64/base64_test.go new file mode 100644 index 000000000..c163dae84 --- /dev/null +++ b/src/pkg/encoding/base64/base64_test.go @@ -0,0 +1,199 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package base64 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +var pairs = []testpair{ + // RFC 3548 examples + {"\x14\xfb\x9c\x03\xd9\x7e", "FPucA9l+"}, + {"\x14\xfb\x9c\x03\xd9", "FPucA9k="}, + {"\x14\xfb\x9c\x03", "FPucAw=="}, + + // RFC 4648 examples + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + + // Wikipedia examples + {"sure.", "c3VyZS4="}, + {"sure", "c3VyZQ=="}, + {"sur", "c3Vy"}, + {"su", "c3U="}, + {"leasure.", "bGVhc3VyZS4="}, + {"easure.", "ZWFzdXJlLg=="}, + {"asure.", "YXN1cmUu"}, + {"sure.", "c3VyZS4="}, +} + +var bigtest = testpair{ + "Twas brillig, and the slithy toves", + "VHdhcyBicmlsbGlnLCBhbmQgdGhlIHNsaXRoeSB0b3Zlcw==", +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestEncode(t *testing.T) { + for _, p := range pairs { + got := StdEncoding.EncodeToString([]byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, got, p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range pairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(bigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(StdEncoding, bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, bigtest.decoded, bb.String(), bigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range pairs { + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, end, err := StdEncoding.decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = length %v, want %v", p.encoded, count, len(p.decoded)) + if len(p.encoded) > 0 { + testEqual(t, "Decode(%q) = end %v, want %v", p.encoded, end, (p.encoded[len(p.encoded)-1] == '=')) + } + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + + dbuf, err = StdEncoding.DecodeString(p.encoded) + testEqual(t, "DecodeString(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "DecodeString(%q) = %q, want %q", string(dbuf), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range pairs { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) + count, err := decoder.Read(dbuf) + if err != nil && err != os.EOF { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, count, len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + if err != os.EOF { + count, err = decoder.Read(dbuf) + } + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + buf := make([]byte, len(bigtest.decoded)+12) + var total int + for total = 0; total < len(bigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"!!!!", 0}, + {"x===", 1}, + {"AA=A", 2}, + {"AAA=AAAA", 3}, + {"AAAAA", 4}, + {"AAAAAA", 4}, + } + + for _, e := range examples { + dbuf := make([]byte, StdEncoding.DecodedLen(len(e.e))) + _, err := StdEncoding.Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(StdEncoding, encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(StdEncoding, encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/binary/Makefile b/src/pkg/encoding/binary/Makefile new file mode 100644 index 000000000..dc46abe90 --- /dev/null +++ b/src/pkg/encoding/binary/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/binary +GOFILES=\ + binary.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/binary/binary.go b/src/pkg/encoding/binary/binary.go new file mode 100644 index 000000000..8e55cb23b --- /dev/null +++ b/src/pkg/encoding/binary/binary.go @@ -0,0 +1,498 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package binary implements translation between +// unsigned integer values and byte sequences +// and the reading and writing of fixed-size values. +package binary + +import ( + "math" + "io" + "os" + "reflect" +) + +// A ByteOrder specifies how to convert byte sequences into +// 16-, 32-, or 64-bit unsigned integers. +type ByteOrder interface { + Uint16(b []byte) uint16 + Uint32(b []byte) uint32 + Uint64(b []byte) uint64 + PutUint16([]byte, uint16) + PutUint32([]byte, uint32) + PutUint64([]byte, uint64) + String() string +} + +// This is byte instead of struct{} so that it can be compared, +// allowing, e.g., order == binary.LittleEndian. +type unused byte + +// LittleEndian is the little-endian implementation of ByteOrder. +var LittleEndian littleEndian + +// BigEndian is the big-endian implementation of ByteOrder. +var BigEndian bigEndian + +type littleEndian unused + +func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 } + +func (littleEndian) PutUint16(b []byte, v uint16) { + b[0] = byte(v) + b[1] = byte(v >> 8) +} + +func (littleEndian) Uint32(b []byte) uint32 { + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func (littleEndian) PutUint32(b []byte, v uint32) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) +} + +func (littleEndian) Uint64(b []byte) uint64 { + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func (littleEndian) PutUint64(b []byte, v uint64) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) + b[4] = byte(v >> 32) + b[5] = byte(v >> 40) + b[6] = byte(v >> 48) + b[7] = byte(v >> 56) +} + +func (littleEndian) String() string { return "LittleEndian" } + +func (littleEndian) GoString() string { return "binary.LittleEndian" } + +type bigEndian unused + +func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 } + +func (bigEndian) PutUint16(b []byte, v uint16) { + b[0] = byte(v >> 8) + b[1] = byte(v) +} + +func (bigEndian) Uint32(b []byte) uint32 { + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func (bigEndian) PutUint32(b []byte, v uint32) { + b[0] = byte(v >> 24) + b[1] = byte(v >> 16) + b[2] = byte(v >> 8) + b[3] = byte(v) +} + +func (bigEndian) Uint64(b []byte) uint64 { + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} + +func (bigEndian) PutUint64(b []byte, v uint64) { + b[0] = byte(v >> 56) + b[1] = byte(v >> 48) + b[2] = byte(v >> 40) + b[3] = byte(v >> 32) + b[4] = byte(v >> 24) + b[5] = byte(v >> 16) + b[6] = byte(v >> 8) + b[7] = byte(v) +} + +func (bigEndian) String() string { return "BigEndian" } + +func (bigEndian) GoString() string { return "binary.BigEndian" } + +// Read reads structured binary data from r into data. +// Data must be a pointer to a fixed-size value or a slice +// of fixed-size values. +// A fixed-size value is either a fixed-size arithmetic +// type (int8, uint8, int16, float32, complex64, ...) +// or an array or struct containing only fixed-size values. +// Bytes read from r are decoded using the specified byte order +// and written to successive fields of the data. +func Read(r io.Reader, order ByteOrder, data interface{}) os.Error { + // Fast path for basic types. + if n := intDestSize(data); n != 0 { + var b [8]byte + bs := b[:n] + if _, err := io.ReadFull(r, bs); err != nil { + return err + } + switch v := data.(type) { + case *int8: + *v = int8(b[0]) + case *uint8: + *v = b[0] + case *int16: + *v = int16(order.Uint16(bs)) + case *uint16: + *v = order.Uint16(bs) + case *int32: + *v = int32(order.Uint32(bs)) + case *uint32: + *v = order.Uint32(bs) + case *int64: + *v = int64(order.Uint64(bs)) + case *uint64: + *v = order.Uint64(bs) + } + return nil + } + + // Fallback to reflect-based. + var v reflect.Value + switch d := reflect.ValueOf(data); d.Kind() { + case reflect.Ptr: + v = d.Elem() + case reflect.Slice: + v = d + default: + return os.NewError("binary.Read: invalid type " + d.Type().String()) + } + size := TotalSize(v) + if size < 0 { + return os.NewError("binary.Read: invalid type " + v.Type().String()) + } + d := &decoder{order: order, buf: make([]byte, size)} + if _, err := io.ReadFull(r, d.buf); err != nil { + return err + } + d.value(v) + return nil +} + +// Write writes the binary representation of data into w. +// Data must be a fixed-size value or a pointer to +// a fixed-size value. +// A fixed-size value is either a fixed-size arithmetic +// type (int8, uint8, int16, float32, complex64, ...) +// or an array or struct containing only fixed-size values. +// Bytes written to w are encoded using the specified byte order +// and read from successive fields of the data. +func Write(w io.Writer, order ByteOrder, data interface{}) os.Error { + // Fast path for basic types. + var b [8]byte + var bs []byte + switch v := data.(type) { + case *int8: + bs = b[:1] + b[0] = byte(*v) + case int8: + bs = b[:1] + b[0] = byte(v) + case *uint8: + bs = b[:1] + b[0] = *v + case uint8: + bs = b[:1] + b[0] = byte(v) + case *int16: + bs = b[:2] + order.PutUint16(bs, uint16(*v)) + case int16: + bs = b[:2] + order.PutUint16(bs, uint16(v)) + case *uint16: + bs = b[:2] + order.PutUint16(bs, *v) + case uint16: + bs = b[:2] + order.PutUint16(bs, v) + case *int32: + bs = b[:4] + order.PutUint32(bs, uint32(*v)) + case int32: + bs = b[:4] + order.PutUint32(bs, uint32(v)) + case *uint32: + bs = b[:4] + order.PutUint32(bs, *v) + case uint32: + bs = b[:4] + order.PutUint32(bs, v) + case *int64: + bs = b[:8] + order.PutUint64(bs, uint64(*v)) + case int64: + bs = b[:8] + order.PutUint64(bs, uint64(v)) + case *uint64: + bs = b[:8] + order.PutUint64(bs, *v) + case uint64: + bs = b[:8] + order.PutUint64(bs, v) + } + if bs != nil { + _, err := w.Write(bs) + return err + } + v := reflect.Indirect(reflect.ValueOf(data)) + size := TotalSize(v) + if size < 0 { + return os.NewError("binary.Write: invalid type " + v.Type().String()) + } + buf := make([]byte, size) + e := &encoder{order: order, buf: buf} + e.value(v) + _, err := w.Write(buf) + return err +} + +func TotalSize(v reflect.Value) int { + if v.Kind() == reflect.Slice { + elem := sizeof(v.Type().Elem()) + if elem < 0 { + return -1 + } + return v.Len() * elem + } + return sizeof(v.Type()) +} + +func sizeof(t reflect.Type) int { + switch t.Kind() { + case reflect.Array: + n := sizeof(t.Elem()) + if n < 0 { + return -1 + } + return t.Len() * n + + case reflect.Struct: + sum := 0 + for i, n := 0, t.NumField(); i < n; i++ { + s := sizeof(t.Field(i).Type) + if s < 0 { + return -1 + } + sum += s + } + return sum + + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: + return int(t.Size()) + } + return -1 +} + +type decoder struct { + order ByteOrder + buf []byte +} + +type encoder struct { + order ByteOrder + buf []byte +} + +func (d *decoder) uint8() uint8 { + x := d.buf[0] + d.buf = d.buf[1:] + return x +} + +func (e *encoder) uint8(x uint8) { + e.buf[0] = x + e.buf = e.buf[1:] +} + +func (d *decoder) uint16() uint16 { + x := d.order.Uint16(d.buf[0:2]) + d.buf = d.buf[2:] + return x +} + +func (e *encoder) uint16(x uint16) { + e.order.PutUint16(e.buf[0:2], x) + e.buf = e.buf[2:] +} + +func (d *decoder) uint32() uint32 { + x := d.order.Uint32(d.buf[0:4]) + d.buf = d.buf[4:] + return x +} + +func (e *encoder) uint32(x uint32) { + e.order.PutUint32(e.buf[0:4], x) + e.buf = e.buf[4:] +} + +func (d *decoder) uint64() uint64 { + x := d.order.Uint64(d.buf[0:8]) + d.buf = d.buf[8:] + return x +} + +func (e *encoder) uint64(x uint64) { + e.order.PutUint64(e.buf[0:8], x) + e.buf = e.buf[8:] +} + +func (d *decoder) int8() int8 { return int8(d.uint8()) } + +func (e *encoder) int8(x int8) { e.uint8(uint8(x)) } + +func (d *decoder) int16() int16 { return int16(d.uint16()) } + +func (e *encoder) int16(x int16) { e.uint16(uint16(x)) } + +func (d *decoder) int32() int32 { return int32(d.uint32()) } + +func (e *encoder) int32(x int32) { e.uint32(uint32(x)) } + +func (d *decoder) int64() int64 { return int64(d.uint64()) } + +func (e *encoder) int64(x int64) { e.uint64(uint64(x)) } + +func (d *decoder) value(v reflect.Value) { + switch v.Kind() { + case reflect.Array: + l := v.Len() + for i := 0; i < l; i++ { + d.value(v.Index(i)) + } + case reflect.Struct: + l := v.NumField() + for i := 0; i < l; i++ { + d.value(v.Field(i)) + } + + case reflect.Slice: + l := v.Len() + for i := 0; i < l; i++ { + d.value(v.Index(i)) + } + + case reflect.Int8: + v.SetInt(int64(d.int8())) + case reflect.Int16: + v.SetInt(int64(d.int16())) + case reflect.Int32: + v.SetInt(int64(d.int32())) + case reflect.Int64: + v.SetInt(d.int64()) + + case reflect.Uint8: + v.SetUint(uint64(d.uint8())) + case reflect.Uint16: + v.SetUint(uint64(d.uint16())) + case reflect.Uint32: + v.SetUint(uint64(d.uint32())) + case reflect.Uint64: + v.SetUint(d.uint64()) + + case reflect.Float32: + v.SetFloat(float64(math.Float32frombits(d.uint32()))) + case reflect.Float64: + v.SetFloat(math.Float64frombits(d.uint64())) + + case reflect.Complex64: + v.SetComplex(complex( + float64(math.Float32frombits(d.uint32())), + float64(math.Float32frombits(d.uint32())), + )) + case reflect.Complex128: + v.SetComplex(complex( + math.Float64frombits(d.uint64()), + math.Float64frombits(d.uint64()), + )) + } +} + +func (e *encoder) value(v reflect.Value) { + switch v.Kind() { + case reflect.Array: + l := v.Len() + for i := 0; i < l; i++ { + e.value(v.Index(i)) + } + case reflect.Struct: + l := v.NumField() + for i := 0; i < l; i++ { + e.value(v.Field(i)) + } + case reflect.Slice: + l := v.Len() + for i := 0; i < l; i++ { + e.value(v.Index(i)) + } + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch v.Type().Kind() { + case reflect.Int8: + e.int8(int8(v.Int())) + case reflect.Int16: + e.int16(int16(v.Int())) + case reflect.Int32: + e.int32(int32(v.Int())) + case reflect.Int64: + e.int64(v.Int()) + } + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch v.Type().Kind() { + case reflect.Uint8: + e.uint8(uint8(v.Uint())) + case reflect.Uint16: + e.uint16(uint16(v.Uint())) + case reflect.Uint32: + e.uint32(uint32(v.Uint())) + case reflect.Uint64: + e.uint64(v.Uint()) + } + + case reflect.Float32, reflect.Float64: + switch v.Type().Kind() { + case reflect.Float32: + e.uint32(math.Float32bits(float32(v.Float()))) + case reflect.Float64: + e.uint64(math.Float64bits(v.Float())) + } + + case reflect.Complex64, reflect.Complex128: + switch v.Type().Kind() { + case reflect.Complex64: + x := v.Complex() + e.uint32(math.Float32bits(float32(real(x)))) + e.uint32(math.Float32bits(float32(imag(x)))) + case reflect.Complex128: + x := v.Complex() + e.uint64(math.Float64bits(real(x))) + e.uint64(math.Float64bits(imag(x))) + } + } +} + +// intDestSize returns the size of the integer that ptrType points to, +// or 0 if the type is not supported. +func intDestSize(ptrType interface{}) int { + switch ptrType.(type) { + case *int8, *uint8: + return 1 + case *int16, *uint16: + return 2 + case *int32, *uint32: + return 4 + case *int64, *uint64: + return 8 + } + return 0 +} diff --git a/src/pkg/encoding/binary/binary_test.go b/src/pkg/encoding/binary/binary_test.go new file mode 100644 index 000000000..b266996f6 --- /dev/null +++ b/src/pkg/encoding/binary/binary_test.go @@ -0,0 +1,235 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binary + +import ( + "io" + "os" + "bytes" + "math" + "reflect" + "testing" +) + +type Struct struct { + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + Complex64 complex64 + Complex128 complex128 + Array [4]uint8 +} + +type T struct { + Int int + Uint uint + Uintptr uintptr + Array [4]int +} + +var s = Struct{ + 0x01, + 0x0203, + 0x04050607, + 0x08090a0b0c0d0e0f, + 0x10, + 0x1112, + 0x13141516, + 0x1718191a1b1c1d1e, + + math.Float32frombits(0x1f202122), + math.Float64frombits(0x232425262728292a), + complex( + math.Float32frombits(0x2b2c2d2e), + math.Float32frombits(0x2f303132), + ), + complex( + math.Float64frombits(0x333435363738393a), + math.Float64frombits(0x3b3c3d3e3f404142), + ), + + [4]uint8{0x43, 0x44, 0x45, 0x46}, +} + +var big = []byte{ + 1, + 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, + 17, 18, + 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, + + 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + + 67, 68, 69, 70, +} + +var little = []byte{ + 1, + 3, 2, + 7, 6, 5, 4, + 15, 14, 13, 12, 11, 10, 9, 8, + 16, + 18, 17, + 22, 21, 20, 19, + 30, 29, 28, 27, 26, 25, 24, 23, + + 34, 33, 32, 31, + 42, 41, 40, 39, 38, 37, 36, 35, + 46, 45, 44, 43, 50, 49, 48, 47, + 58, 57, 56, 55, 54, 53, 52, 51, 66, 65, 64, 63, 62, 61, 60, 59, + + 67, 68, 69, 70, +} + +var src = []byte{1, 2, 3, 4, 5, 6, 7, 8} +var res = []int32{0x01020304, 0x05060708} + +func checkResult(t *testing.T, dir string, order, err os.Error, have, want interface{}) { + if err != nil { + t.Errorf("%v %v: %v", dir, order, err) + return + } + if !reflect.DeepEqual(have, want) { + t.Errorf("%v %v:\n\thave %+v\n\twant %+v", dir, order, have, want) + } +} + +func testRead(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { + var s2 Struct + err := Read(bytes.NewBuffer(b), order, &s2) + checkResult(t, "Read", order, err, s2, s1) +} + +func testWrite(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { + buf := new(bytes.Buffer) + err := Write(buf, order, s1) + checkResult(t, "Write", order, err, buf.Bytes(), b) +} + +func TestBigEndianRead(t *testing.T) { testRead(t, BigEndian, big, s) } + +func TestLittleEndianRead(t *testing.T) { testRead(t, LittleEndian, little, s) } + +func TestBigEndianWrite(t *testing.T) { testWrite(t, BigEndian, big, s) } + +func TestLittleEndianWrite(t *testing.T) { testWrite(t, LittleEndian, little, s) } + +func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) } + +func TestLittleEndianPtrWrite(t *testing.T) { testWrite(t, LittleEndian, little, &s) } + +func TestReadSlice(t *testing.T) { + slice := make([]int32, 2) + err := Read(bytes.NewBuffer(src), BigEndian, slice) + checkResult(t, "ReadSlice", BigEndian, err, slice, res) +} + +func TestWriteSlice(t *testing.T) { + buf := new(bytes.Buffer) + err := Write(buf, BigEndian, res) + checkResult(t, "WriteSlice", BigEndian, err, buf.Bytes(), src) +} + +func TestWriteT(t *testing.T) { + buf := new(bytes.Buffer) + ts := T{} + err := Write(buf, BigEndian, ts) + if err == nil { + t.Errorf("WriteT: have nil, want non-nil") + } + + tv := reflect.Indirect(reflect.ValueOf(ts)) + for i, n := 0, tv.NumField(); i < n; i++ { + err = Write(buf, BigEndian, tv.Field(i).Interface()) + if err == nil { + t.Errorf("WriteT.%v: have nil, want non-nil", tv.Field(i).Type()) + } + } +} + +type byteSliceReader struct { + remain []byte +} + +func (br *byteSliceReader) Read(p []byte) (int, os.Error) { + n := copy(p, br.remain) + br.remain = br.remain[n:] + return n, nil +} + +func BenchmarkRead(b *testing.B) { + var ls Struct + bsr := &byteSliceReader{} + var r io.Reader = bsr + + for i := 0; i < b.N; i++ { + bsr.remain = big + Read(r, BigEndian, &ls.Int8) + Read(r, BigEndian, &ls.Int16) + Read(r, BigEndian, &ls.Int32) + Read(r, BigEndian, &ls.Int64) + Read(r, BigEndian, &ls.Uint8) + Read(r, BigEndian, &ls.Uint16) + Read(r, BigEndian, &ls.Uint32) + Read(r, BigEndian, &ls.Uint64) + } + + want := s + want.Float32 = 0 + want.Float64 = 0 + want.Complex64 = 0 + want.Complex128 = 0 + for i := range want.Array { + want.Array[i] = 0 + } + if !reflect.DeepEqual(ls, want) { + panic("no match") + } +} + +func BenchmarkWrite(b *testing.B) { + buf := new(bytes.Buffer) + var w io.Writer = buf + + for i := 0; i < b.N; i++ { + buf.Reset() + Write(w, BigEndian, &s.Int8) + Write(w, BigEndian, &s.Int16) + Write(w, BigEndian, &s.Int32) + Write(w, BigEndian, &s.Int64) + Write(w, BigEndian, &s.Uint8) + Write(w, BigEndian, &s.Uint16) + Write(w, BigEndian, &s.Uint32) + Write(w, BigEndian, &s.Uint64) + Write(w, BigEndian, s.Int8) + Write(w, BigEndian, s.Int16) + Write(w, BigEndian, s.Int32) + Write(w, BigEndian, s.Int64) + Write(w, BigEndian, s.Uint8) + Write(w, BigEndian, s.Uint16) + Write(w, BigEndian, s.Uint32) + Write(w, BigEndian, s.Uint64) + } + + if !bytes.Equal(buf.Bytes()[:30], big[:30]) { + panic("first half doesn't match") + } + if !bytes.Equal(buf.Bytes()[30:], big[:30]) { + panic("second half doesn't match") + } +} diff --git a/src/pkg/encoding/git85/Makefile b/src/pkg/encoding/git85/Makefile new file mode 100644 index 000000000..fbd003454 --- /dev/null +++ b/src/pkg/encoding/git85/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/git85 +GOFILES=\ + git.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/git85/git.go b/src/pkg/encoding/git85/git.go new file mode 100644 index 000000000..6bb74f46c --- /dev/null +++ b/src/pkg/encoding/git85/git.go @@ -0,0 +1,277 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package git85 implements the radix 85 data encoding +// used in the Git version control system. +package git85 + +import ( + "bytes" + "io" + "os" + "strconv" +) + +type CorruptInputError int64 + +func (e CorruptInputError) String() string { + return "illegal git85 data at input byte " + strconv.Itoa64(int64(e)) +} + +const encode = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~" + +// The decodings are 1+ the actual value, so that the +// default zero value can be used to mean "not valid". +var decode = [256]uint8{ + '0': 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 'A': 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 'a': 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + '!': 63, + '#': 64, 65, 66, 67, + '(': 68, 69, 70, 71, + '-': 72, + ';': 73, + '<': 74, 75, 76, 77, + '@': 78, + '^': 79, 80, 81, + '{': 82, 83, 84, 85, +} + +// Encode encodes src into EncodedLen(len(src)) +// bytes of dst. As a convenience, it returns the number +// of bytes written to dst, but this value is always EncodedLen(len(src)). +// Encode implements the radix 85 encoding used in the +// Git version control tool. +// +// The encoding splits src into chunks of at most 52 bytes +// and encodes each chunk on its own line. +func Encode(dst, src []byte) int { + ndst := 0 + for len(src) > 0 { + n := len(src) + if n > 52 { + n = 52 + } + if n <= 27 { + dst[ndst] = byte('A' + n - 1) + } else { + dst[ndst] = byte('a' + n - 26 - 1) + } + ndst++ + for i := 0; i < n; i += 4 { + var v uint32 + for j := 0; j < 4 && i+j < n; j++ { + v |= uint32(src[i+j]) << uint(24-j*8) + } + for j := 4; j >= 0; j-- { + dst[ndst+j] = encode[v%85] + v /= 85 + } + ndst += 5 + } + dst[ndst] = '\n' + ndst++ + src = src[n:] + } + return ndst +} + +// EncodedLen returns the length of an encoding of n source bytes. +func EncodedLen(n int) int { + if n == 0 { + return 0 + } + // 5 bytes per 4 bytes of input, rounded up. + // 2 extra bytes for each line of 52 src bytes, rounded up. + return (n+3)/4*5 + (n+51)/52*2 +} + +var newline = []byte{'\n'} + +// Decode decodes src into at most MaxDecodedLen(len(src)) +// bytes, returning the actual number of bytes written to dst. +// +// If Decode encounters invalid input, it returns a CorruptInputError. +// +func Decode(dst, src []byte) (n int, err os.Error) { + ndst := 0 + nsrc := 0 + for nsrc < len(src) { + var l int + switch ch := int(src[nsrc]); { + case 'A' <= ch && ch <= 'Z': + l = ch - 'A' + 1 + case 'a' <= ch && ch <= 'z': + l = ch - 'a' + 26 + 1 + default: + return ndst, CorruptInputError(nsrc) + } + if nsrc+1+l > len(src) { + return ndst, CorruptInputError(nsrc) + } + el := (l + 3) / 4 * 5 // encoded len + if nsrc+1+el+1 > len(src) || src[nsrc+1+el] != '\n' { + return ndst, CorruptInputError(nsrc) + } + line := src[nsrc+1 : nsrc+1+el] + for i := 0; i < el; i += 5 { + var v uint32 + for j := 0; j < 5; j++ { + ch := decode[line[i+j]] + if ch == 0 { + return ndst, CorruptInputError(nsrc + 1 + i + j) + } + v = v*85 + uint32(ch-1) + } + for j := 0; j < 4; j++ { + dst[ndst] = byte(v >> 24) + v <<= 8 + ndst++ + } + } + // Last fragment may have run too far (but there was room in dst). + // Back up. + if l%4 != 0 { + ndst -= 4 - l%4 + } + nsrc += 1 + el + 1 + } + return ndst, nil +} + +func MaxDecodedLen(n int) int { return n / 5 * 4 } + +// NewEncoder returns a new Git base85 stream encoder. Data written to +// the returned writer will be encoded and then written to w. +// The Git encoding operates on 52-byte blocks; when finished +// writing, the caller must Close the returned encoder to flush any +// partially written blocks. +func NewEncoder(w io.Writer) io.WriteCloser { return &encoder{w: w} } + +type encoder struct { + w io.Writer + err os.Error + buf [52]byte + nbuf int + out [1024]byte + nout int +} + +func (e *encoder) Write(p []byte) (n int, err os.Error) { + if e.err != nil { + return 0, e.err + } + + // Leading fringe. + if e.nbuf > 0 { + var i int + for i = 0; i < len(p) && e.nbuf < 52; i++ { + e.buf[e.nbuf] = p[i] + e.nbuf++ + } + n += i + p = p[i:] + if e.nbuf < 52 { + return + } + nout := Encode(e.out[0:], e.buf[0:]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + e.nbuf = 0 + } + + // Large interior chunks. + for len(p) >= 52 { + nn := len(e.out) / (1 + 52/4*5 + 1) * 52 + if nn > len(p) { + nn = len(p) / 52 * 52 + } + if nn > 0 { + nout := Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0:nout]); e.err != nil { + return n, e.err + } + } + n += nn + p = p[nn:] + } + + // Trailing fringe. + for i := 0; i < len(p); i++ { + e.buf[i] = p[i] + } + e.nbuf = len(p) + n += len(p) + return +} + +func (e *encoder) Close() os.Error { + // If there's anything left in the buffer, flush it out + if e.err == nil && e.nbuf > 0 { + nout := Encode(e.out[0:], e.buf[0:e.nbuf]) + e.nbuf = 0 + _, e.err = e.w.Write(e.out[0:nout]) + } + return e.err +} + +// NewDecoder returns a new Git base85 stream decoder. +func NewDecoder(r io.Reader) io.Reader { return &decoder{r: r} } + +type decoder struct { + r io.Reader + err os.Error + readErr os.Error + buf [1024]byte + nbuf int + out []byte + outbuf [1024]byte + off int64 +} + +func (d *decoder) Read(p []byte) (n int, err os.Error) { + if len(p) == 0 { + return 0, nil + } + + for { + // Copy leftover output from last decode. + if len(d.out) > 0 { + n = copy(p, d.out) + d.out = d.out[n:] + return + } + + // Out of decoded output. Check errors. + if d.err != nil { + return 0, d.err + } + if d.readErr != nil { + d.err = d.readErr + return 0, d.err + } + + // Read and decode more input. + var nn int + nn, d.readErr = d.r.Read(d.buf[d.nbuf:]) + d.nbuf += nn + + // Send complete lines to Decode. + nl := bytes.LastIndex(d.buf[0:d.nbuf], newline) + if nl < 0 { + continue + } + nn, d.err = Decode(d.outbuf[0:], d.buf[0:nl+1]) + if e, ok := d.err.(CorruptInputError); ok { + d.err = CorruptInputError(int64(e) + d.off) + } + d.out = d.outbuf[0:nn] + d.nbuf = copy(d.buf[0:], d.buf[nl+1:d.nbuf]) + d.off += int64(nl + 1) + } + panic("unreachable") +} diff --git a/src/pkg/encoding/git85/git_test.go b/src/pkg/encoding/git85/git_test.go new file mode 100644 index 000000000..c76385c35 --- /dev/null +++ b/src/pkg/encoding/git85/git_test.go @@ -0,0 +1,194 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package git85 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" +) + +type testpair struct { + decoded, encoded string +} + +func testEqual(t *testing.T, msg string, args ...interface{}) bool { + if args[len(args)-2] != args[len(args)-1] { + t.Errorf(msg, args...) + return false + } + return true +} + +func TestGitTable(t *testing.T) { + var saw [256]bool + for i, c := range encode { + if decode[c] != uint8(i+1) { + t.Errorf("decode['%c'] = %d, want %d", c, decode[c], i+1) + } + saw[c] = true + } + for i, b := range saw { + if !b && decode[i] != 0 { + t.Errorf("decode[%d] = %d, want 0", i, decode[i]) + } + } +} + +var gitPairs = []testpair{ + // Wikipedia example, adapted. + { + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + + "zO<`^zX>%ZCX>)XGZfA9Ab7*B`EFf-gbRchTYDO_b1WctXlY|;AZc?T\n" + + "zVIXXEb95kYW*~HEWgu;7Ze%PVbZB98AYyqSVIXj2a&u*NWpZI|V`U(3W*}r`Y-wj`\n" + + "zbRcPNAarPDAY*TCbZKsNWn>^>Ze$>7Ze(RV>IZ)PBC\n" + + "zZf|#NWn^b%EFfigV`XJzb0BnRWgv5CZ*p`Xc4cT~ZDnp_Wgu^6AYpEKAY);2ZeeU7\n" + + "IaBO8^b9HiME&u=k\n", + }, +} + +var gitBigtest = gitPairs[len(gitPairs)-1] + +func TestEncode(t *testing.T) { + for _, p := range gitPairs { + buf := make([]byte, EncodedLen(len(p.decoded))) + n := Encode(buf, []byte(p.decoded)) + if n != len(buf) { + t.Errorf("EncodedLen does not agree with Encode") + } + buf = buf[0:n] + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + } +} + +func TestEncoder(t *testing.T) { + for _, p := range gitPairs { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + encoder.Write([]byte(p.decoded)) + encoder.Close() + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, bb.String(), p.encoded) + } +} + +func TestEncoderBuffering(t *testing.T) { + input := []byte(gitBigtest.decoded) + for bs := 1; bs <= 12; bs++ { + bb := &bytes.Buffer{} + encoder := NewEncoder(bb) + for pos := 0; pos < len(input); pos += bs { + end := pos + bs + if end > len(input) { + end = len(input) + } + n, err := encoder.Write(input[pos:end]) + testEqual(t, "Write(%q) gave error %v, want %v", input[pos:end], err, os.Error(nil)) + testEqual(t, "Write(%q) gave length %v, want %v", input[pos:end], n, end-pos) + } + err := encoder.Close() + testEqual(t, "Close gave error %v, want %v", err, os.Error(nil)) + testEqual(t, "Encoding/%d of %q = %q, want %q", bs, gitBigtest.decoded, bb.String(), gitBigtest.encoded) + } +} + +func TestDecode(t *testing.T) { + for _, p := range gitPairs { + dbuf := make([]byte, 4*len(p.encoded)) + ndst, err := Decode(dbuf, []byte(p.encoded)) + testEqual(t, "Decode(%q) = error %v, want %v", p.encoded, err, os.Error(nil)) + testEqual(t, "Decode(%q) = ndst %v, want %v", p.encoded, ndst, len(p.decoded)) + testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:ndst]), p.decoded) + } +} + +func TestDecoder(t *testing.T) { + for _, p := range gitPairs { + decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + dbuf, err := ioutil.ReadAll(decoder) + if err != nil { + t.Fatal("Read failed", err) + } + testEqual(t, "Read from %q = length %v, want %v", p.encoded, len(dbuf), len(p.decoded)) + testEqual(t, "Decoding of %q = %q, want %q", p.encoded, string(dbuf), p.decoded) + if err != nil { + testEqual(t, "Read from %q = %v, want %v", p.encoded, err, os.EOF) + } + } +} + +func TestDecoderBuffering(t *testing.T) { + for bs := 1; bs <= 12; bs++ { + decoder := NewDecoder(bytes.NewBufferString(gitBigtest.encoded)) + buf := make([]byte, len(gitBigtest.decoded)+12) + var total int + for total = 0; total < len(gitBigtest.decoded); { + n, err := decoder.Read(buf[total : total+bs]) + testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", gitBigtest.encoded, total, n, err, os.Error(nil)) + total += n + } + testEqual(t, "Decoding/%d of %q = %q, want %q", bs, gitBigtest.encoded, string(buf[0:total]), gitBigtest.decoded) + } +} + +func TestDecodeCorrupt(t *testing.T) { + type corrupt struct { + e string + p int + } + examples := []corrupt{ + {"v", 0}, + {"!z!!!!!!!!!", 0}, + } + + for _, e := range examples { + dbuf := make([]byte, 2*len(e.e)) + _, err := Decode(dbuf, []byte(e.e)) + switch err := err.(type) { + case CorruptInputError: + testEqual(t, "Corruption in %q at offset %v, want %v", e.e, int(err), e.p) + default: + t.Error("Decoder failed to detect corruption in", e) + } + } +} + +func TestGitBig(t *testing.T) { + n := 3*1000 + 1 + raw := make([]byte, n) + const alpha = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + for i := 0; i < n; i++ { + raw[i] = alpha[i%len(alpha)] + } + encoded := new(bytes.Buffer) + w := NewEncoder(encoded) + nn, err := w.Write(raw) + if nn != n || err != nil { + t.Fatalf("Encoder.Write(raw) = %d, %v want %d, nil", nn, err, n) + } + err = w.Close() + if err != nil { + t.Fatalf("Encoder.Close() = %v want nil", err) + } + decoded, err := ioutil.ReadAll(NewDecoder(encoded)) + if err != nil { + t.Fatalf("ioutil.ReadAll(NewDecoder(...)): %v", err) + } + + if !bytes.Equal(raw, decoded) { + var i int + for i = 0; i < len(decoded) && i < len(raw); i++ { + if decoded[i] != raw[i] { + break + } + } + t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) + } +} diff --git a/src/pkg/encoding/hex/Makefile b/src/pkg/encoding/hex/Makefile new file mode 100644 index 000000000..22049f43b --- /dev/null +++ b/src/pkg/encoding/hex/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/hex +GOFILES=\ + hex.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/hex/hex.go b/src/pkg/encoding/hex/hex.go new file mode 100644 index 000000000..e7ea8b0f2 --- /dev/null +++ b/src/pkg/encoding/hex/hex.go @@ -0,0 +1,216 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package hex implements hexadecimal encoding and decoding. +package hex + +import ( + "bytes" + "io" + "os" + "strconv" +) + +const hextable = "0123456789abcdef" + +// EncodedLen returns the length of an encoding of n source bytes. +func EncodedLen(n int) int { return n * 2 } + +// Encode encodes src into EncodedLen(len(src)) +// bytes of dst. As a convenience, it returns the number +// of bytes written to dst, but this value is always EncodedLen(len(src)). +// Encode implements hexadecimal encoding. +func Encode(dst, src []byte) int { + for i, v := range src { + dst[i*2] = hextable[v>>4] + dst[i*2+1] = hextable[v&0x0f] + } + + return len(src) * 2 +} + +// OddLengthInputError results from decoding an odd length slice. +type OddLengthInputError struct{} + +func (OddLengthInputError) String() string { return "odd length hex string" } + +// InvalidHexCharError results from finding an invalid character in a hex string. +type InvalidHexCharError byte + +func (e InvalidHexCharError) String() string { + return "invalid hex char: " + strconv.Itoa(int(e)) +} + +func DecodedLen(x int) int { return x / 2 } + +// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual +// number of bytes written to dst. +// +// If Decode encounters invalid input, it returns an OddLengthInputError or an +// InvalidHexCharError. +func Decode(dst, src []byte) (int, os.Error) { + if len(src)%2 == 1 { + return 0, OddLengthInputError{} + } + + for i := 0; i < len(src)/2; i++ { + a, ok := fromHexChar(src[i*2]) + if !ok { + return 0, InvalidHexCharError(src[i*2]) + } + b, ok := fromHexChar(src[i*2+1]) + if !ok { + return 0, InvalidHexCharError(src[i*2+1]) + } + dst[i] = (a << 4) | b + } + + return len(src) / 2, nil +} + +// fromHexChar converts a hex character into its value and a success flag. +func fromHexChar(c byte) (byte, bool) { + switch { + case '0' <= c && c <= '9': + return c - '0', true + case 'a' <= c && c <= 'f': + return c - 'a' + 10, true + case 'A' <= c && c <= 'F': + return c - 'A' + 10, true + } + + return 0, false +} + +// EncodeToString returns the hexadecimal encoding of src. +func EncodeToString(src []byte) string { + dst := make([]byte, EncodedLen(len(src))) + Encode(dst, src) + return string(dst) +} + +// DecodeString returns the bytes represented by the hexadecimal string s. +func DecodeString(s string) ([]byte, os.Error) { + src := []byte(s) + dst := make([]byte, DecodedLen(len(src))) + _, err := Decode(dst, src) + if err != nil { + return nil, err + } + return dst, nil +} + +// Dump returns a string that contains a hex dump of the given data. The format +// of the hex dump matches the output of `hexdump -C` on the command line. +func Dump(data []byte) string { + buf := bytes.NewBuffer(nil) + dumper := Dumper(buf) + dumper.Write(data) + dumper.Close() + return string(buf.Bytes()) +} + +// Dumper returns a WriteCloser that writes a hex dump of all written data to +// w. The format of the dump matches the output of `hexdump -C` on the command +// line. +func Dumper(w io.Writer) io.WriteCloser { + return &dumper{w: w} +} + +type dumper struct { + w io.Writer + rightChars [18]byte + buf [14]byte + used int // number of bytes in the current line + n uint // number of bytes, total +} + +func toChar(b byte) byte { + if b < 32 || b > 126 { + return '.' + } + return b +} + +func (h *dumper) Write(data []byte) (n int, err os.Error) { + // Output lines look like: + // 00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| + // ^ offset ^ extra space ^ ASCII of line. + for i := range data { + if h.used == 0 { + // At the beginning of a line we print the current + // offset in hex. + h.buf[0] = byte(h.n >> 24) + h.buf[1] = byte(h.n >> 16) + h.buf[2] = byte(h.n >> 8) + h.buf[3] = byte(h.n) + Encode(h.buf[4:], h.buf[:4]) + h.buf[12] = ' ' + h.buf[13] = ' ' + _, err = h.w.Write(h.buf[4:]) + } + Encode(h.buf[:], data[i:i+1]) + h.buf[2] = ' ' + l := 3 + if h.used == 7 { + // There's an additional space after the 8th byte. + h.buf[3] = ' ' + l = 4 + } else if h.used == 15 { + // At the end of the line there's an extra space and + // the bar for the right column. + h.buf[3] = ' ' + h.buf[4] = '|' + l = 5 + } + _, err = h.w.Write(h.buf[:l]) + if err != nil { + return + } + n++ + h.rightChars[h.used] = toChar(data[i]) + h.used++ + h.n++ + if h.used == 16 { + h.rightChars[16] = '|' + h.rightChars[17] = '\n' + _, err = h.w.Write(h.rightChars[:]) + if err != nil { + return + } + h.used = 0 + } + } + return +} + +func (h *dumper) Close() (err os.Error) { + // See the comments in Write() for the details of this format. + if h.used == 0 { + return + } + h.buf[0] = ' ' + h.buf[1] = ' ' + h.buf[2] = ' ' + h.buf[3] = ' ' + h.buf[4] = '|' + nBytes := h.used + for h.used < 16 { + l := 3 + if h.used == 7 { + l = 4 + } else if h.used == 15 { + l = 5 + } + _, err = h.w.Write(h.buf[:l]) + if err != nil { + return + } + h.used++ + } + h.rightChars[nBytes] = '|' + h.rightChars[nBytes+1] = '\n' + _, err = h.w.Write(h.rightChars[:nBytes+2]) + return +} diff --git a/src/pkg/encoding/hex/hex_test.go b/src/pkg/encoding/hex/hex_test.go new file mode 100644 index 000000000..8e1838e51 --- /dev/null +++ b/src/pkg/encoding/hex/hex_test.go @@ -0,0 +1,192 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hex + +import ( + "bytes" + "testing" +) + +type encodeTest struct { + in, out []byte +} + +var encodeTests = []encodeTest{ + {[]byte{}, []byte{}}, + {[]byte{0x01}, []byte{'0', '1'}}, + {[]byte{0xff}, []byte{'f', 'f'}}, + {[]byte{0xff, 00}, []byte{'f', 'f', '0', '0'}}, + {[]byte{0}, []byte{'0', '0'}}, + {[]byte{1}, []byte{'0', '1'}}, + {[]byte{2}, []byte{'0', '2'}}, + {[]byte{3}, []byte{'0', '3'}}, + {[]byte{4}, []byte{'0', '4'}}, + {[]byte{5}, []byte{'0', '5'}}, + {[]byte{6}, []byte{'0', '6'}}, + {[]byte{7}, []byte{'0', '7'}}, + {[]byte{8}, []byte{'0', '8'}}, + {[]byte{9}, []byte{'0', '9'}}, + {[]byte{10}, []byte{'0', 'a'}}, + {[]byte{11}, []byte{'0', 'b'}}, + {[]byte{12}, []byte{'0', 'c'}}, + {[]byte{13}, []byte{'0', 'd'}}, + {[]byte{14}, []byte{'0', 'e'}}, + {[]byte{15}, []byte{'0', 'f'}}, +} + +func TestEncode(t *testing.T) { + for i, test := range encodeTests { + dst := make([]byte, EncodedLen(len(test.in))) + n := Encode(dst, test.in) + if n != len(dst) { + t.Errorf("#%d: bad return value: got: %d want: %d", i, n, len(dst)) + } + if bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } + } +} + +type decodeTest struct { + in, out []byte + ok bool +} + +var decodeTests = []decodeTest{ + {[]byte{}, []byte{}, true}, + {[]byte{'0'}, []byte{}, false}, + {[]byte{'0', 'g'}, []byte{}, false}, + {[]byte{'0', '\x01'}, []byte{}, false}, + {[]byte{'0', '0'}, []byte{0}, true}, + {[]byte{'0', '1'}, []byte{1}, true}, + {[]byte{'0', '2'}, []byte{2}, true}, + {[]byte{'0', '3'}, []byte{3}, true}, + {[]byte{'0', '4'}, []byte{4}, true}, + {[]byte{'0', '5'}, []byte{5}, true}, + {[]byte{'0', '6'}, []byte{6}, true}, + {[]byte{'0', '7'}, []byte{7}, true}, + {[]byte{'0', '8'}, []byte{8}, true}, + {[]byte{'0', '9'}, []byte{9}, true}, + {[]byte{'0', 'a'}, []byte{10}, true}, + {[]byte{'0', 'b'}, []byte{11}, true}, + {[]byte{'0', 'c'}, []byte{12}, true}, + {[]byte{'0', 'd'}, []byte{13}, true}, + {[]byte{'0', 'e'}, []byte{14}, true}, + {[]byte{'0', 'f'}, []byte{15}, true}, + {[]byte{'0', 'A'}, []byte{10}, true}, + {[]byte{'0', 'B'}, []byte{11}, true}, + {[]byte{'0', 'C'}, []byte{12}, true}, + {[]byte{'0', 'D'}, []byte{13}, true}, + {[]byte{'0', 'E'}, []byte{14}, true}, + {[]byte{'0', 'F'}, []byte{15}, true}, +} + +func TestDecode(t *testing.T) { + for i, test := range decodeTests { + dst := make([]byte, DecodedLen(len(test.in))) + n, err := Decode(dst, test.in) + if err == nil && n != len(dst) { + t.Errorf("#%d: bad return value: got:%d want:%d", i, n, len(dst)) + } + if test.ok != (err == nil) { + t.Errorf("#%d: unexpected err value: %s", i, err) + } + if err == nil && bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } + } +} + +type encodeStringTest struct { + in []byte + out string +} + +var encodeStringTests = []encodeStringTest{ + {[]byte{}, ""}, + {[]byte{0}, "00"}, + {[]byte{0, 1}, "0001"}, + {[]byte{0, 1, 255}, "0001ff"}, +} + +func TestEncodeToString(t *testing.T) { + for i, test := range encodeStringTests { + s := EncodeToString(test.in) + if s != test.out { + t.Errorf("#%d got:%s want:%s", i, s, test.out) + } + } +} + +type decodeStringTest struct { + in string + out []byte + ok bool +} + +var decodeStringTests = []decodeStringTest{ + {"", []byte{}, true}, + {"0", []byte{}, false}, + {"00", []byte{0}, true}, + {"0\x01", []byte{}, false}, + {"0g", []byte{}, false}, + {"00ff00", []byte{0, 255, 0}, true}, + {"0000ff", []byte{0, 0, 255}, true}, +} + +func TestDecodeString(t *testing.T) { + for i, test := range decodeStringTests { + dst, err := DecodeString(test.in) + if test.ok != (err == nil) { + t.Errorf("#%d: unexpected err value: %s", i, err) + } + if err == nil && bytes.Compare(dst, test.out) != 0 { + t.Errorf("#%d: got: %#v want: #%v", i, dst, test.out) + } + } +} + +func TestDumper(t *testing.T) { + var in [40]byte + for i := range in { + in[i] = byte(i + 30) + } + + for stride := 1; stride < len(in); stride++ { + out := bytes.NewBuffer(nil) + dumper := Dumper(out) + done := 0 + for done < len(in) { + todo := done + stride + if todo > len(in) { + todo = len(in) + } + dumper.Write(in[done:todo]) + done = todo + } + + dumper.Close() + if !bytes.Equal(out.Bytes(), expectedHexDump) { + t.Errorf("stride: %d failed. got:\n%s\nwant:\n%s", stride, out.Bytes(), expectedHexDump) + } + } +} + +func TestDump(t *testing.T) { + var in [40]byte + for i := range in { + in[i] = byte(i + 30) + } + + out := []byte(Dump(in[:])) + if !bytes.Equal(out, expectedHexDump) { + t.Errorf("got:\n%s\nwant:\n%s", out, expectedHexDump) + } +} + +var expectedHexDump = []byte(`00000000 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d |.. !"#$%&'()*+,-| +00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| +00000020 3e 3f 40 41 42 43 44 45 |>?@ABCDE| +`) diff --git a/src/pkg/encoding/pem/Makefile b/src/pkg/encoding/pem/Makefile new file mode 100644 index 000000000..527670380 --- /dev/null +++ b/src/pkg/encoding/pem/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=encoding/pem +GOFILES=\ + pem.go\ + +include ../../../Make.pkg diff --git a/src/pkg/encoding/pem/pem.go b/src/pkg/encoding/pem/pem.go new file mode 100644 index 000000000..12689b57b --- /dev/null +++ b/src/pkg/encoding/pem/pem.go @@ -0,0 +1,258 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pem implements the PEM data encoding, which originated in Privacy +// Enhanced Mail. The most common use of PEM encoding today is in TLS keys and +// certificates. See RFC 1421. +package pem + +import ( + "bytes" + "encoding/base64" + "io" + "os" +) + +// A Block represents a PEM encoded structure. +// +// The encoded form is: +// -----BEGIN Type----- +// Headers +// base64-encoded Bytes +// -----END Type----- +// where Headers is a possibly empty sequence of Key: Value lines. +type Block struct { + Type string // The type, taken from the preamble (i.e. "RSA PRIVATE KEY"). + Headers map[string]string // Optional headers. + Bytes []byte // The decoded bytes of the contents. Typically a DER encoded ASN.1 structure. +} + +// getLine results the first \r\n or \n delineated line from the given byte +// array. The line does not include the \r\n or \n. The remainder of the byte +// array (also not including the new line bytes) is also returned and this will +// always be smaller than the original argument. +func getLine(data []byte) (line, rest []byte) { + i := bytes.Index(data, []byte{'\n'}) + var j int + if i < 0 { + i = len(data) + j = i + } else { + j = i + 1 + if i > 0 && data[i-1] == '\r' { + i-- + } + } + return data[0:i], data[j:] +} + +// removeWhitespace returns a copy of its input with all spaces, tab and +// newline characters removed. +func removeWhitespace(data []byte) []byte { + result := make([]byte, len(data)) + n := 0 + + for _, b := range data { + if b == ' ' || b == '\t' || b == '\r' || b == '\n' { + continue + } + result[n] = b + n++ + } + + return result[0:n] +} + +var pemStart = []byte("\n-----BEGIN ") +var pemEnd = []byte("\n-----END ") +var pemEndOfLine = []byte("-----") + +// Decode will find the next PEM formatted block (certificate, private key +// etc) in the input. It returns that block and the remainder of the input. If +// no PEM data is found, p is nil and the whole of the input is returned in +// rest. +func Decode(data []byte) (p *Block, rest []byte) { + // pemStart begins with a newline. However, at the very beginning of + // the byte array, we'll accept the start string without it. + rest = data + if bytes.HasPrefix(data, pemStart[1:]) { + rest = rest[len(pemStart)-1 : len(data)] + } else if i := bytes.Index(data, pemStart); i >= 0 { + rest = rest[i+len(pemStart) : len(data)] + } else { + return nil, data + } + + typeLine, rest := getLine(rest) + if !bytes.HasSuffix(typeLine, pemEndOfLine) { + return decodeError(data, rest) + } + typeLine = typeLine[0 : len(typeLine)-len(pemEndOfLine)] + + p = &Block{ + Headers: make(map[string]string), + Type: string(typeLine), + } + + for { + // This loop terminates because getLine's second result is + // always smaller than its argument. + if len(rest) == 0 { + return nil, data + } + line, next := getLine(rest) + + i := bytes.Index(line, []byte{':'}) + if i == -1 { + break + } + + // TODO(agl): need to cope with values that spread across lines. + key, val := line[0:i], line[i+1:] + key = bytes.TrimSpace(key) + val = bytes.TrimSpace(val) + p.Headers[string(key)] = string(val) + rest = next + } + + i := bytes.Index(rest, pemEnd) + if i < 0 { + return decodeError(data, rest) + } + base64Data := removeWhitespace(rest[0:i]) + + p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) + n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) + if err != nil { + return decodeError(data, rest) + } + p.Bytes = p.Bytes[0:n] + + _, rest = getLine(rest[i+len(pemEnd):]) + + return +} + +func decodeError(data, rest []byte) (*Block, []byte) { + // If we get here then we have rejected a likely looking, but + // ultimately invalid PEM block. We need to start over from a new + // position. We have consumed the preamble line and will have consumed + // any lines which could be header lines. However, a valid preamble + // line is not a valid header line, therefore we cannot have consumed + // the preamble line for the any subsequent block. Thus, we will always + // find any valid block, no matter what bytes precede it. + // + // For example, if the input is + // + // -----BEGIN MALFORMED BLOCK----- + // junk that may look like header lines + // or data lines, but no END line + // + // -----BEGIN ACTUAL BLOCK----- + // realdata + // -----END ACTUAL BLOCK----- + // + // we've failed to parse using the first BEGIN line + // and now will try again, using the second BEGIN line. + p, rest := Decode(rest) + if p == nil { + rest = data + } + return p, rest +} + +const pemLineLength = 64 + +type lineBreaker struct { + line [pemLineLength]byte + used int + out io.Writer +} + +func (l *lineBreaker) Write(b []byte) (n int, err os.Error) { + if l.used+len(b) < pemLineLength { + copy(l.line[l.used:], b) + l.used += len(b) + return len(b), nil + } + + n, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + excess := pemLineLength - l.used + l.used = 0 + + n, err = l.out.Write(b[0:excess]) + if err != nil { + return + } + + n, err = l.out.Write([]byte{'\n'}) + if err != nil { + return + } + + return l.Write(b[excess:]) +} + +func (l *lineBreaker) Close() (err os.Error) { + if l.used > 0 { + _, err = l.out.Write(l.line[0:l.used]) + if err != nil { + return + } + _, err = l.out.Write([]byte{'\n'}) + } + + return +} + +func Encode(out io.Writer, b *Block) (err os.Error) { + _, err = out.Write(pemStart[1:]) + if err != nil { + return + } + _, err = out.Write([]byte(b.Type + "-----\n")) + if err != nil { + return + } + + if len(b.Headers) > 0 { + for k, v := range b.Headers { + _, err = out.Write([]byte(k + ": " + v + "\n")) + if err != nil { + return + } + } + _, err = out.Write([]byte{'\n'}) + if err != nil { + return + } + } + + var breaker lineBreaker + breaker.out = out + + b64 := base64.NewEncoder(base64.StdEncoding, &breaker) + _, err = b64.Write(b.Bytes) + if err != nil { + return + } + b64.Close() + breaker.Close() + + _, err = out.Write(pemEnd[1:]) + if err != nil { + return + } + _, err = out.Write([]byte(b.Type + "-----\n")) + return +} + +func EncodeToMemory(b *Block) []byte { + buf := bytes.NewBuffer(nil) + Encode(buf, b) + return buf.Bytes() +} diff --git a/src/pkg/encoding/pem/pem_test.go b/src/pkg/encoding/pem/pem_test.go new file mode 100644 index 000000000..11efe5544 --- /dev/null +++ b/src/pkg/encoding/pem/pem_test.go @@ -0,0 +1,390 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pem + +import ( + "bytes" + "reflect" + "testing" +) + +type GetLineTest struct { + in, out1, out2 string +} + +var getLineTests = []GetLineTest{ + {"abc", "abc", ""}, + {"abc\r", "abc\r", ""}, + {"abc\n", "abc", ""}, + {"abc\r\n", "abc", ""}, + {"abc\nd", "abc", "d"}, + {"abc\r\nd", "abc", "d"}, + {"\nabc", "", "abc"}, + {"\r\nabc", "", "abc"}, +} + +func TestGetLine(t *testing.T) { + for i, test := range getLineTests { + x, y := getLine([]byte(test.in)) + if string(x) != test.out1 || string(y) != test.out2 { + t.Errorf("#%d got:%+v,%+v want:%s,%s", i, x, y, test.out1, test.out2) + } + } +} + +func TestDecode(t *testing.T) { + result, remainder := Decode([]byte(pemData)) + if !reflect.DeepEqual(result, certificate) { + t.Errorf("#0 got:%#v want:%#v", result, certificate) + } + result, remainder = Decode(remainder) + if !reflect.DeepEqual(result, privateKey) { + t.Errorf("#1 got:%#v want:%#v", result, privateKey) + } + result, _ = Decode([]byte(pemPrivateKey)) + if !reflect.DeepEqual(result, privateKey2) { + t.Errorf("#2 got:%#v want:%#v", result, privateKey2) + } +} + +func TestEncode(t *testing.T) { + r := EncodeToMemory(privateKey2) + if string(r) != pemPrivateKey { + t.Errorf("got:%s want:%s", r, pemPrivateKey) + } +} + +type lineBreakerTest struct { + in, out string +} + +const sixtyFourCharString = "0123456789012345678901234567890123456789012345678901234567890123" + +var lineBreakerTests = []lineBreakerTest{ + {"", ""}, + {"a", "a\n"}, + {"ab", "ab\n"}, + {sixtyFourCharString, sixtyFourCharString + "\n"}, + {sixtyFourCharString + "X", sixtyFourCharString + "\nX\n"}, + {sixtyFourCharString + sixtyFourCharString, sixtyFourCharString + "\n" + sixtyFourCharString + "\n"}, +} + +func TestLineBreaker(t *testing.T) { + for i, test := range lineBreakerTests { + buf := bytes.NewBuffer(nil) + var breaker lineBreaker + breaker.out = buf + _, err := breaker.Write([]byte(test.in)) + if err != nil { + t.Errorf("#%d: error from Write: %s", i, err) + continue + } + err = breaker.Close() + if err != nil { + t.Errorf("#%d: error from Close: %s", i, err) + continue + } + + if string(buf.Bytes()) != test.out { + t.Errorf("#%d: got:%s want:%s", i, string(buf.Bytes()), test.out) + } + } + + for i, test := range lineBreakerTests { + buf := bytes.NewBuffer(nil) + var breaker lineBreaker + breaker.out = buf + + for i := 0; i < len(test.in); i++ { + _, err := breaker.Write([]byte(test.in[i : i+1])) + if err != nil { + t.Errorf("#%d: error from Write (byte by byte): %s", i, err) + continue + } + } + err := breaker.Close() + if err != nil { + t.Errorf("#%d: error from Close (byte by byte): %s", i, err) + continue + } + + if string(buf.Bytes()) != test.out { + t.Errorf("#%d: (byte by byte) got:%s want:%s", i, string(buf.Bytes()), test.out) + } + } +} + +var pemData = `verify return:0 +-----BEGIN CERTIFICATE----- +sdlfkjskldfj + -----BEGIN CERTIFICATE----- +--- +Certificate chain + 0 s:/C=AU/ST=Somewhere/L=Someplace/O=Foo Bar/CN=foo.example.com + i:/C=ZA/O=CA Inc./CN=CA Inc +-----BEGIN CERTIFICATE----- +testing +-----BEGIN CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID6TCCA1ICAQEwDQYJKoZIhvcNAQEFBQAwgYsxCzAJBgNVBAYTAlVTMRMwEQYD +VQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQK +EwtHb29nbGUgSW5jLjEMMAoGA1UECxMDRW5nMQwwCgYDVQQDEwNhZ2wxHTAbBgkq +hkiG9w0BCQEWDmFnbEBnb29nbGUuY29tMB4XDTA5MDkwOTIyMDU0M1oXDTEwMDkw +OTIyMDU0M1owajELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAf +BgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEjMCEGA1UEAxMaZXVyb3Bh +LnNmby5jb3JwLmdvb2dsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQC6pgYt7/EibBDumASF+S0qvqdL/f+nouJw2T1Qc8GmXF/iiUcrsgzh/Fd8 +pDhz/T96Qg9IyR4ztuc2MXrmPra+zAuSf5bevFReSqvpIt8Duv0HbDbcqs/XKPfB +uMDe+of7a9GCywvAZ4ZUJcp0thqD9fKTTjUWOBzHY1uNE4RitrhmJCrbBGXbJ249 +bvgmb7jgdInH2PU7PT55hujvOoIsQW2osXBFRur4pF1wmVh4W4lTLD6pjfIMUcML +ICHEXEN73PDic8KS3EtNYCwoIld+tpIBjE1QOb1KOyuJBNW6Esw9ALZn7stWdYcE +qAwvv20egN2tEXqj7Q4/1ccyPZc3PQgC3FJ8Be2mtllM+80qf4dAaQ/fWvCtOrQ5 +pnfe9juQvCo8Y0VGlFcrSys/MzSg9LJ/24jZVgzQved/Qupsp89wVidwIzjt+WdS +fyWfH0/v1aQLvu5cMYuW//C0W2nlYziL5blETntM8My2ybNARy3ICHxCBv2RNtPI +WQVm+E9/W5rwh2IJR4DHn2LHwUVmT/hHNTdBLl5Uhwr4Wc7JhE7AVqb14pVNz1lr +5jxsp//ncIwftb7mZQ3DF03Yna+jJhpzx8CQoeLT6aQCHyzmH68MrHHT4MALPyUs +Pomjn71GNTtDeWAXibjCgdL6iHACCF6Htbl0zGlG0OAK+bdn0QIDAQABMA0GCSqG +SIb3DQEBBQUAA4GBAOKnQDtqBV24vVqvesL5dnmyFpFPXBn3WdFfwD6DzEb21UVG +5krmJiu+ViipORJPGMkgoL6BjU21XI95VQbun5P8vvg8Z+FnFsvRFY3e1CCzAVQY +ZsUkLw2I7zI/dNlWdB8Xp7v+3w9sX5N3J/WuJ1KOO5m26kRlHQo7EzT3974g +-----END CERTIFICATE----- + 1 s:/C=ZA/O=Ca Inc./CN=CA Inc + +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,80C7C7A09690757A + +eQp5ZkH6CyHBz7BZfUPxyLCCmftsBJ7HlqGb8Ld21cSwnzWZ4/SIlhyrUtsfw7VR +2TTwA+odo9ex7GdxOTaH8oZFumIRoiEjHsk8U7Bhntp+ekkPP79xunnN7hb7hkhr +yGDQZgA7s2cQHQ71v3gwT2BACAft26jCjbM1wgNzBnJ8M0Rzn68YWqaPtdBu8qb/ +zVR5JB1mnqvTSbFsfF5yMc6o2WQ9jJCl6KypnMl+BpL+dlvdjYVK4l9lYsB1Hs3d ++zDBbWxos818zzhS8/y6eIfiSG27cqrbhURbmgiSfDXjncK4m/pLcQ7mmBL6mFOr +3Pj4jepzgOiFRL6MKE//h62fZvI1ErYr8VunHEykgKNhChDvb1RO6LEfqKBu+Ivw +TB6fBhW3TCLMnVPYVoYwA+fHNTmZZm8BEonlIMfI+KktjWUg4Oia+NI6vKcPpFox +hSnlGgCtvfEaq5/H4kHJp95eOpnFsLviw2seHNkz/LxJMRP1X428+DpYW/QD/0JU +tJSuC/q9FUHL6RI3u/Asrv8pCb4+D7i1jW/AMIdJTtycOGsbPxQA7yHMWujHmeb1 +BTiHcL3s3KrJu1vDVrshvxfnz71KTeNnZH8UbOqT5i7fPGyXtY1XJddcbI/Q6tXf +wHFsZc20TzSdsVLBtwksUacpbDogcEVMctnNrB8FIrB3vZEv9Q0Z1VeY7nmTpF+6 +a+z2P7acL7j6A6Pr3+q8P9CPiPC7zFonVzuVPyB8GchGR2hytyiOVpuD9+k8hcuw +ZWAaUoVtWIQ52aKS0p19G99hhb+IVANC4akkdHV4SP8i7MVNZhfUmg== +-----END RSA PRIVATE KEY-----` + +var certificate = &Block{Type: "CERTIFICATE", + Headers: map[string]string{}, + Bytes: []uint8{0x30, 0x82, 0x3, 0xe9, 0x30, 0x82, 0x3, 0x52, 0x2, 0x1, + 0x1, 0x30, 0xd, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, + 0x1, 0x1, 0x5, 0x5, 0x0, 0x30, 0x81, 0x8b, 0x31, 0xb, 0x30, + 0x9, 0x6, 0x3, 0x55, 0x4, 0x6, 0x13, 0x2, 0x55, 0x53, 0x31, + 0x13, 0x30, 0x11, 0x6, 0x3, 0x55, 0x4, 0x8, 0x13, 0xa, 0x43, + 0x61, 0x6c, 0x69, 0x66, 0x6f, 0x72, 0x6e, 0x69, 0x61, 0x31, + 0x16, 0x30, 0x14, 0x6, 0x3, 0x55, 0x4, 0x7, 0x13, 0xd, 0x53, + 0x61, 0x6e, 0x20, 0x46, 0x72, 0x61, 0x6e, 0x63, 0x69, 0x73, + 0x63, 0x6f, 0x31, 0x14, 0x30, 0x12, 0x6, 0x3, 0x55, 0x4, 0xa, + 0x13, 0xb, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x20, 0x49, + 0x6e, 0x63, 0x2e, 0x31, 0xc, 0x30, 0xa, 0x6, 0x3, 0x55, 0x4, + 0xb, 0x13, 0x3, 0x45, 0x6e, 0x67, 0x31, 0xc, 0x30, 0xa, 0x6, + 0x3, 0x55, 0x4, 0x3, 0x13, 0x3, 0x61, 0x67, 0x6c, 0x31, 0x1d, + 0x30, 0x1b, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, + 0x9, 0x1, 0x16, 0xe, 0x61, 0x67, 0x6c, 0x40, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x30, 0x1e, 0x17, + 0xd, 0x30, 0x39, 0x30, 0x39, 0x30, 0x39, 0x32, 0x32, 0x30, + 0x35, 0x34, 0x33, 0x5a, 0x17, 0xd, 0x31, 0x30, 0x30, 0x39, + 0x30, 0x39, 0x32, 0x32, 0x30, 0x35, 0x34, 0x33, 0x5a, 0x30, + 0x6a, 0x31, 0xb, 0x30, 0x9, 0x6, 0x3, 0x55, 0x4, 0x6, 0x13, + 0x2, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x6, 0x3, 0x55, 0x4, + 0x8, 0x13, 0xa, 0x53, 0x6f, 0x6d, 0x65, 0x2d, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x31, 0x21, 0x30, 0x1f, 0x6, 0x3, 0x55, 0x4, 0xa, + 0x13, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x20, 0x57, 0x69, 0x64, 0x67, 0x69, 0x74, 0x73, 0x20, 0x50, + 0x74, 0x79, 0x20, 0x4c, 0x74, 0x64, 0x31, 0x23, 0x30, 0x21, + 0x6, 0x3, 0x55, 0x4, 0x3, 0x13, 0x1a, 0x65, 0x75, 0x72, 0x6f, + 0x70, 0x61, 0x2e, 0x73, 0x66, 0x6f, 0x2e, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, + 0x6f, 0x6d, 0x30, 0x82, 0x2, 0x22, 0x30, 0xd, 0x6, 0x9, 0x2a, + 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, 0x1, 0x1, 0x5, 0x0, 0x3, + 0x82, 0x2, 0xf, 0x0, 0x30, 0x82, 0x2, 0xa, 0x2, 0x82, 0x2, 0x1, + 0x0, 0xba, 0xa6, 0x6, 0x2d, 0xef, 0xf1, 0x22, 0x6c, 0x10, 0xee, + 0x98, 0x4, 0x85, 0xf9, 0x2d, 0x2a, 0xbe, 0xa7, 0x4b, 0xfd, + 0xff, 0xa7, 0xa2, 0xe2, 0x70, 0xd9, 0x3d, 0x50, 0x73, 0xc1, + 0xa6, 0x5c, 0x5f, 0xe2, 0x89, 0x47, 0x2b, 0xb2, 0xc, 0xe1, + 0xfc, 0x57, 0x7c, 0xa4, 0x38, 0x73, 0xfd, 0x3f, 0x7a, 0x42, + 0xf, 0x48, 0xc9, 0x1e, 0x33, 0xb6, 0xe7, 0x36, 0x31, 0x7a, + 0xe6, 0x3e, 0xb6, 0xbe, 0xcc, 0xb, 0x92, 0x7f, 0x96, 0xde, + 0xbc, 0x54, 0x5e, 0x4a, 0xab, 0xe9, 0x22, 0xdf, 0x3, 0xba, + 0xfd, 0x7, 0x6c, 0x36, 0xdc, 0xaa, 0xcf, 0xd7, 0x28, 0xf7, + 0xc1, 0xb8, 0xc0, 0xde, 0xfa, 0x87, 0xfb, 0x6b, 0xd1, 0x82, + 0xcb, 0xb, 0xc0, 0x67, 0x86, 0x54, 0x25, 0xca, 0x74, 0xb6, + 0x1a, 0x83, 0xf5, 0xf2, 0x93, 0x4e, 0x35, 0x16, 0x38, 0x1c, + 0xc7, 0x63, 0x5b, 0x8d, 0x13, 0x84, 0x62, 0xb6, 0xb8, 0x66, + 0x24, 0x2a, 0xdb, 0x4, 0x65, 0xdb, 0x27, 0x6e, 0x3d, 0x6e, + 0xf8, 0x26, 0x6f, 0xb8, 0xe0, 0x74, 0x89, 0xc7, 0xd8, 0xf5, + 0x3b, 0x3d, 0x3e, 0x79, 0x86, 0xe8, 0xef, 0x3a, 0x82, 0x2c, + 0x41, 0x6d, 0xa8, 0xb1, 0x70, 0x45, 0x46, 0xea, 0xf8, 0xa4, + 0x5d, 0x70, 0x99, 0x58, 0x78, 0x5b, 0x89, 0x53, 0x2c, 0x3e, + 0xa9, 0x8d, 0xf2, 0xc, 0x51, 0xc3, 0xb, 0x20, 0x21, 0xc4, 0x5c, + 0x43, 0x7b, 0xdc, 0xf0, 0xe2, 0x73, 0xc2, 0x92, 0xdc, 0x4b, + 0x4d, 0x60, 0x2c, 0x28, 0x22, 0x57, 0x7e, 0xb6, 0x92, 0x1, + 0x8c, 0x4d, 0x50, 0x39, 0xbd, 0x4a, 0x3b, 0x2b, 0x89, 0x4, + 0xd5, 0xba, 0x12, 0xcc, 0x3d, 0x0, 0xb6, 0x67, 0xee, 0xcb, + 0x56, 0x75, 0x87, 0x4, 0xa8, 0xc, 0x2f, 0xbf, 0x6d, 0x1e, 0x80, + 0xdd, 0xad, 0x11, 0x7a, 0xa3, 0xed, 0xe, 0x3f, 0xd5, 0xc7, + 0x32, 0x3d, 0x97, 0x37, 0x3d, 0x8, 0x2, 0xdc, 0x52, 0x7c, 0x5, + 0xed, 0xa6, 0xb6, 0x59, 0x4c, 0xfb, 0xcd, 0x2a, 0x7f, 0x87, + 0x40, 0x69, 0xf, 0xdf, 0x5a, 0xf0, 0xad, 0x3a, 0xb4, 0x39, + 0xa6, 0x77, 0xde, 0xf6, 0x3b, 0x90, 0xbc, 0x2a, 0x3c, 0x63, + 0x45, 0x46, 0x94, 0x57, 0x2b, 0x4b, 0x2b, 0x3f, 0x33, 0x34, + 0xa0, 0xf4, 0xb2, 0x7f, 0xdb, 0x88, 0xd9, 0x56, 0xc, 0xd0, + 0xbd, 0xe7, 0x7f, 0x42, 0xea, 0x6c, 0xa7, 0xcf, 0x70, 0x56, + 0x27, 0x70, 0x23, 0x38, 0xed, 0xf9, 0x67, 0x52, 0x7f, 0x25, + 0x9f, 0x1f, 0x4f, 0xef, 0xd5, 0xa4, 0xb, 0xbe, 0xee, 0x5c, + 0x31, 0x8b, 0x96, 0xff, 0xf0, 0xb4, 0x5b, 0x69, 0xe5, 0x63, + 0x38, 0x8b, 0xe5, 0xb9, 0x44, 0x4e, 0x7b, 0x4c, 0xf0, 0xcc, + 0xb6, 0xc9, 0xb3, 0x40, 0x47, 0x2d, 0xc8, 0x8, 0x7c, 0x42, 0x6, + 0xfd, 0x91, 0x36, 0xd3, 0xc8, 0x59, 0x5, 0x66, 0xf8, 0x4f, + 0x7f, 0x5b, 0x9a, 0xf0, 0x87, 0x62, 0x9, 0x47, 0x80, 0xc7, + 0x9f, 0x62, 0xc7, 0xc1, 0x45, 0x66, 0x4f, 0xf8, 0x47, 0x35, + 0x37, 0x41, 0x2e, 0x5e, 0x54, 0x87, 0xa, 0xf8, 0x59, 0xce, + 0xc9, 0x84, 0x4e, 0xc0, 0x56, 0xa6, 0xf5, 0xe2, 0x95, 0x4d, + 0xcf, 0x59, 0x6b, 0xe6, 0x3c, 0x6c, 0xa7, 0xff, 0xe7, 0x70, + 0x8c, 0x1f, 0xb5, 0xbe, 0xe6, 0x65, 0xd, 0xc3, 0x17, 0x4d, + 0xd8, 0x9d, 0xaf, 0xa3, 0x26, 0x1a, 0x73, 0xc7, 0xc0, 0x90, + 0xa1, 0xe2, 0xd3, 0xe9, 0xa4, 0x2, 0x1f, 0x2c, 0xe6, 0x1f, + 0xaf, 0xc, 0xac, 0x71, 0xd3, 0xe0, 0xc0, 0xb, 0x3f, 0x25, 0x2c, + 0x3e, 0x89, 0xa3, 0x9f, 0xbd, 0x46, 0x35, 0x3b, 0x43, 0x79, + 0x60, 0x17, 0x89, 0xb8, 0xc2, 0x81, 0xd2, 0xfa, 0x88, 0x70, + 0x2, 0x8, 0x5e, 0x87, 0xb5, 0xb9, 0x74, 0xcc, 0x69, 0x46, 0xd0, + 0xe0, 0xa, 0xf9, 0xb7, 0x67, 0xd1, 0x2, 0x3, 0x1, 0x0, 0x1, + 0x30, 0xd, 0x6, 0x9, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0xd, 0x1, + 0x1, 0x5, 0x5, 0x0, 0x3, 0x81, 0x81, 0x0, 0xe2, 0xa7, 0x40, + 0x3b, 0x6a, 0x5, 0x5d, 0xb8, 0xbd, 0x5a, 0xaf, 0x7a, 0xc2, + 0xf9, 0x76, 0x79, 0xb2, 0x16, 0x91, 0x4f, 0x5c, 0x19, 0xf7, + 0x59, 0xd1, 0x5f, 0xc0, 0x3e, 0x83, 0xcc, 0x46, 0xf6, 0xd5, + 0x45, 0x46, 0xe6, 0x4a, 0xe6, 0x26, 0x2b, 0xbe, 0x56, 0x28, + 0xa9, 0x39, 0x12, 0x4f, 0x18, 0xc9, 0x20, 0xa0, 0xbe, 0x81, + 0x8d, 0x4d, 0xb5, 0x5c, 0x8f, 0x79, 0x55, 0x6, 0xee, 0x9f, + 0x93, 0xfc, 0xbe, 0xf8, 0x3c, 0x67, 0xe1, 0x67, 0x16, 0xcb, + 0xd1, 0x15, 0x8d, 0xde, 0xd4, 0x20, 0xb3, 0x1, 0x54, 0x18, + 0x66, 0xc5, 0x24, 0x2f, 0xd, 0x88, 0xef, 0x32, 0x3f, 0x74, + 0xd9, 0x56, 0x74, 0x1f, 0x17, 0xa7, 0xbb, 0xfe, 0xdf, 0xf, + 0x6c, 0x5f, 0x93, 0x77, 0x27, 0xf5, 0xae, 0x27, 0x52, 0x8e, + 0x3b, 0x99, 0xb6, 0xea, 0x44, 0x65, 0x1d, 0xa, 0x3b, 0x13, + 0x34, 0xf7, 0xf7, 0xbe, 0x20, + }, +} + +var privateKey = &Block{Type: "RSA PRIVATE KEY", + Headers: map[string]string{"DEK-Info": "DES-EDE3-CBC,80C7C7A09690757A", "Proc-Type": "4,ENCRYPTED"}, + Bytes: []uint8{0x79, 0xa, 0x79, 0x66, 0x41, 0xfa, 0xb, + 0x21, 0xc1, 0xcf, 0xb0, 0x59, 0x7d, 0x43, 0xf1, 0xc8, 0xb0, + 0x82, 0x99, 0xfb, 0x6c, 0x4, 0x9e, 0xc7, 0x96, 0xa1, 0x9b, + 0xf0, 0xb7, 0x76, 0xd5, 0xc4, 0xb0, 0x9f, 0x35, 0x99, 0xe3, + 0xf4, 0x88, 0x96, 0x1c, 0xab, 0x52, 0xdb, 0x1f, 0xc3, 0xb5, + 0x51, 0xd9, 0x34, 0xf0, 0x3, 0xea, 0x1d, 0xa3, 0xd7, 0xb1, + 0xec, 0x67, 0x71, 0x39, 0x36, 0x87, 0xf2, 0x86, 0x45, 0xba, + 0x62, 0x11, 0xa2, 0x21, 0x23, 0x1e, 0xc9, 0x3c, 0x53, 0xb0, + 0x61, 0x9e, 0xda, 0x7e, 0x7a, 0x49, 0xf, 0x3f, 0xbf, 0x71, + 0xba, 0x79, 0xcd, 0xee, 0x16, 0xfb, 0x86, 0x48, 0x6b, 0xc8, + 0x60, 0xd0, 0x66, 0x0, 0x3b, 0xb3, 0x67, 0x10, 0x1d, 0xe, + 0xf5, 0xbf, 0x78, 0x30, 0x4f, 0x60, 0x40, 0x8, 0x7, 0xed, + 0xdb, 0xa8, 0xc2, 0x8d, 0xb3, 0x35, 0xc2, 0x3, 0x73, 0x6, + 0x72, 0x7c, 0x33, 0x44, 0x73, 0x9f, 0xaf, 0x18, 0x5a, 0xa6, + 0x8f, 0xb5, 0xd0, 0x6e, 0xf2, 0xa6, 0xff, 0xcd, 0x54, 0x79, + 0x24, 0x1d, 0x66, 0x9e, 0xab, 0xd3, 0x49, 0xb1, 0x6c, 0x7c, + 0x5e, 0x72, 0x31, 0xce, 0xa8, 0xd9, 0x64, 0x3d, 0x8c, 0x90, + 0xa5, 0xe8, 0xac, 0xa9, 0x9c, 0xc9, 0x7e, 0x6, 0x92, 0xfe, + 0x76, 0x5b, 0xdd, 0x8d, 0x85, 0x4a, 0xe2, 0x5f, 0x65, 0x62, + 0xc0, 0x75, 0x1e, 0xcd, 0xdd, 0xfb, 0x30, 0xc1, 0x6d, 0x6c, + 0x68, 0xb3, 0xcd, 0x7c, 0xcf, 0x38, 0x52, 0xf3, 0xfc, 0xba, + 0x78, 0x87, 0xe2, 0x48, 0x6d, 0xbb, 0x72, 0xaa, 0xdb, 0x85, + 0x44, 0x5b, 0x9a, 0x8, 0x92, 0x7c, 0x35, 0xe3, 0x9d, 0xc2, + 0xb8, 0x9b, 0xfa, 0x4b, 0x71, 0xe, 0xe6, 0x98, 0x12, 0xfa, + 0x98, 0x53, 0xab, 0xdc, 0xf8, 0xf8, 0x8d, 0xea, 0x73, 0x80, + 0xe8, 0x85, 0x44, 0xbe, 0x8c, 0x28, 0x4f, 0xff, 0x87, 0xad, + 0x9f, 0x66, 0xf2, 0x35, 0x12, 0xb6, 0x2b, 0xf1, 0x5b, 0xa7, + 0x1c, 0x4c, 0xa4, 0x80, 0xa3, 0x61, 0xa, 0x10, 0xef, 0x6f, + 0x54, 0x4e, 0xe8, 0xb1, 0x1f, 0xa8, 0xa0, 0x6e, 0xf8, 0x8b, + 0xf0, 0x4c, 0x1e, 0x9f, 0x6, 0x15, 0xb7, 0x4c, 0x22, 0xcc, + 0x9d, 0x53, 0xd8, 0x56, 0x86, 0x30, 0x3, 0xe7, 0xc7, 0x35, + 0x39, 0x99, 0x66, 0x6f, 0x1, 0x12, 0x89, 0xe5, 0x20, 0xc7, + 0xc8, 0xf8, 0xa9, 0x2d, 0x8d, 0x65, 0x20, 0xe0, 0xe8, 0x9a, + 0xf8, 0xd2, 0x3a, 0xbc, 0xa7, 0xf, 0xa4, 0x5a, 0x31, 0x85, + 0x29, 0xe5, 0x1a, 0x0, 0xad, 0xbd, 0xf1, 0x1a, 0xab, 0x9f, + 0xc7, 0xe2, 0x41, 0xc9, 0xa7, 0xde, 0x5e, 0x3a, 0x99, 0xc5, + 0xb0, 0xbb, 0xe2, 0xc3, 0x6b, 0x1e, 0x1c, 0xd9, 0x33, 0xfc, + 0xbc, 0x49, 0x31, 0x13, 0xf5, 0x5f, 0x8d, 0xbc, 0xf8, 0x3a, + 0x58, 0x5b, 0xf4, 0x3, 0xff, 0x42, 0x54, 0xb4, 0x94, 0xae, + 0xb, 0xfa, 0xbd, 0x15, 0x41, 0xcb, 0xe9, 0x12, 0x37, 0xbb, + 0xf0, 0x2c, 0xae, 0xff, 0x29, 0x9, 0xbe, 0x3e, 0xf, 0xb8, + 0xb5, 0x8d, 0x6f, 0xc0, 0x30, 0x87, 0x49, 0x4e, 0xdc, 0x9c, + 0x38, 0x6b, 0x1b, 0x3f, 0x14, 0x0, 0xef, 0x21, 0xcc, 0x5a, + 0xe8, 0xc7, 0x99, 0xe6, 0xf5, 0x5, 0x38, 0x87, 0x70, 0xbd, + 0xec, 0xdc, 0xaa, 0xc9, 0xbb, 0x5b, 0xc3, 0x56, 0xbb, 0x21, + 0xbf, 0x17, 0xe7, 0xcf, 0xbd, 0x4a, 0x4d, 0xe3, 0x67, 0x64, + 0x7f, 0x14, 0x6c, 0xea, 0x93, 0xe6, 0x2e, 0xdf, 0x3c, 0x6c, + 0x97, 0xb5, 0x8d, 0x57, 0x25, 0xd7, 0x5c, 0x6c, 0x8f, 0xd0, + 0xea, 0xd5, 0xdf, 0xc0, 0x71, 0x6c, 0x65, 0xcd, 0xb4, 0x4f, + 0x34, 0x9d, 0xb1, 0x52, 0xc1, 0xb7, 0x9, 0x2c, 0x51, 0xa7, + 0x29, 0x6c, 0x3a, 0x20, 0x70, 0x45, 0x4c, 0x72, 0xd9, 0xcd, + 0xac, 0x1f, 0x5, 0x22, 0xb0, 0x77, 0xbd, 0x91, 0x2f, 0xf5, + 0xd, 0x19, 0xd5, 0x57, 0x98, 0xee, 0x79, 0x93, 0xa4, 0x5f, + 0xba, 0x6b, 0xec, 0xf6, 0x3f, 0xb6, 0x9c, 0x2f, 0xb8, 0xfa, + 0x3, 0xa3, 0xeb, 0xdf, 0xea, 0xbc, 0x3f, 0xd0, 0x8f, 0x88, + 0xf0, 0xbb, 0xcc, 0x5a, 0x27, 0x57, 0x3b, 0x95, 0x3f, 0x20, + 0x7c, 0x19, 0xc8, 0x46, 0x47, 0x68, 0x72, 0xb7, 0x28, 0x8e, + 0x56, 0x9b, 0x83, 0xf7, 0xe9, 0x3c, 0x85, 0xcb, 0xb0, 0x65, + 0x60, 0x1a, 0x52, 0x85, 0x6d, 0x58, 0x84, 0x39, 0xd9, 0xa2, + 0x92, 0xd2, 0x9d, 0x7d, 0x1b, 0xdf, 0x61, 0x85, 0xbf, 0x88, + 0x54, 0x3, 0x42, 0xe1, 0xa9, 0x24, 0x74, 0x75, 0x78, 0x48, + 0xff, 0x22, 0xec, 0xc5, 0x4d, 0x66, 0x17, 0xd4, 0x9a, + }, +} + +var privateKey2 = &Block{Type: "RSA PRIVATE KEY", + Headers: map[string]string{}, + Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2, + 0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c, + 0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13, + 0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79, + 0xa, 0xde, 0x9a, 0x4c, 0xc8, 0x2b, 0x8b, 0x2a, 0x81, 0x74, + 0x7d, 0xde, 0xc0, 0x8b, 0x62, 0x96, 0xe5, 0x3a, 0x8, 0xc3, + 0x31, 0x68, 0x7e, 0xf2, 0x5c, 0x4b, 0xf4, 0x93, 0x6b, 0xa1, + 0xc0, 0xe6, 0x4, 0x1e, 0x9d, 0x15, 0x2, 0x3, 0x1, 0x0, 0x1, + 0x2, 0x41, 0x0, 0x8a, 0xbd, 0x6a, 0x69, 0xf4, 0xd1, 0xa4, + 0xb4, 0x87, 0xf0, 0xab, 0x8d, 0x7a, 0xae, 0xfd, 0x38, 0x60, + 0x94, 0x5, 0xc9, 0x99, 0x98, 0x4e, 0x30, 0xf5, 0x67, 0xe1, + 0xe8, 0xae, 0xef, 0xf4, 0x4e, 0x8b, 0x18, 0xbd, 0xb1, 0xec, + 0x78, 0xdf, 0xa3, 0x1a, 0x55, 0xe3, 0x2a, 0x48, 0xd7, 0xfb, + 0x13, 0x1f, 0x5a, 0xf1, 0xf4, 0x4d, 0x7d, 0x6b, 0x2c, 0xed, + 0x2a, 0x9d, 0xf5, 0xe5, 0xae, 0x45, 0x35, 0x2, 0x21, 0x0, + 0xda, 0xb2, 0xf1, 0x80, 0x48, 0xba, 0xa6, 0x8d, 0xe7, 0xdf, + 0x4, 0xd2, 0xd3, 0x5d, 0x5d, 0x80, 0xe6, 0xe, 0x2d, 0xfa, + 0x42, 0xd5, 0xa, 0x9b, 0x4, 0x21, 0x90, 0x32, 0x71, 0x5e, + 0x46, 0xb3, 0x2, 0x21, 0x0, 0xd1, 0xf, 0x2e, 0x66, 0xb1, + 0xd0, 0xc1, 0x3f, 0x10, 0xef, 0x99, 0x27, 0xbf, 0x53, 0x24, + 0xa3, 0x79, 0xca, 0x21, 0x81, 0x46, 0xcb, 0xf9, 0xca, 0xfc, + 0x79, 0x52, 0x21, 0xf1, 0x6a, 0x31, 0x17, 0x2, 0x20, 0x21, + 0x2, 0x89, 0x79, 0x37, 0x81, 0x14, 0xca, 0xae, 0x88, 0xf7, + 0xd, 0x6b, 0x61, 0xd8, 0x4f, 0x30, 0x6a, 0x4b, 0x7e, 0x4e, + 0xc0, 0x21, 0x4d, 0xac, 0x9d, 0xf4, 0x49, 0xe8, 0xda, 0xb6, + 0x9, 0x2, 0x20, 0x16, 0xb3, 0xec, 0x59, 0x10, 0xa4, 0x57, + 0xe8, 0xe, 0x61, 0xc6, 0xa3, 0xf, 0x5e, 0xeb, 0x12, 0xa9, + 0xae, 0x2e, 0xb7, 0x48, 0x45, 0xec, 0x69, 0x83, 0xc3, 0x75, + 0xc, 0xe4, 0x97, 0xa0, 0x9f, 0x2, 0x20, 0x69, 0x52, 0xb4, + 0x6, 0xe8, 0x50, 0x60, 0x71, 0x4c, 0x3a, 0xb7, 0x66, 0xba, + 0xd, 0x8a, 0xc9, 0xb7, 0xd, 0xa3, 0x8, 0x6c, 0xa3, 0xf2, + 0x62, 0xb0, 0x2a, 0x84, 0xaa, 0x2f, 0xd6, 0x1e, 0x55, + }, +} + +var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 +fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu +/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu +RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ +EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A +IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS +tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +-----END RSA PRIVATE KEY----- +` diff --git a/src/pkg/exec/Makefile b/src/pkg/exec/Makefile new file mode 100644 index 000000000..ba19d0e4d --- /dev/null +++ b/src/pkg/exec/Makefile @@ -0,0 +1,31 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=exec +GOFILES=\ + exec.go\ + +GOFILES_freebsd=\ + lp_unix.go\ + +GOFILES_darwin=\ + lp_unix.go\ + +GOFILES_linux=\ + lp_unix.go\ + +GOFILES_openbsd=\ + lp_unix.go\ + +GOFILES_windows=\ + lp_windows.go\ + +GOFILES_plan9=\ + lp_plan9.go\ + +GOFILES+=$(GOFILES_$(GOOS)) + +include ../../Make.pkg diff --git a/src/pkg/exec/exec.go b/src/pkg/exec/exec.go new file mode 100644 index 000000000..3b20f2008 --- /dev/null +++ b/src/pkg/exec/exec.go @@ -0,0 +1,377 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package exec runs external commands. It wraps os.StartProcess to make it +// easier to remap stdin and stdout, connect I/O with pipes, and do other +// adjustments. +package exec + +import ( + "bytes" + "io" + "os" + "strconv" + "syscall" +) + +// Error records the name of a binary that failed to be be executed +// and the reason it failed. +type Error struct { + Name string + Error os.Error +} + +func (e *Error) String() string { + return "exec: " + strconv.Quote(e.Name) + ": " + e.Error.String() +} + +// Cmd represents an external command being prepared or run. +type Cmd struct { + // Path is the path of the command to run. + // + // This is the only field that must be set to a non-zero + // value. + Path string + + // Args holds command line arguments, including the command as Args[0]. + // If the Args field is empty or nil, Run uses {Path}. + // + // In typical use, both Path and Args are set by calling Command. + Args []string + + // Env specifies the environment of the process. + // If Env is nil, Run uses the current process's environment. + Env []string + + // Dir specifies the working directory of the command. + // If Dir is the empty string, Run runs the command in the + // calling process's current directory. + Dir string + + // Stdin specifies the process's standard input. + // If Stdin is nil, the process reads from DevNull. + Stdin io.Reader + + // Stdout and Stderr specify the process's standard output and error. + // + // If either is nil, Run connects the + // corresponding file descriptor to /dev/null. + // + // If Stdout and Stderr are are the same writer, at most one + // goroutine at a time will call Write. + Stdout io.Writer + Stderr io.Writer + + // SysProcAttr holds optional, operating system-specific attributes. + // Run passes it to os.StartProcess as the os.ProcAttr's Sys field. + SysProcAttr *syscall.SysProcAttr + + // Process is the underlying process, once started. + Process *os.Process + + err os.Error // last error (from LookPath, stdin, stdout, stderr) + finished bool // when Wait was called + childFiles []*os.File + closeAfterStart []io.Closer + closeAfterWait []io.Closer + goroutine []func() os.Error + errch chan os.Error // one send per goroutine +} + +// Command returns the Cmd struct to execute the named program with +// the given arguments. +// +// It sets Path and Args in the returned structure and zeroes the +// other fields. +// +// If name contains no path separators, Command uses LookPath to +// resolve the path to a complete name if possible. Otherwise it uses +// name directly. +// +// The returned Cmd's Args field is constructed from the command name +// followed by the elements of arg, so arg should not include the +// command name itself. For example, Command("echo", "hello") +func Command(name string, arg ...string) *Cmd { + aname, err := LookPath(name) + if err != nil { + aname = name + } + return &Cmd{ + Path: aname, + Args: append([]string{name}, arg...), + err: err, + } +} + +// interfaceEqual protects against panics from doing equality tests on +// two interfaces with non-comparable underlying types +func interfaceEqual(a, b interface{}) bool { + defer func() { + recover() + }() + return a == b +} + +func (c *Cmd) envv() []string { + if c.Env != nil { + return c.Env + } + return os.Environ() +} + +func (c *Cmd) argv() []string { + if len(c.Args) > 0 { + return c.Args + } + return []string{c.Path} +} + +func (c *Cmd) stdin() (f *os.File, err os.Error) { + if c.Stdin == nil { + f, err = os.Open(os.DevNull) + c.closeAfterStart = append(c.closeAfterStart, f) + return + } + + if f, ok := c.Stdin.(*os.File); ok { + return f, nil + } + + pr, pw, err := os.Pipe() + if err != nil { + return + } + + c.closeAfterStart = append(c.closeAfterStart, pr) + c.closeAfterWait = append(c.closeAfterWait, pw) + c.goroutine = append(c.goroutine, func() os.Error { + _, err := io.Copy(pw, c.Stdin) + if err1 := pw.Close(); err == nil { + err = err1 + } + return err + }) + return pr, nil +} + +func (c *Cmd) stdout() (f *os.File, err os.Error) { + return c.writerDescriptor(c.Stdout) +} + +func (c *Cmd) stderr() (f *os.File, err os.Error) { + if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) { + return c.childFiles[1], nil + } + return c.writerDescriptor(c.Stderr) +} + +func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err os.Error) { + if w == nil { + f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0) + c.closeAfterStart = append(c.closeAfterStart, f) + return + } + + if f, ok := w.(*os.File); ok { + return f, nil + } + + pr, pw, err := os.Pipe() + if err != nil { + return + } + + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + c.goroutine = append(c.goroutine, func() os.Error { + _, err := io.Copy(w, pr) + return err + }) + return pw, nil +} + +// Run starts the specified command and waits for it to complete. +// +// The returned error is nil if the command runs, has no problems +// copying stdin, stdout, and stderr, and exits with a zero exit +// status. +// +// If the command fails to run or doesn't complete successfully, the +// error is of type *os.Waitmsg. Other error types may be +// returned for I/O problems. +func (c *Cmd) Run() os.Error { + if err := c.Start(); err != nil { + return err + } + return c.Wait() +} + +// Start starts the specified command but does not wait for it to complete. +func (c *Cmd) Start() os.Error { + if c.err != nil { + return c.err + } + if c.Process != nil { + return os.NewError("exec: already started") + } + + type F func(*Cmd) (*os.File, os.Error) + for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} { + fd, err := setupFd(c) + if err != nil { + return err + } + c.childFiles = append(c.childFiles, fd) + } + + var err os.Error + c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{ + Dir: c.Dir, + Files: c.childFiles, + Env: c.envv(), + Sys: c.SysProcAttr, + }) + if err != nil { + return err + } + + for _, fd := range c.closeAfterStart { + fd.Close() + } + + c.errch = make(chan os.Error, len(c.goroutine)) + for _, fn := range c.goroutine { + go func(fn func() os.Error) { + c.errch <- fn() + }(fn) + } + + return nil +} + +// Wait waits for the command to exit. +// It must have been started by Start. +// +// The returned error is nil if the command runs, has no problems +// copying stdin, stdout, and stderr, and exits with a zero exit +// status. +// +// If the command fails to run or doesn't complete successfully, the +// error is of type *os.Waitmsg. Other error types may be +// returned for I/O problems. +func (c *Cmd) Wait() os.Error { + if c.Process == nil { + return os.NewError("exec: not started") + } + if c.finished { + return os.NewError("exec: Wait was already called") + } + c.finished = true + msg, err := c.Process.Wait(0) + + var copyError os.Error + for _ = range c.goroutine { + if err := <-c.errch; err != nil && copyError == nil { + copyError = err + } + } + + for _, fd := range c.closeAfterWait { + fd.Close() + } + + if err != nil { + return err + } else if !msg.Exited() || msg.ExitStatus() != 0 { + return msg + } + + return copyError +} + +// Output runs the command and returns its standard output. +func (c *Cmd) Output() ([]byte, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } + var b bytes.Buffer + c.Stdout = &b + err := c.Run() + return b.Bytes(), err +} + +// CombinedOutput runs the command and returns its combined standard +// output and standard error. +func (c *Cmd) CombinedOutput() ([]byte, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } + if c.Stderr != nil { + return nil, os.NewError("exec: Stderr already set") + } + var b bytes.Buffer + c.Stdout = &b + c.Stderr = &b + err := c.Run() + return b.Bytes(), err +} + +// StdinPipe returns a pipe that will be connected to the command's +// standard input when the command starts. +func (c *Cmd) StdinPipe() (io.WriteCloser, os.Error) { + if c.Stdin != nil { + return nil, os.NewError("exec: Stdin already set") + } + if c.Process != nil { + return nil, os.NewError("exec: StdinPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stdin = pr + c.closeAfterStart = append(c.closeAfterStart, pr) + c.closeAfterWait = append(c.closeAfterWait, pw) + return pw, nil +} + +// StdoutPipe returns a pipe that will be connected to the command's +// standard output when the command starts. +// The pipe will be closed automatically after Wait sees the command exit. +func (c *Cmd) StdoutPipe() (io.ReadCloser, os.Error) { + if c.Stdout != nil { + return nil, os.NewError("exec: Stdout already set") + } + if c.Process != nil { + return nil, os.NewError("exec: StdoutPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stdout = pw + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + return pr, nil +} + +// StderrPipe returns a pipe that will be connected to the command's +// standard error when the command starts. +// The pipe will be closed automatically after Wait sees the command exit. +func (c *Cmd) StderrPipe() (io.ReadCloser, os.Error) { + if c.Stderr != nil { + return nil, os.NewError("exec: Stderr already set") + } + if c.Process != nil { + return nil, os.NewError("exec: StderrPipe after process started") + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + c.Stderr = pw + c.closeAfterStart = append(c.closeAfterStart, pw) + c.closeAfterWait = append(c.closeAfterWait, pr) + return pr, nil +} diff --git a/src/pkg/exec/exec_test.go b/src/pkg/exec/exec_test.go new file mode 100644 index 000000000..242120faa --- /dev/null +++ b/src/pkg/exec/exec_test.go @@ -0,0 +1,214 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "bufio" + "bytes" + "fmt" + "io" + "testing" + "os" + "strconv" + "strings" +) + +func helperCommand(s ...string) *Cmd { + cs := []string{"-test.run=exec.TestHelperProcess", "--"} + cs = append(cs, s...) + cmd := Command(os.Args[0], cs...) + cmd.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) + return cmd +} + +func TestEcho(t *testing.T) { + bs, err := helperCommand("echo", "foo bar", "baz").Output() + if err != nil { + t.Errorf("echo: %v", err) + } + if g, e := string(bs), "foo bar baz\n"; g != e { + t.Errorf("echo: want %q, got %q", e, g) + } +} + +func TestCatStdin(t *testing.T) { + // Cat, testing stdin and stdout. + input := "Input string\nLine 2" + p := helperCommand("cat") + p.Stdin = strings.NewReader(input) + bs, err := p.Output() + if err != nil { + t.Errorf("cat: %v", err) + } + s := string(bs) + if s != input { + t.Errorf("cat: want %q, got %q", input, s) + } +} + +func TestCatGoodAndBadFile(t *testing.T) { + // Testing combined output and error values. + bs, err := helperCommand("cat", "/bogus/file.foo", "exec_test.go").CombinedOutput() + if _, ok := err.(*os.Waitmsg); !ok { + t.Errorf("expected Waitmsg from cat combined; got %T: %v", err, err) + } + s := string(bs) + sp := strings.SplitN(s, "\n", 2) + if len(sp) != 2 { + t.Fatalf("expected two lines from cat; got %q", s) + } + errLine, body := sp[0], sp[1] + if !strings.HasPrefix(errLine, "Error: open /bogus/file.foo") { + t.Errorf("expected stderr to complain about file; got %q", errLine) + } + if !strings.Contains(body, "func TestHelperProcess(t *testing.T)") { + t.Errorf("expected test code; got %q (len %d)", body, len(body)) + } +} + +func TestNoExistBinary(t *testing.T) { + // Can't run a non-existent binary + err := Command("/no-exist-binary").Run() + if err == nil { + t.Error("expected error from /no-exist-binary") + } +} + +func TestExitStatus(t *testing.T) { + // Test that exit values are returned correctly + err := helperCommand("exit", "42").Run() + if werr, ok := err.(*os.Waitmsg); ok { + if s, e := werr.String(), "exit status 42"; s != e { + t.Errorf("from exit 42 got exit %q, want %q", s, e) + } + } else { + t.Fatalf("expected Waitmsg from exit 42; got %T: %v", err, err) + } +} + +func TestPipes(t *testing.T) { + check := func(what string, err os.Error) { + if err != nil { + t.Fatalf("%s: %v", what, err) + } + } + // Cat, testing stdin and stdout. + c := helperCommand("pipetest") + stdin, err := c.StdinPipe() + check("StdinPipe", err) + stdout, err := c.StdoutPipe() + check("StdoutPipe", err) + stderr, err := c.StderrPipe() + check("StderrPipe", err) + + outbr := bufio.NewReader(stdout) + errbr := bufio.NewReader(stderr) + line := func(what string, br *bufio.Reader) string { + line, _, err := br.ReadLine() + if err != nil { + t.Fatalf("%s: %v", what, err) + } + return string(line) + } + + err = c.Start() + check("Start", err) + + _, err = stdin.Write([]byte("O:I am output\n")) + check("first stdin Write", err) + if g, e := line("first output line", outbr), "O:I am output"; g != e { + t.Errorf("got %q, want %q", g, e) + } + + _, err = stdin.Write([]byte("E:I am error\n")) + check("second stdin Write", err) + if g, e := line("first error line", errbr), "E:I am error"; g != e { + t.Errorf("got %q, want %q", g, e) + } + + _, err = stdin.Write([]byte("O:I am output2\n")) + check("third stdin Write 3", err) + if g, e := line("second output line", outbr), "O:I am output2"; g != e { + t.Errorf("got %q, want %q", g, e) + } + + stdin.Close() + err = c.Wait() + check("Wait", err) +} + +// TestHelperProcess isn't a real test. It's used as a helper process +// for TestParameterRun. +func TestHelperProcess(*testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + + args := os.Args + for len(args) > 0 { + if args[0] == "--" { + args = args[1:] + break + } + args = args[1:] + } + if len(args) == 0 { + fmt.Fprintf(os.Stderr, "No command\n") + os.Exit(2) + } + + cmd, args := args[0], args[1:] + switch cmd { + case "echo": + iargs := []interface{}{} + for _, s := range args { + iargs = append(iargs, s) + } + fmt.Println(iargs...) + case "cat": + if len(args) == 0 { + io.Copy(os.Stdout, os.Stdin) + return + } + exit := 0 + for _, fn := range args { + f, err := os.Open(fn) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + exit = 2 + } else { + defer f.Close() + io.Copy(os.Stdout, f) + } + } + os.Exit(exit) + case "pipetest": + bufr := bufio.NewReader(os.Stdin) + for { + line, _, err := bufr.ReadLine() + if err == os.EOF { + break + } else if err != nil { + os.Exit(1) + } + if bytes.HasPrefix(line, []byte("O:")) { + os.Stdout.Write(line) + os.Stdout.Write([]byte{'\n'}) + } else if bytes.HasPrefix(line, []byte("E:")) { + os.Stderr.Write(line) + os.Stderr.Write([]byte{'\n'}) + } else { + os.Exit(1) + } + } + case "exit": + n, _ := strconv.Atoi(args[0]) + os.Exit(n) + default: + fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmd) + os.Exit(2) + } +} diff --git a/src/pkg/exec/lp_plan9.go b/src/pkg/exec/lp_plan9.go new file mode 100644 index 000000000..e4751a4df --- /dev/null +++ b/src/pkg/exec/lp_plan9.go @@ -0,0 +1,51 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.NewError("executable file not found in $path") + +func findExecutable(file string) os.Error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsRegular() && d.Permission()&0111 != 0 { + return nil + } + return os.EPERM +} + +// LookPath searches for an executable binary named file +// in the directories named by the path environment variable. +// If file begins with "/", "#", "./", or "../", it is tried +// directly and the path is not consulted. +func LookPath(file string) (string, os.Error) { + // skip the path lookup for these prefixes + skip := []string{"/", "#", "./", "../"} + + for _, p := range skip { + if strings.HasPrefix(file, p) { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + } + + path := os.Getenv("path") + for _, dir := range strings.Split(path, "\000") { + if err := findExecutable(dir + "/" + file); err == nil { + return dir + "/" + file, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/src/pkg/exec/lp_test.go b/src/pkg/exec/lp_test.go new file mode 100644 index 000000000..77d8e848c --- /dev/null +++ b/src/pkg/exec/lp_test.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "testing" +) + +var nonExistentPaths = []string{ + "some-non-existent-path", + "non-existent-path/slashed", +} + +func TestLookPathNotFound(t *testing.T) { + for _, name := range nonExistentPaths { + path, err := LookPath(name) + if err == nil { + t.Fatalf("LookPath found %q in $PATH", name) + } + if path != "" { + t.Fatalf("LookPath path == %q when err != nil", path) + } + perr, ok := err.(*Error) + if !ok { + t.Fatal("LookPath error is not an exec.Error") + } + if perr.Name != name { + t.Fatalf("want Error name %q, got %q", name, perr.Name) + } + } +} diff --git a/src/pkg/exec/lp_unix.go b/src/pkg/exec/lp_unix.go new file mode 100644 index 000000000..008fb11a8 --- /dev/null +++ b/src/pkg/exec/lp_unix.go @@ -0,0 +1,52 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.NewError("executable file not found in $PATH") + +func findExecutable(file string) os.Error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsRegular() && d.Permission()&0111 != 0 { + return nil + } + return os.EPERM +} + +// LookPath searches for an executable binary named file +// in the directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +func LookPath(file string) (string, os.Error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + pathenv := os.Getenv("PATH") + for _, dir := range strings.Split(pathenv, ":") { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + if err := findExecutable(dir + "/" + file); err == nil { + return dir + "/" + file, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/src/pkg/exec/lp_windows.go b/src/pkg/exec/lp_windows.go new file mode 100644 index 000000000..7581088eb --- /dev/null +++ b/src/pkg/exec/lp_windows.go @@ -0,0 +1,77 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "os" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = os.NewError("executable file not found in %PATH%") + +func chkStat(file string) os.Error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsRegular() { + return nil + } + return os.EPERM +} + +func findExecutable(file string, exts []string) (string, os.Error) { + if len(exts) == 0 { + return file, chkStat(file) + } + f := strings.ToLower(file) + for _, e := range exts { + if strings.HasSuffix(f, e) { + return file, chkStat(file) + } + } + for _, e := range exts { + if f := file + e; chkStat(f) == nil { + return f, nil + } + } + return ``, os.ENOENT +} + +func LookPath(file string) (f string, err os.Error) { + x := os.Getenv(`PATHEXT`) + if x == `` { + x = `.COM;.EXE;.BAT;.CMD` + } + exts := []string{} + for _, e := range strings.Split(strings.ToLower(x), `;`) { + if e == "" { + continue + } + if e[0] != '.' { + e = "." + e + } + exts = append(exts, e) + } + if strings.IndexAny(file, `:\/`) != -1 { + if f, err = findExecutable(file, exts); err == nil { + return + } + return ``, &Error{file, err} + } + if pathenv := os.Getenv(`PATH`); pathenv == `` { + if f, err = findExecutable(`.\`+file, exts); err == nil { + return + } + } else { + for _, dir := range strings.Split(pathenv, `;`) { + if f, err = findExecutable(dir+`\`+file, exts); err == nil { + return + } + } + } + return ``, &Error{file, ErrNotFound} +} diff --git a/src/pkg/exp/README b/src/pkg/exp/README new file mode 100644 index 000000000..e602e3ac9 --- /dev/null +++ b/src/pkg/exp/README @@ -0,0 +1,3 @@ +This directory tree contains experimental packages and +unfinished code that is subject to even more change than the +rest of the Go tree. diff --git a/src/pkg/exp/datafmt/Makefile b/src/pkg/exp/datafmt/Makefile new file mode 100644 index 000000000..aa9453897 --- /dev/null +++ b/src/pkg/exp/datafmt/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/datafmt +GOFILES=\ + datafmt.go\ + parser.go\ + +include ../../../Make.pkg diff --git a/src/pkg/exp/datafmt/datafmt.go b/src/pkg/exp/datafmt/datafmt.go new file mode 100644 index 000000000..6d7e76442 --- /dev/null +++ b/src/pkg/exp/datafmt/datafmt.go @@ -0,0 +1,710 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* Package datafmt implements syntax-directed, type-driven formatting + of arbitrary data structures. Formatting a data structure consists of + two phases: first, a parser reads a format specification and builds a + "compiled" format. Then, the format can be applied repeatedly to + arbitrary values. Applying a format to a value evaluates to a []byte + containing the formatted value bytes, or nil. + + A format specification is a set of package declarations and format rules: + + Format = [ Entry { ";" Entry } [ ";" ] ] . + Entry = PackageDecl | FormatRule . + + (The syntax of a format specification is presented in the same EBNF + notation as used in the Go language specification. The syntax of white + space, comments, identifiers, and string literals is the same as in Go.) + + A package declaration binds a package name (such as 'ast') to a + package import path (such as '"go/ast"'). Each package used (in + a type name, see below) must be declared once before use. + + PackageDecl = PackageName ImportPath . + PackageName = identifier . + ImportPath = string . + + A format rule binds a rule name to a format expression. A rule name + may be a type name or one of the special names 'default' or '/'. + A type name may be the name of a predeclared type (for example, 'int', + 'float32', etc.), the package-qualified name of a user-defined type + (for example, 'ast.MapType'), or an identifier indicating the structure + of unnamed composite types ('array', 'chan', 'func', 'interface', 'map', + or 'ptr'). Each rule must have a unique name; rules can be declared in + any order. + + FormatRule = RuleName "=" Expression . + RuleName = TypeName | "default" | "/" . + TypeName = [ PackageName "." ] identifier . + + To format a value, the value's type name is used to select the format rule + (there is an override mechanism, see below). The format expression of the + selected rule specifies how the value is formatted. Each format expression, + when applied to a value, evaluates to a byte sequence or nil. + + In its most general form, a format expression is a list of alternatives, + each of which is a sequence of operands: + + Expression = [ Sequence ] { "|" [ Sequence ] } . + Sequence = Operand { Operand } . + + The formatted result produced by an expression is the result of the first + alternative sequence that evaluates to a non-nil result; if there is no + such alternative, the expression evaluates to nil. The result produced by + an operand sequence is the concatenation of the results of its operands. + If any operand in the sequence evaluates to nil, the entire sequence + evaluates to nil. + + There are five kinds of operands: + + Operand = Literal | Field | Group | Option | Repetition . + + Literals evaluate to themselves, with two substitutions. First, + %-formats expand in the manner of fmt.Printf, with the current value + passed as the parameter. Second, the current indentation (see below) + is inserted after every newline or form feed character. + + Literal = string . + + This table shows string literals applied to the value 42 and the + corresponding formatted result: + + "foo" foo + "%x" 2a + "x = %d" x = 42 + "%#x = %d" 0x2a = 42 + + A field operand is a field name optionally followed by an alternate + rule name. The field name may be an identifier or one of the special + names @ or *. + + Field = FieldName [ ":" RuleName ] . + FieldName = identifier | "@" | "*" . + + If the field name is an identifier, the current value must be a struct, + and there must be a field with that name in the struct. The same lookup + rules apply as in the Go language (for instance, the name of an anonymous + field is the unqualified type name). The field name denotes the field + value in the struct. If the field is not found, formatting is aborted + and an error message is returned. (TODO consider changing the semantics + such that if a field is not found, it evaluates to nil). + + The special name '@' denotes the current value. + + The meaning of the special name '*' depends on the type of the current + value: + + array, slice types array, slice element (inside {} only, see below) + interfaces value stored in interface + pointers value pointed to by pointer + + (Implementation restriction: channel, function and map types are not + supported due to missing reflection support). + + Fields are evaluated as follows: If the field value is nil, or an array + or slice element does not exist, the result is nil (see below for details + on array/slice elements). If the value is not nil the field value is + formatted (recursively) using the rule corresponding to its type name, + or the alternate rule name, if given. + + The following example shows a complete format specification for a + struct 'myPackage.Point'. Assume the package + + package myPackage // in directory myDir/myPackage + type Point struct { + name string; + x, y int; + } + + Applying the format specification + + myPackage "myDir/myPackage"; + int = "%d"; + hexInt = "0x%x"; + string = "---%s---"; + myPackage.Point = name "{" x ", " y:hexInt "}"; + + to the value myPackage.Point{"foo", 3, 15} results in + + ---foo---{3, 0xf} + + Finally, an operand may be a grouped, optional, or repeated expression. + A grouped expression ("group") groups a more complex expression (body) + so that it can be used in place of a single operand: + + Group = "(" [ Indentation ">>" ] Body ")" . + Indentation = Expression . + Body = Expression . + + A group body may be prefixed by an indentation expression followed by '>>'. + The indentation expression is applied to the current value like any other + expression and the result, if not nil, is appended to the current indentation + during the evaluation of the body (see also formatting state, below). + + An optional expression ("option") is enclosed in '[]' brackets. + + Option = "[" Body "]" . + + An option evaluates to its body, except that if the body evaluates to nil, + the option expression evaluates to an empty []byte. Thus an option's purpose + is to protect the expression containing the option from a nil operand. + + A repeated expression ("repetition") is enclosed in '{}' braces. + + Repetition = "{" Body [ "/" Separator ] "}" . + Separator = Expression . + + A repeated expression is evaluated as follows: The body is evaluated + repeatedly and its results are concatenated until the body evaluates + to nil. The result of the repetition is the (possibly empty) concatenation, + but it is never nil. An implicit index is supplied for the evaluation of + the body: that index is used to address elements of arrays or slices. If + the corresponding elements do not exist, the field denoting the element + evaluates to nil (which in turn may terminate the repetition). + + The body of a repetition may be followed by a '/' and a "separator" + expression. If the separator is present, it is invoked between repetitions + of the body. + + The following example shows a complete format specification for formatting + a slice of unnamed type. Applying the specification + + int = "%b"; + array = { * / ", " }; // array is the type name for an unnamed slice + + to the value '[]int{2, 3, 5, 7}' results in + + 10, 11, 101, 111 + + Default rule: If a format rule named 'default' is present, it is used for + formatting a value if no other rule was found. A common default rule is + + default = "%v" + + to provide default formatting for basic types without having to specify + a specific rule for each basic type. + + Global separator rule: If a format rule named '/' is present, it is + invoked with the current value between literals. If the separator + expression evaluates to nil, it is ignored. + + For instance, a global separator rule may be used to punctuate a sequence + of values with commas. The rules: + + default = "%v"; + / = ", "; + + will format an argument list by printing each one in its default format, + separated by a comma and a space. +*/ +package datafmt + +import ( + "bytes" + "fmt" + "go/token" + "io" + "os" + "reflect" + "runtime" +) + +// ---------------------------------------------------------------------------- +// Format representation + +// Custom formatters implement the Formatter function type. +// A formatter is invoked with the current formatting state, the +// value to format, and the rule name under which the formatter +// was installed (the same formatter function may be installed +// under different names). The formatter may access the current state +// to guide formatting and use State.Write to append to the state's +// output. +// +// A formatter must return a boolean value indicating if it evaluated +// to a non-nil value (true), or a nil value (false). +// +type Formatter func(state *State, value interface{}, ruleName string) bool + +// A FormatterMap is a set of custom formatters. +// It maps a rule name to a formatter function. +// +type FormatterMap map[string]Formatter + +// A parsed format expression is built from the following nodes. +// +type ( + expr interface{} + + alternatives []expr // x | y | z + + sequence []expr // x y z + + literal [][]byte // a list of string segments, possibly starting with '%' + + field struct { + fieldName string // including "@", "*" + ruleName string // "" if no rule name specified + } + + group struct { + indent, body expr // (indent >> body) + } + + option struct { + body expr // [body] + } + + repetition struct { + body, separator expr // {body / separator} + } + + custom struct { + ruleName string + fun Formatter + } +) + +// A Format is the result of parsing a format specification. +// The format may be applied repeatedly to format values. +// +type Format map[string]expr + +// ---------------------------------------------------------------------------- +// Formatting + +// An application-specific environment may be provided to Format.Apply; +// the environment is available inside custom formatters via State.Env(). +// Environments must implement copying; the Copy method must return an +// complete copy of the receiver. This is necessary so that the formatter +// can save and restore an environment (in case of an absent expression). +// +// If the Environment doesn't change during formatting (this is under +// control of the custom formatters), the Copy function can simply return +// the receiver, and thus can be very light-weight. +// +type Environment interface { + Copy() Environment +} + +// State represents the current formatting state. +// It is provided as argument to custom formatters. +// +type State struct { + fmt Format // format in use + env Environment // user-supplied environment + errors chan os.Error // not chan *Error (errors <- nil would be wrong!) + hasOutput bool // true after the first literal has been written + indent bytes.Buffer // current indentation + output bytes.Buffer // format output + linePos token.Position // position of line beginning (Column == 0) + default_ expr // possibly nil + separator expr // possibly nil +} + +func newState(fmt Format, env Environment, errors chan os.Error) *State { + s := new(State) + s.fmt = fmt + s.env = env + s.errors = errors + s.linePos = token.Position{Line: 1} + + // if we have a default rule, cache its expression for fast access + if x, found := fmt["default"]; found { + s.default_ = x + } + + // if we have a global separator rule, cache its expression for fast access + if x, found := fmt["/"]; found { + s.separator = x + } + + return s +} + +// Env returns the environment passed to Format.Apply. +func (s *State) Env() interface{} { return s.env } + +// LinePos returns the position of the current line beginning +// in the state's output buffer. Line numbers start at 1. +// +func (s *State) LinePos() token.Position { return s.linePos } + +// Pos returns the position of the next byte to be written to the +// output buffer. Line numbers start at 1. +// +func (s *State) Pos() token.Position { + offs := s.output.Len() + return token.Position{Line: s.linePos.Line, Column: offs - s.linePos.Offset, Offset: offs} +} + +// Write writes data to the output buffer, inserting the indentation +// string after each newline or form feed character. It cannot return an error. +// +func (s *State) Write(data []byte) (int, os.Error) { + n := 0 + i0 := 0 + for i, ch := range data { + if ch == '\n' || ch == '\f' { + // write text segment and indentation + n1, _ := s.output.Write(data[i0 : i+1]) + n2, _ := s.output.Write(s.indent.Bytes()) + n += n1 + n2 + i0 = i + 1 + s.linePos.Offset = s.output.Len() + s.linePos.Line++ + } + } + n3, _ := s.output.Write(data[i0:]) + return n + n3, nil +} + +type checkpoint struct { + env Environment + hasOutput bool + outputLen int + linePos token.Position +} + +func (s *State) save() checkpoint { + saved := checkpoint{nil, s.hasOutput, s.output.Len(), s.linePos} + if s.env != nil { + saved.env = s.env.Copy() + } + return saved +} + +func (s *State) restore(m checkpoint) { + s.env = m.env + s.output.Truncate(m.outputLen) +} + +func (s *State) error(msg string) { + s.errors <- os.NewError(msg) + runtime.Goexit() +} + +// TODO At the moment, unnamed types are simply mapped to the default +// names below. For instance, all unnamed arrays are mapped to +// 'array' which is not really sufficient. Eventually one may want +// to be able to specify rules for say an unnamed slice of T. +// + +func typename(typ reflect.Type) string { + switch typ.Kind() { + case reflect.Array: + return "array" + case reflect.Slice: + return "array" + case reflect.Chan: + return "chan" + case reflect.Func: + return "func" + case reflect.Interface: + return "interface" + case reflect.Map: + return "map" + case reflect.Ptr: + return "ptr" + } + return typ.String() +} + +func (s *State) getFormat(name string) expr { + if fexpr, found := s.fmt[name]; found { + return fexpr + } + + if s.default_ != nil { + return s.default_ + } + + s.error(fmt.Sprintf("no format rule for type: '%s'", name)) + return nil +} + +// eval applies a format expression fexpr to a value. If the expression +// evaluates internally to a non-nil []byte, that slice is appended to +// the state's output buffer and eval returns true. Otherwise, eval +// returns false and the state remains unchanged. +// +func (s *State) eval(fexpr expr, value reflect.Value, index int) bool { + // an empty format expression always evaluates + // to a non-nil (but empty) []byte + if fexpr == nil { + return true + } + + switch t := fexpr.(type) { + case alternatives: + // append the result of the first alternative that evaluates to + // a non-nil []byte to the state's output + mark := s.save() + for _, x := range t { + if s.eval(x, value, index) { + return true + } + s.restore(mark) + } + return false + + case sequence: + // append the result of all operands to the state's output + // unless a nil result is encountered + mark := s.save() + for _, x := range t { + if !s.eval(x, value, index) { + s.restore(mark) + return false + } + } + return true + + case literal: + // write separator, if any + if s.hasOutput { + // not the first literal + if s.separator != nil { + sep := s.separator // save current separator + s.separator = nil // and disable it (avoid recursion) + mark := s.save() + if !s.eval(sep, value, index) { + s.restore(mark) + } + s.separator = sep // enable it again + } + } + s.hasOutput = true + // write literal segments + for _, lit := range t { + if len(lit) > 1 && lit[0] == '%' { + // segment contains a %-format at the beginning + if lit[1] == '%' { + // "%%" is printed as a single "%" + s.Write(lit[1:]) + } else { + // use s instead of s.output to get indentation right + fmt.Fprintf(s, string(lit), value.Interface()) + } + } else { + // segment contains no %-formats + s.Write(lit) + } + } + return true // a literal never evaluates to nil + + case *field: + // determine field value + switch t.fieldName { + case "@": + // field value is current value + + case "*": + // indirection: operation is type-specific + switch v := value; v.Kind() { + case reflect.Array: + if v.Len() <= index { + return false + } + value = v.Index(index) + + case reflect.Slice: + if v.IsNil() || v.Len() <= index { + return false + } + value = v.Index(index) + + case reflect.Map: + s.error("reflection support for maps incomplete") + + case reflect.Ptr: + if v.IsNil() { + return false + } + value = v.Elem() + + case reflect.Interface: + if v.IsNil() { + return false + } + value = v.Elem() + + case reflect.Chan: + s.error("reflection support for chans incomplete") + + case reflect.Func: + s.error("reflection support for funcs incomplete") + + default: + s.error(fmt.Sprintf("error: * does not apply to `%s`", value.Type())) + } + + default: + // value is value of named field + var field reflect.Value + if sval := value; sval.Kind() == reflect.Struct { + field = sval.FieldByName(t.fieldName) + if !field.IsValid() { + // TODO consider just returning false in this case + s.error(fmt.Sprintf("error: no field `%s` in `%s`", t.fieldName, value.Type())) + } + } + value = field + } + + // determine rule + ruleName := t.ruleName + if ruleName == "" { + // no alternate rule name, value type determines rule + ruleName = typename(value.Type()) + } + fexpr = s.getFormat(ruleName) + + mark := s.save() + if !s.eval(fexpr, value, index) { + s.restore(mark) + return false + } + return true + + case *group: + // remember current indentation + indentLen := s.indent.Len() + + // update current indentation + mark := s.save() + s.eval(t.indent, value, index) + // if the indentation evaluates to nil, the state's output buffer + // didn't change - either way it's ok to append the difference to + // the current indentation + s.indent.Write(s.output.Bytes()[mark.outputLen:s.output.Len()]) + s.restore(mark) + + // format group body + mark = s.save() + b := true + if !s.eval(t.body, value, index) { + s.restore(mark) + b = false + } + + // reset indentation + s.indent.Truncate(indentLen) + return b + + case *option: + // evaluate the body and append the result to the state's output + // buffer unless the result is nil + mark := s.save() + if !s.eval(t.body, value, 0) { // TODO is 0 index correct? + s.restore(mark) + } + return true // an option never evaluates to nil + + case *repetition: + // evaluate the body and append the result to the state's output + // buffer until a result is nil + for i := 0; ; i++ { + mark := s.save() + // write separator, if any + if i > 0 && t.separator != nil { + // nil result from separator is ignored + mark := s.save() + if !s.eval(t.separator, value, i) { + s.restore(mark) + } + } + if !s.eval(t.body, value, i) { + s.restore(mark) + break + } + } + return true // a repetition never evaluates to nil + + case *custom: + // invoke the custom formatter to obtain the result + mark := s.save() + if !t.fun(s, value.Interface(), t.ruleName) { + s.restore(mark) + return false + } + return true + } + + panic("unreachable") + return false +} + +// Eval formats each argument according to the format +// f and returns the resulting []byte and os.Error. If +// an error occurred, the []byte contains the partially +// formatted result. An environment env may be passed +// in which is available in custom formatters through +// the state parameter. +// +func (f Format) Eval(env Environment, args ...interface{}) ([]byte, os.Error) { + if f == nil { + return nil, os.NewError("format is nil") + } + + errors := make(chan os.Error) + s := newState(f, env, errors) + + go func() { + for _, v := range args { + fld := reflect.ValueOf(v) + if !fld.IsValid() { + errors <- os.NewError("nil argument") + return + } + mark := s.save() + if !s.eval(s.getFormat(typename(fld.Type())), fld, 0) { // TODO is 0 index correct? + s.restore(mark) + } + } + errors <- nil // no errors + }() + + err := <-errors + return s.output.Bytes(), err +} + +// ---------------------------------------------------------------------------- +// Convenience functions + +// Fprint formats each argument according to the format f +// and writes to w. The result is the total number of bytes +// written and an os.Error, if any. +// +func (f Format) Fprint(w io.Writer, env Environment, args ...interface{}) (int, os.Error) { + data, err := f.Eval(env, args...) + if err != nil { + // TODO should we print partial result in case of error? + return 0, err + } + return w.Write(data) +} + +// Print formats each argument according to the format f +// and writes to standard output. The result is the total +// number of bytes written and an os.Error, if any. +// +func (f Format) Print(args ...interface{}) (int, os.Error) { + return f.Fprint(os.Stdout, nil, args...) +} + +// Sprint formats each argument according to the format f +// and returns the resulting string. If an error occurs +// during formatting, the result string contains the +// partially formatted result followed by an error message. +// +func (f Format) Sprint(args ...interface{}) string { + var buf bytes.Buffer + _, err := f.Fprint(&buf, nil, args...) + if err != nil { + var i interface{} = args + fmt.Fprintf(&buf, "--- Sprint(%s) failed: %v", fmt.Sprint(i), err) + } + return buf.String() +} diff --git a/src/pkg/exp/datafmt/datafmt_test.go b/src/pkg/exp/datafmt/datafmt_test.go new file mode 100644 index 000000000..87d071659 --- /dev/null +++ b/src/pkg/exp/datafmt/datafmt_test.go @@ -0,0 +1,330 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafmt + +import ( + "fmt" + "testing" + "go/token" +) + +var fset = token.NewFileSet() + +func parse(t *testing.T, form string, fmap FormatterMap) Format { + f, err := Parse(fset, "", []byte(form), fmap) + if err != nil { + t.Errorf("Parse(%s): %v", form, err) + return nil + } + return f +} + +func verify(t *testing.T, f Format, expected string, args ...interface{}) { + if f == nil { + return // allow other tests to run + } + result := f.Sprint(args...) + if result != expected { + t.Errorf( + "result : `%s`\nexpected: `%s`\n\n", + result, expected) + } +} + +func formatter(s *State, value interface{}, rule_name string) bool { + switch rule_name { + case "/": + fmt.Fprintf(s, "%d %d %d", s.Pos().Line, s.LinePos().Column, s.Pos().Column) + return true + case "blank": + s.Write([]byte{' '}) + return true + case "int": + if value.(int)&1 == 0 { + fmt.Fprint(s, "even ") + } else { + fmt.Fprint(s, "odd ") + } + return true + case "nil": + return false + case "testing.T": + s.Write([]byte("testing.T")) + return true + } + panic("unreachable") + return false +} + +func TestCustomFormatters(t *testing.T) { + fmap0 := FormatterMap{"/": formatter} + fmap1 := FormatterMap{"int": formatter, "blank": formatter, "nil": formatter} + fmap2 := FormatterMap{"testing.T": formatter} + + f := parse(t, `int=`, fmap0) + verify(t, f, ``, 1, 2, 3) + + f = parse(t, `int="#"`, nil) + verify(t, f, `###`, 1, 2, 3) + + f = parse(t, `int="#";string="%s"`, fmap0) + verify(t, f, "#1 0 1#1 0 7#1 0 13\n2 0 0foo2 0 8\n", 1, 2, 3, "\n", "foo", "\n") + + f = parse(t, ``, fmap1) + verify(t, f, `even odd even odd `, 0, 1, 2, 3) + + f = parse(t, `/ =@:blank; float64="#"`, fmap1) + verify(t, f, `# # #`, 0.0, 1.0, 2.0) + + f = parse(t, `float64=@:nil`, fmap1) + verify(t, f, ``, 0.0, 1.0, 2.0) + + f = parse(t, `testing "testing"; ptr=*`, fmap2) + verify(t, f, `testing.T`, t) + + // TODO needs more tests +} + +// ---------------------------------------------------------------------------- +// Formatting of basic and simple composite types + +func check(t *testing.T, form, expected string, args ...interface{}) { + f := parse(t, form, nil) + if f == nil { + return // allow other tests to run + } + result := f.Sprint(args...) + if result != expected { + t.Errorf( + "format : %s\nresult : `%s`\nexpected: `%s`\n\n", + form, result, expected) + } +} + +func TestBasicTypes(t *testing.T) { + check(t, ``, ``) + check(t, `bool=":%v"`, `:true:false`, true, false) + check(t, `int="%b %d %o 0x%x"`, `101010 42 52 0x2a`, 42) + + check(t, `int="%"`, `%`, 42) + check(t, `int="%%"`, `%`, 42) + check(t, `int="**%%**"`, `**%**`, 42) + check(t, `int="%%%%%%"`, `%%%`, 42) + check(t, `int="%%%d%%"`, `%42%`, 42) + + const i = -42 + const is = `-42` + check(t, `int ="%d"`, is, i) + check(t, `int8 ="%d"`, is, int8(i)) + check(t, `int16="%d"`, is, int16(i)) + check(t, `int32="%d"`, is, int32(i)) + check(t, `int64="%d"`, is, int64(i)) + + const u = 42 + const us = `42` + check(t, `uint ="%d"`, us, uint(u)) + check(t, `uint8 ="%d"`, us, uint8(u)) + check(t, `uint16="%d"`, us, uint16(u)) + check(t, `uint32="%d"`, us, uint32(u)) + check(t, `uint64="%d"`, us, uint64(u)) + + const f = 3.141592 + const fs = `3.141592` + check(t, `float64="%g"`, fs, f) + check(t, `float32="%g"`, fs, float32(f)) + check(t, `float64="%g"`, fs, float64(f)) +} + +func TestArrayTypes(t *testing.T) { + var a0 [10]int + check(t, `array="array";`, `array`, a0) + + a1 := [...]int{1, 2, 3} + check(t, `array="array";`, `array`, a1) + check(t, `array={*}; int="%d";`, `123`, a1) + check(t, `array={* / ", "}; int="%d";`, `1, 2, 3`, a1) + check(t, `array={* / *}; int="%d";`, `12233`, a1) + + a2 := []interface{}{42, "foo", 3.14} + check(t, `array={* / ", "}; interface=*; string="bar"; default="%v";`, `42, bar, 3.14`, a2) +} + +func TestChanTypes(t *testing.T) { + var c0 chan int + check(t, `chan="chan"`, `chan`, c0) + + c1 := make(chan int) + go func() { c1 <- 42 }() + check(t, `chan="chan"`, `chan`, c1) + // check(t, `chan=*`, `42`, c1); // reflection support for chans incomplete +} + +func TestFuncTypes(t *testing.T) { + var f0 func() int + check(t, `func="func"`, `func`, f0) + + f1 := func() int { return 42 } + check(t, `func="func"`, `func`, f1) + // check(t, `func=*`, `42`, f1); // reflection support for funcs incomplete +} + +func TestMapTypes(t *testing.T) { + var m0 map[string]int + check(t, `map="map"`, `map`, m0) + + m1 := map[string]int{} + check(t, `map="map"`, `map`, m1) + // check(t, `map=*`, ``, m1); // reflection support for maps incomplete +} + +func TestPointerTypes(t *testing.T) { + var p0 *int + check(t, `ptr="ptr"`, `ptr`, p0) + check(t, `ptr=*`, ``, p0) + check(t, `ptr=*|"nil"`, `nil`, p0) + + x := 99991 + p1 := &x + check(t, `ptr="ptr"`, `ptr`, p1) + check(t, `ptr=*; int="%d"`, `99991`, p1) +} + +func TestDefaultRule(t *testing.T) { + check(t, `default="%v"`, `42foo3.14`, 42, "foo", 3.14) + check(t, `default="%v"; int="%x"`, `abcdef`, 10, 11, 12, 13, 14, 15) + check(t, `default="%v"; int="%x"`, `ab**ef`, 10, 11, "**", 14, 15) + check(t, `default="%x"; int=@:default`, `abcdef`, 10, 11, 12, 13, 14, 15) +} + +func TestGlobalSeparatorRule(t *testing.T) { + check(t, `int="%d"; / ="-"`, `1-2-3-4`, 1, 2, 3, 4) + check(t, `int="%x%x"; / ="*"`, `aa*aa`, 10, 10) +} + +// ---------------------------------------------------------------------------- +// Formatting of a struct + +type T1 struct { + a int +} + +const F1 = `datafmt "datafmt";` + + `int = "%d";` + + `datafmt.T1 = "<" a ">";` + +func TestStruct1(t *testing.T) { check(t, F1, "<42>", T1{42}) } + +// ---------------------------------------------------------------------------- +// Formatting of a struct with an optional field (ptr) + +type T2 struct { + s string + p *T1 +} + +const F2a = F1 + + `string = "%s";` + + `ptr = *;` + + `datafmt.T2 = s ["-" p "-"];` + +const F2b = F1 + + `string = "%s";` + + `ptr = *;` + + `datafmt.T2 = s ("-" p "-" | "empty");` + +func TestStruct2(t *testing.T) { + check(t, F2a, "foo", T2{"foo", nil}) + check(t, F2a, "bar-<17>-", T2{"bar", &T1{17}}) + check(t, F2b, "fooempty", T2{"foo", nil}) +} + +// ---------------------------------------------------------------------------- +// Formatting of a struct with a repetitive field (slice) + +type T3 struct { + s string + a []int +} + +const F3a = `datafmt "datafmt";` + + `default = "%v";` + + `array = *;` + + `datafmt.T3 = s {" " a a / ","};` + +const F3b = `datafmt "datafmt";` + + `int = "%d";` + + `string = "%s";` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T3 = s [a:empty ": " {a / "-"}]` + +func TestStruct3(t *testing.T) { + check(t, F3a, "foo", T3{"foo", nil}) + check(t, F3a, "foo 00, 11, 22", T3{"foo", []int{0, 1, 2}}) + check(t, F3b, "bar", T3{"bar", nil}) + check(t, F3b, "bal: 2-3-5", T3{"bal", []int{2, 3, 5}}) +} + +// ---------------------------------------------------------------------------- +// Formatting of a struct with alternative field + +type T4 struct { + x *int + a []int +} + +const F4a = `datafmt "datafmt";` + + `int = "%d";` + + `ptr = *;` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T4 = "<" (x:empty x | "-") ">" ` + +const F4b = `datafmt "datafmt";` + + `int = "%d";` + + `ptr = *;` + + `array = *;` + + `nil = ;` + + `empty = *:nil;` + + `datafmt.T4 = "<" (a:empty {a / ", "} | "-") ">" ` + +func TestStruct4(t *testing.T) { + x := 7 + check(t, F4a, "<->", T4{nil, nil}) + check(t, F4a, "<7>", T4{&x, nil}) + check(t, F4b, "<->", T4{nil, nil}) + check(t, F4b, "<2, 3, 7>", T4{nil, []int{2, 3, 7}}) +} + +// ---------------------------------------------------------------------------- +// Formatting a struct (documentation example) + +type Point struct { + name string + x, y int +} + +const FPoint = `datafmt "datafmt";` + + `int = "%d";` + + `hexInt = "0x%x";` + + `string = "---%s---";` + + `datafmt.Point = name "{" x ", " y:hexInt "}";` + +func TestStructPoint(t *testing.T) { + p := Point{"foo", 3, 15} + check(t, FPoint, "---foo---{3, 0xf}", p) +} + +// ---------------------------------------------------------------------------- +// Formatting a slice (documentation example) + +const FSlice = `int = "%b";` + + `array = { * / ", " }` + +func TestSlice(t *testing.T) { check(t, FSlice, "10, 11, 101, 111", []int{2, 3, 5, 7}) } + +// TODO add more tests diff --git a/src/pkg/exp/datafmt/parser.go b/src/pkg/exp/datafmt/parser.go new file mode 100644 index 000000000..a2ddd3897 --- /dev/null +++ b/src/pkg/exp/datafmt/parser.go @@ -0,0 +1,368 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafmt + +import ( + "go/scanner" + "go/token" + "os" + "strconv" + "strings" +) + +// ---------------------------------------------------------------------------- +// Parsing + +type parser struct { + scanner.ErrorVector + scanner scanner.Scanner + file *token.File + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + packs map[string]string // PackageName -> ImportPath + rules map[string]expr // RuleName -> Expression +} + +func (p *parser) next() { + p.pos, p.tok, p.lit = p.scanner.Scan() + switch p.tok { + case token.CHAN, token.FUNC, token.INTERFACE, token.MAP, token.STRUCT: + // Go keywords for composite types are type names + // returned by reflect. Accept them as identifiers. + p.tok = token.IDENT // p.lit is already set correctly + } +} + +func (p *parser) init(fset *token.FileSet, filename string, src []byte) { + p.ErrorVector.Reset() + p.file = fset.AddFile(filename, fset.Base(), len(src)) + p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message + p.next() // initializes pos, tok, lit + p.packs = make(map[string]string) + p.rules = make(map[string]expr) +} + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.file.Position(pos), msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress in any case + return pos +} + +func (p *parser) parseIdentifier() string { + name := p.lit + p.expect(token.IDENT) + return name +} + +func (p *parser) parseTypeName() (string, bool) { + pos := p.pos + name, isIdent := p.parseIdentifier(), true + if p.tok == token.PERIOD { + // got a package name, lookup package + if importPath, found := p.packs[name]; found { + name = importPath + } else { + p.error(pos, "package not declared: "+name) + } + p.next() + name, isIdent = name+"."+p.parseIdentifier(), false + } + return name, isIdent +} + +// Parses a rule name and returns it. If the rule name is +// a package-qualified type name, the package name is resolved. +// The 2nd result value is true iff the rule name consists of a +// single identifier only (and thus could be a package name). +// +func (p *parser) parseRuleName() (string, bool) { + name, isIdent := "", false + switch p.tok { + case token.IDENT: + name, isIdent = p.parseTypeName() + case token.DEFAULT: + name = "default" + p.next() + case token.QUO: + name = "/" + p.next() + default: + p.errorExpected(p.pos, "rule name") + p.next() // make progress in any case + } + return name, isIdent +} + +func (p *parser) parseString() string { + s := "" + if p.tok == token.STRING { + s, _ = strconv.Unquote(p.lit) + // Unquote may fail with an error, but only if the scanner found + // an illegal string in the first place. In this case the error + // has already been reported. + p.next() + return s + } else { + p.expect(token.STRING) + } + return s +} + +func (p *parser) parseLiteral() literal { + s := []byte(p.parseString()) + + // A string literal may contain %-format specifiers. To simplify + // and speed up printing of the literal, split it into segments + // that start with "%" possibly followed by a last segment that + // starts with some other character. + var list []interface{} + i0 := 0 + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+1 < len(s) { + // the next segment starts with a % format + if i0 < i { + // the current segment is not empty, split it off + list = append(list, s[i0:i]) + i0 = i + } + i++ // skip %; let loop skip over char after % + } + } + // the final segment may start with any character + // (it is empty iff the string is empty) + list = append(list, s[i0:]) + + // convert list into a literal + lit := make(literal, len(list)) + for i := 0; i < len(list); i++ { + lit[i] = list[i].([]byte) + } + + return lit +} + +func (p *parser) parseField() expr { + var fname string + switch p.tok { + case token.ILLEGAL: + if p.lit != "@" { + return nil + } + fname = "@" + p.next() + case token.MUL: + fname = "*" + p.next() + case token.IDENT: + fname = p.parseIdentifier() + default: + return nil + } + + var ruleName string + if p.tok == token.COLON { + p.next() + ruleName, _ = p.parseRuleName() + } + + return &field{fname, ruleName} +} + +func (p *parser) parseOperand() (x expr) { + switch p.tok { + case token.STRING: + x = p.parseLiteral() + + case token.LPAREN: + p.next() + x = p.parseExpression() + if p.tok == token.SHR { + p.next() + x = &group{x, p.parseExpression()} + } + p.expect(token.RPAREN) + + case token.LBRACK: + p.next() + x = &option{p.parseExpression()} + p.expect(token.RBRACK) + + case token.LBRACE: + p.next() + x = p.parseExpression() + var div expr + if p.tok == token.QUO { + p.next() + div = p.parseExpression() + } + x = &repetition{x, div} + p.expect(token.RBRACE) + + default: + x = p.parseField() // may be nil + } + + return x +} + +func (p *parser) parseSequence() expr { + var list []interface{} + + for x := p.parseOperand(); x != nil; x = p.parseOperand() { + list = append(list, x) + } + + // no need for a sequence if list.Len() < 2 + switch len(list) { + case 0: + return nil + case 1: + return list[0].(expr) + } + + // convert list into a sequence + seq := make(sequence, len(list)) + for i := 0; i < len(list); i++ { + seq[i] = list[i].(expr) + } + return seq +} + +func (p *parser) parseExpression() expr { + var list []interface{} + + for { + x := p.parseSequence() + if x != nil { + list = append(list, x) + } + if p.tok != token.OR { + break + } + p.next() + } + + // no need for an alternatives if list.Len() < 2 + switch len(list) { + case 0: + return nil + case 1: + return list[0].(expr) + } + + // convert list into a alternatives + alt := make(alternatives, len(list)) + for i := 0; i < len(list); i++ { + alt[i] = list[i].(expr) + } + return alt +} + +func (p *parser) parseFormat() { + for p.tok != token.EOF { + pos := p.pos + + name, isIdent := p.parseRuleName() + switch p.tok { + case token.STRING: + // package declaration + importPath := p.parseString() + + // add package declaration + if !isIdent { + p.error(pos, "illegal package name: "+name) + } else if _, found := p.packs[name]; !found { + p.packs[name] = importPath + } else { + p.error(pos, "package already declared: "+name) + } + + case token.ASSIGN: + // format rule + p.next() + x := p.parseExpression() + + // add rule + if _, found := p.rules[name]; !found { + p.rules[name] = x + } else { + p.error(pos, "format rule already declared: "+name) + } + + default: + p.errorExpected(p.pos, "package declaration or format rule") + p.next() // make progress in any case + } + + if p.tok == token.SEMICOLON { + p.next() + } else { + break + } + } + p.expect(token.EOF) +} + +func remap(p *parser, name string) string { + i := strings.Index(name, ".") + if i >= 0 { + packageName, suffix := name[0:i], name[i:] + // lookup package + if importPath, found := p.packs[packageName]; found { + name = importPath + suffix + } else { + var invalidPos token.Position + p.Error(invalidPos, "package not declared: "+packageName) + } + } + return name +} + +// Parse parses a set of format productions from source src. Custom +// formatters may be provided via a map of formatter functions. If +// there are no errors, the result is a Format and the error is nil. +// Otherwise the format is nil and a non-empty ErrorList is returned. +// +func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) { + // parse source + var p parser + p.init(fset, filename, src) + p.parseFormat() + + // add custom formatters, if any + for name, form := range fmap { + name = remap(&p, name) + if _, found := p.rules[name]; !found { + p.rules[name] = &custom{name, form} + } else { + var invalidPos token.Position + p.Error(invalidPos, "formatter already declared: "+name) + } + } + + return p.rules, p.GetError(scanner.NoMultiples) +} diff --git a/src/pkg/exp/gui/Makefile b/src/pkg/exp/gui/Makefile new file mode 100644 index 000000000..af065e4a5 --- /dev/null +++ b/src/pkg/exp/gui/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/gui +GOFILES=\ + gui.go\ + +include ../../../Make.pkg diff --git a/src/pkg/exp/gui/gui.go b/src/pkg/exp/gui/gui.go new file mode 100644 index 000000000..171499186 --- /dev/null +++ b/src/pkg/exp/gui/gui.go @@ -0,0 +1,58 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gui defines a basic graphical user interface programming model. +package gui + +import ( + "image" + "image/draw" + "os" +) + +// A Window represents a single graphics window. +type Window interface { + // Screen returns an editable Image for the window. + Screen() draw.Image + // FlushImage flushes changes made to Screen() back to screen. + FlushImage() + // EventChan returns a channel carrying UI events such as key presses, + // mouse movements and window resizes. + EventChan() <-chan interface{} + // Close closes the window. + Close() os.Error +} + +// A KeyEvent is sent for a key press or release. +type KeyEvent struct { + // The value k represents key k being pressed. + // The value -k represents key k being released. + // The specific set of key values is not specified, + // but ordinary characters represent themselves. + Key int +} + +// A MouseEvent is sent for a button press or release or for a mouse movement. +type MouseEvent struct { + // Buttons is a bit mask of buttons: 1<<0 is left, 1<<1 middle, 1<<2 right. + // It represents button state and not necessarily the state delta: bit 0 + // being on means that the left mouse button is down, but does not imply + // that the same button was up in the previous MouseEvent. + Buttons int + // Loc is the location of the cursor. + Loc image.Point + // Nsec is the event's timestamp. + Nsec int64 +} + +// A ConfigEvent is sent each time the window's color model or size changes. +// The client should respond by calling Window.Screen to obtain a new image. +type ConfigEvent struct { + Config image.Config +} + +// An ErrEvent is sent when an error occurs. +type ErrEvent struct { + Err os.Error +} diff --git a/src/pkg/exp/gui/x11/Makefile b/src/pkg/exp/gui/x11/Makefile new file mode 100644 index 000000000..88cc1e23b --- /dev/null +++ b/src/pkg/exp/gui/x11/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=exp/gui/x11 +GOFILES=\ + auth.go\ + conn.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/exp/gui/x11/auth.go b/src/pkg/exp/gui/x11/auth.go new file mode 100644 index 000000000..d48936ac1 --- /dev/null +++ b/src/pkg/exp/gui/x11/auth.go @@ -0,0 +1,93 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x11 + +import ( + "bufio" + "io" + "os" +) + +// readU16BE reads a big-endian uint16 from r, using b as a scratch buffer. +func readU16BE(r io.Reader, b []byte) (uint16, os.Error) { + _, err := io.ReadFull(r, b[0:2]) + if err != nil { + return 0, err + } + return uint16(b[0])<<8 + uint16(b[1]), nil +} + +// readStr reads a length-prefixed string from r, using b as a scratch buffer. +func readStr(r io.Reader, b []byte) (string, os.Error) { + n, err := readU16BE(r, b) + if err != nil { + return "", err + } + if int(n) > len(b) { + return "", os.NewError("Xauthority entry too long for buffer") + } + _, err = io.ReadFull(r, b[0:n]) + if err != nil { + return "", err + } + return string(b[0:n]), nil +} + +// readAuth reads the X authority file and returns the name/data pair for the display. +// displayStr is the "12" out of a $DISPLAY like ":12.0". +func readAuth(displayStr string) (name, data string, err os.Error) { + // b is a scratch buffer to use and should be at least 256 bytes long + // (i.e. it should be able to hold a hostname). + var b [256]byte + // As per /usr/include/X11/Xauth.h. + const familyLocal = 256 + + fn := os.Getenv("XAUTHORITY") + if fn == "" { + home := os.Getenv("HOME") + if home == "" { + err = os.NewError("Xauthority not found: $XAUTHORITY, $HOME not set") + return + } + fn = home + "/.Xauthority" + } + r, err := os.Open(fn) + if err != nil { + return + } + defer r.Close() + br := bufio.NewReader(r) + + hostname, err := os.Hostname() + if err != nil { + return + } + for { + family, err := readU16BE(br, b[0:2]) + if err != nil { + return + } + addr, err := readStr(br, b[0:]) + if err != nil { + return + } + disp, err := readStr(br, b[0:]) + if err != nil { + return + } + name0, err := readStr(br, b[0:]) + if err != nil { + return + } + data0, err := readStr(br, b[0:]) + if err != nil { + return + } + if family == familyLocal && addr == hostname && disp == displayStr { + return name0, data0, nil + } + } + panic("unreachable") +} diff --git a/src/pkg/exp/gui/x11/conn.go b/src/pkg/exp/gui/x11/conn.go new file mode 100644 index 000000000..1d237816a --- /dev/null +++ b/src/pkg/exp/gui/x11/conn.go @@ -0,0 +1,627 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package x11 implements an X11 backend for the exp/gui package. +// +// The X protocol specification is at ftp://ftp.x.org/pub/X11R7.0/doc/PDF/proto.pdf. +// A summary of the wire format can be found in XCB's xproto.xml. +package x11 + +import ( + "bufio" + "exp/gui" + "image" + "image/draw" + "io" + "log" + "net" + "os" + "strconv" + "strings" + "time" +) + +type resID uint32 // X resource IDs. + +// TODO(nigeltao): Handle window resizes. +const ( + windowHeight = 600 + windowWidth = 800 +) + +const ( + keymapLo = 8 + keymapHi = 255 +) + +type conn struct { + c io.Closer + r *bufio.Reader + w *bufio.Writer + + gc, window, root, visual resID + + img *image.RGBA + eventc chan interface{} + mouseState gui.MouseEvent + + buf [256]byte // General purpose scratch buffer. + + flush chan bool + flushBuf0 [24]byte + flushBuf1 [4 * 1024]byte +} + +// writeSocket runs in its own goroutine, serving both FlushImage calls +// directly from the exp/gui client and indirectly from X expose events. +// It paints c.img to the X server via PutImage requests. +func (c *conn) writeSocket() { + defer c.c.Close() + for _ = range c.flush { + b := c.img.Bounds() + if b.Empty() { + continue + } + // Each X request has a 16-bit length (in terms of 4-byte units). To avoid going over + // this limit, we send PutImage for each row of the image, rather than trying to paint + // the entire image in one X request. This approach could easily be optimized (or the + // X protocol may have an escape sequence to delimit very large requests). + // TODO(nigeltao): See what XCB's xcb_put_image does in this situation. + units := 6 + b.Dx() + if units > 0xffff || b.Dy() > 0xffff { + log.Print("x11: window is too large for PutImage") + return + } + + c.flushBuf0[0] = 0x48 // PutImage opcode. + c.flushBuf0[1] = 0x02 // XCB_IMAGE_FORMAT_Z_PIXMAP. + c.flushBuf0[2] = uint8(units) + c.flushBuf0[3] = uint8(units >> 8) + setU32LE(c.flushBuf0[4:8], uint32(c.window)) + setU32LE(c.flushBuf0[8:12], uint32(c.gc)) + setU32LE(c.flushBuf0[12:16], 1<<16|uint32(b.Dx())) + c.flushBuf0[21] = 0x18 // depth = 24 bits. + + for y := b.Min.Y; y < b.Max.Y; y++ { + setU32LE(c.flushBuf0[16:20], uint32(y<<16)) + if _, err := c.w.Write(c.flushBuf0[:24]); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + p := c.img.Pix[(y-b.Min.Y)*c.img.Stride:] + for x, dx := 0, 4*b.Dx(); x < dx; { + nx := dx - x + if nx > len(c.flushBuf1) { + nx = len(c.flushBuf1) &^ 3 + } + for i := 0; i < nx; i += 4 { + // X11's order is BGRX, not RGBA. + c.flushBuf1[i+0] = p[x+i+2] + c.flushBuf1[i+1] = p[x+i+1] + c.flushBuf1[i+2] = p[x+i+0] + } + x += nx + if _, err := c.w.Write(c.flushBuf1[:nx]); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + } + } + if err := c.w.Flush(); err != nil { + if err != os.EOF { + log.Println("x11:", err.String()) + } + return + } + } +} + +func (c *conn) Screen() draw.Image { return c.img } + +func (c *conn) FlushImage() { + select { + case c.flush <- false: + // Flush notification sent. + default: + // Could not send. + // Flush notification must be pending already. + } +} + +func (c *conn) Close() os.Error { + // Shut down the writeSocket goroutine. This will close the socket to the + // X11 server, which will cause c.eventc to close. + close(c.flush) + for _ = range c.eventc { + // Drain the channel to allow the readSocket goroutine to shut down. + } + return nil +} + +func (c *conn) EventChan() <-chan interface{} { return c.eventc } + +// readSocket runs in its own goroutine, reading X events and sending gui +// events on c's EventChan. +func (c *conn) readSocket() { + var ( + keymap [256][]int + keysymsPerKeycode int + ) + defer close(c.eventc) + for { + // X events are always 32 bytes long. + if _, err := io.ReadFull(c.r, c.buf[:32]); err != nil { + if err != os.EOF { + c.eventc <- gui.ErrEvent{err} + } + return + } + switch c.buf[0] { + case 0x01: // Reply from a request (e.g. GetKeyboardMapping). + cookie := int(c.buf[3])<<8 | int(c.buf[2]) + if cookie != 1 { + // We issued only one request (GetKeyboardMapping) with a cookie of 1, + // so we shouldn't get any other reply from the X server. + c.eventc <- gui.ErrEvent{os.NewError("x11: unexpected cookie")} + return + } + keysymsPerKeycode = int(c.buf[1]) + b := make([]int, 256*keysymsPerKeycode) + for i := range keymap { + keymap[i] = b[i*keysymsPerKeycode : (i+1)*keysymsPerKeycode] + } + for i := keymapLo; i <= keymapHi; i++ { + m := keymap[i] + for j := range m { + u, err := readU32LE(c.r, c.buf[:4]) + if err != nil { + if err != os.EOF { + c.eventc <- gui.ErrEvent{err} + } + return + } + m[j] = int(u) + } + } + case 0x02, 0x03: // Key press, key release. + // X Keyboard Encoding is documented at http://tronche.com/gui/x/xlib/input/keyboard-encoding.html + // TODO(nigeltao): Do we need to implement the "MODE SWITCH / group modifier" feature + // or is that some no-longer-used X construct? + if keysymsPerKeycode < 2 { + // Either we haven't yet received the GetKeyboardMapping reply or + // the X server has sent one that's too short. + continue + } + keycode := int(c.buf[1]) + shift := int(c.buf[28]) & 0x01 + keysym := keymap[keycode][shift] + if keysym == 0 { + keysym = keymap[keycode][0] + } + // TODO(nigeltao): Should we send KeyEvents for Shift/Ctrl/Alt? Should Shift-A send + // the same int down the channel as the sent on just the A key? + // TODO(nigeltao): How should IME events (e.g. key presses that should generate CJK text) work? Or + // is that outside the scope of the gui.Window interface? + if c.buf[0] == 0x03 { + keysym = -keysym + } + c.eventc <- gui.KeyEvent{keysym} + case 0x04, 0x05: // Button press, button release. + mask := 1 << (c.buf[1] - 1) + if c.buf[0] == 0x04 { + c.mouseState.Buttons |= mask + } else { + c.mouseState.Buttons &^= mask + } + c.mouseState.Nsec = time.Nanoseconds() + c.eventc <- c.mouseState + case 0x06: // Motion notify. + c.mouseState.Loc.X = int(int16(c.buf[25])<<8 | int16(c.buf[24])) + c.mouseState.Loc.Y = int(int16(c.buf[27])<<8 | int16(c.buf[26])) + c.mouseState.Nsec = time.Nanoseconds() + c.eventc <- c.mouseState + case 0x0c: // Expose. + // A single user action could trigger multiple expose events (e.g. if moving another + // window with XShape'd rounded corners over our window). In that case, the X server will + // send a uint16 count (in bytes 16-17) of the number of additional expose events coming. + // We could parse each event for the (x, y, width, height) and maintain a minimal dirty + // rectangle, but for now, the simplest approach is to paint the entire window, when + // receiving the final event in the series. + if c.buf[17] == 0 && c.buf[16] == 0 { + // TODO(nigeltao): Should we ignore the very first expose event? A freshly mapped window + // will trigger expose, but until the first c.FlushImage call, there's probably nothing to + // paint but black. For an 800x600 window, at 4 bytes per pixel, each repaint writes about + // 2MB over the socket. + c.FlushImage() + } + // TODO(nigeltao): Should we listen to DestroyNotify (0x11) and ResizeRequest (0x19) events? + // What about EnterNotify (0x07) and LeaveNotify (0x08)? + } + } +} + +// connect connects to the X server given by the full X11 display name (e.g. +// ":12.0") and returns the connection as well as the portion of the full name +// that is the display number (e.g. "12"). +// Examples: +// connect(":1") // calls net.Dial("unix", "", "/tmp/.X11-unix/X1"), displayStr="1" +// connect("/tmp/launch-123/:0") // calls net.Dial("unix", "", "/tmp/launch-123/:0"), displayStr="0" +// connect("hostname:2.1") // calls net.Dial("tcp", "", "hostname:6002"), displayStr="2" +// connect("tcp/hostname:1.0") // calls net.Dial("tcp", "", "hostname:6001"), displayStr="1" +func connect(display string) (conn net.Conn, displayStr string, err os.Error) { + colonIdx := strings.LastIndex(display, ":") + if colonIdx < 0 { + return nil, "", os.NewError("bad display: " + display) + } + // Parse the section before the colon. + var protocol, host, socket string + if display[0] == '/' { + socket = display[:colonIdx] + } else { + if i := strings.LastIndex(display, "/"); i < 0 { + // The default protocol is TCP. + protocol = "tcp" + host = display[:colonIdx] + } else { + protocol = display[:i] + host = display[i+1 : colonIdx] + } + } + // Parse the section after the colon. + after := display[colonIdx+1:] + if after == "" { + return nil, "", os.NewError("bad display: " + display) + } + if i := strings.LastIndex(after, "."); i < 0 { + displayStr = after + } else { + displayStr = after[:i] + } + displayInt, err := strconv.Atoi(displayStr) + if err != nil || displayInt < 0 { + return nil, "", os.NewError("bad display: " + display) + } + // Make the connection. + if socket != "" { + conn, err = net.Dial("unix", socket+":"+displayStr) + } else if host != "" { + conn, err = net.Dial(protocol, host+":"+strconv.Itoa(6000+displayInt)) + } else { + conn, err = net.Dial("unix", "/tmp/.X11-unix/X"+displayStr) + } + if err != nil { + return nil, "", os.NewError("cannot connect to " + display + ": " + err.String()) + } + return +} + +// authenticate authenticates ourselves with the X server. +// displayStr is the "12" out of ":12.0". +func authenticate(w *bufio.Writer, displayStr string) os.Error { + key, value, err := readAuth(displayStr) + if err != nil { + return err + } + // Assume that the authentication protocol is "MIT-MAGIC-COOKIE-1". + if len(key) != 18 || len(value) != 16 { + return os.NewError("unsupported Xauth") + } + // 0x006c means little-endian. 0x000b, 0x0000 means X major version 11, minor version 0. + // 0x0012 and 0x0010 means the auth key and value have lengths 18 and 16. + // The final 0x0000 is padding, so that the string length is a multiple of 4. + _, err = io.WriteString(w, "\x6c\x00\x0b\x00\x00\x00\x12\x00\x10\x00\x00\x00") + if err != nil { + return err + } + _, err = io.WriteString(w, key) + if err != nil { + return err + } + // Again, the 0x0000 is padding. + _, err = io.WriteString(w, "\x00\x00") + if err != nil { + return err + } + _, err = io.WriteString(w, value) + if err != nil { + return err + } + err = w.Flush() + if err != nil { + return err + } + return nil +} + +// readU8 reads a uint8 from r, using b as a scratch buffer. +func readU8(r io.Reader, b []byte) (uint8, os.Error) { + _, err := io.ReadFull(r, b[:1]) + if err != nil { + return 0, err + } + return uint8(b[0]), nil +} + +// readU16LE reads a little-endian uint16 from r, using b as a scratch buffer. +func readU16LE(r io.Reader, b []byte) (uint16, os.Error) { + _, err := io.ReadFull(r, b[:2]) + if err != nil { + return 0, err + } + return uint16(b[0]) | uint16(b[1])<<8, nil +} + +// readU32LE reads a little-endian uint32 from r, using b as a scratch buffer. +func readU32LE(r io.Reader, b []byte) (uint32, os.Error) { + _, err := io.ReadFull(r, b[:4]) + if err != nil { + return 0, err + } + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24, nil +} + +// setU32LE sets b[:4] to be the little-endian representation of u. +func setU32LE(b []byte, u uint32) { + b[0] = byte((u >> 0) & 0xff) + b[1] = byte((u >> 8) & 0xff) + b[2] = byte((u >> 16) & 0xff) + b[3] = byte((u >> 24) & 0xff) +} + +// checkPixmapFormats checks that we have an agreeable X pixmap Format. +func checkPixmapFormats(r io.Reader, b []byte, n int) (agree bool, err os.Error) { + for i := 0; i < n; i++ { + _, err = io.ReadFull(r, b[:8]) + if err != nil { + return + } + // Byte 0 is depth, byte 1 is bits-per-pixel, byte 2 is scanline-pad, the rest (5) is padding. + if b[0] == 24 && b[1] == 32 { + agree = true + } + } + return +} + +// checkDepths checks that we have an agreeable X Depth (i.e. one that has an agreeable X VisualType). +func checkDepths(r io.Reader, b []byte, n int, visual uint32) (agree bool, err os.Error) { + for i := 0; i < n; i++ { + depth, err := readU16LE(r, b) + if err != nil { + return + } + depth &= 0xff + visualsLen, err := readU16LE(r, b) + if err != nil { + return + } + // Ignore 4 bytes of padding. + _, err = io.ReadFull(r, b[:4]) + if err != nil { + return + } + for j := 0; j < int(visualsLen); j++ { + // Read 24 bytes: visual(4), class(1), bits per rgb value(1), colormap entries(2), + // red mask(4), green mask(4), blue mask(4), padding(4). + v, err := readU32LE(r, b) + _, err = readU32LE(r, b) + rm, err := readU32LE(r, b) + gm, err := readU32LE(r, b) + bm, err := readU32LE(r, b) + _, err = readU32LE(r, b) + if err != nil { + return + } + if v == visual && rm == 0xff0000 && gm == 0xff00 && bm == 0xff && depth == 24 { + agree = true + } + } + } + return +} + +// checkScreens checks that we have an agreeable X Screen. +func checkScreens(r io.Reader, b []byte, n int) (root, visual uint32, err os.Error) { + for i := 0; i < n; i++ { + root0, err := readU32LE(r, b) + if err != nil { + return + } + // Ignore the next 7x4 bytes, which is: colormap, whitepixel, blackpixel, current input masks, + // width and height (pixels), width and height (mm), min and max installed maps. + _, err = io.ReadFull(r, b[:28]) + if err != nil { + return + } + visual0, err := readU32LE(r, b) + if err != nil { + return + } + // Next 4 bytes: backing stores, save unders, root depth, allowed depths length. + x, err := readU32LE(r, b) + if err != nil { + return + } + nDepths := int(x >> 24) + agree, err := checkDepths(r, b, nDepths, visual0) + if err != nil { + return + } + if agree && root == 0 { + root = root0 + visual = visual0 + } + } + return +} + +// handshake performs the protocol handshake with the X server, and ensures +// that the server provides a compatible Screen, Depth, etc. +func (c *conn) handshake() os.Error { + _, err := io.ReadFull(c.r, c.buf[:8]) + if err != nil { + return err + } + // Byte 0 should be 1 (success), bytes 2:6 should be 0xb0000000 (major/minor version 11.0). + if c.buf[0] != 1 || c.buf[2] != 11 || c.buf[3] != 0 || c.buf[4] != 0 || c.buf[5] != 0 { + return os.NewError("unsupported X version") + } + // Ignore the release number. + _, err = io.ReadFull(c.r, c.buf[:4]) + if err != nil { + return err + } + // Read the resource ID base. + resourceIdBase, err := readU32LE(c.r, c.buf[:4]) + if err != nil { + return err + } + // Read the resource ID mask. + resourceIdMask, err := readU32LE(c.r, c.buf[:4]) + if err != nil { + return err + } + if resourceIdMask < 256 { + return os.NewError("X resource ID mask is too small") + } + // Ignore the motion buffer size. + _, err = io.ReadFull(c.r, c.buf[:4]) + if err != nil { + return err + } + // Read the vendor length and round it up to a multiple of 4, + // for X11 protocol alignment reasons. + vendorLen, err := readU16LE(c.r, c.buf[:2]) + if err != nil { + return err + } + vendorLen = (vendorLen + 3) &^ 3 + // Read the maximum request length. + maxReqLen, err := readU16LE(c.r, c.buf[:2]) + if err != nil { + return err + } + if maxReqLen != 0xffff { + return os.NewError("unsupported X maximum request length") + } + // Read the roots length. + rootsLen, err := readU8(c.r, c.buf[:1]) + if err != nil { + return err + } + // Read the pixmap formats length. + pixmapFormatsLen, err := readU8(c.r, c.buf[:1]) + if err != nil { + return err + } + // Ignore some things that we don't care about (totaling 10 + vendorLen bytes): + // imageByteOrder(1), bitmapFormatBitOrder(1), bitmapFormatScanlineUnit(1) bitmapFormatScanlinePad(1), + // minKeycode(1), maxKeycode(1), padding(4), vendor (vendorLen). + if 10+int(vendorLen) > cap(c.buf) { + return os.NewError("unsupported X vendor") + } + _, err = io.ReadFull(c.r, c.buf[:10+int(vendorLen)]) + if err != nil { + return err + } + // Check that we have an agreeable pixmap format. + agree, err := checkPixmapFormats(c.r, c.buf[:8], int(pixmapFormatsLen)) + if err != nil { + return err + } + if !agree { + return os.NewError("unsupported X pixmap formats") + } + // Check that we have an agreeable screen. + root, visual, err := checkScreens(c.r, c.buf[:24], int(rootsLen)) + if err != nil { + return err + } + if root == 0 || visual == 0 { + return os.NewError("unsupported X screen") + } + c.gc = resID(resourceIdBase) + c.window = resID(resourceIdBase + 1) + c.root = resID(root) + c.visual = resID(visual) + return nil +} + +// NewWindow calls NewWindowDisplay with $DISPLAY. +func NewWindow() (gui.Window, os.Error) { + display := os.Getenv("DISPLAY") + if len(display) == 0 { + return nil, os.NewError("$DISPLAY not set") + } + return NewWindowDisplay(display) +} + +// NewWindowDisplay returns a new gui.Window, backed by a newly created and +// mapped X11 window. The X server to connect to is specified by the display +// string, such as ":1". +func NewWindowDisplay(display string) (gui.Window, os.Error) { + socket, displayStr, err := connect(display) + if err != nil { + return nil, err + } + c := new(conn) + c.c = socket + c.r = bufio.NewReader(socket) + c.w = bufio.NewWriter(socket) + err = authenticate(c.w, displayStr) + if err != nil { + return nil, err + } + err = c.handshake() + if err != nil { + return nil, err + } + + // Now that we're connected, show a window, via three X protocol messages. + // First, issue a GetKeyboardMapping request. This is the first request, and + // will be associated with a cookie of 1. + setU32LE(c.buf[0:4], 0x00020065) // 0x65 is the GetKeyboardMapping opcode, and the message is 2 x 4 bytes long. + setU32LE(c.buf[4:8], uint32((keymapHi-keymapLo+1)<<8|keymapLo)) + // Second, create a graphics context (GC). + setU32LE(c.buf[8:12], 0x00060037) // 0x37 is the CreateGC opcode, and the message is 6 x 4 bytes long. + setU32LE(c.buf[12:16], uint32(c.gc)) + setU32LE(c.buf[16:20], uint32(c.root)) + setU32LE(c.buf[20:24], 0x00010004) // Bit 2 is XCB_GC_FOREGROUND, bit 16 is XCB_GC_GRAPHICS_EXPOSURES. + setU32LE(c.buf[24:28], 0x00000000) // The Foreground is black. + setU32LE(c.buf[28:32], 0x00000000) // GraphicsExposures' value is unused. + // Third, create the window. + setU32LE(c.buf[32:36], 0x000a0001) // 0x01 is the CreateWindow opcode, and the message is 10 x 4 bytes long. + setU32LE(c.buf[36:40], uint32(c.window)) + setU32LE(c.buf[40:44], uint32(c.root)) + setU32LE(c.buf[44:48], 0x00000000) // Initial (x, y) is (0, 0). + setU32LE(c.buf[48:52], windowHeight<<16|windowWidth) + setU32LE(c.buf[52:56], 0x00010000) // Border width is 0, XCB_WINDOW_CLASS_INPUT_OUTPUT is 1. + setU32LE(c.buf[56:60], uint32(c.visual)) + setU32LE(c.buf[60:64], 0x00000802) // Bit 1 is XCB_CW_BACK_PIXEL, bit 11 is XCB_CW_EVENT_MASK. + setU32LE(c.buf[64:68], 0x00000000) // The Back-Pixel is black. + setU32LE(c.buf[68:72], 0x0000804f) // Key/button press and release, pointer motion, and expose event masks. + // Fourth, map the window. + setU32LE(c.buf[72:76], 0x00020008) // 0x08 is the MapWindow opcode, and the message is 2 x 4 bytes long. + setU32LE(c.buf[76:80], uint32(c.window)) + // Write the bytes. + _, err = c.w.Write(c.buf[:80]) + if err != nil { + return nil, err + } + err = c.w.Flush() + if err != nil { + return nil, err + } + + c.img = image.NewRGBA(windowWidth, windowHeight) + c.eventc = make(chan interface{}, 16) + c.flush = make(chan bool, 1) + go c.readSocket() + go c.writeSocket() + return c, nil +} diff --git a/src/pkg/exp/norm/Makefile b/src/pkg/exp/norm/Makefile new file mode 100644 index 000000000..a4dfb43f7 --- /dev/null +++ b/src/pkg/exp/norm/Makefile @@ -0,0 +1,44 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/norm +GOFILES=\ + composition.go\ + forminfo.go\ + normalize.go\ + tables.go\ + trie.go\ + +include ../../../Make.pkg + +CLEANFILES+=maketables maketesttables + +maketables: maketables.go triegen.go + $(GC) maketables.go triegen.go + $(LD) -o maketables maketables.$O + +maketesttables: maketesttables.go triegen.go + $(GC) maketesttables.go triegen.go + $(LD) -o maketesttables maketesttables.$O + +tables: maketables + ./maketables > tables.go + gofmt -w tables.go + +trietesttables: maketesttables + ./maketesttables > triedata_test.go + gofmt -w triedata_test.go + +# Build (but do not run) maketables during testing, +# just to make sure it still compiles. +testshort: maketables maketesttables + +# Downloads from www.unicode.org, so not part +# of standard test scripts. +test: testtables + +testtables: maketables + ./maketables -test -tables= diff --git a/src/pkg/exp/norm/composition.go b/src/pkg/exp/norm/composition.go new file mode 100644 index 000000000..b2d2abaf6 --- /dev/null +++ b/src/pkg/exp/norm/composition.go @@ -0,0 +1,344 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "utf8" + +const ( + maxCombiningChars = 30 + 2 // +2 to hold CGJ and Hangul overflow. + maxBackRunes = maxCombiningChars - 1 + maxNFCExpansion = 3 // NFC(0x1D160) + maxNFKCExpansion = 18 // NFKC(0xFDFA) + + maxRuneSizeInDecomp = 4 + // Need to multiply by 2 as we don't reuse byte buffer space for recombining. + maxByteBufferSize = 2 * maxRuneSizeInDecomp * maxCombiningChars // 256 +) + +// reorderBuffer is used to normalize a single segment. Characters inserted with +// insert() are decomposed and reordered based on CCC. The compose() method can +// be used to recombine characters. Note that the byte buffer does not hold +// the UTF-8 characters in order. Only the rune array is maintained in sorted +// order. flush() writes the resulting segment to a byte array. +type reorderBuffer struct { + rune [maxCombiningChars]runeInfo // Per character info. + byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. + nrune int // Number of runeInfos. + nbyte uint8 // Number or bytes. + f formInfo +} + +// reset discards all characters from the buffer. +func (rb *reorderBuffer) reset() { + rb.nrune = 0 + rb.nbyte = 0 +} + +// flush appends the normalized segment to out and resets rb. +func (rb *reorderBuffer) flush(out []byte) []byte { + for i := 0; i < rb.nrune; i++ { + start := rb.rune[i].pos + end := start + rb.rune[i].size + out = append(out, rb.byte[start:end]...) + } + rb.reset() + return out +} + +// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. +// It returns false if the buffer is not large enough to hold the rune. +// It is used internally by insert. +func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { + n := rb.nrune + if n >= maxCombiningChars { + return false + } + b := rb.rune[:] + cc := info.ccc + if cc > 0 { + // Find insertion position + move elements to make room. + for ; n > 0; n-- { + if b[n-1].ccc <= cc { + break + } + b[n] = b[n-1] + } + } + rb.nrune += 1 + pos := uint8(rb.nbyte) + rb.nbyte += info.size + info.pos = pos + b[n] = info + return true +} + +// insert inserts the given rune in the buffer ordered by CCC. +// It returns true if the buffer was large enough to hold the decomposed rune. +func (rb *reorderBuffer) insert(src []byte, info runeInfo) bool { + if info.size == 3 && isHangul(src) { + rune, _ := utf8.DecodeRune(src) + return rb.decomposeHangul(uint32(rune)) + } + pos := rb.nbyte + if info.flags.hasDecomposition() { + dcomp := rb.f.decompose(src) + for i := 0; i < len(dcomp); i += int(info.size) { + info = rb.f.info(dcomp[i:]) + if !rb.insertOrdered(info) { + return false + } + } + copy(rb.byte[pos:], dcomp) + } else { + if !rb.insertOrdered(info) { + return false + } + copy(rb.byte[pos:], src[:info.size]) + } + return true +} + +// insertString inserts the given rune in the buffer ordered by CCC. +// It returns true if the buffer was large enough to hold the decomposed rune. +func (rb *reorderBuffer) insertString(src string, info runeInfo) bool { + if info.size == 3 && isHangulString(src) { + rune, _ := utf8.DecodeRuneInString(src) + return rb.decomposeHangul(uint32(rune)) + } + pos := rb.nbyte + dcomp := rb.f.decomposeString(src) + dn := len(dcomp) + if dn != 0 { + for i := 0; i < dn; i += int(info.size) { + info = rb.f.info(dcomp[i:]) + if !rb.insertOrdered(info) { + return false + } + } + copy(rb.byte[pos:], dcomp) + } else { + if !rb.insertOrdered(info) { + return false + } + copy(rb.byte[pos:], src[:info.size]) + } + return true +} + +// appendRune inserts a rune at the end of the buffer. It is used for Hangul. +func (rb *reorderBuffer) appendRune(rune uint32) { + bn := rb.nbyte + sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) + rb.nbyte += uint8(sz) + rb.rune[rb.nrune] = runeInfo{bn, uint8(sz), 0, 0} + rb.nrune++ +} + +// assignRune sets a rune at position pos. It is used for Hangul and recomposition. +func (rb *reorderBuffer) assignRune(pos int, rune uint32) { + bn := rb.nbyte + sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) + rb.rune[pos] = runeInfo{bn, uint8(sz), 0, 0} + rb.nbyte += uint8(sz) +} + +// runeAt returns the rune at position n. It is used for Hangul and recomposition. +func (rb *reorderBuffer) runeAt(n int) uint32 { + inf := rb.rune[n] + rune, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size]) + return uint32(rune) +} + +// bytesAt returns the UTF-8 encoding of the rune at position n. +// It is used for Hangul and recomposition. +func (rb *reorderBuffer) bytesAt(n int) []byte { + inf := rb.rune[n] + return rb.byte[inf.pos : int(inf.pos)+int(inf.size)] +} + +// For Hangul we combine algorithmically, instead of using tables. +const ( + hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80 + hangulBase0 = 0xEA + hangulBase1 = 0xB0 + hangulBase2 = 0x80 + + hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4 + hangulEnd0 = 0xED + hangulEnd1 = 0x9E + hangulEnd2 = 0xA4 + + jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00 + jamoLBase0 = 0xE1 + jamoLBase1 = 0x84 + jamoLEnd = 0x1113 + jamoVBase = 0x1161 + jamoVEnd = 0x1176 + jamoTBase = 0x11A7 + jamoTEnd = 0x11C3 + + jamoTCount = 28 + jamoVCount = 21 + jamoVTCount = 21 * 28 + jamoLVTCount = 19 * 21 * 28 +) + +// Caller must verify that len(b) >= 3. +func isHangul(b []byte) bool { + b0 := b[0] + if b0 < hangulBase0 { + return false + } + b1 := b[1] + switch { + case b0 == hangulBase0: + return b1 >= hangulBase1 + case b0 < hangulEnd0: + return true + case b0 > hangulEnd0: + return false + case b1 < hangulEnd1: + return true + } + return b1 == hangulEnd1 && b[2] < hangulEnd2 +} + +// Caller must verify that len(b) >= 3. +func isHangulString(b string) bool { + b0 := b[0] + if b0 < hangulBase0 { + return false + } + b1 := b[1] + switch { + case b0 == hangulBase0: + return b1 >= hangulBase1 + case b0 < hangulEnd0: + return true + case b0 > hangulEnd0: + return false + case b1 < hangulEnd1: + return true + } + return b1 == hangulEnd1 && b[2] < hangulEnd2 +} + +// Caller must ensure len(b) >= 2. +func isJamoVT(b []byte) bool { + // True if (rune & 0xff00) == jamoLBase + return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1 +} + +func isHangulWithoutJamoT(b []byte) bool { + c, _ := utf8.DecodeRune(b) + c -= hangulBase + return c < jamoLVTCount && c%jamoTCount == 0 +} + +// decomposeHangul algorithmically decomposes a Hangul rune into +// its Jamo components. +// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul. +func (rb *reorderBuffer) decomposeHangul(rune uint32) bool { + b := rb.rune[:] + n := rb.nrune + if n+3 > len(b) { + return false + } + rune -= hangulBase + x := rune % jamoTCount + rune /= jamoTCount + rb.appendRune(jamoLBase + rune/jamoVCount) + rb.appendRune(jamoVBase + rune%jamoVCount) + if x != 0 { + rb.appendRune(jamoTBase + x) + } + return true +} + +// combineHangul algorithmically combines Jamo character components into Hangul. +// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul. +func (rb *reorderBuffer) combineHangul() { + k := 1 + b := rb.rune[:] + bn := rb.nrune + for s, i := 0, 1; i < bn; i++ { + cccB := b[k-1].ccc + cccC := b[i].ccc + if cccB == 0 { + s = k - 1 + } + if s != k-1 && cccB >= cccC { + // b[i] is blocked by greater-equal cccX below it + b[k] = b[i] + k++ + } else { + l := rb.runeAt(s) // also used to compare to hangulBase + v := rb.runeAt(i) // also used to compare to jamoT + switch { + case jamoLBase <= l && l < jamoLEnd && + jamoVBase <= v && v < jamoVEnd: + // 11xx plus 116x to LV + rb.assignRune(s, hangulBase+ + (l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount) + case hangulBase <= l && l < hangulEnd && + jamoTBase < v && v < jamoTEnd && + ((l-hangulBase)%jamoTCount) == 0: + // ACxx plus 11Ax to LVT + rb.assignRune(s, l+v-jamoTBase) + default: + b[k] = b[i] + k++ + } + } + } + rb.nrune = k +} + +// compose recombines the runes in the buffer. +// It should only be used to recompose a single segment, as it will not +// handle alternations between Hangul and non-Hangul characters correctly. +func (rb *reorderBuffer) compose() { + // UAX #15, section X5 , including Corrigendum #5 + // "In any character sequence beginning with starter S, a character C is + // blocked from S if and only if there is some character B between S + // and C, and either B is a starter or it has the same or higher + // combining class as C." + k := 1 + b := rb.rune[:] + bn := rb.nrune + for s, i := 0, 1; i < bn; i++ { + if isJamoVT(rb.bytesAt(i)) { + // Redo from start in Hangul mode. Necessary to support + // U+320E..U+321E in NFKC mode. + rb.combineHangul() + return + } + ii := b[i] + // We can only use combineForward as a filter if we later + // get the info for the combined character. This is more + // expensive than using the filter. Using combinesBackward() + // is safe. + if ii.flags.combinesBackward() { + cccB := b[k-1].ccc + cccC := ii.ccc + blocked := false // b[i] blocked by starter or greater or equal CCC? + if cccB == 0 { + s = k - 1 + } else { + blocked = s != k-1 && cccB >= cccC + } + if !blocked { + combined := combine(rb.runeAt(s), rb.runeAt(i)) + if combined != 0 { + rb.assignRune(s, combined) + continue + } + } + } + b[k] = b[i] + k++ + } + rb.nrune = k +} diff --git a/src/pkg/exp/norm/composition_test.go b/src/pkg/exp/norm/composition_test.go new file mode 100644 index 000000000..195a0c1e8 --- /dev/null +++ b/src/pkg/exp/norm/composition_test.go @@ -0,0 +1,138 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import "testing" + +// TestCase is used for most tests. +type TestCase struct { + in []int + out []int +} + +type insertFunc func(rb *reorderBuffer, rune int) bool + +func insert(rb *reorderBuffer, rune int) bool { + b := []byte(string(rune)) + return rb.insert(b, rb.f.info(b)) +} + +func insertString(rb *reorderBuffer, rune int) bool { + s := string(rune) + return rb.insertString(s, rb.f.infoString(s)) +} + +func runTests(t *testing.T, name string, rb *reorderBuffer, f insertFunc, tests []TestCase) { + for i, test := range tests { + rb.reset() + for j, rune := range test.in { + b := []byte(string(rune)) + if !rb.insert(b, rb.f.info(b)) { + t.Errorf("%s:%d: insert failed for rune %d", name, i, j) + } + } + if rb.f.composing { + rb.compose() + } + if rb.nrune != len(test.out) { + t.Errorf("%s:%d: length = %d; want %d", name, i, rb.nrune, len(test.out)) + continue + } + for j, want := range test.out { + found := int(rb.runeAt(j)) + if found != want { + t.Errorf("%s:%d: runeAt(%d) = %U; want %U", name, i, j, found, want) + } + } + } +} + +func TestFlush(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFC]} + out := make([]byte, 0) + + out = rb.flush(out) + if len(out) != 0 { + t.Errorf("wrote bytes on flush of empty buffer. (len(out) = %d)", len(out)) + } + + for _, r := range []int("world!") { + insert(rb, r) + } + + out = []byte("Hello ") + out = rb.flush(out) + want := "Hello world!" + if string(out) != want { + t.Errorf(`output after flush was "%s"; want "%s"`, string(out), want) + } + if rb.nrune != 0 { + t.Errorf("flush: non-null size of info buffer (rb.nrune == %d)", rb.nrune) + } + if rb.nbyte != 0 { + t.Errorf("flush: non-null size of byte buffer (rb.nbyte == %d)", rb.nbyte) + } +} + +var insertTests = []TestCase{ + {[]int{'a'}, []int{'a'}}, + {[]int{0x300}, []int{0x300}}, + {[]int{0x300, 0x316}, []int{0x316, 0x300}}, // CCC(0x300)==230; CCC(0x316)==220 + {[]int{0x316, 0x300}, []int{0x316, 0x300}}, + {[]int{0x41, 0x316, 0x300}, []int{0x41, 0x316, 0x300}}, + {[]int{0x41, 0x300, 0x316}, []int{0x41, 0x316, 0x300}}, + {[]int{0x300, 0x316, 0x41}, []int{0x316, 0x300, 0x41}}, + {[]int{0x41, 0x300, 0x40, 0x316}, []int{0x41, 0x300, 0x40, 0x316}}, +} + +func TestInsert(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFD]} + runTests(t, "TestInsert", rb, insert, insertTests) +} + +func TestInsertString(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFD]} + runTests(t, "TestInsertString", rb, insertString, insertTests) +} + +var decompositionNFDTest = []TestCase{ + {[]int{0xC0}, []int{0x41, 0x300}}, + {[]int{0xAC00}, []int{0x1100, 0x1161}}, + {[]int{0x01C4}, []int{0x01C4}}, + {[]int{0x320E}, []int{0x320E}}, + {[]int("음ẻ과"), []int{0x110B, 0x1173, 0x11B7, 0x65, 0x309, 0x1100, 0x116A}}, +} + +var decompositionNFKDTest = []TestCase{ + {[]int{0xC0}, []int{0x41, 0x300}}, + {[]int{0xAC00}, []int{0x1100, 0x1161}}, + {[]int{0x01C4}, []int{0x44, 0x5A, 0x030C}}, + {[]int{0x320E}, []int{0x28, 0x1100, 0x1161, 0x29}}, +} + +func TestDecomposition(t *testing.T) { + rb := &reorderBuffer{} + rb.f = *formTable[NFD] + runTests(t, "TestDecompositionNFD", rb, insert, decompositionNFDTest) + rb.f = *formTable[NFKD] + runTests(t, "TestDecompositionNFKD", rb, insert, decompositionNFKDTest) +} + +var compositionTest = []TestCase{ + {[]int{0x41, 0x300}, []int{0xC0}}, + {[]int{0x41, 0x316}, []int{0x41, 0x316}}, + {[]int{0x41, 0x300, 0x35D}, []int{0xC0, 0x35D}}, + {[]int{0x41, 0x316, 0x300}, []int{0xC0, 0x316}}, + // blocking starter + {[]int{0x41, 0x316, 0x40, 0x300}, []int{0x41, 0x316, 0x40, 0x300}}, + {[]int{0x1100, 0x1161}, []int{0xAC00}}, + // parenthesized Hangul, alternate between ASCII and Hangul. + {[]int{0x28, 0x1100, 0x1161, 0x29}, []int{0x28, 0xAC00, 0x29}}, +} + +func TestComposition(t *testing.T) { + rb := &reorderBuffer{f: *formTable[NFC]} + runTests(t, "TestComposition", rb, insert, compositionTest) +} diff --git a/src/pkg/exp/norm/forminfo.go b/src/pkg/exp/norm/forminfo.go new file mode 100644 index 000000000..ee3edb8ea --- /dev/null +++ b/src/pkg/exp/norm/forminfo.go @@ -0,0 +1,188 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +// This file contains Form-specific logic and wrappers for data in tables.go. + +type runeInfo struct { + pos uint8 // start position in reorderBuffer; used in composition.go + size uint8 // length of UTF-8 encoding of this rune + ccc uint8 // canonical combining class + flags qcInfo // quick check flags +} + +// functions dispatchable per form +type boundaryFunc func(f *formInfo, info runeInfo) bool +type lookupFunc func(b []byte) runeInfo +type lookupFuncString func(s string) runeInfo +type decompFunc func(b []byte) []byte +type decompFuncString func(s string) []byte + +// formInfo holds Form-specific functions and tables. +type formInfo struct { + form Form + + composing, compatibility bool // form type + + decompose decompFunc + decomposeString decompFuncString + info lookupFunc + infoString lookupFuncString + boundaryBefore boundaryFunc + boundaryAfter boundaryFunc +} + +var formTable []*formInfo + +func init() { + formTable = make([]*formInfo, 4) + + for i := range formTable { + f := &formInfo{} + formTable[i] = f + f.form = Form(i) + if Form(i) == NFKD || Form(i) == NFKC { + f.compatibility = true + f.decompose = decomposeNFKC + f.decomposeString = decomposeStringNFKC + f.info = lookupInfoNFKC + f.infoString = lookupInfoStringNFKC + } else { + f.decompose = decomposeNFC + f.decomposeString = decomposeStringNFC + f.info = lookupInfoNFC + f.infoString = lookupInfoStringNFC + } + if Form(i) == NFC || Form(i) == NFKC { + f.composing = true + f.boundaryBefore = compBoundaryBefore + f.boundaryAfter = compBoundaryAfter + } else { + f.boundaryBefore = decompBoundary + f.boundaryAfter = decompBoundary + } + } +} + +func decompBoundary(f *formInfo, info runeInfo) bool { + if info.ccc == 0 && info.flags.isYesD() { // Implies isHangul(b) == true + return true + } + // We assume that the CCC of the first character in a decomposition + // is always non-zero if different from info.ccc and that we can return + // false at this point. This is verified by maketables. + return false +} + +func compBoundaryBefore(f *formInfo, info runeInfo) bool { + if info.ccc == 0 && info.flags.isYesC() { + return true + } + // We assume that the CCC of the first character in a decomposition + // is always non-zero if different from info.ccc and that we can return + // false at this point. This is verified by maketables. + return false +} + +func compBoundaryAfter(f *formInfo, info runeInfo) bool { + // This misses values where the last char in a decomposition is a + // boundary such as Hangul with JamoT. + // TODO(mpvl): verify this does not lead to segments that do + // not fit in the reorderBuffer. + return info.flags.isInert() +} + +// We pack quick check data in 4 bits: +// 0: NFD_QC Yes (0) or No (1). No also means there is a decomposition. +// 1..2: NFC_QC Yes(00), No (01), or Maybe (11) +// 3: Combines forward (0 == false, 1 == true) +// +// When all 4 bits are zero, the character is inert, meaning it is never +// influenced by normalization. +// +// We pack the bits for both NFC/D and NFKC/D in one byte. +type qcInfo uint8 + +func (i qcInfo) isYesC() bool { return i&0x2 == 0 } +func (i qcInfo) isNoC() bool { return i&0x6 == 0x2 } +func (i qcInfo) isMaybe() bool { return i&0x4 != 0 } +func (i qcInfo) isYesD() bool { return i&0x1 == 0 } +func (i qcInfo) isNoD() bool { return i&0x1 != 0 } +func (i qcInfo) isInert() bool { return i&0xf == 0 } + +func (i qcInfo) combinesForward() bool { return i&0x8 != 0 } +func (i qcInfo) combinesBackward() bool { return i&0x4 != 0 } // == isMaybe +func (i qcInfo) hasDecomposition() bool { return i&0x1 != 0 } // == isNoD + +// Wrappers for tables.go + +// The 16-bit value of the decompostion tries is an index into a byte +// array of UTF-8 decomposition sequences. The first byte is the number +// of bytes in the decomposition (excluding this length byte). The actual +// sequence starts at the offset+1. +func decomposeNFC(b []byte) []byte { + p := nfcDecompTrie.lookupUnsafe(b) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +func decomposeNFKC(b []byte) []byte { + p := nfkcDecompTrie.lookupUnsafe(b) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +func decomposeStringNFC(s string) []byte { + p := nfcDecompTrie.lookupStringUnsafe(s) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +func decomposeStringNFKC(s string) []byte { + p := nfkcDecompTrie.lookupStringUnsafe(s) + n := decomps[p] + p++ + return decomps[p : p+uint16(n)] +} + +// Recomposition +// We use 32-bit keys instead of 64-bit for the two codepoint keys. +// This clips off the bits of three entries, but we know this will not +// result in a collision. In the unlikely event that changes to +// UnicodeData.txt introduce collisions, the compiler will catch it. +// Note that the recomposition map for NFC and NFKC are identical. + +// combine returns the combined rune or 0 if it doesn't exist. +func combine(a, b uint32) uint32 { + key := uint32(uint16(a))<<16 + uint32(uint16(b)) + return recompMap[key] +} + +// The 16-bit character info has the following bit layout: +// 0..7 CCC value. +// 8..11 qcInfo for NFC/NFD +// 12..15 qcInfo for NFKC/NFKD +func lookupInfoNFC(b []byte) runeInfo { + v, sz := charInfoTrie.lookup(b) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 8)} +} + +func lookupInfoStringNFC(s string) runeInfo { + v, sz := charInfoTrie.lookupString(s) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 8)} +} + +func lookupInfoNFKC(b []byte) runeInfo { + v, sz := charInfoTrie.lookup(b) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 12)} +} + +func lookupInfoStringNFKC(s string) runeInfo { + v, sz := charInfoTrie.lookupString(s) + return runeInfo{0, uint8(sz), uint8(v), qcInfo(v >> 12)} +} diff --git a/src/pkg/exp/norm/maketables.go b/src/pkg/exp/norm/maketables.go new file mode 100644 index 000000000..e3e5700a6 --- /dev/null +++ b/src/pkg/exp/norm/maketables.go @@ -0,0 +1,855 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Normalization table generator. +// Data read from the web. + +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "http" + "io" + "log" + "os" + "regexp" + "strconv" + "strings" +) + +func main() { + flag.Parse() + loadUnicodeData() + loadCompositionExclusions() + completeCharFields(FCanonical) + completeCharFields(FCompatibility) + verifyComputed() + printChars() + makeTables() + testDerived() +} + +var url = flag.String("url", + "http://www.unicode.org/Public/6.0.0/ucd/", + "URL of Unicode database directory") +var tablelist = flag.String("tables", + "all", + "comma-separated list of which tables to generate; "+ + "can be 'decomp', 'recomp', 'info' and 'all'") +var test = flag.Bool("test", + false, + "test existing tables; can be used to compare web data with package data") +var verbose = flag.Bool("verbose", + false, + "write data to stdout as it is parsed") +var localFiles = flag.Bool("local", + false, + "data files have been copied to the current directory; for debugging only") + +var logger = log.New(os.Stderr, "", log.Lshortfile) + +// UnicodeData.txt has form: +// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;; +// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A +// See http://unicode.org/reports/tr44/ for full explanation +// The fields: +const ( + FCodePoint = iota + FName + FGeneralCategory + FCanonicalCombiningClass + FBidiClass + FDecompMapping + FDecimalValue + FDigitValue + FNumericValue + FBidiMirrored + FUnicode1Name + FISOComment + FSimpleUppercaseMapping + FSimpleLowercaseMapping + FSimpleTitlecaseMapping + NumField + + MaxChar = 0x10FFFF // anything above this shouldn't exist +) + +// Quick Check properties of runes allow us to quickly +// determine whether a rune may occur in a normal form. +// For a given normal form, a rune may be guaranteed to occur +// verbatim (QC=Yes), may or may not combine with another +// rune (QC=Maybe), or may not occur (QC=No). +type QCResult int + +const ( + QCUnknown QCResult = iota + QCYes + QCNo + QCMaybe +) + +func (r QCResult) String() string { + switch r { + case QCYes: + return "Yes" + case QCNo: + return "No" + case QCMaybe: + return "Maybe" + } + return "***UNKNOWN***" +} + +const ( + FCanonical = iota // NFC or NFD + FCompatibility // NFKC or NFKD + FNumberOfFormTypes +) + +const ( + MComposed = iota // NFC or NFKC + MDecomposed // NFD or NFKD + MNumberOfModes +) + +// This contains only the properties we're interested in. +type Char struct { + name string + codePoint int // if zero, this index is not a valid code point. + ccc uint8 // canonical combining class + excludeInComp bool // from CompositionExclusions.txt + compatDecomp bool // it has a compatibility expansion + + forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility + + state State +} + +var chars = make([]Char, MaxChar+1) + +func (c Char) String() string { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) + fmt.Fprintf(buf, " ccc: %v\n", c.ccc) + fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) + fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) + fmt.Fprintf(buf, " state: %v\n", c.state) + fmt.Fprintf(buf, " NFC:\n") + fmt.Fprint(buf, c.forms[FCanonical]) + fmt.Fprintf(buf, " NFKC:\n") + fmt.Fprint(buf, c.forms[FCompatibility]) + + return buf.String() +} + +// In UnicodeData.txt, some ranges are marked like this: +// 3400;;Lo;0;L;;;;;N;;;;; +// 4DB5;;Lo;0;L;;;;;N;;;;; +// parseCharacter keeps a state variable indicating the weirdness. +type State int + +const ( + SNormal State = iota // known to be zero for the type + SFirst + SLast + SMissing +) + +var lastChar int = 0 + +func (c Char) isValid() bool { + return c.codePoint != 0 && c.state != SMissing +} + +type FormInfo struct { + quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed + verified [MNumberOfModes]bool // index: MComposed or MDecomposed + + combinesForward bool // May combine with rune on the right + combinesBackward bool // May combine with rune on the left + isOneWay bool // Never appears in result + inDecomp bool // Some decompositions result in this char. + decomp Decomposition + expandedDecomp Decomposition +} + +func (f FormInfo) String() string { + buf := bytes.NewBuffer(make([]byte, 0)) + + fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) + fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) + fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) + fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) + fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) + fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) + fmt.Fprintf(buf, " decomposition: %v\n", f.decomp) + fmt.Fprintf(buf, " expandedDecomp: %v\n", f.expandedDecomp) + + return buf.String() +} + +type Decomposition []int + +func (d Decomposition) String() string { + return fmt.Sprintf("%.4X", d) +} + +func openReader(file string) (input io.ReadCloser) { + if *localFiles { + f, err := os.Open(file) + if err != nil { + logger.Fatal(err) + } + input = f + } else { + path := *url + file + resp, err := http.Get(path) + if err != nil { + logger.Fatal(err) + } + if resp.StatusCode != 200 { + logger.Fatal("bad GET status for "+file, resp.Status) + } + input = resp.Body + } + return +} + +func parseDecomposition(s string, skipfirst bool) (a []int, e os.Error) { + decomp := strings.Split(s, " ") + if len(decomp) > 0 && skipfirst { + decomp = decomp[1:] + } + for _, d := range decomp { + point, err := strconv.Btoui64(d, 16) + if err != nil { + return a, err + } + a = append(a, int(point)) + } + return a, nil +} + +func parseCharacter(line string) { + field := strings.Split(line, ";") + if len(field) != NumField { + logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField) + } + x, err := strconv.Btoui64(field[FCodePoint], 16) + point := int(x) + if err != nil { + logger.Fatalf("%.5s...: %s", line, err) + } + if point == 0 { + return // not interesting and we use 0 as unset + } + if point > MaxChar { + logger.Fatalf("%5s: Rune %X > MaxChar (%X)", line, point, MaxChar) + return + } + state := SNormal + switch { + case strings.Index(field[FName], ", First>") > 0: + state = SFirst + case strings.Index(field[FName], ", Last>") > 0: + state = SLast + } + firstChar := lastChar + 1 + lastChar = int(point) + if state != SLast { + firstChar = lastChar + } + x, err = strconv.Atoui64(field[FCanonicalCombiningClass]) + if err != nil { + logger.Fatalf("%U: bad ccc field: %s", int(x), err) + } + ccc := uint8(x) + decmap := field[FDecompMapping] + exp, e := parseDecomposition(decmap, false) + isCompat := false + if e != nil { + if len(decmap) > 0 { + exp, e = parseDecomposition(decmap, true) + if e != nil { + logger.Fatalf(`%U: bad decomp |%v|: "%s"`, int(x), decmap, e) + } + isCompat = true + } + } + for i := firstChar; i <= lastChar; i++ { + char := &chars[i] + char.name = field[FName] + char.codePoint = i + char.forms[FCompatibility].decomp = exp + if !isCompat { + char.forms[FCanonical].decomp = exp + } else { + char.compatDecomp = true + } + if len(decmap) > 0 { + char.forms[FCompatibility].decomp = exp + } + char.ccc = ccc + char.state = SMissing + if i == lastChar { + char.state = state + } + } + return +} + +func loadUnicodeData() { + f := openReader("UnicodeData.txt") + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + parseCharacter(line[0 : len(line)-1]) + } +} + +var singlePointRe = regexp.MustCompile(`^([0-9A-F]+) *$`) + +// CompositionExclusions.txt has form: +// 0958 # ... +// See http://unicode.org/reports/tr44/ for full explanation +func parseExclusion(line string) int { + comment := strings.Index(line, "#") + if comment >= 0 { + line = line[0:comment] + } + if len(line) == 0 { + return 0 + } + matches := singlePointRe.FindStringSubmatch(line) + if len(matches) != 2 { + logger.Fatalf("%s: %d matches (expected 1)\n", line, len(matches)) + } + point, err := strconv.Btoui64(matches[1], 16) + if err != nil { + logger.Fatalf("%.5s...: %s", line, err) + } + return int(point) +} + +func loadCompositionExclusions() { + f := openReader("CompositionExclusions.txt") + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + point := parseExclusion(line[0 : len(line)-1]) + if point == 0 { + continue + } + c := &chars[point] + if c.excludeInComp { + logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) + } + c.excludeInComp = true + } +} + +// hasCompatDecomp returns true if any of the recursive +// decompositions contains a compatibility expansion. +// In this case, the character may not occur in NFK*. +func hasCompatDecomp(rune int) bool { + c := &chars[rune] + if c.compatDecomp { + return true + } + for _, d := range c.forms[FCompatibility].decomp { + if hasCompatDecomp(d) { + return true + } + } + return false +} + +// Hangul related constants. +const ( + HangulBase = 0xAC00 + HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) + + JamoLBase = 0x1100 + JamoLEnd = 0x1113 + JamoVBase = 0x1161 + JamoVEnd = 0x1176 + JamoTBase = 0x11A8 + JamoTEnd = 0x11C3 +) + +func isHangul(rune int) bool { + return HangulBase <= rune && rune < HangulEnd +} + +func ccc(rune int) uint8 { + return chars[rune].ccc +} + +// Insert a rune in a buffer, ordered by Canonical Combining Class. +func insertOrdered(b Decomposition, rune int) Decomposition { + n := len(b) + b = append(b, 0) + cc := ccc(rune) + if cc > 0 { + // Use bubble sort. + for ; n > 0; n-- { + if ccc(b[n-1]) <= cc { + break + } + b[n] = b[n-1] + } + } + b[n] = rune + return b +} + +// Recursively decompose. +func decomposeRecursive(form int, rune int, d Decomposition) Decomposition { + if isHangul(rune) { + return d + } + dcomp := chars[rune].forms[form].decomp + if len(dcomp) == 0 { + return insertOrdered(d, rune) + } + for _, c := range dcomp { + d = decomposeRecursive(form, c, d) + } + return d +} + +func completeCharFields(form int) { + // Phase 0: pre-expand decomposition. + for i := range chars { + f := &chars[i].forms[form] + if len(f.decomp) == 0 { + continue + } + exp := make(Decomposition, 0) + for _, c := range f.decomp { + exp = decomposeRecursive(form, c, exp) + } + f.expandedDecomp = exp + } + + // Phase 1: composition exclusion, mark decomposition. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + // Marks script-specific exclusions and version restricted. + f.isOneWay = c.excludeInComp + + // Singletons + f.isOneWay = f.isOneWay || len(f.decomp) == 1 + + // Non-starter decompositions + if len(f.decomp) > 1 { + chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 + f.isOneWay = f.isOneWay || chk + } + + // Runes that decompose into more than two runes. + f.isOneWay = f.isOneWay || len(f.decomp) > 2 + + if form == FCompatibility { + f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) + } + + for _, rune := range f.decomp { + chars[rune].forms[form].inDecomp = true + } + } + + // Phase 2: forward and backward combining. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + if !f.isOneWay && len(f.decomp) == 2 { + f0 := &chars[f.decomp[0]].forms[form] + f1 := &chars[f.decomp[1]].forms[form] + if !f0.isOneWay { + f0.combinesForward = true + } + if !f1.isOneWay { + f1.combinesBackward = true + } + } + } + + // Phase 3: quick check values. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + switch { + case len(f.decomp) > 0: + f.quickCheck[MDecomposed] = QCNo + case isHangul(i): + f.quickCheck[MDecomposed] = QCNo + default: + f.quickCheck[MDecomposed] = QCYes + } + switch { + case f.isOneWay: + f.quickCheck[MComposed] = QCNo + case (i & 0xffff00) == JamoLBase: + f.quickCheck[MComposed] = QCYes + if JamoVBase <= i && i < JamoVEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + if JamoTBase <= i && i < JamoTEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + case !f.combinesBackward: + f.quickCheck[MComposed] = QCYes + default: + f.quickCheck[MComposed] = QCMaybe + } + } +} + +func printBytes(b []byte, name string) { + fmt.Printf("// %s: %d bytes\n", name, len(b)) + fmt.Printf("var %s = [...]byte {", name) + for i, c := range b { + switch { + case i%64 == 0: + fmt.Printf("\n// Bytes %x - %x\n", i, i+63) + case i%8 == 0: + fmt.Printf("\n") + } + fmt.Printf("0x%.2X, ", c) + } + fmt.Print("\n}\n\n") +} + +// See forminfo.go for format. +func makeEntry(f *FormInfo) uint16 { + e := uint16(0) + if f.combinesForward { + e |= 0x8 + } + if f.quickCheck[MDecomposed] == QCNo { + e |= 0x1 + } + switch f.quickCheck[MComposed] { + case QCYes: + case QCNo: + e |= 0x2 + case QCMaybe: + e |= 0x6 + default: + log.Fatalf("Illegal quickcheck value %d.", f.quickCheck[MComposed]) + } + return e +} + +// Bits +// 0..8: CCC +// 9..12: NF(C|D) qc bits. +// 13..16: NFK(C|D) qc bits. +func makeCharInfo(c Char) uint16 { + e := makeEntry(&c.forms[FCompatibility]) + e = e<<4 | makeEntry(&c.forms[FCanonical]) + e = e<<8 | uint16(c.ccc) + return e +} + +func printCharInfoTables() int { + // Quick Check + CCC trie. + t := newNode() + for i, char := range chars { + v := makeCharInfo(char) + if v != 0 { + t.insert(i, v) + } + } + return t.printTables("charInfo") +} + +func printDecompositionTables() int { + decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) + size := 0 + + // Map decompositions + positionMap := make(map[string]uint16) + + // Store the uniqued decompositions in a byte buffer, + // preceded by their byte length. + for _, c := range chars { + for f := 0; f < 2; f++ { + d := c.forms[f].expandedDecomp + s := string([]int(d)) + if _, ok := positionMap[s]; !ok { + p := decompositions.Len() + decompositions.WriteByte(uint8(len(s))) + decompositions.WriteString(s) + positionMap[s] = uint16(p) + } + } + } + b := decompositions.Bytes() + printBytes(b, "decomps") + size += len(b) + + nfcT := newNode() + nfkcT := newNode() + for i, c := range chars { + d := c.forms[FCanonical].expandedDecomp + if len(d) != 0 { + nfcT.insert(i, positionMap[string([]int(d))]) + if ccc(c.codePoint) != ccc(d[0]) { + // We assume the lead ccc of a decomposition is !=0 in this case. + if ccc(d[0]) == 0 { + logger.Fatal("Expected differing CCC to be non-zero.") + } + } + } + d = c.forms[FCompatibility].expandedDecomp + if len(d) != 0 { + nfkcT.insert(i, positionMap[string([]int(d))]) + if ccc(c.codePoint) != ccc(d[0]) { + // We assume the lead ccc of a decomposition is !=0 in this case. + if ccc(d[0]) == 0 { + logger.Fatal("Expected differing CCC to be non-zero.") + } + } + } + } + size += nfcT.printTables("nfcDecomp") + size += nfkcT.printTables("nfkcDecomp") + return size +} + +func contains(sa []string, s string) bool { + for _, a := range sa { + if a == s { + return true + } + } + return false +} + +// Extract the version number from the URL. +func version() string { + // From http://www.unicode.org/standard/versions/#Version_Numbering: + // for the later Unicode versions, data files are located in + // versioned directories. + fields := strings.Split(*url, "/") + for _, f := range fields { + if match, _ := regexp.MatchString(`[0-9]\.[0-9]\.[0-9]`, f); match { + return f + } + } + logger.Fatal("unknown version") + return "Unknown" +} + +const fileHeader = `// Generated by running +// maketables --tables=%s --url=%s +// DO NOT EDIT + +package norm + +` + +func makeTables() { + size := 0 + if *tablelist == "" { + return + } + list := strings.Split(*tablelist, ",") + if *tablelist == "all" { + list = []string{"decomp", "recomp", "info"} + } + fmt.Printf(fileHeader, *tablelist, *url) + + fmt.Println("// Version is the Unicode edition from which the tables are derived.") + fmt.Printf("const Version = %q\n\n", version()) + + if contains(list, "decomp") { + size += printDecompositionTables() + } + + if contains(list, "recomp") { + // Note that we use 32 bit keys, instead of 64 bit. + // This clips the bits of three entries, but we know + // this won't cause a collision. The compiler will catch + // any changes made to UnicodeData.txt that introduces + // a collision. + // Note that the recomposition map for NFC and NFKC + // are identical. + + // Recomposition map + nrentries := 0 + for _, c := range chars { + f := c.forms[FCanonical] + if !f.isOneWay && len(f.decomp) > 0 { + nrentries++ + } + } + sz := nrentries * 8 + size += sz + fmt.Printf("// recompMap: %d bytes (entries only)\n", sz) + fmt.Println("var recompMap = map[uint32]uint32{") + for i, c := range chars { + f := c.forms[FCanonical] + d := f.decomp + if !f.isOneWay && len(d) > 0 { + key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) + fmt.Printf("0x%.8X: 0x%.4X,\n", key, i) + } + } + fmt.Printf("}\n\n") + } + + if contains(list, "info") { + size += printCharInfoTables() + } + fmt.Printf("// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) +} + +func printChars() { + if *verbose { + for _, c := range chars { + if !c.isValid() || c.state == SMissing { + continue + } + fmt.Println(c) + } + } +} + +// verifyComputed does various consistency tests. +func verifyComputed() { + for i, c := range chars { + for _, f := range c.forms { + isNo := (f.quickCheck[MDecomposed] == QCNo) + if (len(f.decomp) > 0) != isNo && !isHangul(i) { + log.Fatalf("%U: NF*D must be no if rune decomposes", i) + } + + isMaybe := f.quickCheck[MComposed] == QCMaybe + if f.combinesBackward != isMaybe { + log.Fatalf("%U: NF*C must be maybe if combinesBackward", i) + } + } + } +} + +var qcRe = regexp.MustCompile(`^([0-9A-F\.]+) *; (NF.*_QC); ([YNM]) #.*$`) + +// Use values in DerivedNormalizationProps.txt to compare against the +// values we computed. +// DerivedNormalizationProps.txt has form: +// 00C0..00C5 ; NFD_QC; N # ... +// 0374 ; NFD_QC; N # ... +// See http://unicode.org/reports/tr44/ for full explanation +func testDerived() { + if !*test { + return + } + f := openReader("DerivedNormalizationProps.txt") + defer f.Close() + input := bufio.NewReader(f) + for { + line, err := input.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + logger.Fatal(err) + } + qc := qcRe.FindStringSubmatch(line) + if qc == nil { + continue + } + rng := strings.Split(qc[1], "..") + i, err := strconv.Btoui64(rng[0], 16) + if err != nil { + log.Fatal(err) + } + j := i + if len(rng) > 1 { + j, err = strconv.Btoui64(rng[1], 16) + if err != nil { + log.Fatal(err) + } + } + var ftype, mode int + qt := strings.TrimSpace(qc[2]) + switch qt { + case "NFC_QC": + ftype, mode = FCanonical, MComposed + case "NFD_QC": + ftype, mode = FCanonical, MDecomposed + case "NFKC_QC": + ftype, mode = FCompatibility, MComposed + case "NFKD_QC": + ftype, mode = FCompatibility, MDecomposed + default: + log.Fatalf(`Unexpected quick check type "%s"`, qt) + } + var qr QCResult + switch qc[3] { + case "Y": + qr = QCYes + case "N": + qr = QCNo + case "M": + qr = QCMaybe + default: + log.Fatalf(`Unexpected quick check value "%s"`, qc[3]) + } + var lastFailed bool + // Verify current + for ; i <= j; i++ { + c := &chars[int(i)] + c.forms[ftype].verified[mode] = true + curqr := c.forms[ftype].quickCheck[mode] + if curqr != qr { + if !lastFailed { + logger.Printf("%s: %.4X..%.4X -- %s\n", + qt, int(i), int(j), line[0:50]) + } + logger.Printf("%U: FAILED %s (was %v need %v)\n", + int(i), qt, curqr, qr) + lastFailed = true + } + } + } + // Any unspecified value must be QCYes. Verify this. + for i, c := range chars { + for j, fd := range c.forms { + for k, qr := range fd.quickCheck { + if !fd.verified[k] && qr != QCYes { + m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" + logger.Printf(m, i, j, k, qr, c.name) + } + } + } + } +} diff --git a/src/pkg/exp/norm/maketesttables.go b/src/pkg/exp/norm/maketesttables.go new file mode 100644 index 000000000..c5f6a6436 --- /dev/null +++ b/src/pkg/exp/norm/maketesttables.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Generate test data for trie code. + +package main + +import ( + "fmt" +) + +func main() { + printTestTables() +} + +// We take the smallest, largest and an arbitrary value for each +// of the UTF-8 sequence lengths. +var testRunes = []int{ + 0x01, 0x0C, 0x7F, // 1-byte sequences + 0x80, 0x100, 0x7FF, // 2-byte sequences + 0x800, 0x999, 0xFFFF, // 3-byte sequences + 0x10000, 0x10101, 0x10FFFF, // 4-byte sequences +} + +const fileHeader = `// Generated by running +// maketesttables +// DO NOT EDIT + +package norm + +` + +func printTestTables() { + fmt.Print(fileHeader) + fmt.Printf("var testRunes = %#v\n\n", testRunes) + t := newNode() + for i, r := range testRunes { + t.insert(r, uint16(i)) + } + t.printTables("testdata") +} diff --git a/src/pkg/exp/norm/norm_test.go b/src/pkg/exp/norm/norm_test.go new file mode 100644 index 000000000..12dacfcf3 --- /dev/null +++ b/src/pkg/exp/norm/norm_test.go @@ -0,0 +1,14 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm_test + +import ( + "testing" +) + +func TestPlaceHolder(t *testing.T) { + // Does nothing, just allows the Makefile to be canonical + // while waiting for the package itself to be written. +} diff --git a/src/pkg/exp/norm/normalize.go b/src/pkg/exp/norm/normalize.go new file mode 100644 index 000000000..e9d18dd9e --- /dev/null +++ b/src/pkg/exp/norm/normalize.go @@ -0,0 +1,99 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package norm contains types and functions for normalizing Unicode strings. +package norm + +// A Form denotes a canonical representation of Unicode code points. +// The Unicode-defined normalization and equivalence forms are: +// +// NFC Unicode Normalization Form C +// NFD Unicode Normalization Form D +// NFKC Unicode Normalization Form KC +// NFKD Unicode Normalization Form KD +// +// For a Form f, this documentation uses the notation f(x) to mean +// the bytes or string x converted to the given form. +// A position n in x is called a boundary if conversion to the form can +// proceed independently on both sides: +// f(x) == append(f(x[0:n]), f(x[n:])...) +// +// References: http://unicode.org/reports/tr15/ and +// http://unicode.org/notes/tn5/. +type Form int + +const ( + NFC Form = iota + NFD + NFKC + NFKD +) + +// Bytes returns f(b). May return b if f(b) = b. +func (f Form) Bytes(b []byte) []byte { + panic("not implemented") +} + +// String returns f(s). +func (f Form) String(s string) string { + panic("not implemented") +} + +// IsNormal returns true if b == f(b). +func (f Form) IsNormal(b []byte) bool { + panic("not implemented") +} + +// IsNormalString returns true if s == f(s). +func (f Form) IsNormalString(s string) bool { + panic("not implemented") +} + +// Append returns f(append(out, b...)). +// The buffer out must be empty or equal to f(out). +func (f Form) Append(out, b []byte) []byte { + panic("not implemented") +} + +// AppendString returns f(append(out, []byte(s))). +// The buffer out must be empty or equal to f(out). +func (f Form) AppendString(out []byte, s string) []byte { + panic("not implemented") +} + +// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) QuickSpan(b []byte) int { + panic("not implemented") +} + +// QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]). +// It is not guaranteed to return the largest such n. +func (f Form) QuickSpanString(s string) int { + panic("not implemented") +} + +// FirstBoundary returns the position i of the first boundary in b. +// It returns len(b), false if b contains no boundaries. +func (f Form) FirstBoundary(b []byte) (i int, ok bool) { + panic("not implemented") +} + +// FirstBoundaryInString return the position i of the first boundary in s. +// It returns len(s), false if s contains no boundaries. +func (f Form) FirstBoundaryInString(s string) (i int, ok bool) { + panic("not implemented") +} + +// LastBoundaryIn returns the position i of the last boundary in b. +// It returns 0, false if b contains no boundary. +func (f Form) LastBoundary(b []byte) (i int, ok bool) { + panic("not implemented") +} + +// LastBoundaryInString returns the position i of the last boundary in s. +// It returns 0, false if s contains no boundary. +func (f Form) LastBoundaryInString(s string) (i int, ok bool) { + panic("not implemented") +} diff --git a/src/pkg/exp/norm/tables.go b/src/pkg/exp/norm/tables.go new file mode 100644 index 000000000..76995c2fa --- /dev/null +++ b/src/pkg/exp/norm/tables.go @@ -0,0 +1,6580 @@ +// Generated by running +// maketables --tables=all --url=http://www.unicode.org/Public/6.0.0/ucd/ +// DO NOT EDIT + +package norm + +// Version is the Unicode edition from which the tables are derived. +const Version = "6.0.0" + +// decomps: 17618 bytes +var decomps = [...]byte{ + // Bytes 0 - 3f + 0x00, 0x01, 0x20, 0x03, 0x20, 0xCC, 0x88, 0x01, + 0x61, 0x03, 0x20, 0xCC, 0x84, 0x01, 0x32, 0x01, + 0x33, 0x03, 0x20, 0xCC, 0x81, 0x02, 0xCE, 0xBC, + 0x03, 0x20, 0xCC, 0xA7, 0x01, 0x31, 0x01, 0x6F, + 0x05, 0x31, 0xE2, 0x81, 0x84, 0x34, 0x05, 0x31, + 0xE2, 0x81, 0x84, 0x32, 0x05, 0x33, 0xE2, 0x81, + 0x84, 0x34, 0x03, 0x41, 0xCC, 0x80, 0x03, 0x41, + 0xCC, 0x81, 0x03, 0x41, 0xCC, 0x82, 0x03, 0x41, + // Bytes 40 - 7f + 0xCC, 0x83, 0x03, 0x41, 0xCC, 0x88, 0x03, 0x41, + 0xCC, 0x8A, 0x03, 0x43, 0xCC, 0xA7, 0x03, 0x45, + 0xCC, 0x80, 0x03, 0x45, 0xCC, 0x81, 0x03, 0x45, + 0xCC, 0x82, 0x03, 0x45, 0xCC, 0x88, 0x03, 0x49, + 0xCC, 0x80, 0x03, 0x49, 0xCC, 0x81, 0x03, 0x49, + 0xCC, 0x82, 0x03, 0x49, 0xCC, 0x88, 0x03, 0x4E, + 0xCC, 0x83, 0x03, 0x4F, 0xCC, 0x80, 0x03, 0x4F, + 0xCC, 0x81, 0x03, 0x4F, 0xCC, 0x82, 0x03, 0x4F, + // Bytes 80 - bf + 0xCC, 0x83, 0x03, 0x4F, 0xCC, 0x88, 0x03, 0x55, + 0xCC, 0x80, 0x03, 0x55, 0xCC, 0x81, 0x03, 0x55, + 0xCC, 0x82, 0x03, 0x55, 0xCC, 0x88, 0x03, 0x59, + 0xCC, 0x81, 0x03, 0x61, 0xCC, 0x80, 0x03, 0x61, + 0xCC, 0x81, 0x03, 0x61, 0xCC, 0x82, 0x03, 0x61, + 0xCC, 0x83, 0x03, 0x61, 0xCC, 0x88, 0x03, 0x61, + 0xCC, 0x8A, 0x03, 0x63, 0xCC, 0xA7, 0x03, 0x65, + 0xCC, 0x80, 0x03, 0x65, 0xCC, 0x81, 0x03, 0x65, + // Bytes c0 - ff + 0xCC, 0x82, 0x03, 0x65, 0xCC, 0x88, 0x03, 0x69, + 0xCC, 0x80, 0x03, 0x69, 0xCC, 0x81, 0x03, 0x69, + 0xCC, 0x82, 0x03, 0x69, 0xCC, 0x88, 0x03, 0x6E, + 0xCC, 0x83, 0x03, 0x6F, 0xCC, 0x80, 0x03, 0x6F, + 0xCC, 0x81, 0x03, 0x6F, 0xCC, 0x82, 0x03, 0x6F, + 0xCC, 0x83, 0x03, 0x6F, 0xCC, 0x88, 0x03, 0x75, + 0xCC, 0x80, 0x03, 0x75, 0xCC, 0x81, 0x03, 0x75, + 0xCC, 0x82, 0x03, 0x75, 0xCC, 0x88, 0x03, 0x79, + // Bytes 100 - 13f + 0xCC, 0x81, 0x03, 0x79, 0xCC, 0x88, 0x03, 0x41, + 0xCC, 0x84, 0x03, 0x61, 0xCC, 0x84, 0x03, 0x41, + 0xCC, 0x86, 0x03, 0x61, 0xCC, 0x86, 0x03, 0x41, + 0xCC, 0xA8, 0x03, 0x61, 0xCC, 0xA8, 0x03, 0x43, + 0xCC, 0x81, 0x03, 0x63, 0xCC, 0x81, 0x03, 0x43, + 0xCC, 0x82, 0x03, 0x63, 0xCC, 0x82, 0x03, 0x43, + 0xCC, 0x87, 0x03, 0x63, 0xCC, 0x87, 0x03, 0x43, + 0xCC, 0x8C, 0x03, 0x63, 0xCC, 0x8C, 0x03, 0x44, + // Bytes 140 - 17f + 0xCC, 0x8C, 0x03, 0x64, 0xCC, 0x8C, 0x03, 0x45, + 0xCC, 0x84, 0x03, 0x65, 0xCC, 0x84, 0x03, 0x45, + 0xCC, 0x86, 0x03, 0x65, 0xCC, 0x86, 0x03, 0x45, + 0xCC, 0x87, 0x03, 0x65, 0xCC, 0x87, 0x03, 0x45, + 0xCC, 0xA8, 0x03, 0x65, 0xCC, 0xA8, 0x03, 0x45, + 0xCC, 0x8C, 0x03, 0x65, 0xCC, 0x8C, 0x03, 0x47, + 0xCC, 0x82, 0x03, 0x67, 0xCC, 0x82, 0x03, 0x47, + 0xCC, 0x86, 0x03, 0x67, 0xCC, 0x86, 0x03, 0x47, + // Bytes 180 - 1bf + 0xCC, 0x87, 0x03, 0x67, 0xCC, 0x87, 0x03, 0x47, + 0xCC, 0xA7, 0x03, 0x67, 0xCC, 0xA7, 0x03, 0x48, + 0xCC, 0x82, 0x03, 0x68, 0xCC, 0x82, 0x03, 0x49, + 0xCC, 0x83, 0x03, 0x69, 0xCC, 0x83, 0x03, 0x49, + 0xCC, 0x84, 0x03, 0x69, 0xCC, 0x84, 0x03, 0x49, + 0xCC, 0x86, 0x03, 0x69, 0xCC, 0x86, 0x03, 0x49, + 0xCC, 0xA8, 0x03, 0x69, 0xCC, 0xA8, 0x03, 0x49, + 0xCC, 0x87, 0x02, 0x49, 0x4A, 0x02, 0x69, 0x6A, + // Bytes 1c0 - 1ff + 0x03, 0x4A, 0xCC, 0x82, 0x03, 0x6A, 0xCC, 0x82, + 0x03, 0x4B, 0xCC, 0xA7, 0x03, 0x6B, 0xCC, 0xA7, + 0x03, 0x4C, 0xCC, 0x81, 0x03, 0x6C, 0xCC, 0x81, + 0x03, 0x4C, 0xCC, 0xA7, 0x03, 0x6C, 0xCC, 0xA7, + 0x03, 0x4C, 0xCC, 0x8C, 0x03, 0x6C, 0xCC, 0x8C, + 0x03, 0x4C, 0xC2, 0xB7, 0x03, 0x6C, 0xC2, 0xB7, + 0x03, 0x4E, 0xCC, 0x81, 0x03, 0x6E, 0xCC, 0x81, + 0x03, 0x4E, 0xCC, 0xA7, 0x03, 0x6E, 0xCC, 0xA7, + // Bytes 200 - 23f + 0x03, 0x4E, 0xCC, 0x8C, 0x03, 0x6E, 0xCC, 0x8C, + 0x03, 0xCA, 0xBC, 0x6E, 0x03, 0x4F, 0xCC, 0x84, + 0x03, 0x6F, 0xCC, 0x84, 0x03, 0x4F, 0xCC, 0x86, + 0x03, 0x6F, 0xCC, 0x86, 0x03, 0x4F, 0xCC, 0x8B, + 0x03, 0x6F, 0xCC, 0x8B, 0x03, 0x52, 0xCC, 0x81, + 0x03, 0x72, 0xCC, 0x81, 0x03, 0x52, 0xCC, 0xA7, + 0x03, 0x72, 0xCC, 0xA7, 0x03, 0x52, 0xCC, 0x8C, + 0x03, 0x72, 0xCC, 0x8C, 0x03, 0x53, 0xCC, 0x81, + // Bytes 240 - 27f + 0x03, 0x73, 0xCC, 0x81, 0x03, 0x53, 0xCC, 0x82, + 0x03, 0x73, 0xCC, 0x82, 0x03, 0x53, 0xCC, 0xA7, + 0x03, 0x73, 0xCC, 0xA7, 0x03, 0x53, 0xCC, 0x8C, + 0x03, 0x73, 0xCC, 0x8C, 0x03, 0x54, 0xCC, 0xA7, + 0x03, 0x74, 0xCC, 0xA7, 0x03, 0x54, 0xCC, 0x8C, + 0x03, 0x74, 0xCC, 0x8C, 0x03, 0x55, 0xCC, 0x83, + 0x03, 0x75, 0xCC, 0x83, 0x03, 0x55, 0xCC, 0x84, + 0x03, 0x75, 0xCC, 0x84, 0x03, 0x55, 0xCC, 0x86, + // Bytes 280 - 2bf + 0x03, 0x75, 0xCC, 0x86, 0x03, 0x55, 0xCC, 0x8A, + 0x03, 0x75, 0xCC, 0x8A, 0x03, 0x55, 0xCC, 0x8B, + 0x03, 0x75, 0xCC, 0x8B, 0x03, 0x55, 0xCC, 0xA8, + 0x03, 0x75, 0xCC, 0xA8, 0x03, 0x57, 0xCC, 0x82, + 0x03, 0x77, 0xCC, 0x82, 0x03, 0x59, 0xCC, 0x82, + 0x03, 0x79, 0xCC, 0x82, 0x03, 0x59, 0xCC, 0x88, + 0x03, 0x5A, 0xCC, 0x81, 0x03, 0x7A, 0xCC, 0x81, + 0x03, 0x5A, 0xCC, 0x87, 0x03, 0x7A, 0xCC, 0x87, + // Bytes 2c0 - 2ff + 0x03, 0x5A, 0xCC, 0x8C, 0x03, 0x7A, 0xCC, 0x8C, + 0x01, 0x73, 0x03, 0x4F, 0xCC, 0x9B, 0x03, 0x6F, + 0xCC, 0x9B, 0x03, 0x55, 0xCC, 0x9B, 0x03, 0x75, + 0xCC, 0x9B, 0x04, 0x44, 0x5A, 0xCC, 0x8C, 0x04, + 0x44, 0x7A, 0xCC, 0x8C, 0x04, 0x64, 0x7A, 0xCC, + 0x8C, 0x02, 0x4C, 0x4A, 0x02, 0x4C, 0x6A, 0x02, + 0x6C, 0x6A, 0x02, 0x4E, 0x4A, 0x02, 0x4E, 0x6A, + 0x02, 0x6E, 0x6A, 0x03, 0x41, 0xCC, 0x8C, 0x03, + // Bytes 300 - 33f + 0x61, 0xCC, 0x8C, 0x03, 0x49, 0xCC, 0x8C, 0x03, + 0x69, 0xCC, 0x8C, 0x03, 0x4F, 0xCC, 0x8C, 0x03, + 0x6F, 0xCC, 0x8C, 0x03, 0x55, 0xCC, 0x8C, 0x03, + 0x75, 0xCC, 0x8C, 0x05, 0x55, 0xCC, 0x88, 0xCC, + 0x84, 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x84, 0x05, + 0x55, 0xCC, 0x88, 0xCC, 0x81, 0x05, 0x75, 0xCC, + 0x88, 0xCC, 0x81, 0x05, 0x55, 0xCC, 0x88, 0xCC, + 0x8C, 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x8C, 0x05, + // Bytes 340 - 37f + 0x55, 0xCC, 0x88, 0xCC, 0x80, 0x05, 0x75, 0xCC, + 0x88, 0xCC, 0x80, 0x05, 0x41, 0xCC, 0x88, 0xCC, + 0x84, 0x05, 0x61, 0xCC, 0x88, 0xCC, 0x84, 0x05, + 0x41, 0xCC, 0x87, 0xCC, 0x84, 0x05, 0x61, 0xCC, + 0x87, 0xCC, 0x84, 0x04, 0xC3, 0x86, 0xCC, 0x84, + 0x04, 0xC3, 0xA6, 0xCC, 0x84, 0x03, 0x47, 0xCC, + 0x8C, 0x03, 0x67, 0xCC, 0x8C, 0x03, 0x4B, 0xCC, + 0x8C, 0x03, 0x6B, 0xCC, 0x8C, 0x03, 0x4F, 0xCC, + // Bytes 380 - 3bf + 0xA8, 0x03, 0x6F, 0xCC, 0xA8, 0x05, 0x4F, 0xCC, + 0xA8, 0xCC, 0x84, 0x05, 0x6F, 0xCC, 0xA8, 0xCC, + 0x84, 0x04, 0xC6, 0xB7, 0xCC, 0x8C, 0x04, 0xCA, + 0x92, 0xCC, 0x8C, 0x03, 0x6A, 0xCC, 0x8C, 0x02, + 0x44, 0x5A, 0x02, 0x44, 0x7A, 0x02, 0x64, 0x7A, + 0x03, 0x47, 0xCC, 0x81, 0x03, 0x67, 0xCC, 0x81, + 0x03, 0x4E, 0xCC, 0x80, 0x03, 0x6E, 0xCC, 0x80, + 0x05, 0x41, 0xCC, 0x8A, 0xCC, 0x81, 0x05, 0x61, + // Bytes 3c0 - 3ff + 0xCC, 0x8A, 0xCC, 0x81, 0x04, 0xC3, 0x86, 0xCC, + 0x81, 0x04, 0xC3, 0xA6, 0xCC, 0x81, 0x04, 0xC3, + 0x98, 0xCC, 0x81, 0x04, 0xC3, 0xB8, 0xCC, 0x81, + 0x03, 0x41, 0xCC, 0x8F, 0x03, 0x61, 0xCC, 0x8F, + 0x03, 0x41, 0xCC, 0x91, 0x03, 0x61, 0xCC, 0x91, + 0x03, 0x45, 0xCC, 0x8F, 0x03, 0x65, 0xCC, 0x8F, + 0x03, 0x45, 0xCC, 0x91, 0x03, 0x65, 0xCC, 0x91, + 0x03, 0x49, 0xCC, 0x8F, 0x03, 0x69, 0xCC, 0x8F, + // Bytes 400 - 43f + 0x03, 0x49, 0xCC, 0x91, 0x03, 0x69, 0xCC, 0x91, + 0x03, 0x4F, 0xCC, 0x8F, 0x03, 0x6F, 0xCC, 0x8F, + 0x03, 0x4F, 0xCC, 0x91, 0x03, 0x6F, 0xCC, 0x91, + 0x03, 0x52, 0xCC, 0x8F, 0x03, 0x72, 0xCC, 0x8F, + 0x03, 0x52, 0xCC, 0x91, 0x03, 0x72, 0xCC, 0x91, + 0x03, 0x55, 0xCC, 0x8F, 0x03, 0x75, 0xCC, 0x8F, + 0x03, 0x55, 0xCC, 0x91, 0x03, 0x75, 0xCC, 0x91, + 0x03, 0x53, 0xCC, 0xA6, 0x03, 0x73, 0xCC, 0xA6, + // Bytes 440 - 47f + 0x03, 0x54, 0xCC, 0xA6, 0x03, 0x74, 0xCC, 0xA6, + 0x03, 0x48, 0xCC, 0x8C, 0x03, 0x68, 0xCC, 0x8C, + 0x03, 0x41, 0xCC, 0x87, 0x03, 0x61, 0xCC, 0x87, + 0x03, 0x45, 0xCC, 0xA7, 0x03, 0x65, 0xCC, 0xA7, + 0x05, 0x4F, 0xCC, 0x88, 0xCC, 0x84, 0x05, 0x6F, + 0xCC, 0x88, 0xCC, 0x84, 0x05, 0x4F, 0xCC, 0x83, + 0xCC, 0x84, 0x05, 0x6F, 0xCC, 0x83, 0xCC, 0x84, + 0x03, 0x4F, 0xCC, 0x87, 0x03, 0x6F, 0xCC, 0x87, + // Bytes 480 - 4bf + 0x05, 0x4F, 0xCC, 0x87, 0xCC, 0x84, 0x05, 0x6F, + 0xCC, 0x87, 0xCC, 0x84, 0x03, 0x59, 0xCC, 0x84, + 0x03, 0x79, 0xCC, 0x84, 0x01, 0x68, 0x02, 0xC9, + 0xA6, 0x01, 0x6A, 0x01, 0x72, 0x02, 0xC9, 0xB9, + 0x02, 0xC9, 0xBB, 0x02, 0xCA, 0x81, 0x01, 0x77, + 0x01, 0x79, 0x03, 0x20, 0xCC, 0x86, 0x03, 0x20, + 0xCC, 0x87, 0x03, 0x20, 0xCC, 0x8A, 0x03, 0x20, + 0xCC, 0xA8, 0x03, 0x20, 0xCC, 0x83, 0x03, 0x20, + // Bytes 4c0 - 4ff + 0xCC, 0x8B, 0x02, 0xC9, 0xA3, 0x01, 0x6C, 0x01, + 0x78, 0x02, 0xCA, 0x95, 0x02, 0xCC, 0x80, 0x02, + 0xCC, 0x81, 0x02, 0xCC, 0x93, 0x04, 0xCC, 0x88, + 0xCC, 0x81, 0x02, 0xCA, 0xB9, 0x03, 0x20, 0xCD, + 0x85, 0x01, 0x3B, 0x04, 0xC2, 0xA8, 0xCC, 0x81, + 0x05, 0x20, 0xCC, 0x88, 0xCC, 0x81, 0x04, 0xCE, + 0x91, 0xCC, 0x81, 0x02, 0xC2, 0xB7, 0x04, 0xCE, + 0x95, 0xCC, 0x81, 0x04, 0xCE, 0x97, 0xCC, 0x81, + // Bytes 500 - 53f + 0x04, 0xCE, 0x99, 0xCC, 0x81, 0x04, 0xCE, 0x9F, + 0xCC, 0x81, 0x04, 0xCE, 0xA5, 0xCC, 0x81, 0x04, + 0xCE, 0xA9, 0xCC, 0x81, 0x06, 0xCE, 0xB9, 0xCC, + 0x88, 0xCC, 0x81, 0x04, 0xCE, 0x99, 0xCC, 0x88, + 0x04, 0xCE, 0xA5, 0xCC, 0x88, 0x04, 0xCE, 0xB1, + 0xCC, 0x81, 0x04, 0xCE, 0xB5, 0xCC, 0x81, 0x04, + 0xCE, 0xB7, 0xCC, 0x81, 0x04, 0xCE, 0xB9, 0xCC, + 0x81, 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCC, 0x81, + // Bytes 540 - 57f + 0x04, 0xCE, 0xB9, 0xCC, 0x88, 0x04, 0xCF, 0x85, + 0xCC, 0x88, 0x04, 0xCE, 0xBF, 0xCC, 0x81, 0x04, + 0xCF, 0x85, 0xCC, 0x81, 0x04, 0xCF, 0x89, 0xCC, + 0x81, 0x02, 0xCE, 0xB2, 0x02, 0xCE, 0xB8, 0x02, + 0xCE, 0xA5, 0x04, 0xCF, 0x92, 0xCC, 0x81, 0x04, + 0xCF, 0x92, 0xCC, 0x88, 0x02, 0xCF, 0x86, 0x02, + 0xCF, 0x80, 0x02, 0xCE, 0xBA, 0x02, 0xCF, 0x81, + 0x02, 0xCF, 0x82, 0x02, 0xCE, 0x98, 0x02, 0xCE, + // Bytes 580 - 5bf + 0xB5, 0x02, 0xCE, 0xA3, 0x04, 0xD0, 0x95, 0xCC, + 0x80, 0x04, 0xD0, 0x95, 0xCC, 0x88, 0x04, 0xD0, + 0x93, 0xCC, 0x81, 0x04, 0xD0, 0x86, 0xCC, 0x88, + 0x04, 0xD0, 0x9A, 0xCC, 0x81, 0x04, 0xD0, 0x98, + 0xCC, 0x80, 0x04, 0xD0, 0xA3, 0xCC, 0x86, 0x04, + 0xD0, 0x98, 0xCC, 0x86, 0x04, 0xD0, 0xB8, 0xCC, + 0x86, 0x04, 0xD0, 0xB5, 0xCC, 0x80, 0x04, 0xD0, + 0xB5, 0xCC, 0x88, 0x04, 0xD0, 0xB3, 0xCC, 0x81, + // Bytes 5c0 - 5ff + 0x04, 0xD1, 0x96, 0xCC, 0x88, 0x04, 0xD0, 0xBA, + 0xCC, 0x81, 0x04, 0xD0, 0xB8, 0xCC, 0x80, 0x04, + 0xD1, 0x83, 0xCC, 0x86, 0x04, 0xD1, 0xB4, 0xCC, + 0x8F, 0x04, 0xD1, 0xB5, 0xCC, 0x8F, 0x04, 0xD0, + 0x96, 0xCC, 0x86, 0x04, 0xD0, 0xB6, 0xCC, 0x86, + 0x04, 0xD0, 0x90, 0xCC, 0x86, 0x04, 0xD0, 0xB0, + 0xCC, 0x86, 0x04, 0xD0, 0x90, 0xCC, 0x88, 0x04, + 0xD0, 0xB0, 0xCC, 0x88, 0x04, 0xD0, 0x95, 0xCC, + // Bytes 600 - 63f + 0x86, 0x04, 0xD0, 0xB5, 0xCC, 0x86, 0x04, 0xD3, + 0x98, 0xCC, 0x88, 0x04, 0xD3, 0x99, 0xCC, 0x88, + 0x04, 0xD0, 0x96, 0xCC, 0x88, 0x04, 0xD0, 0xB6, + 0xCC, 0x88, 0x04, 0xD0, 0x97, 0xCC, 0x88, 0x04, + 0xD0, 0xB7, 0xCC, 0x88, 0x04, 0xD0, 0x98, 0xCC, + 0x84, 0x04, 0xD0, 0xB8, 0xCC, 0x84, 0x04, 0xD0, + 0x98, 0xCC, 0x88, 0x04, 0xD0, 0xB8, 0xCC, 0x88, + 0x04, 0xD0, 0x9E, 0xCC, 0x88, 0x04, 0xD0, 0xBE, + // Bytes 640 - 67f + 0xCC, 0x88, 0x04, 0xD3, 0xA8, 0xCC, 0x88, 0x04, + 0xD3, 0xA9, 0xCC, 0x88, 0x04, 0xD0, 0xAD, 0xCC, + 0x88, 0x04, 0xD1, 0x8D, 0xCC, 0x88, 0x04, 0xD0, + 0xA3, 0xCC, 0x84, 0x04, 0xD1, 0x83, 0xCC, 0x84, + 0x04, 0xD0, 0xA3, 0xCC, 0x88, 0x04, 0xD1, 0x83, + 0xCC, 0x88, 0x04, 0xD0, 0xA3, 0xCC, 0x8B, 0x04, + 0xD1, 0x83, 0xCC, 0x8B, 0x04, 0xD0, 0xA7, 0xCC, + 0x88, 0x04, 0xD1, 0x87, 0xCC, 0x88, 0x04, 0xD0, + // Bytes 680 - 6bf + 0xAB, 0xCC, 0x88, 0x04, 0xD1, 0x8B, 0xCC, 0x88, + 0x04, 0xD5, 0xA5, 0xD6, 0x82, 0x04, 0xD8, 0xA7, + 0xD9, 0x93, 0x04, 0xD8, 0xA7, 0xD9, 0x94, 0x04, + 0xD9, 0x88, 0xD9, 0x94, 0x04, 0xD8, 0xA7, 0xD9, + 0x95, 0x04, 0xD9, 0x8A, 0xD9, 0x94, 0x04, 0xD8, + 0xA7, 0xD9, 0xB4, 0x04, 0xD9, 0x88, 0xD9, 0xB4, + 0x04, 0xDB, 0x87, 0xD9, 0xB4, 0x04, 0xD9, 0x8A, + 0xD9, 0xB4, 0x04, 0xDB, 0x95, 0xD9, 0x94, 0x04, + // Bytes 6c0 - 6ff + 0xDB, 0x81, 0xD9, 0x94, 0x04, 0xDB, 0x92, 0xD9, + 0x94, 0x06, 0xE0, 0xA4, 0xA8, 0xE0, 0xA4, 0xBC, + 0x06, 0xE0, 0xA4, 0xB0, 0xE0, 0xA4, 0xBC, 0x06, + 0xE0, 0xA4, 0xB3, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, + 0xA4, 0x95, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, 0xA4, + 0x96, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, 0xA4, 0x97, + 0xE0, 0xA4, 0xBC, 0x06, 0xE0, 0xA4, 0x9C, 0xE0, + 0xA4, 0xBC, 0x06, 0xE0, 0xA4, 0xA1, 0xE0, 0xA4, + // Bytes 700 - 73f + 0xBC, 0x06, 0xE0, 0xA4, 0xA2, 0xE0, 0xA4, 0xBC, + 0x06, 0xE0, 0xA4, 0xAB, 0xE0, 0xA4, 0xBC, 0x06, + 0xE0, 0xA4, 0xAF, 0xE0, 0xA4, 0xBC, 0x06, 0xE0, + 0xA7, 0x87, 0xE0, 0xA6, 0xBE, 0x06, 0xE0, 0xA7, + 0x87, 0xE0, 0xA7, 0x97, 0x06, 0xE0, 0xA6, 0xA1, + 0xE0, 0xA6, 0xBC, 0x06, 0xE0, 0xA6, 0xA2, 0xE0, + 0xA6, 0xBC, 0x06, 0xE0, 0xA6, 0xAF, 0xE0, 0xA6, + 0xBC, 0x06, 0xE0, 0xA8, 0xB2, 0xE0, 0xA8, 0xBC, + // Bytes 740 - 77f + 0x06, 0xE0, 0xA8, 0xB8, 0xE0, 0xA8, 0xBC, 0x06, + 0xE0, 0xA8, 0x96, 0xE0, 0xA8, 0xBC, 0x06, 0xE0, + 0xA8, 0x97, 0xE0, 0xA8, 0xBC, 0x06, 0xE0, 0xA8, + 0x9C, 0xE0, 0xA8, 0xBC, 0x06, 0xE0, 0xA8, 0xAB, + 0xE0, 0xA8, 0xBC, 0x06, 0xE0, 0xAD, 0x87, 0xE0, + 0xAD, 0x96, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAC, + 0xBE, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAD, 0x97, + 0x06, 0xE0, 0xAC, 0xA1, 0xE0, 0xAC, 0xBC, 0x06, + // Bytes 780 - 7bf + 0xE0, 0xAC, 0xA2, 0xE0, 0xAC, 0xBC, 0x06, 0xE0, + 0xAE, 0x92, 0xE0, 0xAF, 0x97, 0x06, 0xE0, 0xAF, + 0x86, 0xE0, 0xAE, 0xBE, 0x06, 0xE0, 0xAF, 0x87, + 0xE0, 0xAE, 0xBE, 0x06, 0xE0, 0xAF, 0x86, 0xE0, + 0xAF, 0x97, 0x06, 0xE0, 0xB1, 0x86, 0xE0, 0xB1, + 0x96, 0x06, 0xE0, 0xB2, 0xBF, 0xE0, 0xB3, 0x95, + 0x06, 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x95, 0x06, + 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x96, 0x06, 0xE0, + // Bytes 7c0 - 7ff + 0xB3, 0x86, 0xE0, 0xB3, 0x82, 0x09, 0xE0, 0xB3, + 0x86, 0xE0, 0xB3, 0x82, 0xE0, 0xB3, 0x95, 0x06, + 0xE0, 0xB5, 0x86, 0xE0, 0xB4, 0xBE, 0x06, 0xE0, + 0xB5, 0x87, 0xE0, 0xB4, 0xBE, 0x06, 0xE0, 0xB5, + 0x86, 0xE0, 0xB5, 0x97, 0x06, 0xE0, 0xB7, 0x99, + 0xE0, 0xB7, 0x8A, 0x06, 0xE0, 0xB7, 0x99, 0xE0, + 0xB7, 0x8F, 0x09, 0xE0, 0xB7, 0x99, 0xE0, 0xB7, + 0x8F, 0xE0, 0xB7, 0x8A, 0x06, 0xE0, 0xB7, 0x99, + // Bytes 800 - 83f + 0xE0, 0xB7, 0x9F, 0x06, 0xE0, 0xB9, 0x8D, 0xE0, + 0xB8, 0xB2, 0x06, 0xE0, 0xBB, 0x8D, 0xE0, 0xBA, + 0xB2, 0x06, 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, 0x99, + 0x06, 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, 0xA1, 0x03, + 0xE0, 0xBC, 0x8B, 0x06, 0xE0, 0xBD, 0x82, 0xE0, + 0xBE, 0xB7, 0x06, 0xE0, 0xBD, 0x8C, 0xE0, 0xBE, + 0xB7, 0x06, 0xE0, 0xBD, 0x91, 0xE0, 0xBE, 0xB7, + 0x06, 0xE0, 0xBD, 0x96, 0xE0, 0xBE, 0xB7, 0x06, + // Bytes 840 - 87f + 0xE0, 0xBD, 0x9B, 0xE0, 0xBE, 0xB7, 0x06, 0xE0, + 0xBD, 0x80, 0xE0, 0xBE, 0xB5, 0x06, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBD, 0xB2, 0x06, 0xE0, 0xBD, 0xB1, + 0xE0, 0xBD, 0xB4, 0x06, 0xE0, 0xBE, 0xB2, 0xE0, + 0xBE, 0x80, 0x09, 0xE0, 0xBE, 0xB2, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBE, 0x80, 0x06, 0xE0, 0xBE, 0xB3, + 0xE0, 0xBE, 0x80, 0x09, 0xE0, 0xBE, 0xB3, 0xE0, + 0xBD, 0xB1, 0xE0, 0xBE, 0x80, 0x06, 0xE0, 0xBD, + // Bytes 880 - 8bf + 0xB1, 0xE0, 0xBE, 0x80, 0x06, 0xE0, 0xBE, 0x92, + 0xE0, 0xBE, 0xB7, 0x06, 0xE0, 0xBE, 0x9C, 0xE0, + 0xBE, 0xB7, 0x06, 0xE0, 0xBE, 0xA1, 0xE0, 0xBE, + 0xB7, 0x06, 0xE0, 0xBE, 0xA6, 0xE0, 0xBE, 0xB7, + 0x06, 0xE0, 0xBE, 0xAB, 0xE0, 0xBE, 0xB7, 0x06, + 0xE0, 0xBE, 0x90, 0xE0, 0xBE, 0xB5, 0x06, 0xE1, + 0x80, 0xA5, 0xE1, 0x80, 0xAE, 0x03, 0xE1, 0x83, + 0x9C, 0x06, 0xE1, 0xAC, 0x85, 0xE1, 0xAC, 0xB5, + // Bytes 8c0 - 8ff + 0x06, 0xE1, 0xAC, 0x87, 0xE1, 0xAC, 0xB5, 0x06, + 0xE1, 0xAC, 0x89, 0xE1, 0xAC, 0xB5, 0x06, 0xE1, + 0xAC, 0x8B, 0xE1, 0xAC, 0xB5, 0x06, 0xE1, 0xAC, + 0x8D, 0xE1, 0xAC, 0xB5, 0x06, 0xE1, 0xAC, 0x91, + 0xE1, 0xAC, 0xB5, 0x06, 0xE1, 0xAC, 0xBA, 0xE1, + 0xAC, 0xB5, 0x06, 0xE1, 0xAC, 0xBC, 0xE1, 0xAC, + 0xB5, 0x06, 0xE1, 0xAC, 0xBE, 0xE1, 0xAC, 0xB5, + 0x06, 0xE1, 0xAC, 0xBF, 0xE1, 0xAC, 0xB5, 0x06, + // Bytes 900 - 93f + 0xE1, 0xAD, 0x82, 0xE1, 0xAC, 0xB5, 0x01, 0x41, + 0x02, 0xC3, 0x86, 0x01, 0x42, 0x01, 0x44, 0x01, + 0x45, 0x02, 0xC6, 0x8E, 0x01, 0x47, 0x01, 0x48, + 0x01, 0x49, 0x01, 0x4A, 0x01, 0x4B, 0x01, 0x4C, + 0x01, 0x4D, 0x01, 0x4E, 0x01, 0x4F, 0x02, 0xC8, + 0xA2, 0x01, 0x50, 0x01, 0x52, 0x01, 0x54, 0x01, + 0x55, 0x01, 0x57, 0x02, 0xC9, 0x90, 0x02, 0xC9, + 0x91, 0x03, 0xE1, 0xB4, 0x82, 0x01, 0x62, 0x01, + // Bytes 940 - 97f + 0x64, 0x01, 0x65, 0x02, 0xC9, 0x99, 0x02, 0xC9, + 0x9B, 0x02, 0xC9, 0x9C, 0x01, 0x67, 0x01, 0x6B, + 0x01, 0x6D, 0x02, 0xC5, 0x8B, 0x02, 0xC9, 0x94, + 0x03, 0xE1, 0xB4, 0x96, 0x03, 0xE1, 0xB4, 0x97, + 0x01, 0x70, 0x01, 0x74, 0x01, 0x75, 0x03, 0xE1, + 0xB4, 0x9D, 0x02, 0xC9, 0xAF, 0x01, 0x76, 0x03, + 0xE1, 0xB4, 0xA5, 0x02, 0xCE, 0xB3, 0x02, 0xCE, + 0xB4, 0x02, 0xCF, 0x87, 0x01, 0x69, 0x02, 0xD0, + // Bytes 980 - 9bf + 0xBD, 0x02, 0xC9, 0x92, 0x01, 0x63, 0x02, 0xC9, + 0x95, 0x02, 0xC3, 0xB0, 0x01, 0x66, 0x02, 0xC9, + 0x9F, 0x02, 0xC9, 0xA1, 0x02, 0xC9, 0xA5, 0x02, + 0xC9, 0xA8, 0x02, 0xC9, 0xA9, 0x02, 0xC9, 0xAA, + 0x03, 0xE1, 0xB5, 0xBB, 0x02, 0xCA, 0x9D, 0x02, + 0xC9, 0xAD, 0x03, 0xE1, 0xB6, 0x85, 0x02, 0xCA, + 0x9F, 0x02, 0xC9, 0xB1, 0x02, 0xC9, 0xB0, 0x02, + 0xC9, 0xB2, 0x02, 0xC9, 0xB3, 0x02, 0xC9, 0xB4, + // Bytes 9c0 - 9ff + 0x02, 0xC9, 0xB5, 0x02, 0xC9, 0xB8, 0x02, 0xCA, + 0x82, 0x02, 0xCA, 0x83, 0x02, 0xC6, 0xAB, 0x02, + 0xCA, 0x89, 0x02, 0xCA, 0x8A, 0x03, 0xE1, 0xB4, + 0x9C, 0x02, 0xCA, 0x8B, 0x02, 0xCA, 0x8C, 0x01, + 0x7A, 0x02, 0xCA, 0x90, 0x02, 0xCA, 0x91, 0x02, + 0xCA, 0x92, 0x03, 0x41, 0xCC, 0xA5, 0x03, 0x61, + 0xCC, 0xA5, 0x03, 0x42, 0xCC, 0x87, 0x03, 0x62, + 0xCC, 0x87, 0x03, 0x42, 0xCC, 0xA3, 0x03, 0x62, + // Bytes a00 - a3f + 0xCC, 0xA3, 0x03, 0x42, 0xCC, 0xB1, 0x03, 0x62, + 0xCC, 0xB1, 0x05, 0x43, 0xCC, 0xA7, 0xCC, 0x81, + 0x05, 0x63, 0xCC, 0xA7, 0xCC, 0x81, 0x03, 0x44, + 0xCC, 0x87, 0x03, 0x64, 0xCC, 0x87, 0x03, 0x44, + 0xCC, 0xA3, 0x03, 0x64, 0xCC, 0xA3, 0x03, 0x44, + 0xCC, 0xB1, 0x03, 0x64, 0xCC, 0xB1, 0x03, 0x44, + 0xCC, 0xA7, 0x03, 0x64, 0xCC, 0xA7, 0x03, 0x44, + 0xCC, 0xAD, 0x03, 0x64, 0xCC, 0xAD, 0x05, 0x45, + // Bytes a40 - a7f + 0xCC, 0x84, 0xCC, 0x80, 0x05, 0x65, 0xCC, 0x84, + 0xCC, 0x80, 0x05, 0x45, 0xCC, 0x84, 0xCC, 0x81, + 0x05, 0x65, 0xCC, 0x84, 0xCC, 0x81, 0x03, 0x45, + 0xCC, 0xAD, 0x03, 0x65, 0xCC, 0xAD, 0x03, 0x45, + 0xCC, 0xB0, 0x03, 0x65, 0xCC, 0xB0, 0x05, 0x45, + 0xCC, 0xA7, 0xCC, 0x86, 0x05, 0x65, 0xCC, 0xA7, + 0xCC, 0x86, 0x03, 0x46, 0xCC, 0x87, 0x03, 0x66, + 0xCC, 0x87, 0x03, 0x47, 0xCC, 0x84, 0x03, 0x67, + // Bytes a80 - abf + 0xCC, 0x84, 0x03, 0x48, 0xCC, 0x87, 0x03, 0x68, + 0xCC, 0x87, 0x03, 0x48, 0xCC, 0xA3, 0x03, 0x68, + 0xCC, 0xA3, 0x03, 0x48, 0xCC, 0x88, 0x03, 0x68, + 0xCC, 0x88, 0x03, 0x48, 0xCC, 0xA7, 0x03, 0x68, + 0xCC, 0xA7, 0x03, 0x48, 0xCC, 0xAE, 0x03, 0x68, + 0xCC, 0xAE, 0x03, 0x49, 0xCC, 0xB0, 0x03, 0x69, + 0xCC, 0xB0, 0x05, 0x49, 0xCC, 0x88, 0xCC, 0x81, + 0x05, 0x69, 0xCC, 0x88, 0xCC, 0x81, 0x03, 0x4B, + // Bytes ac0 - aff + 0xCC, 0x81, 0x03, 0x6B, 0xCC, 0x81, 0x03, 0x4B, + 0xCC, 0xA3, 0x03, 0x6B, 0xCC, 0xA3, 0x03, 0x4B, + 0xCC, 0xB1, 0x03, 0x6B, 0xCC, 0xB1, 0x03, 0x4C, + 0xCC, 0xA3, 0x03, 0x6C, 0xCC, 0xA3, 0x05, 0x4C, + 0xCC, 0xA3, 0xCC, 0x84, 0x05, 0x6C, 0xCC, 0xA3, + 0xCC, 0x84, 0x03, 0x4C, 0xCC, 0xB1, 0x03, 0x6C, + 0xCC, 0xB1, 0x03, 0x4C, 0xCC, 0xAD, 0x03, 0x6C, + 0xCC, 0xAD, 0x03, 0x4D, 0xCC, 0x81, 0x03, 0x6D, + // Bytes b00 - b3f + 0xCC, 0x81, 0x03, 0x4D, 0xCC, 0x87, 0x03, 0x6D, + 0xCC, 0x87, 0x03, 0x4D, 0xCC, 0xA3, 0x03, 0x6D, + 0xCC, 0xA3, 0x03, 0x4E, 0xCC, 0x87, 0x03, 0x6E, + 0xCC, 0x87, 0x03, 0x4E, 0xCC, 0xA3, 0x03, 0x6E, + 0xCC, 0xA3, 0x03, 0x4E, 0xCC, 0xB1, 0x03, 0x6E, + 0xCC, 0xB1, 0x03, 0x4E, 0xCC, 0xAD, 0x03, 0x6E, + 0xCC, 0xAD, 0x05, 0x4F, 0xCC, 0x83, 0xCC, 0x81, + 0x05, 0x6F, 0xCC, 0x83, 0xCC, 0x81, 0x05, 0x4F, + // Bytes b40 - b7f + 0xCC, 0x83, 0xCC, 0x88, 0x05, 0x6F, 0xCC, 0x83, + 0xCC, 0x88, 0x05, 0x4F, 0xCC, 0x84, 0xCC, 0x80, + 0x05, 0x6F, 0xCC, 0x84, 0xCC, 0x80, 0x05, 0x4F, + 0xCC, 0x84, 0xCC, 0x81, 0x05, 0x6F, 0xCC, 0x84, + 0xCC, 0x81, 0x03, 0x50, 0xCC, 0x81, 0x03, 0x70, + 0xCC, 0x81, 0x03, 0x50, 0xCC, 0x87, 0x03, 0x70, + 0xCC, 0x87, 0x03, 0x52, 0xCC, 0x87, 0x03, 0x72, + 0xCC, 0x87, 0x03, 0x52, 0xCC, 0xA3, 0x03, 0x72, + // Bytes b80 - bbf + 0xCC, 0xA3, 0x05, 0x52, 0xCC, 0xA3, 0xCC, 0x84, + 0x05, 0x72, 0xCC, 0xA3, 0xCC, 0x84, 0x03, 0x52, + 0xCC, 0xB1, 0x03, 0x72, 0xCC, 0xB1, 0x03, 0x53, + 0xCC, 0x87, 0x03, 0x73, 0xCC, 0x87, 0x03, 0x53, + 0xCC, 0xA3, 0x03, 0x73, 0xCC, 0xA3, 0x05, 0x53, + 0xCC, 0x81, 0xCC, 0x87, 0x05, 0x73, 0xCC, 0x81, + 0xCC, 0x87, 0x05, 0x53, 0xCC, 0x8C, 0xCC, 0x87, + 0x05, 0x73, 0xCC, 0x8C, 0xCC, 0x87, 0x05, 0x53, + // Bytes bc0 - bff + 0xCC, 0xA3, 0xCC, 0x87, 0x05, 0x73, 0xCC, 0xA3, + 0xCC, 0x87, 0x03, 0x54, 0xCC, 0x87, 0x03, 0x74, + 0xCC, 0x87, 0x03, 0x54, 0xCC, 0xA3, 0x03, 0x74, + 0xCC, 0xA3, 0x03, 0x54, 0xCC, 0xB1, 0x03, 0x74, + 0xCC, 0xB1, 0x03, 0x54, 0xCC, 0xAD, 0x03, 0x74, + 0xCC, 0xAD, 0x03, 0x55, 0xCC, 0xA4, 0x03, 0x75, + 0xCC, 0xA4, 0x03, 0x55, 0xCC, 0xB0, 0x03, 0x75, + 0xCC, 0xB0, 0x03, 0x55, 0xCC, 0xAD, 0x03, 0x75, + // Bytes c00 - c3f + 0xCC, 0xAD, 0x05, 0x55, 0xCC, 0x83, 0xCC, 0x81, + 0x05, 0x75, 0xCC, 0x83, 0xCC, 0x81, 0x05, 0x55, + 0xCC, 0x84, 0xCC, 0x88, 0x05, 0x75, 0xCC, 0x84, + 0xCC, 0x88, 0x03, 0x56, 0xCC, 0x83, 0x03, 0x76, + 0xCC, 0x83, 0x03, 0x56, 0xCC, 0xA3, 0x03, 0x76, + 0xCC, 0xA3, 0x03, 0x57, 0xCC, 0x80, 0x03, 0x77, + 0xCC, 0x80, 0x03, 0x57, 0xCC, 0x81, 0x03, 0x77, + 0xCC, 0x81, 0x03, 0x57, 0xCC, 0x88, 0x03, 0x77, + // Bytes c40 - c7f + 0xCC, 0x88, 0x03, 0x57, 0xCC, 0x87, 0x03, 0x77, + 0xCC, 0x87, 0x03, 0x57, 0xCC, 0xA3, 0x03, 0x77, + 0xCC, 0xA3, 0x03, 0x58, 0xCC, 0x87, 0x03, 0x78, + 0xCC, 0x87, 0x03, 0x58, 0xCC, 0x88, 0x03, 0x78, + 0xCC, 0x88, 0x03, 0x59, 0xCC, 0x87, 0x03, 0x79, + 0xCC, 0x87, 0x03, 0x5A, 0xCC, 0x82, 0x03, 0x7A, + 0xCC, 0x82, 0x03, 0x5A, 0xCC, 0xA3, 0x03, 0x7A, + 0xCC, 0xA3, 0x03, 0x5A, 0xCC, 0xB1, 0x03, 0x7A, + // Bytes c80 - cbf + 0xCC, 0xB1, 0x03, 0x68, 0xCC, 0xB1, 0x03, 0x74, + 0xCC, 0x88, 0x03, 0x77, 0xCC, 0x8A, 0x03, 0x79, + 0xCC, 0x8A, 0x03, 0x61, 0xCA, 0xBE, 0x04, 0xC5, + 0xBF, 0xCC, 0x87, 0x03, 0x41, 0xCC, 0xA3, 0x03, + 0x61, 0xCC, 0xA3, 0x03, 0x41, 0xCC, 0x89, 0x03, + 0x61, 0xCC, 0x89, 0x05, 0x41, 0xCC, 0x82, 0xCC, + 0x81, 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x81, 0x05, + 0x41, 0xCC, 0x82, 0xCC, 0x80, 0x05, 0x61, 0xCC, + // Bytes cc0 - cff + 0x82, 0xCC, 0x80, 0x05, 0x41, 0xCC, 0x82, 0xCC, + 0x89, 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x89, 0x05, + 0x41, 0xCC, 0x82, 0xCC, 0x83, 0x05, 0x61, 0xCC, + 0x82, 0xCC, 0x83, 0x05, 0x41, 0xCC, 0xA3, 0xCC, + 0x82, 0x05, 0x61, 0xCC, 0xA3, 0xCC, 0x82, 0x05, + 0x41, 0xCC, 0x86, 0xCC, 0x81, 0x05, 0x61, 0xCC, + 0x86, 0xCC, 0x81, 0x05, 0x41, 0xCC, 0x86, 0xCC, + 0x80, 0x05, 0x61, 0xCC, 0x86, 0xCC, 0x80, 0x05, + // Bytes d00 - d3f + 0x41, 0xCC, 0x86, 0xCC, 0x89, 0x05, 0x61, 0xCC, + 0x86, 0xCC, 0x89, 0x05, 0x41, 0xCC, 0x86, 0xCC, + 0x83, 0x05, 0x61, 0xCC, 0x86, 0xCC, 0x83, 0x05, + 0x41, 0xCC, 0xA3, 0xCC, 0x86, 0x05, 0x61, 0xCC, + 0xA3, 0xCC, 0x86, 0x03, 0x45, 0xCC, 0xA3, 0x03, + 0x65, 0xCC, 0xA3, 0x03, 0x45, 0xCC, 0x89, 0x03, + 0x65, 0xCC, 0x89, 0x03, 0x45, 0xCC, 0x83, 0x03, + 0x65, 0xCC, 0x83, 0x05, 0x45, 0xCC, 0x82, 0xCC, + // Bytes d40 - d7f + 0x81, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x81, 0x05, + 0x45, 0xCC, 0x82, 0xCC, 0x80, 0x05, 0x65, 0xCC, + 0x82, 0xCC, 0x80, 0x05, 0x45, 0xCC, 0x82, 0xCC, + 0x89, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x89, 0x05, + 0x45, 0xCC, 0x82, 0xCC, 0x83, 0x05, 0x65, 0xCC, + 0x82, 0xCC, 0x83, 0x05, 0x45, 0xCC, 0xA3, 0xCC, + 0x82, 0x05, 0x65, 0xCC, 0xA3, 0xCC, 0x82, 0x03, + 0x49, 0xCC, 0x89, 0x03, 0x69, 0xCC, 0x89, 0x03, + // Bytes d80 - dbf + 0x49, 0xCC, 0xA3, 0x03, 0x69, 0xCC, 0xA3, 0x03, + 0x4F, 0xCC, 0xA3, 0x03, 0x6F, 0xCC, 0xA3, 0x03, + 0x4F, 0xCC, 0x89, 0x03, 0x6F, 0xCC, 0x89, 0x05, + 0x4F, 0xCC, 0x82, 0xCC, 0x81, 0x05, 0x6F, 0xCC, + 0x82, 0xCC, 0x81, 0x05, 0x4F, 0xCC, 0x82, 0xCC, + 0x80, 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x80, 0x05, + 0x4F, 0xCC, 0x82, 0xCC, 0x89, 0x05, 0x6F, 0xCC, + 0x82, 0xCC, 0x89, 0x05, 0x4F, 0xCC, 0x82, 0xCC, + // Bytes dc0 - dff + 0x83, 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x83, 0x05, + 0x4F, 0xCC, 0xA3, 0xCC, 0x82, 0x05, 0x6F, 0xCC, + 0xA3, 0xCC, 0x82, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + 0x81, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0x81, 0x05, + 0x4F, 0xCC, 0x9B, 0xCC, 0x80, 0x05, 0x6F, 0xCC, + 0x9B, 0xCC, 0x80, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + 0x89, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0x89, 0x05, + 0x4F, 0xCC, 0x9B, 0xCC, 0x83, 0x05, 0x6F, 0xCC, + // Bytes e00 - e3f + 0x9B, 0xCC, 0x83, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, + 0xA3, 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0xA3, 0x03, + 0x55, 0xCC, 0xA3, 0x03, 0x75, 0xCC, 0xA3, 0x03, + 0x55, 0xCC, 0x89, 0x03, 0x75, 0xCC, 0x89, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0x81, 0x05, 0x75, 0xCC, + 0x9B, 0xCC, 0x81, 0x05, 0x55, 0xCC, 0x9B, 0xCC, + 0x80, 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0x80, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0x89, 0x05, 0x75, 0xCC, + // Bytes e40 - e7f + 0x9B, 0xCC, 0x89, 0x05, 0x55, 0xCC, 0x9B, 0xCC, + 0x83, 0x05, 0x75, 0xCC, 0x9B, 0xCC, 0x83, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0xA3, 0x05, 0x75, 0xCC, + 0x9B, 0xCC, 0xA3, 0x03, 0x59, 0xCC, 0x80, 0x03, + 0x79, 0xCC, 0x80, 0x03, 0x59, 0xCC, 0xA3, 0x03, + 0x79, 0xCC, 0xA3, 0x03, 0x59, 0xCC, 0x89, 0x03, + 0x79, 0xCC, 0x89, 0x03, 0x59, 0xCC, 0x83, 0x03, + 0x79, 0xCC, 0x83, 0x04, 0xCE, 0xB1, 0xCC, 0x93, + // Bytes e80 - ebf + 0x04, 0xCE, 0xB1, 0xCC, 0x94, 0x06, 0xCE, 0xB1, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0xB1, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xB1, 0xCC, 0x93, + 0xCC, 0x81, 0x06, 0xCE, 0xB1, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCE, 0xB1, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x82, 0x04, + 0xCE, 0x91, 0xCC, 0x93, 0x04, 0xCE, 0x91, 0xCC, + 0x94, 0x06, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x80, + // Bytes ec0 - eff + 0x06, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0x91, 0xCC, 0x94, 0xCC, 0x81, 0x06, 0xCE, 0x91, + 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCE, 0x91, 0xCC, + 0x94, 0xCD, 0x82, 0x04, 0xCE, 0xB5, 0xCC, 0x93, + 0x04, 0xCE, 0xB5, 0xCC, 0x94, 0x06, 0xCE, 0xB5, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0xB5, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xB5, 0xCC, 0x93, + // Bytes f00 - f3f + 0xCC, 0x81, 0x06, 0xCE, 0xB5, 0xCC, 0x94, 0xCC, + 0x81, 0x04, 0xCE, 0x95, 0xCC, 0x93, 0x04, 0xCE, + 0x95, 0xCC, 0x94, 0x06, 0xCE, 0x95, 0xCC, 0x93, + 0xCC, 0x80, 0x06, 0xCE, 0x95, 0xCC, 0x94, 0xCC, + 0x80, 0x06, 0xCE, 0x95, 0xCC, 0x93, 0xCC, 0x81, + 0x06, 0xCE, 0x95, 0xCC, 0x94, 0xCC, 0x81, 0x04, + 0xCE, 0xB7, 0xCC, 0x93, 0x04, 0xCE, 0xB7, 0xCC, + 0x94, 0x06, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x80, + // Bytes f40 - f7f + 0x06, 0xCE, 0xB7, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0xB7, 0xCC, 0x94, 0xCC, 0x81, 0x06, 0xCE, 0xB7, + 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCE, 0xB7, 0xCC, + 0x94, 0xCD, 0x82, 0x04, 0xCE, 0x97, 0xCC, 0x93, + 0x04, 0xCE, 0x97, 0xCC, 0x94, 0x06, 0xCE, 0x97, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0x97, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0x97, 0xCC, 0x93, + // Bytes f80 - fbf + 0xCC, 0x81, 0x06, 0xCE, 0x97, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x82, 0x04, + 0xCE, 0xB9, 0xCC, 0x93, 0x04, 0xCE, 0xB9, 0xCC, + 0x94, 0x06, 0xCE, 0xB9, 0xCC, 0x93, 0xCC, 0x80, + 0x06, 0xCE, 0xB9, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0xB9, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0xB9, 0xCC, 0x94, 0xCC, 0x81, 0x06, 0xCE, 0xB9, + // Bytes fc0 - fff + 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCE, 0xB9, 0xCC, + 0x94, 0xCD, 0x82, 0x04, 0xCE, 0x99, 0xCC, 0x93, + 0x04, 0xCE, 0x99, 0xCC, 0x94, 0x06, 0xCE, 0x99, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, 0x99, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0x99, 0xCC, 0x93, + 0xCC, 0x81, 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCE, 0x99, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCE, 0x99, 0xCC, 0x94, 0xCD, 0x82, 0x04, + // Bytes 1000 - 103f + 0xCE, 0xBF, 0xCC, 0x93, 0x04, 0xCE, 0xBF, 0xCC, + 0x94, 0x06, 0xCE, 0xBF, 0xCC, 0x93, 0xCC, 0x80, + 0x06, 0xCE, 0xBF, 0xCC, 0x94, 0xCC, 0x80, 0x06, + 0xCE, 0xBF, 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, + 0xBF, 0xCC, 0x94, 0xCC, 0x81, 0x04, 0xCE, 0x9F, + 0xCC, 0x93, 0x04, 0xCE, 0x9F, 0xCC, 0x94, 0x06, + 0xCE, 0x9F, 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, + 0x9F, 0xCC, 0x94, 0xCC, 0x80, 0x06, 0xCE, 0x9F, + // Bytes 1040 - 107f + 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, 0x9F, 0xCC, + 0x94, 0xCC, 0x81, 0x04, 0xCF, 0x85, 0xCC, 0x93, + 0x04, 0xCF, 0x85, 0xCC, 0x94, 0x06, 0xCF, 0x85, + 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCF, 0x85, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCF, 0x85, 0xCC, 0x93, + 0xCC, 0x81, 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCC, + 0x81, 0x06, 0xCF, 0x85, 0xCC, 0x93, 0xCD, 0x82, + 0x06, 0xCF, 0x85, 0xCC, 0x94, 0xCD, 0x82, 0x04, + // Bytes 1080 - 10bf + 0xCE, 0xA5, 0xCC, 0x94, 0x06, 0xCE, 0xA5, 0xCC, + 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xA5, 0xCC, 0x94, + 0xCC, 0x81, 0x06, 0xCE, 0xA5, 0xCC, 0x94, 0xCD, + 0x82, 0x04, 0xCF, 0x89, 0xCC, 0x93, 0x04, 0xCF, + 0x89, 0xCC, 0x94, 0x06, 0xCF, 0x89, 0xCC, 0x93, + 0xCC, 0x80, 0x06, 0xCF, 0x89, 0xCC, 0x94, 0xCC, + 0x80, 0x06, 0xCF, 0x89, 0xCC, 0x93, 0xCC, 0x81, + 0x06, 0xCF, 0x89, 0xCC, 0x94, 0xCC, 0x81, 0x06, + // Bytes 10c0 - 10ff + 0xCF, 0x89, 0xCC, 0x93, 0xCD, 0x82, 0x06, 0xCF, + 0x89, 0xCC, 0x94, 0xCD, 0x82, 0x04, 0xCE, 0xA9, + 0xCC, 0x93, 0x04, 0xCE, 0xA9, 0xCC, 0x94, 0x06, + 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x80, 0x06, 0xCE, + 0xA9, 0xCC, 0x94, 0xCC, 0x80, 0x06, 0xCE, 0xA9, + 0xCC, 0x93, 0xCC, 0x81, 0x06, 0xCE, 0xA9, 0xCC, + 0x94, 0xCC, 0x81, 0x06, 0xCE, 0xA9, 0xCC, 0x93, + 0xCD, 0x82, 0x06, 0xCE, 0xA9, 0xCC, 0x94, 0xCD, + // Bytes 1100 - 113f + 0x82, 0x04, 0xCE, 0xB1, 0xCC, 0x80, 0x04, 0xCE, + 0xB5, 0xCC, 0x80, 0x04, 0xCE, 0xB7, 0xCC, 0x80, + 0x04, 0xCE, 0xB9, 0xCC, 0x80, 0x04, 0xCE, 0xBF, + 0xCC, 0x80, 0x04, 0xCF, 0x85, 0xCC, 0x80, 0x04, + 0xCF, 0x89, 0xCC, 0x80, 0x06, 0xCE, 0xB1, 0xCC, + 0x93, 0xCD, 0x85, 0x06, 0xCE, 0xB1, 0xCC, 0x94, + 0xCD, 0x85, 0x08, 0xCE, 0xB1, 0xCC, 0x93, 0xCC, + 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB1, 0xCC, 0x94, + // Bytes 1140 - 117f + 0xCC, 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB1, 0xCC, + 0x93, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, 0xB1, + 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, + 0xB1, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, 0x08, + 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, + 0x06, 0xCE, 0x91, 0xCC, 0x93, 0xCD, 0x85, 0x06, + 0xCE, 0x91, 0xCC, 0x94, 0xCD, 0x85, 0x08, 0xCE, + 0x91, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, 0x08, + // Bytes 1180 - 11bf + 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, + 0x08, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x81, 0xCD, + 0x85, 0x08, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x81, + 0xCD, 0x85, 0x08, 0xCE, 0x91, 0xCC, 0x93, 0xCD, + 0x82, 0xCD, 0x85, 0x08, 0xCE, 0x91, 0xCC, 0x94, + 0xCD, 0x82, 0xCD, 0x85, 0x06, 0xCE, 0xB7, 0xCC, + 0x93, 0xCD, 0x85, 0x06, 0xCE, 0xB7, 0xCC, 0x94, + 0xCD, 0x85, 0x08, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, + // Bytes 11c0 - 11ff + 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB7, 0xCC, 0x94, + 0xCC, 0x80, 0xCD, 0x85, 0x08, 0xCE, 0xB7, 0xCC, + 0x93, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, 0xB7, + 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCE, + 0xB7, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, 0x08, + 0xCE, 0xB7, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, + 0x06, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x85, 0x06, + 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x85, 0x08, 0xCE, + // Bytes 1200 - 123f + 0x97, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, 0x08, + 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, + 0x08, 0xCE, 0x97, 0xCC, 0x93, 0xCC, 0x81, 0xCD, + 0x85, 0x08, 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x81, + 0xCD, 0x85, 0x08, 0xCE, 0x97, 0xCC, 0x93, 0xCD, + 0x82, 0xCD, 0x85, 0x08, 0xCE, 0x97, 0xCC, 0x94, + 0xCD, 0x82, 0xCD, 0x85, 0x06, 0xCF, 0x89, 0xCC, + 0x93, 0xCD, 0x85, 0x06, 0xCF, 0x89, 0xCC, 0x94, + // Bytes 1240 - 127f + 0xCD, 0x85, 0x08, 0xCF, 0x89, 0xCC, 0x93, 0xCC, + 0x80, 0xCD, 0x85, 0x08, 0xCF, 0x89, 0xCC, 0x94, + 0xCC, 0x80, 0xCD, 0x85, 0x08, 0xCF, 0x89, 0xCC, + 0x93, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCF, 0x89, + 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, 0x08, 0xCF, + 0x89, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, 0x08, + 0xCF, 0x89, 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, + 0x06, 0xCE, 0xA9, 0xCC, 0x93, 0xCD, 0x85, 0x06, + // Bytes 1280 - 12bf + 0xCE, 0xA9, 0xCC, 0x94, 0xCD, 0x85, 0x08, 0xCE, + 0xA9, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, 0x08, + 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, + 0x08, 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x81, 0xCD, + 0x85, 0x08, 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x81, + 0xCD, 0x85, 0x08, 0xCE, 0xA9, 0xCC, 0x93, 0xCD, + 0x82, 0xCD, 0x85, 0x08, 0xCE, 0xA9, 0xCC, 0x94, + 0xCD, 0x82, 0xCD, 0x85, 0x04, 0xCE, 0xB1, 0xCC, + // Bytes 12c0 - 12ff + 0x86, 0x04, 0xCE, 0xB1, 0xCC, 0x84, 0x06, 0xCE, + 0xB1, 0xCC, 0x80, 0xCD, 0x85, 0x04, 0xCE, 0xB1, + 0xCD, 0x85, 0x06, 0xCE, 0xB1, 0xCC, 0x81, 0xCD, + 0x85, 0x04, 0xCE, 0xB1, 0xCD, 0x82, 0x06, 0xCE, + 0xB1, 0xCD, 0x82, 0xCD, 0x85, 0x04, 0xCE, 0x91, + 0xCC, 0x86, 0x04, 0xCE, 0x91, 0xCC, 0x84, 0x04, + 0xCE, 0x91, 0xCC, 0x80, 0x04, 0xCE, 0x91, 0xCD, + 0x85, 0x03, 0x20, 0xCC, 0x93, 0x02, 0xCE, 0xB9, + // Bytes 1300 - 133f + 0x03, 0x20, 0xCD, 0x82, 0x04, 0xC2, 0xA8, 0xCD, + 0x82, 0x05, 0x20, 0xCC, 0x88, 0xCD, 0x82, 0x06, + 0xCE, 0xB7, 0xCC, 0x80, 0xCD, 0x85, 0x04, 0xCE, + 0xB7, 0xCD, 0x85, 0x06, 0xCE, 0xB7, 0xCC, 0x81, + 0xCD, 0x85, 0x04, 0xCE, 0xB7, 0xCD, 0x82, 0x06, + 0xCE, 0xB7, 0xCD, 0x82, 0xCD, 0x85, 0x04, 0xCE, + 0x95, 0xCC, 0x80, 0x04, 0xCE, 0x97, 0xCC, 0x80, + 0x04, 0xCE, 0x97, 0xCD, 0x85, 0x05, 0xE1, 0xBE, + // Bytes 1340 - 137f + 0xBF, 0xCC, 0x80, 0x05, 0x20, 0xCC, 0x93, 0xCC, + 0x80, 0x05, 0xE1, 0xBE, 0xBF, 0xCC, 0x81, 0x05, + 0x20, 0xCC, 0x93, 0xCC, 0x81, 0x05, 0xE1, 0xBE, + 0xBF, 0xCD, 0x82, 0x05, 0x20, 0xCC, 0x93, 0xCD, + 0x82, 0x04, 0xCE, 0xB9, 0xCC, 0x86, 0x04, 0xCE, + 0xB9, 0xCC, 0x84, 0x06, 0xCE, 0xB9, 0xCC, 0x88, + 0xCC, 0x80, 0x04, 0xCE, 0xB9, 0xCD, 0x82, 0x06, + 0xCE, 0xB9, 0xCC, 0x88, 0xCD, 0x82, 0x04, 0xCE, + // Bytes 1380 - 13bf + 0x99, 0xCC, 0x86, 0x04, 0xCE, 0x99, 0xCC, 0x84, + 0x04, 0xCE, 0x99, 0xCC, 0x80, 0x05, 0xE1, 0xBF, + 0xBE, 0xCC, 0x80, 0x05, 0x20, 0xCC, 0x94, 0xCC, + 0x80, 0x05, 0xE1, 0xBF, 0xBE, 0xCC, 0x81, 0x05, + 0x20, 0xCC, 0x94, 0xCC, 0x81, 0x05, 0xE1, 0xBF, + 0xBE, 0xCD, 0x82, 0x05, 0x20, 0xCC, 0x94, 0xCD, + 0x82, 0x04, 0xCF, 0x85, 0xCC, 0x86, 0x04, 0xCF, + 0x85, 0xCC, 0x84, 0x06, 0xCF, 0x85, 0xCC, 0x88, + // Bytes 13c0 - 13ff + 0xCC, 0x80, 0x04, 0xCF, 0x81, 0xCC, 0x93, 0x04, + 0xCF, 0x81, 0xCC, 0x94, 0x04, 0xCF, 0x85, 0xCD, + 0x82, 0x06, 0xCF, 0x85, 0xCC, 0x88, 0xCD, 0x82, + 0x04, 0xCE, 0xA5, 0xCC, 0x86, 0x04, 0xCE, 0xA5, + 0xCC, 0x84, 0x04, 0xCE, 0xA5, 0xCC, 0x80, 0x04, + 0xCE, 0xA1, 0xCC, 0x94, 0x04, 0xC2, 0xA8, 0xCC, + 0x80, 0x05, 0x20, 0xCC, 0x88, 0xCC, 0x80, 0x01, + 0x60, 0x06, 0xCF, 0x89, 0xCC, 0x80, 0xCD, 0x85, + // Bytes 1400 - 143f + 0x04, 0xCF, 0x89, 0xCD, 0x85, 0x06, 0xCF, 0x89, + 0xCC, 0x81, 0xCD, 0x85, 0x04, 0xCF, 0x89, 0xCD, + 0x82, 0x06, 0xCF, 0x89, 0xCD, 0x82, 0xCD, 0x85, + 0x04, 0xCE, 0x9F, 0xCC, 0x80, 0x04, 0xCE, 0xA9, + 0xCC, 0x80, 0x04, 0xCE, 0xA9, 0xCD, 0x85, 0x02, + 0xC2, 0xB4, 0x03, 0x20, 0xCC, 0x94, 0x03, 0xE2, + 0x80, 0x82, 0x03, 0xE2, 0x80, 0x83, 0x03, 0xE2, + 0x80, 0x90, 0x03, 0x20, 0xCC, 0xB3, 0x01, 0x2E, + // Bytes 1440 - 147f + 0x02, 0x2E, 0x2E, 0x03, 0x2E, 0x2E, 0x2E, 0x06, + 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0x09, 0xE2, + 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0x06, 0xE2, 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0x09, + 0xE2, 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0xE2, 0x80, + 0xB5, 0x02, 0x21, 0x21, 0x03, 0x20, 0xCC, 0x85, + 0x02, 0x3F, 0x3F, 0x02, 0x3F, 0x21, 0x02, 0x21, + 0x3F, 0x0C, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + // Bytes 1480 - 14bf + 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0x01, 0x30, + 0x01, 0x34, 0x01, 0x35, 0x01, 0x36, 0x01, 0x37, + 0x01, 0x38, 0x01, 0x39, 0x01, 0x2B, 0x03, 0xE2, + 0x88, 0x92, 0x01, 0x3D, 0x01, 0x28, 0x01, 0x29, + 0x01, 0x6E, 0x02, 0x52, 0x73, 0x03, 0x61, 0x2F, + 0x63, 0x03, 0x61, 0x2F, 0x73, 0x01, 0x43, 0x03, + 0xC2, 0xB0, 0x43, 0x03, 0x63, 0x2F, 0x6F, 0x03, + 0x63, 0x2F, 0x75, 0x02, 0xC6, 0x90, 0x03, 0xC2, + // Bytes 14c0 - 14ff + 0xB0, 0x46, 0x02, 0xC4, 0xA7, 0x02, 0x4E, 0x6F, + 0x01, 0x51, 0x02, 0x53, 0x4D, 0x03, 0x54, 0x45, + 0x4C, 0x02, 0x54, 0x4D, 0x01, 0x5A, 0x02, 0xCE, + 0xA9, 0x01, 0x46, 0x02, 0xD7, 0x90, 0x02, 0xD7, + 0x91, 0x02, 0xD7, 0x92, 0x02, 0xD7, 0x93, 0x03, + 0x46, 0x41, 0x58, 0x02, 0xCE, 0x93, 0x02, 0xCE, + 0xA0, 0x03, 0xE2, 0x88, 0x91, 0x05, 0x31, 0xE2, + 0x81, 0x84, 0x37, 0x05, 0x31, 0xE2, 0x81, 0x84, + // Bytes 1500 - 153f + 0x39, 0x06, 0x31, 0xE2, 0x81, 0x84, 0x31, 0x30, + 0x05, 0x31, 0xE2, 0x81, 0x84, 0x33, 0x05, 0x32, + 0xE2, 0x81, 0x84, 0x33, 0x05, 0x31, 0xE2, 0x81, + 0x84, 0x35, 0x05, 0x32, 0xE2, 0x81, 0x84, 0x35, + 0x05, 0x33, 0xE2, 0x81, 0x84, 0x35, 0x05, 0x34, + 0xE2, 0x81, 0x84, 0x35, 0x05, 0x31, 0xE2, 0x81, + 0x84, 0x36, 0x05, 0x35, 0xE2, 0x81, 0x84, 0x36, + 0x05, 0x31, 0xE2, 0x81, 0x84, 0x38, 0x05, 0x33, + // Bytes 1540 - 157f + 0xE2, 0x81, 0x84, 0x38, 0x05, 0x35, 0xE2, 0x81, + 0x84, 0x38, 0x05, 0x37, 0xE2, 0x81, 0x84, 0x38, + 0x04, 0x31, 0xE2, 0x81, 0x84, 0x02, 0x49, 0x49, + 0x03, 0x49, 0x49, 0x49, 0x02, 0x49, 0x56, 0x01, + 0x56, 0x02, 0x56, 0x49, 0x03, 0x56, 0x49, 0x49, + 0x04, 0x56, 0x49, 0x49, 0x49, 0x02, 0x49, 0x58, + 0x01, 0x58, 0x02, 0x58, 0x49, 0x03, 0x58, 0x49, + 0x49, 0x02, 0x69, 0x69, 0x03, 0x69, 0x69, 0x69, + // Bytes 1580 - 15bf + 0x02, 0x69, 0x76, 0x02, 0x76, 0x69, 0x03, 0x76, + 0x69, 0x69, 0x04, 0x76, 0x69, 0x69, 0x69, 0x02, + 0x69, 0x78, 0x02, 0x78, 0x69, 0x03, 0x78, 0x69, + 0x69, 0x05, 0x30, 0xE2, 0x81, 0x84, 0x33, 0x05, + 0xE2, 0x86, 0x90, 0xCC, 0xB8, 0x05, 0xE2, 0x86, + 0x92, 0xCC, 0xB8, 0x05, 0xE2, 0x86, 0x94, 0xCC, + 0xB8, 0x05, 0xE2, 0x87, 0x90, 0xCC, 0xB8, 0x05, + 0xE2, 0x87, 0x94, 0xCC, 0xB8, 0x05, 0xE2, 0x87, + // Bytes 15c0 - 15ff + 0x92, 0xCC, 0xB8, 0x05, 0xE2, 0x88, 0x83, 0xCC, + 0xB8, 0x05, 0xE2, 0x88, 0x88, 0xCC, 0xB8, 0x05, + 0xE2, 0x88, 0x8B, 0xCC, 0xB8, 0x05, 0xE2, 0x88, + 0xA3, 0xCC, 0xB8, 0x05, 0xE2, 0x88, 0xA5, 0xCC, + 0xB8, 0x06, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, + 0x09, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0xE2, + 0x88, 0xAB, 0x06, 0xE2, 0x88, 0xAE, 0xE2, 0x88, + 0xAE, 0x09, 0xE2, 0x88, 0xAE, 0xE2, 0x88, 0xAE, + // Bytes 1600 - 163f + 0xE2, 0x88, 0xAE, 0x05, 0xE2, 0x88, 0xBC, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0x83, 0xCC, 0xB8, 0x05, + 0xE2, 0x89, 0x85, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0x88, 0xCC, 0xB8, 0x03, 0x3D, 0xCC, 0xB8, 0x05, + 0xE2, 0x89, 0xA1, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0x8D, 0xCC, 0xB8, 0x03, 0x3C, 0xCC, 0xB8, 0x03, + 0x3E, 0xCC, 0xB8, 0x05, 0xE2, 0x89, 0xA4, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0xA5, 0xCC, 0xB8, 0x05, + // Bytes 1640 - 167f + 0xE2, 0x89, 0xB2, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0xB3, 0xCC, 0xB8, 0x05, 0xE2, 0x89, 0xB6, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0xB7, 0xCC, 0xB8, 0x05, + 0xE2, 0x89, 0xBA, 0xCC, 0xB8, 0x05, 0xE2, 0x89, + 0xBB, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, 0x82, 0xCC, + 0xB8, 0x05, 0xE2, 0x8A, 0x83, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0x86, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + 0x87, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, 0xA2, 0xCC, + // Bytes 1680 - 16bf + 0xB8, 0x05, 0xE2, 0x8A, 0xA8, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0xA9, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + 0xAB, 0xCC, 0xB8, 0x05, 0xE2, 0x89, 0xBC, 0xCC, + 0xB8, 0x05, 0xE2, 0x89, 0xBD, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0x91, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + 0x92, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, 0xB2, 0xCC, + 0xB8, 0x05, 0xE2, 0x8A, 0xB3, 0xCC, 0xB8, 0x05, + 0xE2, 0x8A, 0xB4, 0xCC, 0xB8, 0x05, 0xE2, 0x8A, + // Bytes 16c0 - 16ff + 0xB5, 0xCC, 0xB8, 0x03, 0xE3, 0x80, 0x88, 0x03, + 0xE3, 0x80, 0x89, 0x02, 0x31, 0x30, 0x02, 0x31, + 0x31, 0x02, 0x31, 0x32, 0x02, 0x31, 0x33, 0x02, + 0x31, 0x34, 0x02, 0x31, 0x35, 0x02, 0x31, 0x36, + 0x02, 0x31, 0x37, 0x02, 0x31, 0x38, 0x02, 0x31, + 0x39, 0x02, 0x32, 0x30, 0x03, 0x28, 0x31, 0x29, + 0x03, 0x28, 0x32, 0x29, 0x03, 0x28, 0x33, 0x29, + 0x03, 0x28, 0x34, 0x29, 0x03, 0x28, 0x35, 0x29, + // Bytes 1700 - 173f + 0x03, 0x28, 0x36, 0x29, 0x03, 0x28, 0x37, 0x29, + 0x03, 0x28, 0x38, 0x29, 0x03, 0x28, 0x39, 0x29, + 0x04, 0x28, 0x31, 0x30, 0x29, 0x04, 0x28, 0x31, + 0x31, 0x29, 0x04, 0x28, 0x31, 0x32, 0x29, 0x04, + 0x28, 0x31, 0x33, 0x29, 0x04, 0x28, 0x31, 0x34, + 0x29, 0x04, 0x28, 0x31, 0x35, 0x29, 0x04, 0x28, + 0x31, 0x36, 0x29, 0x04, 0x28, 0x31, 0x37, 0x29, + 0x04, 0x28, 0x31, 0x38, 0x29, 0x04, 0x28, 0x31, + // Bytes 1740 - 177f + 0x39, 0x29, 0x04, 0x28, 0x32, 0x30, 0x29, 0x02, + 0x31, 0x2E, 0x02, 0x32, 0x2E, 0x02, 0x33, 0x2E, + 0x02, 0x34, 0x2E, 0x02, 0x35, 0x2E, 0x02, 0x36, + 0x2E, 0x02, 0x37, 0x2E, 0x02, 0x38, 0x2E, 0x02, + 0x39, 0x2E, 0x03, 0x31, 0x30, 0x2E, 0x03, 0x31, + 0x31, 0x2E, 0x03, 0x31, 0x32, 0x2E, 0x03, 0x31, + 0x33, 0x2E, 0x03, 0x31, 0x34, 0x2E, 0x03, 0x31, + 0x35, 0x2E, 0x03, 0x31, 0x36, 0x2E, 0x03, 0x31, + // Bytes 1780 - 17bf + 0x37, 0x2E, 0x03, 0x31, 0x38, 0x2E, 0x03, 0x31, + 0x39, 0x2E, 0x03, 0x32, 0x30, 0x2E, 0x03, 0x28, + 0x61, 0x29, 0x03, 0x28, 0x62, 0x29, 0x03, 0x28, + 0x63, 0x29, 0x03, 0x28, 0x64, 0x29, 0x03, 0x28, + 0x65, 0x29, 0x03, 0x28, 0x66, 0x29, 0x03, 0x28, + 0x67, 0x29, 0x03, 0x28, 0x68, 0x29, 0x03, 0x28, + 0x69, 0x29, 0x03, 0x28, 0x6A, 0x29, 0x03, 0x28, + 0x6B, 0x29, 0x03, 0x28, 0x6C, 0x29, 0x03, 0x28, + // Bytes 17c0 - 17ff + 0x6D, 0x29, 0x03, 0x28, 0x6E, 0x29, 0x03, 0x28, + 0x6F, 0x29, 0x03, 0x28, 0x70, 0x29, 0x03, 0x28, + 0x71, 0x29, 0x03, 0x28, 0x72, 0x29, 0x03, 0x28, + 0x73, 0x29, 0x03, 0x28, 0x74, 0x29, 0x03, 0x28, + 0x75, 0x29, 0x03, 0x28, 0x76, 0x29, 0x03, 0x28, + 0x77, 0x29, 0x03, 0x28, 0x78, 0x29, 0x03, 0x28, + 0x79, 0x29, 0x03, 0x28, 0x7A, 0x29, 0x01, 0x53, + 0x01, 0x59, 0x01, 0x71, 0x0C, 0xE2, 0x88, 0xAB, + // Bytes 1800 - 183f + 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0xE2, 0x88, + 0xAB, 0x03, 0x3A, 0x3A, 0x3D, 0x02, 0x3D, 0x3D, + 0x03, 0x3D, 0x3D, 0x3D, 0x05, 0xE2, 0xAB, 0x9D, + 0xCC, 0xB8, 0x03, 0xE2, 0xB5, 0xA1, 0x03, 0xE6, + 0xAF, 0x8D, 0x03, 0xE9, 0xBE, 0x9F, 0x03, 0xE4, + 0xB8, 0x80, 0x03, 0xE4, 0xB8, 0xA8, 0x03, 0xE4, + 0xB8, 0xB6, 0x03, 0xE4, 0xB8, 0xBF, 0x03, 0xE4, + 0xB9, 0x99, 0x03, 0xE4, 0xBA, 0x85, 0x03, 0xE4, + // Bytes 1840 - 187f + 0xBA, 0x8C, 0x03, 0xE4, 0xBA, 0xA0, 0x03, 0xE4, + 0xBA, 0xBA, 0x03, 0xE5, 0x84, 0xBF, 0x03, 0xE5, + 0x85, 0xA5, 0x03, 0xE5, 0x85, 0xAB, 0x03, 0xE5, + 0x86, 0x82, 0x03, 0xE5, 0x86, 0x96, 0x03, 0xE5, + 0x86, 0xAB, 0x03, 0xE5, 0x87, 0xA0, 0x03, 0xE5, + 0x87, 0xB5, 0x03, 0xE5, 0x88, 0x80, 0x03, 0xE5, + 0x8A, 0x9B, 0x03, 0xE5, 0x8B, 0xB9, 0x03, 0xE5, + 0x8C, 0x95, 0x03, 0xE5, 0x8C, 0x9A, 0x03, 0xE5, + // Bytes 1880 - 18bf + 0x8C, 0xB8, 0x03, 0xE5, 0x8D, 0x81, 0x03, 0xE5, + 0x8D, 0x9C, 0x03, 0xE5, 0x8D, 0xA9, 0x03, 0xE5, + 0x8E, 0x82, 0x03, 0xE5, 0x8E, 0xB6, 0x03, 0xE5, + 0x8F, 0x88, 0x03, 0xE5, 0x8F, 0xA3, 0x03, 0xE5, + 0x9B, 0x97, 0x03, 0xE5, 0x9C, 0x9F, 0x03, 0xE5, + 0xA3, 0xAB, 0x03, 0xE5, 0xA4, 0x82, 0x03, 0xE5, + 0xA4, 0x8A, 0x03, 0xE5, 0xA4, 0x95, 0x03, 0xE5, + 0xA4, 0xA7, 0x03, 0xE5, 0xA5, 0xB3, 0x03, 0xE5, + // Bytes 18c0 - 18ff + 0xAD, 0x90, 0x03, 0xE5, 0xAE, 0x80, 0x03, 0xE5, + 0xAF, 0xB8, 0x03, 0xE5, 0xB0, 0x8F, 0x03, 0xE5, + 0xB0, 0xA2, 0x03, 0xE5, 0xB0, 0xB8, 0x03, 0xE5, + 0xB1, 0xAE, 0x03, 0xE5, 0xB1, 0xB1, 0x03, 0xE5, + 0xB7, 0x9B, 0x03, 0xE5, 0xB7, 0xA5, 0x03, 0xE5, + 0xB7, 0xB1, 0x03, 0xE5, 0xB7, 0xBE, 0x03, 0xE5, + 0xB9, 0xB2, 0x03, 0xE5, 0xB9, 0xBA, 0x03, 0xE5, + 0xB9, 0xBF, 0x03, 0xE5, 0xBB, 0xB4, 0x03, 0xE5, + // Bytes 1900 - 193f + 0xBB, 0xBE, 0x03, 0xE5, 0xBC, 0x8B, 0x03, 0xE5, + 0xBC, 0x93, 0x03, 0xE5, 0xBD, 0x90, 0x03, 0xE5, + 0xBD, 0xA1, 0x03, 0xE5, 0xBD, 0xB3, 0x03, 0xE5, + 0xBF, 0x83, 0x03, 0xE6, 0x88, 0x88, 0x03, 0xE6, + 0x88, 0xB6, 0x03, 0xE6, 0x89, 0x8B, 0x03, 0xE6, + 0x94, 0xAF, 0x03, 0xE6, 0x94, 0xB4, 0x03, 0xE6, + 0x96, 0x87, 0x03, 0xE6, 0x96, 0x97, 0x03, 0xE6, + 0x96, 0xA4, 0x03, 0xE6, 0x96, 0xB9, 0x03, 0xE6, + // Bytes 1940 - 197f + 0x97, 0xA0, 0x03, 0xE6, 0x97, 0xA5, 0x03, 0xE6, + 0x9B, 0xB0, 0x03, 0xE6, 0x9C, 0x88, 0x03, 0xE6, + 0x9C, 0xA8, 0x03, 0xE6, 0xAC, 0xA0, 0x03, 0xE6, + 0xAD, 0xA2, 0x03, 0xE6, 0xAD, 0xB9, 0x03, 0xE6, + 0xAE, 0xB3, 0x03, 0xE6, 0xAF, 0x8B, 0x03, 0xE6, + 0xAF, 0x94, 0x03, 0xE6, 0xAF, 0x9B, 0x03, 0xE6, + 0xB0, 0x8F, 0x03, 0xE6, 0xB0, 0x94, 0x03, 0xE6, + 0xB0, 0xB4, 0x03, 0xE7, 0x81, 0xAB, 0x03, 0xE7, + // Bytes 1980 - 19bf + 0x88, 0xAA, 0x03, 0xE7, 0x88, 0xB6, 0x03, 0xE7, + 0x88, 0xBB, 0x03, 0xE7, 0x88, 0xBF, 0x03, 0xE7, + 0x89, 0x87, 0x03, 0xE7, 0x89, 0x99, 0x03, 0xE7, + 0x89, 0x9B, 0x03, 0xE7, 0x8A, 0xAC, 0x03, 0xE7, + 0x8E, 0x84, 0x03, 0xE7, 0x8E, 0x89, 0x03, 0xE7, + 0x93, 0x9C, 0x03, 0xE7, 0x93, 0xA6, 0x03, 0xE7, + 0x94, 0x98, 0x03, 0xE7, 0x94, 0x9F, 0x03, 0xE7, + 0x94, 0xA8, 0x03, 0xE7, 0x94, 0xB0, 0x03, 0xE7, + // Bytes 19c0 - 19ff + 0x96, 0x8B, 0x03, 0xE7, 0x96, 0x92, 0x03, 0xE7, + 0x99, 0xB6, 0x03, 0xE7, 0x99, 0xBD, 0x03, 0xE7, + 0x9A, 0xAE, 0x03, 0xE7, 0x9A, 0xBF, 0x03, 0xE7, + 0x9B, 0xAE, 0x03, 0xE7, 0x9F, 0x9B, 0x03, 0xE7, + 0x9F, 0xA2, 0x03, 0xE7, 0x9F, 0xB3, 0x03, 0xE7, + 0xA4, 0xBA, 0x03, 0xE7, 0xA6, 0xB8, 0x03, 0xE7, + 0xA6, 0xBE, 0x03, 0xE7, 0xA9, 0xB4, 0x03, 0xE7, + 0xAB, 0x8B, 0x03, 0xE7, 0xAB, 0xB9, 0x03, 0xE7, + // Bytes 1a00 - 1a3f + 0xB1, 0xB3, 0x03, 0xE7, 0xB3, 0xB8, 0x03, 0xE7, + 0xBC, 0xB6, 0x03, 0xE7, 0xBD, 0x91, 0x03, 0xE7, + 0xBE, 0x8A, 0x03, 0xE7, 0xBE, 0xBD, 0x03, 0xE8, + 0x80, 0x81, 0x03, 0xE8, 0x80, 0x8C, 0x03, 0xE8, + 0x80, 0x92, 0x03, 0xE8, 0x80, 0xB3, 0x03, 0xE8, + 0x81, 0xBF, 0x03, 0xE8, 0x82, 0x89, 0x03, 0xE8, + 0x87, 0xA3, 0x03, 0xE8, 0x87, 0xAA, 0x03, 0xE8, + 0x87, 0xB3, 0x03, 0xE8, 0x87, 0xBC, 0x03, 0xE8, + // Bytes 1a40 - 1a7f + 0x88, 0x8C, 0x03, 0xE8, 0x88, 0x9B, 0x03, 0xE8, + 0x88, 0x9F, 0x03, 0xE8, 0x89, 0xAE, 0x03, 0xE8, + 0x89, 0xB2, 0x03, 0xE8, 0x89, 0xB8, 0x03, 0xE8, + 0x99, 0x8D, 0x03, 0xE8, 0x99, 0xAB, 0x03, 0xE8, + 0xA1, 0x80, 0x03, 0xE8, 0xA1, 0x8C, 0x03, 0xE8, + 0xA1, 0xA3, 0x03, 0xE8, 0xA5, 0xBE, 0x03, 0xE8, + 0xA6, 0x8B, 0x03, 0xE8, 0xA7, 0x92, 0x03, 0xE8, + 0xA8, 0x80, 0x03, 0xE8, 0xB0, 0xB7, 0x03, 0xE8, + // Bytes 1a80 - 1abf + 0xB1, 0x86, 0x03, 0xE8, 0xB1, 0x95, 0x03, 0xE8, + 0xB1, 0xB8, 0x03, 0xE8, 0xB2, 0x9D, 0x03, 0xE8, + 0xB5, 0xA4, 0x03, 0xE8, 0xB5, 0xB0, 0x03, 0xE8, + 0xB6, 0xB3, 0x03, 0xE8, 0xBA, 0xAB, 0x03, 0xE8, + 0xBB, 0x8A, 0x03, 0xE8, 0xBE, 0x9B, 0x03, 0xE8, + 0xBE, 0xB0, 0x03, 0xE8, 0xBE, 0xB5, 0x03, 0xE9, + 0x82, 0x91, 0x03, 0xE9, 0x85, 0x89, 0x03, 0xE9, + 0x87, 0x86, 0x03, 0xE9, 0x87, 0x8C, 0x03, 0xE9, + // Bytes 1ac0 - 1aff + 0x87, 0x91, 0x03, 0xE9, 0x95, 0xB7, 0x03, 0xE9, + 0x96, 0x80, 0x03, 0xE9, 0x98, 0x9C, 0x03, 0xE9, + 0x9A, 0xB6, 0x03, 0xE9, 0x9A, 0xB9, 0x03, 0xE9, + 0x9B, 0xA8, 0x03, 0xE9, 0x9D, 0x91, 0x03, 0xE9, + 0x9D, 0x9E, 0x03, 0xE9, 0x9D, 0xA2, 0x03, 0xE9, + 0x9D, 0xA9, 0x03, 0xE9, 0x9F, 0x8B, 0x03, 0xE9, + 0x9F, 0xAD, 0x03, 0xE9, 0x9F, 0xB3, 0x03, 0xE9, + 0xA0, 0x81, 0x03, 0xE9, 0xA2, 0xA8, 0x03, 0xE9, + // Bytes 1b00 - 1b3f + 0xA3, 0x9B, 0x03, 0xE9, 0xA3, 0x9F, 0x03, 0xE9, + 0xA6, 0x96, 0x03, 0xE9, 0xA6, 0x99, 0x03, 0xE9, + 0xA6, 0xAC, 0x03, 0xE9, 0xAA, 0xA8, 0x03, 0xE9, + 0xAB, 0x98, 0x03, 0xE9, 0xAB, 0x9F, 0x03, 0xE9, + 0xAC, 0xA5, 0x03, 0xE9, 0xAC, 0xAF, 0x03, 0xE9, + 0xAC, 0xB2, 0x03, 0xE9, 0xAC, 0xBC, 0x03, 0xE9, + 0xAD, 0x9A, 0x03, 0xE9, 0xB3, 0xA5, 0x03, 0xE9, + 0xB9, 0xB5, 0x03, 0xE9, 0xB9, 0xBF, 0x03, 0xE9, + // Bytes 1b40 - 1b7f + 0xBA, 0xA5, 0x03, 0xE9, 0xBA, 0xBB, 0x03, 0xE9, + 0xBB, 0x83, 0x03, 0xE9, 0xBB, 0x8D, 0x03, 0xE9, + 0xBB, 0x91, 0x03, 0xE9, 0xBB, 0xB9, 0x03, 0xE9, + 0xBB, 0xBD, 0x03, 0xE9, 0xBC, 0x8E, 0x03, 0xE9, + 0xBC, 0x93, 0x03, 0xE9, 0xBC, 0xA0, 0x03, 0xE9, + 0xBC, 0xBB, 0x03, 0xE9, 0xBD, 0x8A, 0x03, 0xE9, + 0xBD, 0x92, 0x03, 0xE9, 0xBE, 0x8D, 0x03, 0xE9, + 0xBE, 0x9C, 0x03, 0xE9, 0xBE, 0xA0, 0x03, 0xE3, + // Bytes 1b80 - 1bbf + 0x80, 0x92, 0x03, 0xE5, 0x8D, 0x84, 0x03, 0xE5, + 0x8D, 0x85, 0x06, 0xE3, 0x81, 0x8B, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x81, 0x8D, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x81, 0x8F, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x81, 0x91, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x81, 0x93, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, + 0x95, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0x97, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0x99, 0xE3, + // Bytes 1bc0 - 1bff + 0x82, 0x99, 0x06, 0xE3, 0x81, 0x9B, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x81, 0x9D, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x81, 0x9F, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x81, 0xA4, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, + 0xA6, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0xA8, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0xAF, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x81, 0xAF, 0xE3, 0x82, + // Bytes 1c00 - 1c3f + 0x9A, 0x06, 0xE3, 0x81, 0xB2, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x81, 0xB2, 0xE3, 0x82, 0x9A, 0x06, + 0xE3, 0x81, 0xB5, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x81, 0xB5, 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x81, + 0xB8, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x81, 0xB8, + 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x81, 0xBB, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x81, 0xBB, 0xE3, 0x82, + 0x9A, 0x06, 0xE3, 0x81, 0x86, 0xE3, 0x82, 0x99, + // Bytes 1c40 - 1c7f + 0x04, 0x20, 0xE3, 0x82, 0x99, 0x04, 0x20, 0xE3, + 0x82, 0x9A, 0x06, 0xE3, 0x82, 0x9D, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x82, 0x88, 0xE3, 0x82, 0x8A, + 0x06, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x82, + 0xB1, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x82, 0xB3, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x82, 0xB5, 0xE3, + // Bytes 1c80 - 1cbf + 0x82, 0x99, 0x06, 0xE3, 0x82, 0xB7, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x82, 0xB9, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x82, 0xBB, 0xE3, 0x82, 0x99, 0x06, + 0xE3, 0x82, 0xBD, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x82, 0xBF, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, + 0x81, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0x84, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0x86, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x83, 0x88, 0xE3, 0x82, + // Bytes 1cc0 - 1cff + 0x99, 0x06, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, + 0x06, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0x06, + 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x83, + 0x95, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0x95, + 0xE3, 0x82, 0x9A, 0x06, 0xE3, 0x83, 0x98, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x83, 0x98, 0xE3, 0x82, + 0x9A, 0x06, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x99, + // Bytes 1d00 - 1d3f + 0x06, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0x06, + 0xE3, 0x82, 0xA6, 0xE3, 0x82, 0x99, 0x06, 0xE3, + 0x83, 0xAF, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, + 0xB0, 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0xB1, + 0xE3, 0x82, 0x99, 0x06, 0xE3, 0x83, 0xB2, 0xE3, + 0x82, 0x99, 0x06, 0xE3, 0x83, 0xBD, 0xE3, 0x82, + 0x99, 0x06, 0xE3, 0x82, 0xB3, 0xE3, 0x83, 0x88, + 0x03, 0xE1, 0x84, 0x80, 0x03, 0xE1, 0x84, 0x81, + // Bytes 1d40 - 1d7f + 0x03, 0xE1, 0x86, 0xAA, 0x03, 0xE1, 0x84, 0x82, + 0x03, 0xE1, 0x86, 0xAC, 0x03, 0xE1, 0x86, 0xAD, + 0x03, 0xE1, 0x84, 0x83, 0x03, 0xE1, 0x84, 0x84, + 0x03, 0xE1, 0x84, 0x85, 0x03, 0xE1, 0x86, 0xB0, + 0x03, 0xE1, 0x86, 0xB1, 0x03, 0xE1, 0x86, 0xB2, + 0x03, 0xE1, 0x86, 0xB3, 0x03, 0xE1, 0x86, 0xB4, + 0x03, 0xE1, 0x86, 0xB5, 0x03, 0xE1, 0x84, 0x9A, + 0x03, 0xE1, 0x84, 0x86, 0x03, 0xE1, 0x84, 0x87, + // Bytes 1d80 - 1dbf + 0x03, 0xE1, 0x84, 0x88, 0x03, 0xE1, 0x84, 0xA1, + 0x03, 0xE1, 0x84, 0x89, 0x03, 0xE1, 0x84, 0x8A, + 0x03, 0xE1, 0x84, 0x8B, 0x03, 0xE1, 0x84, 0x8C, + 0x03, 0xE1, 0x84, 0x8D, 0x03, 0xE1, 0x84, 0x8E, + 0x03, 0xE1, 0x84, 0x8F, 0x03, 0xE1, 0x84, 0x90, + 0x03, 0xE1, 0x84, 0x91, 0x03, 0xE1, 0x84, 0x92, + 0x03, 0xE1, 0x85, 0xA1, 0x03, 0xE1, 0x85, 0xA2, + 0x03, 0xE1, 0x85, 0xA3, 0x03, 0xE1, 0x85, 0xA4, + // Bytes 1dc0 - 1dff + 0x03, 0xE1, 0x85, 0xA5, 0x03, 0xE1, 0x85, 0xA6, + 0x03, 0xE1, 0x85, 0xA7, 0x03, 0xE1, 0x85, 0xA8, + 0x03, 0xE1, 0x85, 0xA9, 0x03, 0xE1, 0x85, 0xAA, + 0x03, 0xE1, 0x85, 0xAB, 0x03, 0xE1, 0x85, 0xAC, + 0x03, 0xE1, 0x85, 0xAD, 0x03, 0xE1, 0x85, 0xAE, + 0x03, 0xE1, 0x85, 0xAF, 0x03, 0xE1, 0x85, 0xB0, + 0x03, 0xE1, 0x85, 0xB1, 0x03, 0xE1, 0x85, 0xB2, + 0x03, 0xE1, 0x85, 0xB3, 0x03, 0xE1, 0x85, 0xB4, + // Bytes 1e00 - 1e3f + 0x03, 0xE1, 0x85, 0xB5, 0x03, 0xE1, 0x85, 0xA0, + 0x03, 0xE1, 0x84, 0x94, 0x03, 0xE1, 0x84, 0x95, + 0x03, 0xE1, 0x87, 0x87, 0x03, 0xE1, 0x87, 0x88, + 0x03, 0xE1, 0x87, 0x8C, 0x03, 0xE1, 0x87, 0x8E, + 0x03, 0xE1, 0x87, 0x93, 0x03, 0xE1, 0x87, 0x97, + 0x03, 0xE1, 0x87, 0x99, 0x03, 0xE1, 0x84, 0x9C, + 0x03, 0xE1, 0x87, 0x9D, 0x03, 0xE1, 0x87, 0x9F, + 0x03, 0xE1, 0x84, 0x9D, 0x03, 0xE1, 0x84, 0x9E, + // Bytes 1e40 - 1e7f + 0x03, 0xE1, 0x84, 0xA0, 0x03, 0xE1, 0x84, 0xA2, + 0x03, 0xE1, 0x84, 0xA3, 0x03, 0xE1, 0x84, 0xA7, + 0x03, 0xE1, 0x84, 0xA9, 0x03, 0xE1, 0x84, 0xAB, + 0x03, 0xE1, 0x84, 0xAC, 0x03, 0xE1, 0x84, 0xAD, + 0x03, 0xE1, 0x84, 0xAE, 0x03, 0xE1, 0x84, 0xAF, + 0x03, 0xE1, 0x84, 0xB2, 0x03, 0xE1, 0x84, 0xB6, + 0x03, 0xE1, 0x85, 0x80, 0x03, 0xE1, 0x85, 0x87, + 0x03, 0xE1, 0x85, 0x8C, 0x03, 0xE1, 0x87, 0xB1, + // Bytes 1e80 - 1ebf + 0x03, 0xE1, 0x87, 0xB2, 0x03, 0xE1, 0x85, 0x97, + 0x03, 0xE1, 0x85, 0x98, 0x03, 0xE1, 0x85, 0x99, + 0x03, 0xE1, 0x86, 0x84, 0x03, 0xE1, 0x86, 0x85, + 0x03, 0xE1, 0x86, 0x88, 0x03, 0xE1, 0x86, 0x91, + 0x03, 0xE1, 0x86, 0x92, 0x03, 0xE1, 0x86, 0x94, + 0x03, 0xE1, 0x86, 0x9E, 0x03, 0xE1, 0x86, 0xA1, + 0x03, 0xE4, 0xB8, 0x89, 0x03, 0xE5, 0x9B, 0x9B, + 0x03, 0xE4, 0xB8, 0x8A, 0x03, 0xE4, 0xB8, 0xAD, + // Bytes 1ec0 - 1eff + 0x03, 0xE4, 0xB8, 0x8B, 0x03, 0xE7, 0x94, 0xB2, + 0x03, 0xE4, 0xB8, 0x99, 0x03, 0xE4, 0xB8, 0x81, + 0x03, 0xE5, 0xA4, 0xA9, 0x03, 0xE5, 0x9C, 0xB0, + 0x05, 0x28, 0xE1, 0x84, 0x80, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x82, 0x29, 0x05, 0x28, 0xE1, 0x84, + 0x83, 0x29, 0x05, 0x28, 0xE1, 0x84, 0x85, 0x29, + 0x05, 0x28, 0xE1, 0x84, 0x86, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x87, 0x29, 0x05, 0x28, 0xE1, 0x84, + // Bytes 1f00 - 1f3f + 0x89, 0x29, 0x05, 0x28, 0xE1, 0x84, 0x8B, 0x29, + 0x05, 0x28, 0xE1, 0x84, 0x8C, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x8E, 0x29, 0x05, 0x28, 0xE1, 0x84, + 0x8F, 0x29, 0x05, 0x28, 0xE1, 0x84, 0x90, 0x29, + 0x05, 0x28, 0xE1, 0x84, 0x91, 0x29, 0x05, 0x28, + 0xE1, 0x84, 0x92, 0x29, 0x08, 0x28, 0xE1, 0x84, + 0x80, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, + 0x84, 0x82, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, + // Bytes 1f40 - 1f7f + 0xE1, 0x84, 0x83, 0xE1, 0x85, 0xA1, 0x29, 0x08, + 0x28, 0xE1, 0x84, 0x85, 0xE1, 0x85, 0xA1, 0x29, + 0x08, 0x28, 0xE1, 0x84, 0x86, 0xE1, 0x85, 0xA1, + 0x29, 0x08, 0x28, 0xE1, 0x84, 0x87, 0xE1, 0x85, + 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, 0x89, 0xE1, + 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, 0x8B, + 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, + 0x8C, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, 0xE1, + // Bytes 1f80 - 1fbf + 0x84, 0x8E, 0xE1, 0x85, 0xA1, 0x29, 0x08, 0x28, + 0xE1, 0x84, 0x8F, 0xE1, 0x85, 0xA1, 0x29, 0x08, + 0x28, 0xE1, 0x84, 0x90, 0xE1, 0x85, 0xA1, 0x29, + 0x08, 0x28, 0xE1, 0x84, 0x91, 0xE1, 0x85, 0xA1, + 0x29, 0x08, 0x28, 0xE1, 0x84, 0x92, 0xE1, 0x85, + 0xA1, 0x29, 0x08, 0x28, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xAE, 0x29, 0x11, 0x28, 0xE1, 0x84, 0x8B, + 0xE1, 0x85, 0xA9, 0xE1, 0x84, 0x8C, 0xE1, 0x85, + // Bytes 1fc0 - 1fff + 0xA5, 0xE1, 0x86, 0xAB, 0x29, 0x0E, 0x28, 0xE1, + 0x84, 0x8B, 0xE1, 0x85, 0xA9, 0xE1, 0x84, 0x92, + 0xE1, 0x85, 0xAE, 0x29, 0x05, 0x28, 0xE4, 0xB8, + 0x80, 0x29, 0x05, 0x28, 0xE4, 0xBA, 0x8C, 0x29, + 0x05, 0x28, 0xE4, 0xB8, 0x89, 0x29, 0x05, 0x28, + 0xE5, 0x9B, 0x9B, 0x29, 0x05, 0x28, 0xE4, 0xBA, + 0x94, 0x29, 0x05, 0x28, 0xE5, 0x85, 0xAD, 0x29, + 0x05, 0x28, 0xE4, 0xB8, 0x83, 0x29, 0x05, 0x28, + // Bytes 2000 - 203f + 0xE5, 0x85, 0xAB, 0x29, 0x05, 0x28, 0xE4, 0xB9, + 0x9D, 0x29, 0x05, 0x28, 0xE5, 0x8D, 0x81, 0x29, + 0x05, 0x28, 0xE6, 0x9C, 0x88, 0x29, 0x05, 0x28, + 0xE7, 0x81, 0xAB, 0x29, 0x05, 0x28, 0xE6, 0xB0, + 0xB4, 0x29, 0x05, 0x28, 0xE6, 0x9C, 0xA8, 0x29, + 0x05, 0x28, 0xE9, 0x87, 0x91, 0x29, 0x05, 0x28, + 0xE5, 0x9C, 0x9F, 0x29, 0x05, 0x28, 0xE6, 0x97, + 0xA5, 0x29, 0x05, 0x28, 0xE6, 0xA0, 0xAA, 0x29, + // Bytes 2040 - 207f + 0x05, 0x28, 0xE6, 0x9C, 0x89, 0x29, 0x05, 0x28, + 0xE7, 0xA4, 0xBE, 0x29, 0x05, 0x28, 0xE5, 0x90, + 0x8D, 0x29, 0x05, 0x28, 0xE7, 0x89, 0xB9, 0x29, + 0x05, 0x28, 0xE8, 0xB2, 0xA1, 0x29, 0x05, 0x28, + 0xE7, 0xA5, 0x9D, 0x29, 0x05, 0x28, 0xE5, 0x8A, + 0xB4, 0x29, 0x05, 0x28, 0xE4, 0xBB, 0xA3, 0x29, + 0x05, 0x28, 0xE5, 0x91, 0xBC, 0x29, 0x05, 0x28, + 0xE5, 0xAD, 0xA6, 0x29, 0x05, 0x28, 0xE7, 0x9B, + // Bytes 2080 - 20bf + 0xA3, 0x29, 0x05, 0x28, 0xE4, 0xBC, 0x81, 0x29, + 0x05, 0x28, 0xE8, 0xB3, 0x87, 0x29, 0x05, 0x28, + 0xE5, 0x8D, 0x94, 0x29, 0x05, 0x28, 0xE7, 0xA5, + 0xAD, 0x29, 0x05, 0x28, 0xE4, 0xBC, 0x91, 0x29, + 0x05, 0x28, 0xE8, 0x87, 0xAA, 0x29, 0x05, 0x28, + 0xE8, 0x87, 0xB3, 0x29, 0x03, 0xE5, 0x95, 0x8F, + 0x03, 0xE5, 0xB9, 0xBC, 0x03, 0xE7, 0xAE, 0x8F, + 0x03, 0x50, 0x54, 0x45, 0x02, 0x32, 0x31, 0x02, + // Bytes 20c0 - 20ff + 0x32, 0x32, 0x02, 0x32, 0x33, 0x02, 0x32, 0x34, + 0x02, 0x32, 0x35, 0x02, 0x32, 0x36, 0x02, 0x32, + 0x37, 0x02, 0x32, 0x38, 0x02, 0x32, 0x39, 0x02, + 0x33, 0x30, 0x02, 0x33, 0x31, 0x02, 0x33, 0x32, + 0x02, 0x33, 0x33, 0x02, 0x33, 0x34, 0x02, 0x33, + 0x35, 0x06, 0xE1, 0x84, 0x80, 0xE1, 0x85, 0xA1, + 0x06, 0xE1, 0x84, 0x82, 0xE1, 0x85, 0xA1, 0x06, + 0xE1, 0x84, 0x83, 0xE1, 0x85, 0xA1, 0x06, 0xE1, + // Bytes 2100 - 213f + 0x84, 0x85, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, + 0x86, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x87, + 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x89, 0xE1, + 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x8B, 0xE1, 0x85, + 0xA1, 0x06, 0xE1, 0x84, 0x8C, 0xE1, 0x85, 0xA1, + 0x06, 0xE1, 0x84, 0x8E, 0xE1, 0x85, 0xA1, 0x06, + 0xE1, 0x84, 0x8F, 0xE1, 0x85, 0xA1, 0x06, 0xE1, + 0x84, 0x90, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, + // Bytes 2140 - 217f + 0x91, 0xE1, 0x85, 0xA1, 0x06, 0xE1, 0x84, 0x92, + 0xE1, 0x85, 0xA1, 0x0F, 0xE1, 0x84, 0x8E, 0xE1, + 0x85, 0xA1, 0xE1, 0x86, 0xB7, 0xE1, 0x84, 0x80, + 0xE1, 0x85, 0xA9, 0x0C, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xAE, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xB4, + 0x06, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xAE, 0x03, + 0xE4, 0xBA, 0x94, 0x03, 0xE5, 0x85, 0xAD, 0x03, + 0xE4, 0xB8, 0x83, 0x03, 0xE4, 0xB9, 0x9D, 0x03, + // Bytes 2180 - 21bf + 0xE6, 0xA0, 0xAA, 0x03, 0xE6, 0x9C, 0x89, 0x03, + 0xE7, 0xA4, 0xBE, 0x03, 0xE5, 0x90, 0x8D, 0x03, + 0xE7, 0x89, 0xB9, 0x03, 0xE8, 0xB2, 0xA1, 0x03, + 0xE7, 0xA5, 0x9D, 0x03, 0xE5, 0x8A, 0xB4, 0x03, + 0xE7, 0xA7, 0x98, 0x03, 0xE7, 0x94, 0xB7, 0x03, + 0xE9, 0x81, 0xA9, 0x03, 0xE5, 0x84, 0xAA, 0x03, + 0xE5, 0x8D, 0xB0, 0x03, 0xE6, 0xB3, 0xA8, 0x03, + 0xE9, 0xA0, 0x85, 0x03, 0xE4, 0xBC, 0x91, 0x03, + // Bytes 21c0 - 21ff + 0xE5, 0x86, 0x99, 0x03, 0xE6, 0xAD, 0xA3, 0x03, + 0xE5, 0xB7, 0xA6, 0x03, 0xE5, 0x8F, 0xB3, 0x03, + 0xE5, 0x8C, 0xBB, 0x03, 0xE5, 0xAE, 0x97, 0x03, + 0xE5, 0xAD, 0xA6, 0x03, 0xE7, 0x9B, 0xA3, 0x03, + 0xE4, 0xBC, 0x81, 0x03, 0xE8, 0xB3, 0x87, 0x03, + 0xE5, 0x8D, 0x94, 0x03, 0xE5, 0xA4, 0x9C, 0x02, + 0x33, 0x36, 0x02, 0x33, 0x37, 0x02, 0x33, 0x38, + 0x02, 0x33, 0x39, 0x02, 0x34, 0x30, 0x02, 0x34, + // Bytes 2200 - 223f + 0x31, 0x02, 0x34, 0x32, 0x02, 0x34, 0x33, 0x02, + 0x34, 0x34, 0x02, 0x34, 0x35, 0x02, 0x34, 0x36, + 0x02, 0x34, 0x37, 0x02, 0x34, 0x38, 0x02, 0x34, + 0x39, 0x02, 0x35, 0x30, 0x04, 0x31, 0xE6, 0x9C, + 0x88, 0x04, 0x32, 0xE6, 0x9C, 0x88, 0x04, 0x33, + 0xE6, 0x9C, 0x88, 0x04, 0x34, 0xE6, 0x9C, 0x88, + 0x04, 0x35, 0xE6, 0x9C, 0x88, 0x04, 0x36, 0xE6, + 0x9C, 0x88, 0x04, 0x37, 0xE6, 0x9C, 0x88, 0x04, + // Bytes 2240 - 227f + 0x38, 0xE6, 0x9C, 0x88, 0x04, 0x39, 0xE6, 0x9C, + 0x88, 0x05, 0x31, 0x30, 0xE6, 0x9C, 0x88, 0x05, + 0x31, 0x31, 0xE6, 0x9C, 0x88, 0x05, 0x31, 0x32, + 0xE6, 0x9C, 0x88, 0x02, 0x48, 0x67, 0x03, 0x65, + 0x72, 0x67, 0x02, 0x65, 0x56, 0x03, 0x4C, 0x54, + 0x44, 0x03, 0xE3, 0x82, 0xA2, 0x03, 0xE3, 0x82, + 0xA4, 0x03, 0xE3, 0x82, 0xA6, 0x03, 0xE3, 0x82, + 0xA8, 0x03, 0xE3, 0x82, 0xAA, 0x03, 0xE3, 0x82, + // Bytes 2280 - 22bf + 0xAB, 0x03, 0xE3, 0x82, 0xAD, 0x03, 0xE3, 0x82, + 0xAF, 0x03, 0xE3, 0x82, 0xB1, 0x03, 0xE3, 0x82, + 0xB3, 0x03, 0xE3, 0x82, 0xB5, 0x03, 0xE3, 0x82, + 0xB7, 0x03, 0xE3, 0x82, 0xB9, 0x03, 0xE3, 0x82, + 0xBB, 0x03, 0xE3, 0x82, 0xBD, 0x03, 0xE3, 0x82, + 0xBF, 0x03, 0xE3, 0x83, 0x81, 0x03, 0xE3, 0x83, + 0x84, 0x03, 0xE3, 0x83, 0x86, 0x03, 0xE3, 0x83, + 0x88, 0x03, 0xE3, 0x83, 0x8A, 0x03, 0xE3, 0x83, + // Bytes 22c0 - 22ff + 0x8B, 0x03, 0xE3, 0x83, 0x8C, 0x03, 0xE3, 0x83, + 0x8D, 0x03, 0xE3, 0x83, 0x8E, 0x03, 0xE3, 0x83, + 0x8F, 0x03, 0xE3, 0x83, 0x92, 0x03, 0xE3, 0x83, + 0x95, 0x03, 0xE3, 0x83, 0x98, 0x03, 0xE3, 0x83, + 0x9B, 0x03, 0xE3, 0x83, 0x9E, 0x03, 0xE3, 0x83, + 0x9F, 0x03, 0xE3, 0x83, 0xA0, 0x03, 0xE3, 0x83, + 0xA1, 0x03, 0xE3, 0x83, 0xA2, 0x03, 0xE3, 0x83, + 0xA4, 0x03, 0xE3, 0x83, 0xA6, 0x03, 0xE3, 0x83, + // Bytes 2300 - 233f + 0xA8, 0x03, 0xE3, 0x83, 0xA9, 0x03, 0xE3, 0x83, + 0xAA, 0x03, 0xE3, 0x83, 0xAB, 0x03, 0xE3, 0x83, + 0xAC, 0x03, 0xE3, 0x83, 0xAD, 0x03, 0xE3, 0x83, + 0xAF, 0x03, 0xE3, 0x83, 0xB0, 0x03, 0xE3, 0x83, + 0xB1, 0x03, 0xE3, 0x83, 0xB2, 0x0F, 0xE3, 0x82, + 0xA2, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0x88, 0x0C, 0xE3, 0x82, + 0xA2, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x95, 0xE3, + // Bytes 2340 - 237f + 0x82, 0xA1, 0x0F, 0xE3, 0x82, 0xA2, 0xE3, 0x83, + 0xB3, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, + 0x82, 0xA2, 0x09, 0xE3, 0x82, 0xA2, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAB, 0x0F, 0xE3, 0x82, 0xA4, + 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0xB3, 0xE3, 0x82, + 0xAF, 0xE3, 0x82, 0x99, 0x09, 0xE3, 0x82, 0xA4, + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x81, 0x09, 0xE3, + 0x82, 0xA6, 0xE3, 0x82, 0xA9, 0xE3, 0x83, 0xB3, + // Bytes 2380 - 23bf + 0x12, 0xE3, 0x82, 0xA8, 0xE3, 0x82, 0xB9, 0xE3, + 0x82, 0xAF, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, + 0xE3, 0x82, 0x99, 0x0C, 0xE3, 0x82, 0xA8, 0xE3, + 0x83, 0xBC, 0xE3, 0x82, 0xAB, 0xE3, 0x83, 0xBC, + 0x09, 0xE3, 0x82, 0xAA, 0xE3, 0x83, 0xB3, 0xE3, + 0x82, 0xB9, 0x09, 0xE3, 0x82, 0xAA, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xA0, 0x09, 0xE3, 0x82, 0xAB, + 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xAA, 0x0C, 0xE3, + // Bytes 23c0 - 23ff + 0x82, 0xAB, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0x83, + 0xE3, 0x83, 0x88, 0x0C, 0xE3, 0x82, 0xAB, 0xE3, + 0x83, 0xAD, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xBC, + 0x0C, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0xE3, + 0x83, 0xAD, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, 0x82, + 0xAB, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xB3, 0xE3, + 0x83, 0x9E, 0x0C, 0xE3, 0x82, 0xAD, 0xE3, 0x82, + 0x99, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0C, + // Bytes 2400 - 243f + 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0x8B, 0xE3, 0x83, 0xBC, 0x0C, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xA5, 0xE3, 0x83, 0xAA, 0xE3, 0x83, + 0xBC, 0x12, 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0xAB, 0xE3, 0x82, 0xBF, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xBC, 0x06, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xAD, 0x12, 0xE3, 0x82, 0xAD, 0xE3, + 0x83, 0xAD, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, + // Bytes 2440 - 247f + 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0x12, 0xE3, + 0x82, 0xAD, 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xA1, + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x0F, 0xE3, 0x82, 0xAD, 0xE3, 0x83, 0xAD, + 0xE3, 0x83, 0xAF, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x88, 0x0C, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0x12, 0xE3, + 0x82, 0xAF, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xA9, + // Bytes 2480 - 24bf + 0xE3, 0x83, 0xA0, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xB3, 0x12, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, + 0xE3, 0x82, 0xBB, 0xE3, 0x82, 0x99, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0xAD, 0x0C, 0xE3, 0x82, 0xAF, + 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + 0x8D, 0x09, 0xE3, 0x82, 0xB1, 0xE3, 0x83, 0xBC, + 0xE3, 0x82, 0xB9, 0x09, 0xE3, 0x82, 0xB3, 0xE3, + 0x83, 0xAB, 0xE3, 0x83, 0x8A, 0x0C, 0xE3, 0x82, + // Bytes 24c0 - 24ff + 0xB3, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x9B, 0xE3, + 0x82, 0x9A, 0x0C, 0xE3, 0x82, 0xB5, 0xE3, 0x82, + 0xA4, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, 0x0F, + 0xE3, 0x82, 0xB5, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + 0x81, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xA0, 0x0F, + 0xE3, 0x82, 0xB7, 0xE3, 0x83, 0xAA, 0xE3, 0x83, + 0xB3, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x09, + 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + // Bytes 2500 - 253f + 0x81, 0x09, 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, + 0xE3, 0x83, 0x88, 0x0C, 0xE3, 0x82, 0xBF, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xB9, + 0x09, 0xE3, 0x83, 0x86, 0xE3, 0x82, 0x99, 0xE3, + 0x82, 0xB7, 0x09, 0xE3, 0x83, 0x88, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAB, 0x06, 0xE3, 0x83, 0x88, + 0xE3, 0x83, 0xB3, 0x06, 0xE3, 0x83, 0x8A, 0xE3, + 0x83, 0x8E, 0x09, 0xE3, 0x83, 0x8E, 0xE3, 0x83, + // Bytes 2540 - 257f + 0x83, 0xE3, 0x83, 0x88, 0x09, 0xE3, 0x83, 0x8F, + 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0x84, 0x12, 0xE3, + 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, + 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + 0x88, 0x0C, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x84, 0x0F, 0xE3, + 0x83, 0x8F, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, + 0xE3, 0x83, 0xAC, 0xE3, 0x83, 0xAB, 0x12, 0xE3, + // Bytes 2580 - 25bf + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xA2, + 0xE3, 0x82, 0xB9, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x0C, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, + 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAB, 0x09, 0xE3, + 0x83, 0x92, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xB3, + 0x09, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x99, 0xE3, + 0x83, 0xAB, 0x12, 0xE3, 0x83, 0x95, 0xE3, 0x82, + 0xA1, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0x83, 0xE3, + // Bytes 25c0 - 25ff + 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0C, 0xE3, 0x83, + 0x95, 0xE3, 0x82, 0xA3, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0x88, 0x12, 0xE3, 0x83, 0x95, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0x83, 0xE3, 0x82, 0xB7, 0xE3, + 0x82, 0xA7, 0xE3, 0x83, 0xAB, 0x09, 0xE3, 0x83, + 0x95, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xB3, 0x0F, + 0xE3, 0x83, 0x98, 0xE3, 0x82, 0xAF, 0xE3, 0x82, + 0xBF, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xAB, 0x09, + // Bytes 2600 - 263f + 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x82, + 0xBD, 0x0C, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, + 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0x92, 0x09, 0xE3, + 0x83, 0x98, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x84, + 0x0C, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, + 0x83, 0xB3, 0xE3, 0x82, 0xB9, 0x0F, 0xE3, 0x83, + 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xBC, 0xE3, + 0x82, 0xB7, 0xE3, 0x82, 0x99, 0x0C, 0xE3, 0x83, + // Bytes 2640 - 267f + 0x98, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, 0xE3, + 0x82, 0xBF, 0x0F, 0xE3, 0x83, 0x9B, 0xE3, 0x82, + 0x9A, 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xB3, 0xE3, + 0x83, 0x88, 0x0C, 0xE3, 0x83, 0x9B, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x88, 0x06, + 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xB3, 0x0F, 0xE3, + 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0xE3, 0x83, 0xB3, + 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, 0x09, 0xE3, + // Bytes 2680 - 26bf + 0x83, 0x9B, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xAB, + 0x09, 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xB3, 0x0C, 0xE3, 0x83, 0x9E, 0xE3, 0x82, + 0xA4, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0x09, + 0xE3, 0x83, 0x9E, 0xE3, 0x82, 0xA4, 0xE3, 0x83, + 0xAB, 0x09, 0xE3, 0x83, 0x9E, 0xE3, 0x83, 0x83, + 0xE3, 0x83, 0x8F, 0x09, 0xE3, 0x83, 0x9E, 0xE3, + 0x83, 0xAB, 0xE3, 0x82, 0xAF, 0x0F, 0xE3, 0x83, + // Bytes 26c0 - 26ff + 0x9E, 0xE3, 0x83, 0xB3, 0xE3, 0x82, 0xB7, 0xE3, + 0x83, 0xA7, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, 0x83, + 0x9F, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0xE3, + 0x83, 0xB3, 0x06, 0xE3, 0x83, 0x9F, 0xE3, 0x83, + 0xAA, 0x12, 0xE3, 0x83, 0x9F, 0xE3, 0x83, 0xAA, + 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAB, 0x09, 0xE3, 0x83, 0xA1, + 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0F, 0xE3, + // Bytes 2700 - 273f + 0x83, 0xA1, 0xE3, 0x82, 0xAB, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, + 0x83, 0xA1, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, + 0xE3, 0x83, 0xAB, 0x0C, 0xE3, 0x83, 0xA4, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, + 0x09, 0xE3, 0x83, 0xA4, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0xAB, 0x09, 0xE3, 0x83, 0xA6, 0xE3, 0x82, + 0xA2, 0xE3, 0x83, 0xB3, 0x0C, 0xE3, 0x83, 0xAA, + // Bytes 2740 - 277f + 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x06, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xA9, + 0x0C, 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0x92, 0xE3, + 0x82, 0x9A, 0xE3, 0x83, 0xBC, 0x0F, 0xE3, 0x83, + 0xAB, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x95, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0xAB, 0x06, 0xE3, 0x83, + 0xAC, 0xE3, 0x83, 0xA0, 0x12, 0xE3, 0x83, 0xAC, + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, 0xE3, 0x82, + // Bytes 2780 - 27bf + 0xB1, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xB3, 0x09, + 0xE3, 0x83, 0xAF, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x88, 0x04, 0x30, 0xE7, 0x82, 0xB9, 0x04, 0x31, + 0xE7, 0x82, 0xB9, 0x04, 0x32, 0xE7, 0x82, 0xB9, + 0x04, 0x33, 0xE7, 0x82, 0xB9, 0x04, 0x34, 0xE7, + 0x82, 0xB9, 0x04, 0x35, 0xE7, 0x82, 0xB9, 0x04, + 0x36, 0xE7, 0x82, 0xB9, 0x04, 0x37, 0xE7, 0x82, + 0xB9, 0x04, 0x38, 0xE7, 0x82, 0xB9, 0x04, 0x39, + // Bytes 27c0 - 27ff + 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x30, 0xE7, 0x82, + 0xB9, 0x05, 0x31, 0x31, 0xE7, 0x82, 0xB9, 0x05, + 0x31, 0x32, 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x33, + 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x34, 0xE7, 0x82, + 0xB9, 0x05, 0x31, 0x35, 0xE7, 0x82, 0xB9, 0x05, + 0x31, 0x36, 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x37, + 0xE7, 0x82, 0xB9, 0x05, 0x31, 0x38, 0xE7, 0x82, + 0xB9, 0x05, 0x31, 0x39, 0xE7, 0x82, 0xB9, 0x05, + // Bytes 2800 - 283f + 0x32, 0x30, 0xE7, 0x82, 0xB9, 0x05, 0x32, 0x31, + 0xE7, 0x82, 0xB9, 0x05, 0x32, 0x32, 0xE7, 0x82, + 0xB9, 0x05, 0x32, 0x33, 0xE7, 0x82, 0xB9, 0x05, + 0x32, 0x34, 0xE7, 0x82, 0xB9, 0x03, 0x68, 0x50, + 0x61, 0x02, 0x64, 0x61, 0x02, 0x41, 0x55, 0x03, + 0x62, 0x61, 0x72, 0x02, 0x6F, 0x56, 0x02, 0x70, + 0x63, 0x02, 0x64, 0x6D, 0x03, 0x64, 0x6D, 0x32, + 0x03, 0x64, 0x6D, 0x33, 0x02, 0x49, 0x55, 0x06, + // Bytes 2840 - 287f + 0xE5, 0xB9, 0xB3, 0xE6, 0x88, 0x90, 0x06, 0xE6, + 0x98, 0xAD, 0xE5, 0x92, 0x8C, 0x06, 0xE5, 0xA4, + 0xA7, 0xE6, 0xAD, 0xA3, 0x06, 0xE6, 0x98, 0x8E, + 0xE6, 0xB2, 0xBB, 0x0C, 0xE6, 0xA0, 0xAA, 0xE5, + 0xBC, 0x8F, 0xE4, 0xBC, 0x9A, 0xE7, 0xA4, 0xBE, + 0x02, 0x70, 0x41, 0x02, 0x6E, 0x41, 0x03, 0xCE, + 0xBC, 0x41, 0x02, 0x6D, 0x41, 0x02, 0x6B, 0x41, + 0x02, 0x4B, 0x42, 0x02, 0x4D, 0x42, 0x02, 0x47, + // Bytes 2880 - 28bf + 0x42, 0x03, 0x63, 0x61, 0x6C, 0x04, 0x6B, 0x63, + 0x61, 0x6C, 0x02, 0x70, 0x46, 0x02, 0x6E, 0x46, + 0x03, 0xCE, 0xBC, 0x46, 0x03, 0xCE, 0xBC, 0x67, + 0x02, 0x6D, 0x67, 0x02, 0x6B, 0x67, 0x02, 0x48, + 0x7A, 0x03, 0x6B, 0x48, 0x7A, 0x03, 0x4D, 0x48, + 0x7A, 0x03, 0x47, 0x48, 0x7A, 0x03, 0x54, 0x48, + 0x7A, 0x03, 0xCE, 0xBC, 0x6C, 0x02, 0x6D, 0x6C, + 0x02, 0x64, 0x6C, 0x02, 0x6B, 0x6C, 0x02, 0x66, + // Bytes 28c0 - 28ff + 0x6D, 0x02, 0x6E, 0x6D, 0x03, 0xCE, 0xBC, 0x6D, + 0x02, 0x6D, 0x6D, 0x02, 0x63, 0x6D, 0x02, 0x6B, + 0x6D, 0x03, 0x6D, 0x6D, 0x32, 0x03, 0x63, 0x6D, + 0x32, 0x02, 0x6D, 0x32, 0x03, 0x6B, 0x6D, 0x32, + 0x03, 0x6D, 0x6D, 0x33, 0x03, 0x63, 0x6D, 0x33, + 0x02, 0x6D, 0x33, 0x03, 0x6B, 0x6D, 0x33, 0x05, + 0x6D, 0xE2, 0x88, 0x95, 0x73, 0x06, 0x6D, 0xE2, + 0x88, 0x95, 0x73, 0x32, 0x02, 0x50, 0x61, 0x03, + // Bytes 2900 - 293f + 0x6B, 0x50, 0x61, 0x03, 0x4D, 0x50, 0x61, 0x03, + 0x47, 0x50, 0x61, 0x03, 0x72, 0x61, 0x64, 0x07, + 0x72, 0x61, 0x64, 0xE2, 0x88, 0x95, 0x73, 0x08, + 0x72, 0x61, 0x64, 0xE2, 0x88, 0x95, 0x73, 0x32, + 0x02, 0x70, 0x73, 0x02, 0x6E, 0x73, 0x03, 0xCE, + 0xBC, 0x73, 0x02, 0x6D, 0x73, 0x02, 0x70, 0x56, + 0x02, 0x6E, 0x56, 0x03, 0xCE, 0xBC, 0x56, 0x02, + 0x6D, 0x56, 0x02, 0x6B, 0x56, 0x02, 0x4D, 0x56, + // Bytes 2940 - 297f + 0x02, 0x70, 0x57, 0x02, 0x6E, 0x57, 0x03, 0xCE, + 0xBC, 0x57, 0x02, 0x6D, 0x57, 0x02, 0x6B, 0x57, + 0x02, 0x4D, 0x57, 0x03, 0x6B, 0xCE, 0xA9, 0x03, + 0x4D, 0xCE, 0xA9, 0x04, 0x61, 0x2E, 0x6D, 0x2E, + 0x02, 0x42, 0x71, 0x02, 0x63, 0x63, 0x02, 0x63, + 0x64, 0x06, 0x43, 0xE2, 0x88, 0x95, 0x6B, 0x67, + 0x03, 0x43, 0x6F, 0x2E, 0x02, 0x64, 0x42, 0x02, + 0x47, 0x79, 0x02, 0x68, 0x61, 0x02, 0x48, 0x50, + // Bytes 2980 - 29bf + 0x02, 0x69, 0x6E, 0x02, 0x4B, 0x4B, 0x02, 0x4B, + 0x4D, 0x02, 0x6B, 0x74, 0x02, 0x6C, 0x6D, 0x02, + 0x6C, 0x6E, 0x03, 0x6C, 0x6F, 0x67, 0x02, 0x6C, + 0x78, 0x02, 0x6D, 0x62, 0x03, 0x6D, 0x69, 0x6C, + 0x03, 0x6D, 0x6F, 0x6C, 0x02, 0x50, 0x48, 0x04, + 0x70, 0x2E, 0x6D, 0x2E, 0x03, 0x50, 0x50, 0x4D, + 0x02, 0x50, 0x52, 0x02, 0x73, 0x72, 0x02, 0x53, + 0x76, 0x02, 0x57, 0x62, 0x05, 0x56, 0xE2, 0x88, + // Bytes 29c0 - 29ff + 0x95, 0x6D, 0x05, 0x41, 0xE2, 0x88, 0x95, 0x6D, + 0x04, 0x31, 0xE6, 0x97, 0xA5, 0x04, 0x32, 0xE6, + 0x97, 0xA5, 0x04, 0x33, 0xE6, 0x97, 0xA5, 0x04, + 0x34, 0xE6, 0x97, 0xA5, 0x04, 0x35, 0xE6, 0x97, + 0xA5, 0x04, 0x36, 0xE6, 0x97, 0xA5, 0x04, 0x37, + 0xE6, 0x97, 0xA5, 0x04, 0x38, 0xE6, 0x97, 0xA5, + 0x04, 0x39, 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x30, + 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x31, 0xE6, 0x97, + // Bytes 2a00 - 2a3f + 0xA5, 0x05, 0x31, 0x32, 0xE6, 0x97, 0xA5, 0x05, + 0x31, 0x33, 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x34, + 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x35, 0xE6, 0x97, + 0xA5, 0x05, 0x31, 0x36, 0xE6, 0x97, 0xA5, 0x05, + 0x31, 0x37, 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x38, + 0xE6, 0x97, 0xA5, 0x05, 0x31, 0x39, 0xE6, 0x97, + 0xA5, 0x05, 0x32, 0x30, 0xE6, 0x97, 0xA5, 0x05, + 0x32, 0x31, 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x32, + // Bytes 2a40 - 2a7f + 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x33, 0xE6, 0x97, + 0xA5, 0x05, 0x32, 0x34, 0xE6, 0x97, 0xA5, 0x05, + 0x32, 0x35, 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x36, + 0xE6, 0x97, 0xA5, 0x05, 0x32, 0x37, 0xE6, 0x97, + 0xA5, 0x05, 0x32, 0x38, 0xE6, 0x97, 0xA5, 0x05, + 0x32, 0x39, 0xE6, 0x97, 0xA5, 0x05, 0x33, 0x30, + 0xE6, 0x97, 0xA5, 0x05, 0x33, 0x31, 0xE6, 0x97, + 0xA5, 0x03, 0x67, 0x61, 0x6C, 0x03, 0xEA, 0x9D, + // Bytes 2a80 - 2abf + 0xAF, 0x03, 0xE8, 0xB1, 0x88, 0x03, 0xE6, 0x9B, + 0xB4, 0x03, 0xE8, 0xB3, 0x88, 0x03, 0xE6, 0xBB, + 0x91, 0x03, 0xE4, 0xB8, 0xB2, 0x03, 0xE5, 0x8F, + 0xA5, 0x03, 0xE5, 0xA5, 0x91, 0x03, 0xE5, 0x96, + 0x87, 0x03, 0xE5, 0xA5, 0x88, 0x03, 0xE6, 0x87, + 0xB6, 0x03, 0xE7, 0x99, 0xA9, 0x03, 0xE7, 0xBE, + 0x85, 0x03, 0xE8, 0x98, 0xBF, 0x03, 0xE8, 0x9E, + 0xBA, 0x03, 0xE8, 0xA3, 0xB8, 0x03, 0xE9, 0x82, + // Bytes 2ac0 - 2aff + 0x8F, 0x03, 0xE6, 0xA8, 0x82, 0x03, 0xE6, 0xB4, + 0x9B, 0x03, 0xE7, 0x83, 0x99, 0x03, 0xE7, 0x8F, + 0x9E, 0x03, 0xE8, 0x90, 0xBD, 0x03, 0xE9, 0x85, + 0xAA, 0x03, 0xE9, 0xA7, 0xB1, 0x03, 0xE4, 0xBA, + 0x82, 0x03, 0xE5, 0x8D, 0xB5, 0x03, 0xE6, 0xAC, + 0x84, 0x03, 0xE7, 0x88, 0x9B, 0x03, 0xE8, 0x98, + 0xAD, 0x03, 0xE9, 0xB8, 0x9E, 0x03, 0xE5, 0xB5, + 0x90, 0x03, 0xE6, 0xBF, 0xAB, 0x03, 0xE8, 0x97, + // Bytes 2b00 - 2b3f + 0x8D, 0x03, 0xE8, 0xA5, 0xA4, 0x03, 0xE6, 0x8B, + 0x89, 0x03, 0xE8, 0x87, 0x98, 0x03, 0xE8, 0xA0, + 0x9F, 0x03, 0xE5, 0xBB, 0x8A, 0x03, 0xE6, 0x9C, + 0x97, 0x03, 0xE6, 0xB5, 0xAA, 0x03, 0xE7, 0x8B, + 0xBC, 0x03, 0xE9, 0x83, 0x8E, 0x03, 0xE4, 0xBE, + 0x86, 0x03, 0xE5, 0x86, 0xB7, 0x03, 0xE5, 0x8B, + 0x9E, 0x03, 0xE6, 0x93, 0x84, 0x03, 0xE6, 0xAB, + 0x93, 0x03, 0xE7, 0x88, 0x90, 0x03, 0xE7, 0x9B, + // Bytes 2b40 - 2b7f + 0xA7, 0x03, 0xE8, 0x98, 0x86, 0x03, 0xE8, 0x99, + 0x9C, 0x03, 0xE8, 0xB7, 0xAF, 0x03, 0xE9, 0x9C, + 0xB2, 0x03, 0xE9, 0xAD, 0xAF, 0x03, 0xE9, 0xB7, + 0xBA, 0x03, 0xE7, 0xA2, 0x8C, 0x03, 0xE7, 0xA5, + 0xBF, 0x03, 0xE7, 0xB6, 0xA0, 0x03, 0xE8, 0x8F, + 0x89, 0x03, 0xE9, 0x8C, 0x84, 0x03, 0xE8, 0xAB, + 0x96, 0x03, 0xE5, 0xA3, 0x9F, 0x03, 0xE5, 0xBC, + 0x84, 0x03, 0xE7, 0xB1, 0xA0, 0x03, 0xE8, 0x81, + // Bytes 2b80 - 2bbf + 0xBE, 0x03, 0xE7, 0x89, 0xA2, 0x03, 0xE7, 0xA3, + 0x8A, 0x03, 0xE8, 0xB3, 0x82, 0x03, 0xE9, 0x9B, + 0xB7, 0x03, 0xE5, 0xA3, 0x98, 0x03, 0xE5, 0xB1, + 0xA2, 0x03, 0xE6, 0xA8, 0x93, 0x03, 0xE6, 0xB7, + 0x9A, 0x03, 0xE6, 0xBC, 0x8F, 0x03, 0xE7, 0xB4, + 0xAF, 0x03, 0xE7, 0xB8, 0xB7, 0x03, 0xE9, 0x99, + 0x8B, 0x03, 0xE5, 0x8B, 0x92, 0x03, 0xE8, 0x82, + 0x8B, 0x03, 0xE5, 0x87, 0x9C, 0x03, 0xE5, 0x87, + // Bytes 2bc0 - 2bff + 0x8C, 0x03, 0xE7, 0xA8, 0x9C, 0x03, 0xE7, 0xB6, + 0xBE, 0x03, 0xE8, 0x8F, 0xB1, 0x03, 0xE9, 0x99, + 0xB5, 0x03, 0xE8, 0xAE, 0x80, 0x03, 0xE6, 0x8B, + 0x8F, 0x03, 0xE8, 0xAB, 0xBE, 0x03, 0xE4, 0xB8, + 0xB9, 0x03, 0xE5, 0xAF, 0xA7, 0x03, 0xE6, 0x80, + 0x92, 0x03, 0xE7, 0x8E, 0x87, 0x03, 0xE7, 0x95, + 0xB0, 0x03, 0xE5, 0x8C, 0x97, 0x03, 0xE7, 0xA3, + 0xBB, 0x03, 0xE4, 0xBE, 0xBF, 0x03, 0xE5, 0xBE, + // Bytes 2c00 - 2c3f + 0xA9, 0x03, 0xE4, 0xB8, 0x8D, 0x03, 0xE6, 0xB3, + 0x8C, 0x03, 0xE6, 0x95, 0xB8, 0x03, 0xE7, 0xB4, + 0xA2, 0x03, 0xE5, 0x8F, 0x83, 0x03, 0xE5, 0xA1, + 0x9E, 0x03, 0xE7, 0x9C, 0x81, 0x03, 0xE8, 0x91, + 0x89, 0x03, 0xE8, 0xAA, 0xAA, 0x03, 0xE6, 0xAE, + 0xBA, 0x03, 0xE6, 0xB2, 0x88, 0x03, 0xE6, 0x8B, + 0xBE, 0x03, 0xE8, 0x8B, 0xA5, 0x03, 0xE6, 0x8E, + 0xA0, 0x03, 0xE7, 0x95, 0xA5, 0x03, 0xE4, 0xBA, + // Bytes 2c40 - 2c7f + 0xAE, 0x03, 0xE5, 0x85, 0xA9, 0x03, 0xE5, 0x87, + 0x89, 0x03, 0xE6, 0xA2, 0x81, 0x03, 0xE7, 0xB3, + 0xA7, 0x03, 0xE8, 0x89, 0xAF, 0x03, 0xE8, 0xAB, + 0x92, 0x03, 0xE9, 0x87, 0x8F, 0x03, 0xE5, 0x8B, + 0xB5, 0x03, 0xE5, 0x91, 0x82, 0x03, 0xE5, 0xBB, + 0xAC, 0x03, 0xE6, 0x97, 0x85, 0x03, 0xE6, 0xBF, + 0xBE, 0x03, 0xE7, 0xA4, 0xAA, 0x03, 0xE9, 0x96, + 0xAD, 0x03, 0xE9, 0xA9, 0xAA, 0x03, 0xE9, 0xBA, + // Bytes 2c80 - 2cbf + 0x97, 0x03, 0xE9, 0xBB, 0x8E, 0x03, 0xE6, 0x9B, + 0x86, 0x03, 0xE6, 0xAD, 0xB7, 0x03, 0xE8, 0xBD, + 0xA2, 0x03, 0xE5, 0xB9, 0xB4, 0x03, 0xE6, 0x86, + 0x90, 0x03, 0xE6, 0x88, 0x80, 0x03, 0xE6, 0x92, + 0x9A, 0x03, 0xE6, 0xBC, 0xA3, 0x03, 0xE7, 0x85, + 0x89, 0x03, 0xE7, 0x92, 0x89, 0x03, 0xE7, 0xA7, + 0x8A, 0x03, 0xE7, 0xB7, 0xB4, 0x03, 0xE8, 0x81, + 0xAF, 0x03, 0xE8, 0xBC, 0xA6, 0x03, 0xE8, 0x93, + // Bytes 2cc0 - 2cff + 0xAE, 0x03, 0xE9, 0x80, 0xA3, 0x03, 0xE9, 0x8D, + 0x8A, 0x03, 0xE5, 0x88, 0x97, 0x03, 0xE5, 0x8A, + 0xA3, 0x03, 0xE5, 0x92, 0xBD, 0x03, 0xE7, 0x83, + 0x88, 0x03, 0xE8, 0xA3, 0x82, 0x03, 0xE5, 0xBB, + 0x89, 0x03, 0xE5, 0xBF, 0xB5, 0x03, 0xE6, 0x8D, + 0xBB, 0x03, 0xE6, 0xAE, 0xAE, 0x03, 0xE7, 0xB0, + 0xBE, 0x03, 0xE7, 0x8D, 0xB5, 0x03, 0xE4, 0xBB, + 0xA4, 0x03, 0xE5, 0x9B, 0xB9, 0x03, 0xE5, 0xB6, + // Bytes 2d00 - 2d3f + 0xBA, 0x03, 0xE6, 0x80, 0x9C, 0x03, 0xE7, 0x8E, + 0xB2, 0x03, 0xE7, 0x91, 0xA9, 0x03, 0xE7, 0xBE, + 0x9A, 0x03, 0xE8, 0x81, 0x86, 0x03, 0xE9, 0x88, + 0xB4, 0x03, 0xE9, 0x9B, 0xB6, 0x03, 0xE9, 0x9D, + 0x88, 0x03, 0xE9, 0xA0, 0x98, 0x03, 0xE4, 0xBE, + 0x8B, 0x03, 0xE7, 0xA6, 0xAE, 0x03, 0xE9, 0x86, + 0xB4, 0x03, 0xE9, 0x9A, 0xB8, 0x03, 0xE6, 0x83, + 0xA1, 0x03, 0xE4, 0xBA, 0x86, 0x03, 0xE5, 0x83, + // Bytes 2d40 - 2d7f + 0x9A, 0x03, 0xE5, 0xAF, 0xAE, 0x03, 0xE5, 0xB0, + 0xBF, 0x03, 0xE6, 0x96, 0x99, 0x03, 0xE7, 0x87, + 0x8E, 0x03, 0xE7, 0x99, 0x82, 0x03, 0xE8, 0x93, + 0xBC, 0x03, 0xE9, 0x81, 0xBC, 0x03, 0xE6, 0x9A, + 0x88, 0x03, 0xE9, 0x98, 0xAE, 0x03, 0xE5, 0x8A, + 0x89, 0x03, 0xE6, 0x9D, 0xBB, 0x03, 0xE6, 0x9F, + 0xB3, 0x03, 0xE6, 0xB5, 0x81, 0x03, 0xE6, 0xBA, + 0x9C, 0x03, 0xE7, 0x90, 0x89, 0x03, 0xE7, 0x95, + // Bytes 2d80 - 2dbf + 0x99, 0x03, 0xE7, 0xA1, 0xAB, 0x03, 0xE7, 0xB4, + 0x90, 0x03, 0xE9, 0xA1, 0x9E, 0x03, 0xE6, 0x88, + 0xAE, 0x03, 0xE9, 0x99, 0xB8, 0x03, 0xE5, 0x80, + 0xAB, 0x03, 0xE5, 0xB4, 0x99, 0x03, 0xE6, 0xB7, + 0xAA, 0x03, 0xE8, 0xBC, 0xAA, 0x03, 0xE5, 0xBE, + 0x8B, 0x03, 0xE6, 0x85, 0x84, 0x03, 0xE6, 0xA0, + 0x97, 0x03, 0xE9, 0x9A, 0x86, 0x03, 0xE5, 0x88, + 0xA9, 0x03, 0xE5, 0x90, 0x8F, 0x03, 0xE5, 0xB1, + // Bytes 2dc0 - 2dff + 0xA5, 0x03, 0xE6, 0x98, 0x93, 0x03, 0xE6, 0x9D, + 0x8E, 0x03, 0xE6, 0xA2, 0xA8, 0x03, 0xE6, 0xB3, + 0xA5, 0x03, 0xE7, 0x90, 0x86, 0x03, 0xE7, 0x97, + 0xA2, 0x03, 0xE7, 0xBD, 0xB9, 0x03, 0xE8, 0xA3, + 0x8F, 0x03, 0xE8, 0xA3, 0xA1, 0x03, 0xE9, 0x9B, + 0xA2, 0x03, 0xE5, 0x8C, 0xBF, 0x03, 0xE6, 0xBA, + 0xBA, 0x03, 0xE5, 0x90, 0x9D, 0x03, 0xE7, 0x87, + 0x90, 0x03, 0xE7, 0x92, 0x98, 0x03, 0xE8, 0x97, + // Bytes 2e00 - 2e3f + 0xBA, 0x03, 0xE9, 0x9A, 0xA3, 0x03, 0xE9, 0xB1, + 0x97, 0x03, 0xE9, 0xBA, 0x9F, 0x03, 0xE6, 0x9E, + 0x97, 0x03, 0xE6, 0xB7, 0x8B, 0x03, 0xE8, 0x87, + 0xA8, 0x03, 0xE7, 0xAC, 0xA0, 0x03, 0xE7, 0xB2, + 0x92, 0x03, 0xE7, 0x8B, 0x80, 0x03, 0xE7, 0x82, + 0x99, 0x03, 0xE8, 0xAD, 0x98, 0x03, 0xE4, 0xBB, + 0x80, 0x03, 0xE8, 0x8C, 0xB6, 0x03, 0xE5, 0x88, + 0xBA, 0x03, 0xE5, 0x88, 0x87, 0x03, 0xE5, 0xBA, + // Bytes 2e40 - 2e7f + 0xA6, 0x03, 0xE6, 0x8B, 0x93, 0x03, 0xE7, 0xB3, + 0x96, 0x03, 0xE5, 0xAE, 0x85, 0x03, 0xE6, 0xB4, + 0x9E, 0x03, 0xE6, 0x9A, 0xB4, 0x03, 0xE8, 0xBC, + 0xBB, 0x03, 0xE9, 0x99, 0x8D, 0x03, 0xE5, 0xBB, + 0x93, 0x03, 0xE5, 0x85, 0x80, 0x03, 0xE5, 0x97, + 0x80, 0x03, 0xE5, 0xA1, 0x9A, 0x03, 0xE6, 0x99, + 0xB4, 0x03, 0xE5, 0x87, 0x9E, 0x03, 0xE7, 0x8C, + 0xAA, 0x03, 0xE7, 0x9B, 0x8A, 0x03, 0xE7, 0xA4, + // Bytes 2e80 - 2ebf + 0xBC, 0x03, 0xE7, 0xA5, 0x9E, 0x03, 0xE7, 0xA5, + 0xA5, 0x03, 0xE7, 0xA6, 0x8F, 0x03, 0xE9, 0x9D, + 0x96, 0x03, 0xE7, 0xB2, 0xBE, 0x03, 0xE8, 0x98, + 0x92, 0x03, 0xE8, 0xAB, 0xB8, 0x03, 0xE9, 0x80, + 0xB8, 0x03, 0xE9, 0x83, 0xBD, 0x03, 0xE9, 0xA3, + 0xAF, 0x03, 0xE9, 0xA3, 0xBC, 0x03, 0xE9, 0xA4, + 0xA8, 0x03, 0xE9, 0xB6, 0xB4, 0x03, 0xE4, 0xBE, + 0xAE, 0x03, 0xE5, 0x83, 0xA7, 0x03, 0xE5, 0x85, + // Bytes 2ec0 - 2eff + 0x8D, 0x03, 0xE5, 0x8B, 0x89, 0x03, 0xE5, 0x8B, + 0xA4, 0x03, 0xE5, 0x8D, 0x91, 0x03, 0xE5, 0x96, + 0x9D, 0x03, 0xE5, 0x98, 0x86, 0x03, 0xE5, 0x99, + 0xA8, 0x03, 0xE5, 0xA1, 0x80, 0x03, 0xE5, 0xA2, + 0xA8, 0x03, 0xE5, 0xB1, 0xA4, 0x03, 0xE6, 0x82, + 0x94, 0x03, 0xE6, 0x85, 0xA8, 0x03, 0xE6, 0x86, + 0x8E, 0x03, 0xE6, 0x87, 0xB2, 0x03, 0xE6, 0x95, + 0x8F, 0x03, 0xE6, 0x97, 0xA2, 0x03, 0xE6, 0x9A, + // Bytes 2f00 - 2f3f + 0x91, 0x03, 0xE6, 0xA2, 0x85, 0x03, 0xE6, 0xB5, + 0xB7, 0x03, 0xE6, 0xB8, 0x9A, 0x03, 0xE6, 0xBC, + 0xA2, 0x03, 0xE7, 0x85, 0xAE, 0x03, 0xE7, 0x88, + 0xAB, 0x03, 0xE7, 0x90, 0xA2, 0x03, 0xE7, 0xA2, + 0x91, 0x03, 0xE7, 0xA5, 0x89, 0x03, 0xE7, 0xA5, + 0x88, 0x03, 0xE7, 0xA5, 0x90, 0x03, 0xE7, 0xA5, + 0x96, 0x03, 0xE7, 0xA6, 0x8D, 0x03, 0xE7, 0xA6, + 0x8E, 0x03, 0xE7, 0xA9, 0x80, 0x03, 0xE7, 0xAA, + // Bytes 2f40 - 2f7f + 0x81, 0x03, 0xE7, 0xAF, 0x80, 0x03, 0xE7, 0xB8, + 0x89, 0x03, 0xE7, 0xB9, 0x81, 0x03, 0xE7, 0xBD, + 0xB2, 0x03, 0xE8, 0x80, 0x85, 0x03, 0xE8, 0x87, + 0xAD, 0x03, 0xE8, 0x89, 0xB9, 0x03, 0xE8, 0x91, + 0x97, 0x03, 0xE8, 0xA4, 0x90, 0x03, 0xE8, 0xA6, + 0x96, 0x03, 0xE8, 0xAC, 0x81, 0x03, 0xE8, 0xAC, + 0xB9, 0x03, 0xE8, 0xB3, 0x93, 0x03, 0xE8, 0xB4, + 0x88, 0x03, 0xE8, 0xBE, 0xB6, 0x03, 0xE9, 0x9B, + // Bytes 2f80 - 2fbf + 0xA3, 0x03, 0xE9, 0x9F, 0xBF, 0x03, 0xE9, 0xA0, + 0xBB, 0x03, 0xE6, 0x81, 0xB5, 0x04, 0xF0, 0xA4, + 0x8B, 0xAE, 0x03, 0xE8, 0x88, 0x98, 0x03, 0xE4, + 0xB8, 0xA6, 0x03, 0xE5, 0x86, 0xB5, 0x03, 0xE5, + 0x85, 0xA8, 0x03, 0xE4, 0xBE, 0x80, 0x03, 0xE5, + 0x85, 0x85, 0x03, 0xE5, 0x86, 0x80, 0x03, 0xE5, + 0x8B, 0x87, 0x03, 0xE5, 0x8B, 0xBA, 0x03, 0xE5, + 0x95, 0x95, 0x03, 0xE5, 0x96, 0x99, 0x03, 0xE5, + // Bytes 2fc0 - 2fff + 0x97, 0xA2, 0x03, 0xE5, 0xA2, 0xB3, 0x03, 0xE5, + 0xA5, 0x84, 0x03, 0xE5, 0xA5, 0x94, 0x03, 0xE5, + 0xA9, 0xA2, 0x03, 0xE5, 0xAC, 0xA8, 0x03, 0xE5, + 0xBB, 0x92, 0x03, 0xE5, 0xBB, 0x99, 0x03, 0xE5, + 0xBD, 0xA9, 0x03, 0xE5, 0xBE, 0xAD, 0x03, 0xE6, + 0x83, 0x98, 0x03, 0xE6, 0x85, 0x8E, 0x03, 0xE6, + 0x84, 0x88, 0x03, 0xE6, 0x85, 0xA0, 0x03, 0xE6, + 0x88, 0xB4, 0x03, 0xE6, 0x8F, 0x84, 0x03, 0xE6, + // Bytes 3000 - 303f + 0x90, 0x9C, 0x03, 0xE6, 0x91, 0x92, 0x03, 0xE6, + 0x95, 0x96, 0x03, 0xE6, 0x9C, 0x9B, 0x03, 0xE6, + 0x9D, 0x96, 0x03, 0xE6, 0xBB, 0x9B, 0x03, 0xE6, + 0xBB, 0x8B, 0x03, 0xE7, 0x80, 0x9E, 0x03, 0xE7, + 0x9E, 0xA7, 0x03, 0xE7, 0x88, 0xB5, 0x03, 0xE7, + 0x8A, 0xAF, 0x03, 0xE7, 0x91, 0xB1, 0x03, 0xE7, + 0x94, 0x86, 0x03, 0xE7, 0x94, 0xBB, 0x03, 0xE7, + 0x98, 0x9D, 0x03, 0xE7, 0x98, 0x9F, 0x03, 0xE7, + // Bytes 3040 - 307f + 0x9B, 0x9B, 0x03, 0xE7, 0x9B, 0xB4, 0x03, 0xE7, + 0x9D, 0x8A, 0x03, 0xE7, 0x9D, 0x80, 0x03, 0xE7, + 0xA3, 0x8C, 0x03, 0xE7, 0xAA, 0xB1, 0x03, 0xE7, + 0xB1, 0xBB, 0x03, 0xE7, 0xB5, 0x9B, 0x03, 0xE7, + 0xBC, 0xBE, 0x03, 0xE8, 0x8D, 0x92, 0x03, 0xE8, + 0x8F, 0xAF, 0x03, 0xE8, 0x9D, 0xB9, 0x03, 0xE8, + 0xA5, 0x81, 0x03, 0xE8, 0xA6, 0x86, 0x03, 0xE8, + 0xAA, 0xBF, 0x03, 0xE8, 0xAB, 0x8B, 0x03, 0xE8, + // Bytes 3080 - 30bf + 0xAB, 0xAD, 0x03, 0xE8, 0xAE, 0x8A, 0x03, 0xE8, + 0xBC, 0xB8, 0x03, 0xE9, 0x81, 0xB2, 0x03, 0xE9, + 0x86, 0x99, 0x03, 0xE9, 0x89, 0xB6, 0x03, 0xE9, + 0x99, 0xBC, 0x03, 0xE9, 0x9F, 0x9B, 0x03, 0xE9, + 0xA0, 0x8B, 0x03, 0xE9, 0xAC, 0x92, 0x04, 0xF0, + 0xA2, 0xA1, 0x8A, 0x04, 0xF0, 0xA2, 0xA1, 0x84, + 0x04, 0xF0, 0xA3, 0x8F, 0x95, 0x03, 0xE3, 0xAE, + 0x9D, 0x03, 0xE4, 0x80, 0x98, 0x03, 0xE4, 0x80, + // Bytes 30c0 - 30ff + 0xB9, 0x04, 0xF0, 0xA5, 0x89, 0x89, 0x04, 0xF0, + 0xA5, 0xB3, 0x90, 0x04, 0xF0, 0xA7, 0xBB, 0x93, + 0x03, 0xE9, 0xBD, 0x83, 0x03, 0xE9, 0xBE, 0x8E, + 0x02, 0x66, 0x66, 0x02, 0x66, 0x69, 0x02, 0x66, + 0x6C, 0x03, 0x66, 0x66, 0x69, 0x03, 0x66, 0x66, + 0x6C, 0x02, 0x73, 0x74, 0x04, 0xD5, 0xB4, 0xD5, + 0xB6, 0x04, 0xD5, 0xB4, 0xD5, 0xA5, 0x04, 0xD5, + 0xB4, 0xD5, 0xAB, 0x04, 0xD5, 0xBE, 0xD5, 0xB6, + // Bytes 3100 - 313f + 0x04, 0xD5, 0xB4, 0xD5, 0xAD, 0x04, 0xD7, 0x99, + 0xD6, 0xB4, 0x04, 0xD7, 0xB2, 0xD6, 0xB7, 0x02, + 0xD7, 0xA2, 0x02, 0xD7, 0x94, 0x02, 0xD7, 0x9B, + 0x02, 0xD7, 0x9C, 0x02, 0xD7, 0x9D, 0x02, 0xD7, + 0xA8, 0x02, 0xD7, 0xAA, 0x04, 0xD7, 0xA9, 0xD7, + 0x81, 0x04, 0xD7, 0xA9, 0xD7, 0x82, 0x06, 0xD7, + 0xA9, 0xD6, 0xBC, 0xD7, 0x81, 0x06, 0xD7, 0xA9, + 0xD6, 0xBC, 0xD7, 0x82, 0x04, 0xD7, 0x90, 0xD6, + // Bytes 3140 - 317f + 0xB7, 0x04, 0xD7, 0x90, 0xD6, 0xB8, 0x04, 0xD7, + 0x90, 0xD6, 0xBC, 0x04, 0xD7, 0x91, 0xD6, 0xBC, + 0x04, 0xD7, 0x92, 0xD6, 0xBC, 0x04, 0xD7, 0x93, + 0xD6, 0xBC, 0x04, 0xD7, 0x94, 0xD6, 0xBC, 0x04, + 0xD7, 0x95, 0xD6, 0xBC, 0x04, 0xD7, 0x96, 0xD6, + 0xBC, 0x04, 0xD7, 0x98, 0xD6, 0xBC, 0x04, 0xD7, + 0x99, 0xD6, 0xBC, 0x04, 0xD7, 0x9A, 0xD6, 0xBC, + 0x04, 0xD7, 0x9B, 0xD6, 0xBC, 0x04, 0xD7, 0x9C, + // Bytes 3180 - 31bf + 0xD6, 0xBC, 0x04, 0xD7, 0x9E, 0xD6, 0xBC, 0x04, + 0xD7, 0xA0, 0xD6, 0xBC, 0x04, 0xD7, 0xA1, 0xD6, + 0xBC, 0x04, 0xD7, 0xA3, 0xD6, 0xBC, 0x04, 0xD7, + 0xA4, 0xD6, 0xBC, 0x04, 0xD7, 0xA6, 0xD6, 0xBC, + 0x04, 0xD7, 0xA7, 0xD6, 0xBC, 0x04, 0xD7, 0xA8, + 0xD6, 0xBC, 0x04, 0xD7, 0xA9, 0xD6, 0xBC, 0x04, + 0xD7, 0xAA, 0xD6, 0xBC, 0x04, 0xD7, 0x95, 0xD6, + 0xB9, 0x04, 0xD7, 0x91, 0xD6, 0xBF, 0x04, 0xD7, + // Bytes 31c0 - 31ff + 0x9B, 0xD6, 0xBF, 0x04, 0xD7, 0xA4, 0xD6, 0xBF, + 0x04, 0xD7, 0x90, 0xD7, 0x9C, 0x02, 0xD9, 0xB1, + 0x02, 0xD9, 0xBB, 0x02, 0xD9, 0xBE, 0x02, 0xDA, + 0x80, 0x02, 0xD9, 0xBA, 0x02, 0xD9, 0xBF, 0x02, + 0xD9, 0xB9, 0x02, 0xDA, 0xA4, 0x02, 0xDA, 0xA6, + 0x02, 0xDA, 0x84, 0x02, 0xDA, 0x83, 0x02, 0xDA, + 0x86, 0x02, 0xDA, 0x87, 0x02, 0xDA, 0x8D, 0x02, + 0xDA, 0x8C, 0x02, 0xDA, 0x8E, 0x02, 0xDA, 0x88, + // Bytes 3200 - 323f + 0x02, 0xDA, 0x98, 0x02, 0xDA, 0x91, 0x02, 0xDA, + 0xA9, 0x02, 0xDA, 0xAF, 0x02, 0xDA, 0xB3, 0x02, + 0xDA, 0xB1, 0x02, 0xDA, 0xBA, 0x02, 0xDA, 0xBB, + 0x02, 0xDB, 0x81, 0x02, 0xDA, 0xBE, 0x02, 0xDB, + 0x92, 0x02, 0xDA, 0xAD, 0x02, 0xDB, 0x87, 0x02, + 0xDB, 0x86, 0x02, 0xDB, 0x88, 0x02, 0xDB, 0x8B, + 0x02, 0xDB, 0x85, 0x02, 0xDB, 0x89, 0x02, 0xDB, + 0x90, 0x02, 0xD9, 0x89, 0x06, 0xD9, 0x8A, 0xD9, + // Bytes 3240 - 327f + 0x94, 0xD8, 0xA7, 0x06, 0xD9, 0x8A, 0xD9, 0x94, + 0xDB, 0x95, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, + 0x88, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x87, + 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x86, 0x06, + 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x88, 0x06, 0xD9, + 0x8A, 0xD9, 0x94, 0xDB, 0x90, 0x06, 0xD9, 0x8A, + 0xD9, 0x94, 0xD9, 0x89, 0x02, 0xDB, 0x8C, 0x06, + 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAC, 0x06, 0xD9, + // Bytes 3280 - 32bf + 0x8A, 0xD9, 0x94, 0xD8, 0xAD, 0x06, 0xD9, 0x8A, + 0xD9, 0x94, 0xD9, 0x85, 0x06, 0xD9, 0x8A, 0xD9, + 0x94, 0xD9, 0x8A, 0x04, 0xD8, 0xA8, 0xD8, 0xAC, + 0x04, 0xD8, 0xA8, 0xD8, 0xAD, 0x04, 0xD8, 0xA8, + 0xD8, 0xAE, 0x04, 0xD8, 0xA8, 0xD9, 0x85, 0x04, + 0xD8, 0xA8, 0xD9, 0x89, 0x04, 0xD8, 0xA8, 0xD9, + 0x8A, 0x04, 0xD8, 0xAA, 0xD8, 0xAC, 0x04, 0xD8, + 0xAA, 0xD8, 0xAD, 0x04, 0xD8, 0xAA, 0xD8, 0xAE, + // Bytes 32c0 - 32ff + 0x04, 0xD8, 0xAA, 0xD9, 0x85, 0x04, 0xD8, 0xAA, + 0xD9, 0x89, 0x04, 0xD8, 0xAA, 0xD9, 0x8A, 0x04, + 0xD8, 0xAB, 0xD8, 0xAC, 0x04, 0xD8, 0xAB, 0xD9, + 0x85, 0x04, 0xD8, 0xAB, 0xD9, 0x89, 0x04, 0xD8, + 0xAB, 0xD9, 0x8A, 0x04, 0xD8, 0xAC, 0xD8, 0xAD, + 0x04, 0xD8, 0xAC, 0xD9, 0x85, 0x04, 0xD8, 0xAD, + 0xD8, 0xAC, 0x04, 0xD8, 0xAD, 0xD9, 0x85, 0x04, + 0xD8, 0xAE, 0xD8, 0xAC, 0x04, 0xD8, 0xAE, 0xD8, + // Bytes 3300 - 333f + 0xAD, 0x04, 0xD8, 0xAE, 0xD9, 0x85, 0x04, 0xD8, + 0xB3, 0xD8, 0xAC, 0x04, 0xD8, 0xB3, 0xD8, 0xAD, + 0x04, 0xD8, 0xB3, 0xD8, 0xAE, 0x04, 0xD8, 0xB3, + 0xD9, 0x85, 0x04, 0xD8, 0xB5, 0xD8, 0xAD, 0x04, + 0xD8, 0xB5, 0xD9, 0x85, 0x04, 0xD8, 0xB6, 0xD8, + 0xAC, 0x04, 0xD8, 0xB6, 0xD8, 0xAD, 0x04, 0xD8, + 0xB6, 0xD8, 0xAE, 0x04, 0xD8, 0xB6, 0xD9, 0x85, + 0x04, 0xD8, 0xB7, 0xD8, 0xAD, 0x04, 0xD8, 0xB7, + // Bytes 3340 - 337f + 0xD9, 0x85, 0x04, 0xD8, 0xB8, 0xD9, 0x85, 0x04, + 0xD8, 0xB9, 0xD8, 0xAC, 0x04, 0xD8, 0xB9, 0xD9, + 0x85, 0x04, 0xD8, 0xBA, 0xD8, 0xAC, 0x04, 0xD8, + 0xBA, 0xD9, 0x85, 0x04, 0xD9, 0x81, 0xD8, 0xAC, + 0x04, 0xD9, 0x81, 0xD8, 0xAD, 0x04, 0xD9, 0x81, + 0xD8, 0xAE, 0x04, 0xD9, 0x81, 0xD9, 0x85, 0x04, + 0xD9, 0x81, 0xD9, 0x89, 0x04, 0xD9, 0x81, 0xD9, + 0x8A, 0x04, 0xD9, 0x82, 0xD8, 0xAD, 0x04, 0xD9, + // Bytes 3380 - 33bf + 0x82, 0xD9, 0x85, 0x04, 0xD9, 0x82, 0xD9, 0x89, + 0x04, 0xD9, 0x82, 0xD9, 0x8A, 0x04, 0xD9, 0x83, + 0xD8, 0xA7, 0x04, 0xD9, 0x83, 0xD8, 0xAC, 0x04, + 0xD9, 0x83, 0xD8, 0xAD, 0x04, 0xD9, 0x83, 0xD8, + 0xAE, 0x04, 0xD9, 0x83, 0xD9, 0x84, 0x04, 0xD9, + 0x83, 0xD9, 0x85, 0x04, 0xD9, 0x83, 0xD9, 0x89, + 0x04, 0xD9, 0x83, 0xD9, 0x8A, 0x04, 0xD9, 0x84, + 0xD8, 0xAC, 0x04, 0xD9, 0x84, 0xD8, 0xAD, 0x04, + // Bytes 33c0 - 33ff + 0xD9, 0x84, 0xD8, 0xAE, 0x04, 0xD9, 0x84, 0xD9, + 0x85, 0x04, 0xD9, 0x84, 0xD9, 0x89, 0x04, 0xD9, + 0x84, 0xD9, 0x8A, 0x04, 0xD9, 0x85, 0xD8, 0xAC, + 0x04, 0xD9, 0x85, 0xD8, 0xAD, 0x04, 0xD9, 0x85, + 0xD8, 0xAE, 0x04, 0xD9, 0x85, 0xD9, 0x85, 0x04, + 0xD9, 0x85, 0xD9, 0x89, 0x04, 0xD9, 0x85, 0xD9, + 0x8A, 0x04, 0xD9, 0x86, 0xD8, 0xAC, 0x04, 0xD9, + 0x86, 0xD8, 0xAD, 0x04, 0xD9, 0x86, 0xD8, 0xAE, + // Bytes 3400 - 343f + 0x04, 0xD9, 0x86, 0xD9, 0x85, 0x04, 0xD9, 0x86, + 0xD9, 0x89, 0x04, 0xD9, 0x86, 0xD9, 0x8A, 0x04, + 0xD9, 0x87, 0xD8, 0xAC, 0x04, 0xD9, 0x87, 0xD9, + 0x85, 0x04, 0xD9, 0x87, 0xD9, 0x89, 0x04, 0xD9, + 0x87, 0xD9, 0x8A, 0x04, 0xD9, 0x8A, 0xD8, 0xAC, + 0x04, 0xD9, 0x8A, 0xD8, 0xAD, 0x04, 0xD9, 0x8A, + 0xD8, 0xAE, 0x04, 0xD9, 0x8A, 0xD9, 0x85, 0x04, + 0xD9, 0x8A, 0xD9, 0x89, 0x04, 0xD9, 0x8A, 0xD9, + // Bytes 3440 - 347f + 0x8A, 0x04, 0xD8, 0xB0, 0xD9, 0xB0, 0x04, 0xD8, + 0xB1, 0xD9, 0xB0, 0x04, 0xD9, 0x89, 0xD9, 0xB0, + 0x05, 0x20, 0xD9, 0x8C, 0xD9, 0x91, 0x05, 0x20, + 0xD9, 0x8D, 0xD9, 0x91, 0x05, 0x20, 0xD9, 0x8E, + 0xD9, 0x91, 0x05, 0x20, 0xD9, 0x8F, 0xD9, 0x91, + 0x05, 0x20, 0xD9, 0x90, 0xD9, 0x91, 0x05, 0x20, + 0xD9, 0x91, 0xD9, 0xB0, 0x06, 0xD9, 0x8A, 0xD9, + 0x94, 0xD8, 0xB1, 0x06, 0xD9, 0x8A, 0xD9, 0x94, + // Bytes 3480 - 34bf + 0xD8, 0xB2, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, + 0x86, 0x04, 0xD8, 0xA8, 0xD8, 0xB1, 0x04, 0xD8, + 0xA8, 0xD8, 0xB2, 0x04, 0xD8, 0xA8, 0xD9, 0x86, + 0x04, 0xD8, 0xAA, 0xD8, 0xB1, 0x04, 0xD8, 0xAA, + 0xD8, 0xB2, 0x04, 0xD8, 0xAA, 0xD9, 0x86, 0x04, + 0xD8, 0xAB, 0xD8, 0xB1, 0x04, 0xD8, 0xAB, 0xD8, + 0xB2, 0x04, 0xD8, 0xAB, 0xD9, 0x86, 0x04, 0xD9, + 0x85, 0xD8, 0xA7, 0x04, 0xD9, 0x86, 0xD8, 0xB1, + // Bytes 34c0 - 34ff + 0x04, 0xD9, 0x86, 0xD8, 0xB2, 0x04, 0xD9, 0x86, + 0xD9, 0x86, 0x04, 0xD9, 0x8A, 0xD8, 0xB1, 0x04, + 0xD9, 0x8A, 0xD8, 0xB2, 0x04, 0xD9, 0x8A, 0xD9, + 0x86, 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xAE, + 0x06, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, 0x87, 0x04, + 0xD8, 0xA8, 0xD9, 0x87, 0x04, 0xD8, 0xAA, 0xD9, + 0x87, 0x04, 0xD8, 0xB5, 0xD8, 0xAE, 0x04, 0xD9, + 0x84, 0xD9, 0x87, 0x04, 0xD9, 0x86, 0xD9, 0x87, + // Bytes 3500 - 353f + 0x04, 0xD9, 0x87, 0xD9, 0xB0, 0x04, 0xD9, 0x8A, + 0xD9, 0x87, 0x04, 0xD8, 0xAB, 0xD9, 0x87, 0x04, + 0xD8, 0xB3, 0xD9, 0x87, 0x04, 0xD8, 0xB4, 0xD9, + 0x85, 0x04, 0xD8, 0xB4, 0xD9, 0x87, 0x06, 0xD9, + 0x80, 0xD9, 0x8E, 0xD9, 0x91, 0x06, 0xD9, 0x80, + 0xD9, 0x8F, 0xD9, 0x91, 0x06, 0xD9, 0x80, 0xD9, + 0x90, 0xD9, 0x91, 0x04, 0xD8, 0xB7, 0xD9, 0x89, + 0x04, 0xD8, 0xB7, 0xD9, 0x8A, 0x04, 0xD8, 0xB9, + // Bytes 3540 - 357f + 0xD9, 0x89, 0x04, 0xD8, 0xB9, 0xD9, 0x8A, 0x04, + 0xD8, 0xBA, 0xD9, 0x89, 0x04, 0xD8, 0xBA, 0xD9, + 0x8A, 0x04, 0xD8, 0xB3, 0xD9, 0x89, 0x04, 0xD8, + 0xB3, 0xD9, 0x8A, 0x04, 0xD8, 0xB4, 0xD9, 0x89, + 0x04, 0xD8, 0xB4, 0xD9, 0x8A, 0x04, 0xD8, 0xAD, + 0xD9, 0x89, 0x04, 0xD8, 0xAD, 0xD9, 0x8A, 0x04, + 0xD8, 0xAC, 0xD9, 0x89, 0x04, 0xD8, 0xAC, 0xD9, + 0x8A, 0x04, 0xD8, 0xAE, 0xD9, 0x89, 0x04, 0xD8, + // Bytes 3580 - 35bf + 0xAE, 0xD9, 0x8A, 0x04, 0xD8, 0xB5, 0xD9, 0x89, + 0x04, 0xD8, 0xB5, 0xD9, 0x8A, 0x04, 0xD8, 0xB6, + 0xD9, 0x89, 0x04, 0xD8, 0xB6, 0xD9, 0x8A, 0x04, + 0xD8, 0xB4, 0xD8, 0xAC, 0x04, 0xD8, 0xB4, 0xD8, + 0xAD, 0x04, 0xD8, 0xB4, 0xD8, 0xAE, 0x04, 0xD8, + 0xB4, 0xD8, 0xB1, 0x04, 0xD8, 0xB3, 0xD8, 0xB1, + 0x04, 0xD8, 0xB5, 0xD8, 0xB1, 0x04, 0xD8, 0xB6, + 0xD8, 0xB1, 0x04, 0xD8, 0xA7, 0xD9, 0x8B, 0x06, + // Bytes 35c0 - 35ff + 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, 0x85, 0x06, 0xD8, + 0xAA, 0xD8, 0xAD, 0xD8, 0xAC, 0x06, 0xD8, 0xAA, + 0xD8, 0xAD, 0xD9, 0x85, 0x06, 0xD8, 0xAA, 0xD8, + 0xAE, 0xD9, 0x85, 0x06, 0xD8, 0xAA, 0xD9, 0x85, + 0xD8, 0xAC, 0x06, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, + 0xAD, 0x06, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, 0xAE, + 0x06, 0xD8, 0xAC, 0xD9, 0x85, 0xD8, 0xAD, 0x06, + 0xD8, 0xAD, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, + // Bytes 3600 - 363f + 0xAD, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD8, 0xB3, + 0xD8, 0xAD, 0xD8, 0xAC, 0x06, 0xD8, 0xB3, 0xD8, + 0xAC, 0xD8, 0xAD, 0x06, 0xD8, 0xB3, 0xD8, 0xAC, + 0xD9, 0x89, 0x06, 0xD8, 0xB3, 0xD9, 0x85, 0xD8, + 0xAD, 0x06, 0xD8, 0xB3, 0xD9, 0x85, 0xD8, 0xAC, + 0x06, 0xD8, 0xB3, 0xD9, 0x85, 0xD9, 0x85, 0x06, + 0xD8, 0xB5, 0xD8, 0xAD, 0xD8, 0xAD, 0x06, 0xD8, + 0xB5, 0xD9, 0x85, 0xD9, 0x85, 0x06, 0xD8, 0xB4, + // Bytes 3640 - 367f + 0xD8, 0xAD, 0xD9, 0x85, 0x06, 0xD8, 0xB4, 0xD8, + 0xAC, 0xD9, 0x8A, 0x06, 0xD8, 0xB4, 0xD9, 0x85, + 0xD8, 0xAE, 0x06, 0xD8, 0xB4, 0xD9, 0x85, 0xD9, + 0x85, 0x06, 0xD8, 0xB6, 0xD8, 0xAD, 0xD9, 0x89, + 0x06, 0xD8, 0xB6, 0xD8, 0xAE, 0xD9, 0x85, 0x06, + 0xD8, 0xB7, 0xD9, 0x85, 0xD8, 0xAD, 0x06, 0xD8, + 0xB7, 0xD9, 0x85, 0xD9, 0x85, 0x06, 0xD8, 0xB7, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, 0xB9, 0xD8, + // Bytes 3680 - 36bf + 0xAC, 0xD9, 0x85, 0x06, 0xD8, 0xB9, 0xD9, 0x85, + 0xD9, 0x85, 0x06, 0xD8, 0xB9, 0xD9, 0x85, 0xD9, + 0x89, 0x06, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x85, + 0x06, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x8A, 0x06, + 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD9, + 0x81, 0xD8, 0xAE, 0xD9, 0x85, 0x06, 0xD9, 0x82, + 0xD9, 0x85, 0xD8, 0xAD, 0x06, 0xD9, 0x82, 0xD9, + 0x85, 0xD9, 0x85, 0x06, 0xD9, 0x84, 0xD8, 0xAD, + // Bytes 36c0 - 36ff + 0xD9, 0x85, 0x06, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, + 0x8A, 0x06, 0xD9, 0x84, 0xD8, 0xAD, 0xD9, 0x89, + 0x06, 0xD9, 0x84, 0xD8, 0xAC, 0xD8, 0xAC, 0x06, + 0xD9, 0x84, 0xD8, 0xAE, 0xD9, 0x85, 0x06, 0xD9, + 0x84, 0xD9, 0x85, 0xD8, 0xAD, 0x06, 0xD9, 0x85, + 0xD8, 0xAD, 0xD8, 0xAC, 0x06, 0xD9, 0x85, 0xD8, + 0xAD, 0xD9, 0x85, 0x06, 0xD9, 0x85, 0xD8, 0xAD, + 0xD9, 0x8A, 0x06, 0xD9, 0x85, 0xD8, 0xAC, 0xD8, + // Bytes 3700 - 373f + 0xAD, 0x06, 0xD9, 0x85, 0xD8, 0xAC, 0xD9, 0x85, + 0x06, 0xD9, 0x85, 0xD8, 0xAE, 0xD8, 0xAC, 0x06, + 0xD9, 0x85, 0xD8, 0xAE, 0xD9, 0x85, 0x06, 0xD9, + 0x85, 0xD8, 0xAC, 0xD8, 0xAE, 0x06, 0xD9, 0x87, + 0xD9, 0x85, 0xD8, 0xAC, 0x06, 0xD9, 0x87, 0xD9, + 0x85, 0xD9, 0x85, 0x06, 0xD9, 0x86, 0xD8, 0xAD, + 0xD9, 0x85, 0x06, 0xD9, 0x86, 0xD8, 0xAD, 0xD9, + 0x89, 0x06, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x85, + // Bytes 3740 - 377f + 0x06, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x89, 0x06, + 0xD9, 0x86, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, + 0x86, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD9, 0x8A, + 0xD9, 0x85, 0xD9, 0x85, 0x06, 0xD8, 0xA8, 0xD8, + 0xAE, 0xD9, 0x8A, 0x06, 0xD8, 0xAA, 0xD8, 0xAC, + 0xD9, 0x8A, 0x06, 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, + 0x89, 0x06, 0xD8, 0xAA, 0xD8, 0xAE, 0xD9, 0x8A, + 0x06, 0xD8, 0xAA, 0xD8, 0xAE, 0xD9, 0x89, 0x06, + // Bytes 3780 - 37bf + 0xD8, 0xAA, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, + 0xAA, 0xD9, 0x85, 0xD9, 0x89, 0x06, 0xD8, 0xAC, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, 0xAC, 0xD8, + 0xAD, 0xD9, 0x89, 0x06, 0xD8, 0xAC, 0xD9, 0x85, + 0xD9, 0x89, 0x06, 0xD8, 0xB3, 0xD8, 0xAE, 0xD9, + 0x89, 0x06, 0xD8, 0xB5, 0xD8, 0xAD, 0xD9, 0x8A, + 0x06, 0xD8, 0xB4, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, + 0xD8, 0xB6, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, 0xD9, + // Bytes 37c0 - 37ff + 0x84, 0xD8, 0xAC, 0xD9, 0x8A, 0x06, 0xD9, 0x84, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, 0x8A, 0xD8, + 0xAD, 0xD9, 0x8A, 0x06, 0xD9, 0x8A, 0xD8, 0xAC, + 0xD9, 0x8A, 0x06, 0xD9, 0x8A, 0xD9, 0x85, 0xD9, + 0x8A, 0x06, 0xD9, 0x85, 0xD9, 0x85, 0xD9, 0x8A, + 0x06, 0xD9, 0x82, 0xD9, 0x85, 0xD9, 0x8A, 0x06, + 0xD9, 0x86, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, 0xD8, + 0xB9, 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, 0x83, + // Bytes 3800 - 383f + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD9, 0x86, 0xD8, + 0xAC, 0xD8, 0xAD, 0x06, 0xD9, 0x85, 0xD8, 0xAE, + 0xD9, 0x8A, 0x06, 0xD9, 0x84, 0xD8, 0xAC, 0xD9, + 0x85, 0x06, 0xD9, 0x83, 0xD9, 0x85, 0xD9, 0x85, + 0x06, 0xD8, 0xAC, 0xD8, 0xAD, 0xD9, 0x8A, 0x06, + 0xD8, 0xAD, 0xD8, 0xAC, 0xD9, 0x8A, 0x06, 0xD9, + 0x85, 0xD8, 0xAC, 0xD9, 0x8A, 0x06, 0xD9, 0x81, + 0xD9, 0x85, 0xD9, 0x8A, 0x06, 0xD8, 0xA8, 0xD8, + // Bytes 3840 - 387f + 0xAD, 0xD9, 0x8A, 0x06, 0xD8, 0xB3, 0xD8, 0xAE, + 0xD9, 0x8A, 0x06, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, + 0x8A, 0x06, 0xD8, 0xB5, 0xD9, 0x84, 0xDB, 0x92, + 0x06, 0xD9, 0x82, 0xD9, 0x84, 0xDB, 0x92, 0x08, + 0xD8, 0xA7, 0xD9, 0x84, 0xD9, 0x84, 0xD9, 0x87, + 0x08, 0xD8, 0xA7, 0xD9, 0x83, 0xD8, 0xA8, 0xD8, + 0xB1, 0x08, 0xD9, 0x85, 0xD8, 0xAD, 0xD9, 0x85, + 0xD8, 0xAF, 0x08, 0xD8, 0xB5, 0xD9, 0x84, 0xD8, + // Bytes 3880 - 38bf + 0xB9, 0xD9, 0x85, 0x08, 0xD8, 0xB1, 0xD8, 0xB3, + 0xD9, 0x88, 0xD9, 0x84, 0x08, 0xD8, 0xB9, 0xD9, + 0x84, 0xD9, 0x8A, 0xD9, 0x87, 0x08, 0xD9, 0x88, + 0xD8, 0xB3, 0xD9, 0x84, 0xD9, 0x85, 0x06, 0xD8, + 0xB5, 0xD9, 0x84, 0xD9, 0x89, 0x21, 0xD8, 0xB5, + 0xD9, 0x84, 0xD9, 0x89, 0x20, 0xD8, 0xA7, 0xD9, + 0x84, 0xD9, 0x84, 0xD9, 0x87, 0x20, 0xD8, 0xB9, + 0xD9, 0x84, 0xD9, 0x8A, 0xD9, 0x87, 0x20, 0xD9, + // Bytes 38c0 - 38ff + 0x88, 0xD8, 0xB3, 0xD9, 0x84, 0xD9, 0x85, 0x0F, + 0xD8, 0xAC, 0xD9, 0x84, 0x20, 0xD8, 0xAC, 0xD9, + 0x84, 0xD8, 0xA7, 0xD9, 0x84, 0xD9, 0x87, 0x08, + 0xD8, 0xB1, 0xDB, 0x8C, 0xD8, 0xA7, 0xD9, 0x84, + 0x01, 0x2C, 0x03, 0xE3, 0x80, 0x81, 0x03, 0xE3, + 0x80, 0x82, 0x01, 0x3A, 0x01, 0x21, 0x01, 0x3F, + 0x03, 0xE3, 0x80, 0x96, 0x03, 0xE3, 0x80, 0x97, + 0x03, 0xE2, 0x80, 0x94, 0x03, 0xE2, 0x80, 0x93, + // Bytes 3900 - 393f + 0x01, 0x5F, 0x01, 0x7B, 0x01, 0x7D, 0x03, 0xE3, + 0x80, 0x94, 0x03, 0xE3, 0x80, 0x95, 0x03, 0xE3, + 0x80, 0x90, 0x03, 0xE3, 0x80, 0x91, 0x03, 0xE3, + 0x80, 0x8A, 0x03, 0xE3, 0x80, 0x8B, 0x03, 0xE3, + 0x80, 0x8C, 0x03, 0xE3, 0x80, 0x8D, 0x03, 0xE3, + 0x80, 0x8E, 0x03, 0xE3, 0x80, 0x8F, 0x01, 0x5B, + 0x01, 0x5D, 0x01, 0x23, 0x01, 0x26, 0x01, 0x2A, + 0x01, 0x2D, 0x01, 0x3C, 0x01, 0x3E, 0x01, 0x5C, + // Bytes 3940 - 397f + 0x01, 0x24, 0x01, 0x25, 0x01, 0x40, 0x03, 0x20, + 0xD9, 0x8B, 0x04, 0xD9, 0x80, 0xD9, 0x8B, 0x03, + 0x20, 0xD9, 0x8C, 0x03, 0x20, 0xD9, 0x8D, 0x03, + 0x20, 0xD9, 0x8E, 0x04, 0xD9, 0x80, 0xD9, 0x8E, + 0x03, 0x20, 0xD9, 0x8F, 0x04, 0xD9, 0x80, 0xD9, + 0x8F, 0x03, 0x20, 0xD9, 0x90, 0x04, 0xD9, 0x80, + 0xD9, 0x90, 0x03, 0x20, 0xD9, 0x91, 0x04, 0xD9, + 0x80, 0xD9, 0x91, 0x03, 0x20, 0xD9, 0x92, 0x04, + // Bytes 3980 - 39bf + 0xD9, 0x80, 0xD9, 0x92, 0x02, 0xD8, 0xA1, 0x02, + 0xD8, 0xA7, 0x02, 0xD8, 0xA8, 0x02, 0xD8, 0xA9, + 0x02, 0xD8, 0xAA, 0x02, 0xD8, 0xAB, 0x02, 0xD8, + 0xAC, 0x02, 0xD8, 0xAD, 0x02, 0xD8, 0xAE, 0x02, + 0xD8, 0xAF, 0x02, 0xD8, 0xB0, 0x02, 0xD8, 0xB1, + 0x02, 0xD8, 0xB2, 0x02, 0xD8, 0xB3, 0x02, 0xD8, + 0xB4, 0x02, 0xD8, 0xB5, 0x02, 0xD8, 0xB6, 0x02, + 0xD8, 0xB7, 0x02, 0xD8, 0xB8, 0x02, 0xD8, 0xB9, + // Bytes 39c0 - 39ff + 0x02, 0xD8, 0xBA, 0x02, 0xD9, 0x81, 0x02, 0xD9, + 0x82, 0x02, 0xD9, 0x83, 0x02, 0xD9, 0x84, 0x02, + 0xD9, 0x85, 0x02, 0xD9, 0x86, 0x02, 0xD9, 0x87, + 0x02, 0xD9, 0x88, 0x02, 0xD9, 0x8A, 0x06, 0xD9, + 0x84, 0xD8, 0xA7, 0xD9, 0x93, 0x06, 0xD9, 0x84, + 0xD8, 0xA7, 0xD9, 0x94, 0x06, 0xD9, 0x84, 0xD8, + 0xA7, 0xD9, 0x95, 0x04, 0xD9, 0x84, 0xD8, 0xA7, + 0x01, 0x22, 0x01, 0x27, 0x01, 0x2F, 0x01, 0x5E, + // Bytes 3a00 - 3a3f + 0x01, 0x7C, 0x01, 0x7E, 0x03, 0xE2, 0xA6, 0x85, + 0x03, 0xE2, 0xA6, 0x86, 0x03, 0xE3, 0x83, 0xBB, + 0x03, 0xE3, 0x82, 0xA1, 0x03, 0xE3, 0x82, 0xA3, + 0x03, 0xE3, 0x82, 0xA5, 0x03, 0xE3, 0x82, 0xA7, + 0x03, 0xE3, 0x82, 0xA9, 0x03, 0xE3, 0x83, 0xA3, + 0x03, 0xE3, 0x83, 0xA5, 0x03, 0xE3, 0x83, 0xA7, + 0x03, 0xE3, 0x83, 0x83, 0x03, 0xE3, 0x83, 0xBC, + 0x03, 0xE3, 0x83, 0xB3, 0x03, 0xE3, 0x82, 0x99, + // Bytes 3a40 - 3a7f + 0x03, 0xE3, 0x82, 0x9A, 0x02, 0xC2, 0xA2, 0x02, + 0xC2, 0xA3, 0x02, 0xC2, 0xAC, 0x02, 0xC2, 0xA6, + 0x02, 0xC2, 0xA5, 0x03, 0xE2, 0x82, 0xA9, 0x03, + 0xE2, 0x94, 0x82, 0x03, 0xE2, 0x86, 0x90, 0x03, + 0xE2, 0x86, 0x91, 0x03, 0xE2, 0x86, 0x92, 0x03, + 0xE2, 0x86, 0x93, 0x03, 0xE2, 0x96, 0xA0, 0x03, + 0xE2, 0x97, 0x8B, 0x08, 0xF0, 0x91, 0x82, 0x99, + 0xF0, 0x91, 0x82, 0xBA, 0x08, 0xF0, 0x91, 0x82, + // Bytes 3a80 - 3abf + 0x9B, 0xF0, 0x91, 0x82, 0xBA, 0x08, 0xF0, 0x91, + 0x82, 0xA5, 0xF0, 0x91, 0x82, 0xBA, 0x08, 0xF0, + 0x9D, 0x85, 0x97, 0xF0, 0x9D, 0x85, 0xA5, 0x08, + 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, + 0x0C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0x0C, 0xF0, 0x9D, + 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xAF, 0x0C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, + // Bytes 3ac0 - 3aff + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xB0, 0x0C, + 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, + 0xF0, 0x9D, 0x85, 0xB1, 0x0C, 0xF0, 0x9D, 0x85, + 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, + 0xB2, 0x08, 0xF0, 0x9D, 0x86, 0xB9, 0xF0, 0x9D, + 0x85, 0xA5, 0x08, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + 0x9D, 0x85, 0xA5, 0x0C, 0xF0, 0x9D, 0x86, 0xB9, + 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAE, + // Bytes 3b00 - 3b3f + 0x0C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, + 0xA5, 0xF0, 0x9D, 0x85, 0xAE, 0x0C, 0xF0, 0x9D, + 0x86, 0xB9, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xAF, 0x0C, 0xF0, 0x9D, 0x86, 0xBA, 0xF0, + 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0x02, + 0xC4, 0xB1, 0x02, 0xC8, 0xB7, 0x02, 0xCE, 0x91, + 0x02, 0xCE, 0x92, 0x02, 0xCE, 0x94, 0x02, 0xCE, + 0x95, 0x02, 0xCE, 0x96, 0x02, 0xCE, 0x97, 0x02, + // Bytes 3b40 - 3b7f + 0xCE, 0x99, 0x02, 0xCE, 0x9A, 0x02, 0xCE, 0x9B, + 0x02, 0xCE, 0x9C, 0x02, 0xCE, 0x9D, 0x02, 0xCE, + 0x9E, 0x02, 0xCE, 0x9F, 0x02, 0xCE, 0xA1, 0x02, + 0xCE, 0xA4, 0x02, 0xCE, 0xA6, 0x02, 0xCE, 0xA7, + 0x02, 0xCE, 0xA8, 0x03, 0xE2, 0x88, 0x87, 0x02, + 0xCE, 0xB1, 0x02, 0xCE, 0xB6, 0x02, 0xCE, 0xB7, + 0x02, 0xCE, 0xBB, 0x02, 0xCE, 0xBD, 0x02, 0xCE, + 0xBE, 0x02, 0xCE, 0xBF, 0x02, 0xCF, 0x83, 0x02, + // Bytes 3b80 - 3bbf + 0xCF, 0x84, 0x02, 0xCF, 0x85, 0x02, 0xCF, 0x88, + 0x02, 0xCF, 0x89, 0x03, 0xE2, 0x88, 0x82, 0x02, + 0xCF, 0x9C, 0x02, 0xCF, 0x9D, 0x02, 0x30, 0x2E, + 0x02, 0x30, 0x2C, 0x02, 0x31, 0x2C, 0x02, 0x32, + 0x2C, 0x02, 0x33, 0x2C, 0x02, 0x34, 0x2C, 0x02, + 0x35, 0x2C, 0x02, 0x36, 0x2C, 0x02, 0x37, 0x2C, + 0x02, 0x38, 0x2C, 0x02, 0x39, 0x2C, 0x03, 0x28, + 0x41, 0x29, 0x03, 0x28, 0x42, 0x29, 0x03, 0x28, + // Bytes 3bc0 - 3bff + 0x43, 0x29, 0x03, 0x28, 0x44, 0x29, 0x03, 0x28, + 0x45, 0x29, 0x03, 0x28, 0x46, 0x29, 0x03, 0x28, + 0x47, 0x29, 0x03, 0x28, 0x48, 0x29, 0x03, 0x28, + 0x49, 0x29, 0x03, 0x28, 0x4A, 0x29, 0x03, 0x28, + 0x4B, 0x29, 0x03, 0x28, 0x4C, 0x29, 0x03, 0x28, + 0x4D, 0x29, 0x03, 0x28, 0x4E, 0x29, 0x03, 0x28, + 0x4F, 0x29, 0x03, 0x28, 0x50, 0x29, 0x03, 0x28, + 0x51, 0x29, 0x03, 0x28, 0x52, 0x29, 0x03, 0x28, + // Bytes 3c00 - 3c3f + 0x53, 0x29, 0x03, 0x28, 0x54, 0x29, 0x03, 0x28, + 0x55, 0x29, 0x03, 0x28, 0x56, 0x29, 0x03, 0x28, + 0x57, 0x29, 0x03, 0x28, 0x58, 0x29, 0x03, 0x28, + 0x59, 0x29, 0x03, 0x28, 0x5A, 0x29, 0x07, 0xE3, + 0x80, 0x94, 0x53, 0xE3, 0x80, 0x95, 0x02, 0x43, + 0x44, 0x02, 0x57, 0x5A, 0x02, 0x48, 0x56, 0x02, + 0x53, 0x44, 0x02, 0x53, 0x53, 0x03, 0x50, 0x50, + 0x56, 0x02, 0x57, 0x43, 0x02, 0x44, 0x4A, 0x06, + // Bytes 3c40 - 3c7f + 0xE3, 0x81, 0xBB, 0xE3, 0x81, 0x8B, 0x06, 0xE3, + 0x82, 0xB3, 0xE3, 0x82, 0xB3, 0x03, 0xE5, 0xAD, + 0x97, 0x03, 0xE5, 0x8F, 0x8C, 0x03, 0xE5, 0xA4, + 0x9A, 0x03, 0xE8, 0xA7, 0xA3, 0x03, 0xE4, 0xBA, + 0xA4, 0x03, 0xE6, 0x98, 0xA0, 0x03, 0xE7, 0x84, + 0xA1, 0x03, 0xE5, 0x89, 0x8D, 0x03, 0xE5, 0xBE, + 0x8C, 0x03, 0xE5, 0x86, 0x8D, 0x03, 0xE6, 0x96, + 0xB0, 0x03, 0xE5, 0x88, 0x9D, 0x03, 0xE7, 0xB5, + // Bytes 3c80 - 3cbf + 0x82, 0x03, 0xE8, 0xB2, 0xA9, 0x03, 0xE5, 0xA3, + 0xB0, 0x03, 0xE5, 0x90, 0xB9, 0x03, 0xE6, 0xBC, + 0x94, 0x03, 0xE6, 0x8A, 0x95, 0x03, 0xE6, 0x8D, + 0x95, 0x03, 0xE9, 0x81, 0x8A, 0x03, 0xE6, 0x8C, + 0x87, 0x03, 0xE6, 0x89, 0x93, 0x03, 0xE7, 0xA6, + 0x81, 0x03, 0xE7, 0xA9, 0xBA, 0x03, 0xE5, 0x90, + 0x88, 0x03, 0xE6, 0xBA, 0x80, 0x03, 0xE7, 0x94, + 0xB3, 0x03, 0xE5, 0x89, 0xB2, 0x03, 0xE5, 0x96, + // Bytes 3cc0 - 3cff + 0xB6, 0x09, 0xE3, 0x80, 0x94, 0xE6, 0x9C, 0xAC, + 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE4, + 0xB8, 0x89, 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, + 0x94, 0xE4, 0xBA, 0x8C, 0xE3, 0x80, 0x95, 0x09, + 0xE3, 0x80, 0x94, 0xE5, 0xAE, 0x89, 0xE3, 0x80, + 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE7, 0x82, 0xB9, + 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE6, + 0x89, 0x93, 0xE3, 0x80, 0x95, 0x09, 0xE3, 0x80, + // Bytes 3d00 - 3d3f + 0x94, 0xE7, 0x9B, 0x97, 0xE3, 0x80, 0x95, 0x09, + 0xE3, 0x80, 0x94, 0xE5, 0x8B, 0x9D, 0xE3, 0x80, + 0x95, 0x09, 0xE3, 0x80, 0x94, 0xE6, 0x95, 0x97, + 0xE3, 0x80, 0x95, 0x03, 0xE5, 0xBE, 0x97, 0x03, + 0xE5, 0x8F, 0xAF, 0x03, 0xE4, 0xB8, 0xBD, 0x03, + 0xE4, 0xB8, 0xB8, 0x03, 0xE4, 0xB9, 0x81, 0x04, + 0xF0, 0xA0, 0x84, 0xA2, 0x03, 0xE4, 0xBD, 0xA0, + 0x03, 0xE4, 0xBE, 0xBB, 0x03, 0xE5, 0x80, 0x82, + // Bytes 3d40 - 3d7f + 0x03, 0xE5, 0x81, 0xBA, 0x03, 0xE5, 0x82, 0x99, + 0x03, 0xE5, 0x83, 0x8F, 0x03, 0xE3, 0x92, 0x9E, + 0x04, 0xF0, 0xA0, 0x98, 0xBA, 0x03, 0xE5, 0x85, + 0x94, 0x03, 0xE5, 0x85, 0xA4, 0x03, 0xE5, 0x85, + 0xB7, 0x04, 0xF0, 0xA0, 0x94, 0x9C, 0x03, 0xE3, + 0x92, 0xB9, 0x03, 0xE5, 0x85, 0xA7, 0x04, 0xF0, + 0xA0, 0x95, 0x8B, 0x03, 0xE5, 0x86, 0x97, 0x03, + 0xE5, 0x86, 0xA4, 0x03, 0xE4, 0xBB, 0x8C, 0x03, + // Bytes 3d80 - 3dbf + 0xE5, 0x86, 0xAC, 0x04, 0xF0, 0xA9, 0x87, 0x9F, + 0x03, 0xE5, 0x88, 0x83, 0x03, 0xE3, 0x93, 0x9F, + 0x03, 0xE5, 0x88, 0xBB, 0x03, 0xE5, 0x89, 0x86, + 0x03, 0xE5, 0x89, 0xB7, 0x03, 0xE3, 0x94, 0x95, + 0x03, 0xE5, 0x8C, 0x85, 0x03, 0xE5, 0x8C, 0x86, + 0x03, 0xE5, 0x8D, 0x89, 0x03, 0xE5, 0x8D, 0x9A, + 0x03, 0xE5, 0x8D, 0xB3, 0x03, 0xE5, 0x8D, 0xBD, + 0x03, 0xE5, 0x8D, 0xBF, 0x04, 0xF0, 0xA0, 0xA8, + // Bytes 3dc0 - 3dff + 0xAC, 0x03, 0xE7, 0x81, 0xB0, 0x03, 0xE5, 0x8F, + 0x8A, 0x03, 0xE5, 0x8F, 0x9F, 0x04, 0xF0, 0xA0, + 0xAD, 0xA3, 0x03, 0xE5, 0x8F, 0xAB, 0x03, 0xE5, + 0x8F, 0xB1, 0x03, 0xE5, 0x90, 0x86, 0x03, 0xE5, + 0x92, 0x9E, 0x03, 0xE5, 0x90, 0xB8, 0x03, 0xE5, + 0x91, 0x88, 0x03, 0xE5, 0x91, 0xA8, 0x03, 0xE5, + 0x92, 0xA2, 0x03, 0xE5, 0x93, 0xB6, 0x03, 0xE5, + 0x94, 0x90, 0x03, 0xE5, 0x95, 0x93, 0x03, 0xE5, + // Bytes 3e00 - 3e3f + 0x95, 0xA3, 0x03, 0xE5, 0x96, 0x84, 0x03, 0xE5, + 0x96, 0xAB, 0x03, 0xE5, 0x96, 0xB3, 0x03, 0xE5, + 0x97, 0x82, 0x03, 0xE5, 0x9C, 0x96, 0x03, 0xE5, + 0x9C, 0x97, 0x03, 0xE5, 0x99, 0x91, 0x03, 0xE5, + 0x99, 0xB4, 0x03, 0xE5, 0xA3, 0xAE, 0x03, 0xE5, + 0x9F, 0x8E, 0x03, 0xE5, 0x9F, 0xB4, 0x03, 0xE5, + 0xA0, 0x8D, 0x03, 0xE5, 0x9E, 0x8B, 0x03, 0xE5, + 0xA0, 0xB2, 0x03, 0xE5, 0xA0, 0xB1, 0x03, 0xE5, + // Bytes 3e40 - 3e7f + 0xA2, 0xAC, 0x04, 0xF0, 0xA1, 0x93, 0xA4, 0x03, + 0xE5, 0xA3, 0xB2, 0x03, 0xE5, 0xA3, 0xB7, 0x03, + 0xE5, 0xA4, 0x86, 0x03, 0xE5, 0xA4, 0xA2, 0x03, + 0xE5, 0xA5, 0xA2, 0x04, 0xF0, 0xA1, 0x9A, 0xA8, + 0x04, 0xF0, 0xA1, 0x9B, 0xAA, 0x03, 0xE5, 0xA7, + 0xAC, 0x03, 0xE5, 0xA8, 0x9B, 0x03, 0xE5, 0xA8, + 0xA7, 0x03, 0xE5, 0xA7, 0x98, 0x03, 0xE5, 0xA9, + 0xA6, 0x03, 0xE3, 0x9B, 0xAE, 0x03, 0xE3, 0x9B, + // Bytes 3e80 - 3ebf + 0xBC, 0x03, 0xE5, 0xAC, 0x88, 0x03, 0xE5, 0xAC, + 0xBE, 0x04, 0xF0, 0xA1, 0xA7, 0x88, 0x03, 0xE5, + 0xAF, 0x83, 0x03, 0xE5, 0xAF, 0x98, 0x03, 0xE5, + 0xAF, 0xB3, 0x04, 0xF0, 0xA1, 0xAC, 0x98, 0x03, + 0xE5, 0xAF, 0xBF, 0x03, 0xE5, 0xB0, 0x86, 0x03, + 0xE5, 0xBD, 0x93, 0x03, 0xE3, 0x9E, 0x81, 0x03, + 0xE5, 0xB1, 0xA0, 0x03, 0xE5, 0xB3, 0x80, 0x03, + 0xE5, 0xB2, 0x8D, 0x04, 0xF0, 0xA1, 0xB7, 0xA4, + // Bytes 3ec0 - 3eff + 0x03, 0xE5, 0xB5, 0x83, 0x04, 0xF0, 0xA1, 0xB7, + 0xA6, 0x03, 0xE5, 0xB5, 0xAE, 0x03, 0xE5, 0xB5, + 0xAB, 0x03, 0xE5, 0xB5, 0xBC, 0x03, 0xE5, 0xB7, + 0xA1, 0x03, 0xE5, 0xB7, 0xA2, 0x03, 0xE3, 0xA0, + 0xAF, 0x03, 0xE5, 0xB7, 0xBD, 0x03, 0xE5, 0xB8, + 0xA8, 0x03, 0xE5, 0xB8, 0xBD, 0x03, 0xE5, 0xB9, + 0xA9, 0x03, 0xE3, 0xA1, 0xA2, 0x04, 0xF0, 0xA2, + 0x86, 0x83, 0x03, 0xE3, 0xA1, 0xBC, 0x03, 0xE5, + // Bytes 3f00 - 3f3f + 0xBA, 0xB0, 0x03, 0xE5, 0xBA, 0xB3, 0x03, 0xE5, + 0xBA, 0xB6, 0x04, 0xF0, 0xAA, 0x8E, 0x92, 0x04, + 0xF0, 0xA2, 0x8C, 0xB1, 0x03, 0xE8, 0x88, 0x81, + 0x03, 0xE5, 0xBC, 0xA2, 0x03, 0xE3, 0xA3, 0x87, + 0x04, 0xF0, 0xA3, 0x8A, 0xB8, 0x04, 0xF0, 0xA6, + 0x87, 0x9A, 0x03, 0xE5, 0xBD, 0xA2, 0x03, 0xE5, + 0xBD, 0xAB, 0x03, 0xE3, 0xA3, 0xA3, 0x03, 0xE5, + 0xBE, 0x9A, 0x03, 0xE5, 0xBF, 0x8D, 0x03, 0xE5, + // Bytes 3f40 - 3f7f + 0xBF, 0x97, 0x03, 0xE5, 0xBF, 0xB9, 0x03, 0xE6, + 0x82, 0x81, 0x03, 0xE3, 0xA4, 0xBA, 0x03, 0xE3, + 0xA4, 0x9C, 0x04, 0xF0, 0xA2, 0x9B, 0x94, 0x03, + 0xE6, 0x83, 0x87, 0x03, 0xE6, 0x85, 0x88, 0x03, + 0xE6, 0x85, 0x8C, 0x03, 0xE6, 0x85, 0xBA, 0x03, + 0xE6, 0x86, 0xB2, 0x03, 0xE6, 0x86, 0xA4, 0x03, + 0xE6, 0x86, 0xAF, 0x03, 0xE6, 0x87, 0x9E, 0x03, + 0xE6, 0x88, 0x90, 0x03, 0xE6, 0x88, 0x9B, 0x03, + // Bytes 3f80 - 3fbf + 0xE6, 0x89, 0x9D, 0x03, 0xE6, 0x8A, 0xB1, 0x03, + 0xE6, 0x8B, 0x94, 0x03, 0xE6, 0x8D, 0x90, 0x04, + 0xF0, 0xA2, 0xAC, 0x8C, 0x03, 0xE6, 0x8C, 0xBD, + 0x03, 0xE6, 0x8B, 0xBC, 0x03, 0xE6, 0x8D, 0xA8, + 0x03, 0xE6, 0x8E, 0x83, 0x03, 0xE6, 0x8F, 0xA4, + 0x04, 0xF0, 0xA2, 0xAF, 0xB1, 0x03, 0xE6, 0x90, + 0xA2, 0x03, 0xE6, 0x8F, 0x85, 0x03, 0xE6, 0x8E, + 0xA9, 0x03, 0xE3, 0xA8, 0xAE, 0x03, 0xE6, 0x91, + // Bytes 3fc0 - 3fff + 0xA9, 0x03, 0xE6, 0x91, 0xBE, 0x03, 0xE6, 0x92, + 0x9D, 0x03, 0xE6, 0x91, 0xB7, 0x03, 0xE3, 0xA9, + 0xAC, 0x03, 0xE6, 0x95, 0xAC, 0x04, 0xF0, 0xA3, + 0x80, 0x8A, 0x03, 0xE6, 0x97, 0xA3, 0x03, 0xE6, + 0x9B, 0xB8, 0x03, 0xE6, 0x99, 0x89, 0x03, 0xE3, + 0xAC, 0x99, 0x03, 0xE3, 0xAC, 0x88, 0x03, 0xE3, + 0xAB, 0xA4, 0x03, 0xE5, 0x86, 0x92, 0x03, 0xE5, + 0x86, 0x95, 0x03, 0xE6, 0x9C, 0x80, 0x03, 0xE6, + // Bytes 4000 - 403f + 0x9A, 0x9C, 0x03, 0xE8, 0x82, 0xAD, 0x03, 0xE4, + 0x8F, 0x99, 0x03, 0xE6, 0x9C, 0xA1, 0x03, 0xE6, + 0x9D, 0x9E, 0x03, 0xE6, 0x9D, 0x93, 0x04, 0xF0, + 0xA3, 0x8F, 0x83, 0x03, 0xE3, 0xAD, 0x89, 0x03, + 0xE6, 0x9F, 0xBA, 0x03, 0xE6, 0x9E, 0x85, 0x03, + 0xE6, 0xA1, 0x92, 0x04, 0xF0, 0xA3, 0x91, 0xAD, + 0x03, 0xE6, 0xA2, 0x8E, 0x03, 0xE6, 0xA0, 0x9F, + 0x03, 0xE6, 0xA4, 0x94, 0x03, 0xE6, 0xA5, 0x82, + // Bytes 4040 - 407f + 0x03, 0xE6, 0xA6, 0xA3, 0x03, 0xE6, 0xA7, 0xAA, + 0x03, 0xE6, 0xAA, 0xA8, 0x04, 0xF0, 0xA3, 0x9A, + 0xA3, 0x03, 0xE6, 0xAB, 0x9B, 0x03, 0xE3, 0xB0, + 0x98, 0x03, 0xE6, 0xAC, 0xA1, 0x04, 0xF0, 0xA3, + 0xA2, 0xA7, 0x03, 0xE6, 0xAD, 0x94, 0x03, 0xE3, + 0xB1, 0x8E, 0x03, 0xE6, 0xAD, 0xB2, 0x03, 0xE6, + 0xAE, 0x9F, 0x03, 0xE6, 0xAE, 0xBB, 0x04, 0xF0, + 0xA3, 0xAA, 0x8D, 0x04, 0xF0, 0xA1, 0xB4, 0x8B, + // Bytes 4080 - 40bf + 0x04, 0xF0, 0xA3, 0xAB, 0xBA, 0x03, 0xE6, 0xB1, + 0x8E, 0x04, 0xF0, 0xA3, 0xB2, 0xBC, 0x03, 0xE6, + 0xB2, 0xBF, 0x03, 0xE6, 0xB3, 0x8D, 0x03, 0xE6, + 0xB1, 0xA7, 0x03, 0xE6, 0xB4, 0x96, 0x03, 0xE6, + 0xB4, 0xBE, 0x03, 0xE6, 0xB5, 0xA9, 0x03, 0xE6, + 0xB5, 0xB8, 0x03, 0xE6, 0xB6, 0x85, 0x04, 0xF0, + 0xA3, 0xB4, 0x9E, 0x03, 0xE6, 0xB4, 0xB4, 0x03, + 0xE6, 0xB8, 0xAF, 0x03, 0xE6, 0xB9, 0xAE, 0x03, + // Bytes 40c0 - 40ff + 0xE3, 0xB4, 0xB3, 0x03, 0xE6, 0xBB, 0x87, 0x04, + 0xF0, 0xA3, 0xBB, 0x91, 0x03, 0xE6, 0xB7, 0xB9, + 0x03, 0xE6, 0xBD, 0xAE, 0x04, 0xF0, 0xA3, 0xBD, + 0x9E, 0x04, 0xF0, 0xA3, 0xBE, 0x8E, 0x03, 0xE6, + 0xBF, 0x86, 0x03, 0xE7, 0x80, 0xB9, 0x03, 0xE7, + 0x80, 0x9B, 0x03, 0xE3, 0xB6, 0x96, 0x03, 0xE7, + 0x81, 0x8A, 0x03, 0xE7, 0x81, 0xBD, 0x03, 0xE7, + 0x81, 0xB7, 0x03, 0xE7, 0x82, 0xAD, 0x04, 0xF0, + // Bytes 4100 - 413f + 0xA0, 0x94, 0xA5, 0x03, 0xE7, 0x85, 0x85, 0x04, + 0xF0, 0xA4, 0x89, 0xA3, 0x03, 0xE7, 0x86, 0x9C, + 0x04, 0xF0, 0xA4, 0x8E, 0xAB, 0x03, 0xE7, 0x88, + 0xA8, 0x03, 0xE7, 0x89, 0x90, 0x04, 0xF0, 0xA4, + 0x98, 0x88, 0x03, 0xE7, 0x8A, 0x80, 0x03, 0xE7, + 0x8A, 0x95, 0x04, 0xF0, 0xA4, 0x9C, 0xB5, 0x04, + 0xF0, 0xA4, 0xA0, 0x94, 0x03, 0xE7, 0x8D, 0xBA, + 0x03, 0xE7, 0x8E, 0x8B, 0x03, 0xE3, 0xBA, 0xAC, + // Bytes 4140 - 417f + 0x03, 0xE7, 0x8E, 0xA5, 0x03, 0xE3, 0xBA, 0xB8, + 0x03, 0xE7, 0x91, 0x87, 0x03, 0xE7, 0x91, 0x9C, + 0x03, 0xE7, 0x92, 0x85, 0x03, 0xE7, 0x93, 0x8A, + 0x03, 0xE3, 0xBC, 0x9B, 0x03, 0xE7, 0x94, 0xA4, + 0x04, 0xF0, 0xA4, 0xB0, 0xB6, 0x03, 0xE7, 0x94, + 0xBE, 0x04, 0xF0, 0xA4, 0xB2, 0x92, 0x04, 0xF0, + 0xA2, 0x86, 0x9F, 0x03, 0xE7, 0x98, 0x90, 0x04, + 0xF0, 0xA4, 0xBE, 0xA1, 0x04, 0xF0, 0xA4, 0xBE, + // Bytes 4180 - 41bf + 0xB8, 0x04, 0xF0, 0xA5, 0x81, 0x84, 0x03, 0xE3, + 0xBF, 0xBC, 0x03, 0xE4, 0x80, 0x88, 0x04, 0xF0, + 0xA5, 0x83, 0xB3, 0x04, 0xF0, 0xA5, 0x83, 0xB2, + 0x04, 0xF0, 0xA5, 0x84, 0x99, 0x04, 0xF0, 0xA5, + 0x84, 0xB3, 0x03, 0xE7, 0x9C, 0x9E, 0x03, 0xE7, + 0x9C, 0x9F, 0x03, 0xE7, 0x9E, 0x8B, 0x03, 0xE4, + 0x81, 0x86, 0x03, 0xE4, 0x82, 0x96, 0x04, 0xF0, + 0xA5, 0x90, 0x9D, 0x03, 0xE7, 0xA1, 0x8E, 0x03, + // Bytes 41c0 - 41ff + 0xE4, 0x83, 0xA3, 0x04, 0xF0, 0xA5, 0x98, 0xA6, + 0x04, 0xF0, 0xA5, 0x9A, 0x9A, 0x04, 0xF0, 0xA5, + 0x9B, 0x85, 0x03, 0xE7, 0xA7, 0xAB, 0x03, 0xE4, + 0x84, 0xAF, 0x03, 0xE7, 0xA9, 0x8A, 0x03, 0xE7, + 0xA9, 0x8F, 0x04, 0xF0, 0xA5, 0xA5, 0xBC, 0x04, + 0xF0, 0xA5, 0xAA, 0xA7, 0x03, 0xE7, 0xAB, 0xAE, + 0x03, 0xE4, 0x88, 0x82, 0x04, 0xF0, 0xA5, 0xAE, + 0xAB, 0x03, 0xE7, 0xAF, 0x86, 0x03, 0xE7, 0xAF, + // Bytes 4200 - 423f + 0x89, 0x03, 0xE4, 0x88, 0xA7, 0x04, 0xF0, 0xA5, + 0xB2, 0x80, 0x03, 0xE7, 0xB3, 0x92, 0x03, 0xE4, + 0x8A, 0xA0, 0x03, 0xE7, 0xB3, 0xA8, 0x03, 0xE7, + 0xB3, 0xA3, 0x03, 0xE7, 0xB4, 0x80, 0x04, 0xF0, + 0xA5, 0xBE, 0x86, 0x03, 0xE7, 0xB5, 0xA3, 0x03, + 0xE4, 0x8C, 0x81, 0x03, 0xE7, 0xB7, 0x87, 0x03, + 0xE7, 0xB8, 0x82, 0x03, 0xE7, 0xB9, 0x85, 0x03, + 0xE4, 0x8C, 0xB4, 0x04, 0xF0, 0xA6, 0x88, 0xA8, + // Bytes 4240 - 427f + 0x04, 0xF0, 0xA6, 0x89, 0x87, 0x03, 0xE4, 0x8D, + 0x99, 0x04, 0xF0, 0xA6, 0x8B, 0x99, 0x03, 0xE7, + 0xBD, 0xBA, 0x04, 0xF0, 0xA6, 0x8C, 0xBE, 0x03, + 0xE7, 0xBE, 0x95, 0x03, 0xE7, 0xBF, 0xBA, 0x04, + 0xF0, 0xA6, 0x93, 0x9A, 0x04, 0xF0, 0xA6, 0x94, + 0xA3, 0x03, 0xE8, 0x81, 0xA0, 0x04, 0xF0, 0xA6, + 0x96, 0xA8, 0x03, 0xE8, 0x81, 0xB0, 0x04, 0xF0, + 0xA3, 0x8D, 0x9F, 0x03, 0xE4, 0x8F, 0x95, 0x03, + // Bytes 4280 - 42bf + 0xE8, 0x82, 0xB2, 0x03, 0xE8, 0x84, 0x83, 0x03, + 0xE4, 0x90, 0x8B, 0x03, 0xE8, 0x84, 0xBE, 0x03, + 0xE5, 0xAA, 0xB5, 0x04, 0xF0, 0xA6, 0x9E, 0xA7, + 0x04, 0xF0, 0xA6, 0x9E, 0xB5, 0x04, 0xF0, 0xA3, + 0x8E, 0x93, 0x04, 0xF0, 0xA3, 0x8E, 0x9C, 0x03, + 0xE8, 0x88, 0x84, 0x03, 0xE8, 0xBE, 0x9E, 0x03, + 0xE4, 0x91, 0xAB, 0x03, 0xE8, 0x8A, 0x91, 0x03, + 0xE8, 0x8A, 0x8B, 0x03, 0xE8, 0x8A, 0x9D, 0x03, + // Bytes 42c0 - 42ff + 0xE5, 0x8A, 0xB3, 0x03, 0xE8, 0x8A, 0xB1, 0x03, + 0xE8, 0x8A, 0xB3, 0x03, 0xE8, 0x8A, 0xBD, 0x03, + 0xE8, 0x8B, 0xA6, 0x04, 0xF0, 0xA6, 0xAC, 0xBC, + 0x03, 0xE8, 0x8C, 0x9D, 0x03, 0xE8, 0x8D, 0xA3, + 0x03, 0xE8, 0x8E, 0xAD, 0x03, 0xE8, 0x8C, 0xA3, + 0x03, 0xE8, 0x8E, 0xBD, 0x03, 0xE8, 0x8F, 0xA7, + 0x03, 0xE8, 0x8D, 0x93, 0x03, 0xE8, 0x8F, 0x8A, + 0x03, 0xE8, 0x8F, 0x8C, 0x03, 0xE8, 0x8F, 0x9C, + // Bytes 4300 - 433f + 0x04, 0xF0, 0xA6, 0xB0, 0xB6, 0x04, 0xF0, 0xA6, + 0xB5, 0xAB, 0x04, 0xF0, 0xA6, 0xB3, 0x95, 0x03, + 0xE4, 0x94, 0xAB, 0x03, 0xE8, 0x93, 0xB1, 0x03, + 0xE8, 0x93, 0xB3, 0x03, 0xE8, 0x94, 0x96, 0x04, + 0xF0, 0xA7, 0x8F, 0x8A, 0x03, 0xE8, 0x95, 0xA4, + 0x04, 0xF0, 0xA6, 0xBC, 0xAC, 0x03, 0xE4, 0x95, + 0x9D, 0x03, 0xE4, 0x95, 0xA1, 0x04, 0xF0, 0xA6, + 0xBE, 0xB1, 0x04, 0xF0, 0xA7, 0x83, 0x92, 0x03, + // Bytes 4340 - 437f + 0xE4, 0x95, 0xAB, 0x03, 0xE8, 0x99, 0x90, 0x03, + 0xE8, 0x99, 0xA7, 0x03, 0xE8, 0x99, 0xA9, 0x03, + 0xE8, 0x9A, 0xA9, 0x03, 0xE8, 0x9A, 0x88, 0x03, + 0xE8, 0x9C, 0x8E, 0x03, 0xE8, 0x9B, 0xA2, 0x03, + 0xE8, 0x9C, 0xA8, 0x03, 0xE8, 0x9D, 0xAB, 0x03, + 0xE8, 0x9E, 0x86, 0x03, 0xE4, 0x97, 0x97, 0x03, + 0xE8, 0x9F, 0xA1, 0x03, 0xE8, 0xA0, 0x81, 0x03, + 0xE4, 0x97, 0xB9, 0x03, 0xE8, 0xA1, 0xA0, 0x04, + // Bytes 4380 - 43bf + 0xF0, 0xA7, 0x99, 0xA7, 0x03, 0xE8, 0xA3, 0x97, + 0x03, 0xE8, 0xA3, 0x9E, 0x03, 0xE4, 0x98, 0xB5, + 0x03, 0xE8, 0xA3, 0xBA, 0x03, 0xE3, 0x92, 0xBB, + 0x04, 0xF0, 0xA7, 0xA2, 0xAE, 0x04, 0xF0, 0xA7, + 0xA5, 0xA6, 0x03, 0xE4, 0x9A, 0xBE, 0x03, 0xE4, + 0x9B, 0x87, 0x03, 0xE8, 0xAA, 0xA0, 0x04, 0xF0, + 0xA7, 0xB2, 0xA8, 0x03, 0xE8, 0xB2, 0xAB, 0x03, + 0xE8, 0xB3, 0x81, 0x03, 0xE8, 0xB4, 0x9B, 0x03, + // Bytes 43c0 - 43ff + 0xE8, 0xB5, 0xB7, 0x04, 0xF0, 0xA7, 0xBC, 0xAF, + 0x04, 0xF0, 0xA0, 0xA0, 0x84, 0x03, 0xE8, 0xB7, + 0x8B, 0x03, 0xE8, 0xB6, 0xBC, 0x03, 0xE8, 0xB7, + 0xB0, 0x04, 0xF0, 0xA0, 0xA3, 0x9E, 0x03, 0xE8, + 0xBB, 0x94, 0x04, 0xF0, 0xA8, 0x97, 0x92, 0x04, + 0xF0, 0xA8, 0x97, 0xAD, 0x03, 0xE9, 0x82, 0x94, + 0x03, 0xE9, 0x83, 0xB1, 0x03, 0xE9, 0x84, 0x91, + 0x04, 0xF0, 0xA8, 0x9C, 0xAE, 0x03, 0xE9, 0x84, + // Bytes 4400 - 443f + 0x9B, 0x03, 0xE9, 0x88, 0xB8, 0x03, 0xE9, 0x8B, + 0x97, 0x03, 0xE9, 0x8B, 0x98, 0x03, 0xE9, 0x89, + 0xBC, 0x03, 0xE9, 0x8F, 0xB9, 0x03, 0xE9, 0x90, + 0x95, 0x04, 0xF0, 0xA8, 0xAF, 0xBA, 0x03, 0xE9, + 0x96, 0x8B, 0x03, 0xE4, 0xA6, 0x95, 0x03, 0xE9, + 0x96, 0xB7, 0x04, 0xF0, 0xA8, 0xB5, 0xB7, 0x03, + 0xE4, 0xA7, 0xA6, 0x03, 0xE9, 0x9B, 0x83, 0x03, + 0xE5, 0xB6, 0xB2, 0x03, 0xE9, 0x9C, 0xA3, 0x04, + // Bytes 4440 - 447f + 0xF0, 0xA9, 0x85, 0x85, 0x04, 0xF0, 0xA9, 0x88, + 0x9A, 0x03, 0xE4, 0xA9, 0xAE, 0x03, 0xE4, 0xA9, + 0xB6, 0x03, 0xE9, 0x9F, 0xA0, 0x04, 0xF0, 0xA9, + 0x90, 0x8A, 0x03, 0xE4, 0xAA, 0xB2, 0x04, 0xF0, + 0xA9, 0x92, 0x96, 0x03, 0xE9, 0xA0, 0xA9, 0x04, + 0xF0, 0xA9, 0x96, 0xB6, 0x03, 0xE9, 0xA3, 0xA2, + 0x03, 0xE4, 0xAC, 0xB3, 0x03, 0xE9, 0xA4, 0xA9, + 0x03, 0xE9, 0xA6, 0xA7, 0x03, 0xE9, 0xA7, 0x82, + // Bytes 4480 - 44bf + 0x03, 0xE9, 0xA7, 0xBE, 0x03, 0xE4, 0xAF, 0x8E, + 0x04, 0xF0, 0xA9, 0xAC, 0xB0, 0x03, 0xE9, 0xB1, + 0x80, 0x03, 0xE9, 0xB3, 0xBD, 0x03, 0xE4, 0xB3, + 0x8E, 0x03, 0xE4, 0xB3, 0xAD, 0x03, 0xE9, 0xB5, + 0xA7, 0x04, 0xF0, 0xAA, 0x83, 0x8E, 0x03, 0xE4, + 0xB3, 0xB8, 0x04, 0xF0, 0xAA, 0x84, 0x85, 0x04, + 0xF0, 0xAA, 0x88, 0x8E, 0x04, 0xF0, 0xAA, 0x8A, + 0x91, 0x03, 0xE4, 0xB5, 0x96, 0x03, 0xE9, 0xBB, + // Bytes 44c0 - 44ff + 0xBE, 0x03, 0xE9, 0xBC, 0x85, 0x03, 0xE9, 0xBC, + 0x8F, 0x03, 0xE9, 0xBC, 0x96, 0x04, 0xF0, 0xAA, + 0x98, 0x80, +} + +// nfcDecompValues: 4992 entries, 9984 bytes +// Block 2 is the null block. +var nfcDecompValues = [4992]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00c0: 0x0032, 0x00c1: 0x0036, 0x00c2: 0x003a, 0x00c3: 0x003e, 0x00c4: 0x0042, 0x00c5: 0x0046, + 0x00c7: 0x004a, 0x00c8: 0x004e, 0x00c9: 0x0052, 0x00ca: 0x0056, 0x00cb: 0x005a, + 0x00cc: 0x005e, 0x00cd: 0x0062, 0x00ce: 0x0066, 0x00cf: 0x006a, 0x00d1: 0x006e, + 0x00d2: 0x0072, 0x00d3: 0x0076, 0x00d4: 0x007a, 0x00d5: 0x007e, 0x00d6: 0x0082, + 0x00d9: 0x0086, 0x00da: 0x008a, 0x00db: 0x008e, 0x00dc: 0x0092, 0x00dd: 0x0096, + 0x00e0: 0x009a, 0x00e1: 0x009e, 0x00e2: 0x00a2, 0x00e3: 0x00a6, + 0x00e4: 0x00aa, 0x00e5: 0x00ae, 0x00e7: 0x00b2, 0x00e8: 0x00b6, 0x00e9: 0x00ba, + 0x00ea: 0x00be, 0x00eb: 0x00c2, 0x00ec: 0x00c6, 0x00ed: 0x00ca, 0x00ee: 0x00ce, 0x00ef: 0x00d2, + 0x00f1: 0x00d6, 0x00f2: 0x00da, 0x00f3: 0x00de, 0x00f4: 0x00e2, 0x00f5: 0x00e6, + 0x00f6: 0x00ea, 0x00f9: 0x00ee, 0x00fa: 0x00f2, 0x00fb: 0x00f6, + 0x00fc: 0x00fa, 0x00fd: 0x00fe, 0x00ff: 0x0102, + // Block 0x4, offset 0x100 + 0x0100: 0x0106, 0x0101: 0x010a, 0x0102: 0x010e, 0x0103: 0x0112, 0x0104: 0x0116, 0x0105: 0x011a, + 0x0106: 0x011e, 0x0107: 0x0122, 0x0108: 0x0126, 0x0109: 0x012a, 0x010a: 0x012e, 0x010b: 0x0132, + 0x010c: 0x0136, 0x010d: 0x013a, 0x010e: 0x013e, 0x010f: 0x0142, + 0x0112: 0x0146, 0x0113: 0x014a, 0x0114: 0x014e, 0x0115: 0x0152, 0x0116: 0x0156, 0x0117: 0x015a, + 0x0118: 0x015e, 0x0119: 0x0162, 0x011a: 0x0166, 0x011b: 0x016a, 0x011c: 0x016e, 0x011d: 0x0172, + 0x011e: 0x0176, 0x011f: 0x017a, 0x0120: 0x017e, 0x0121: 0x0182, 0x0122: 0x0186, 0x0123: 0x018a, + 0x0124: 0x018e, 0x0125: 0x0192, 0x0128: 0x0196, 0x0129: 0x019a, + 0x012a: 0x019e, 0x012b: 0x01a2, 0x012c: 0x01a6, 0x012d: 0x01aa, 0x012e: 0x01ae, 0x012f: 0x01b2, + 0x0130: 0x01b6, 0x0134: 0x01c0, 0x0135: 0x01c4, + 0x0136: 0x01c8, 0x0137: 0x01cc, 0x0139: 0x01d0, 0x013a: 0x01d4, 0x013b: 0x01d8, + 0x013c: 0x01dc, 0x013d: 0x01e0, 0x013e: 0x01e4, + // Block 0x5, offset 0x140 + 0x0143: 0x01f0, 0x0144: 0x01f4, 0x0145: 0x01f8, + 0x0146: 0x01fc, 0x0147: 0x0200, 0x0148: 0x0204, + 0x014c: 0x020c, 0x014d: 0x0210, 0x014e: 0x0214, 0x014f: 0x0218, 0x0150: 0x021c, 0x0151: 0x0220, + 0x0154: 0x0224, 0x0155: 0x0228, 0x0156: 0x022c, 0x0157: 0x0230, + 0x0158: 0x0234, 0x0159: 0x0238, 0x015a: 0x023c, 0x015b: 0x0240, 0x015c: 0x0244, 0x015d: 0x0248, + 0x015e: 0x024c, 0x015f: 0x0250, 0x0160: 0x0254, 0x0161: 0x0258, 0x0162: 0x025c, 0x0163: 0x0260, + 0x0164: 0x0264, 0x0165: 0x0268, 0x0168: 0x026c, 0x0169: 0x0270, + 0x016a: 0x0274, 0x016b: 0x0278, 0x016c: 0x027c, 0x016d: 0x0280, 0x016e: 0x0284, 0x016f: 0x0288, + 0x0170: 0x028c, 0x0171: 0x0290, 0x0172: 0x0294, 0x0173: 0x0298, 0x0174: 0x029c, 0x0175: 0x02a0, + 0x0176: 0x02a4, 0x0177: 0x02a8, 0x0178: 0x02ac, 0x0179: 0x02b0, 0x017a: 0x02b4, 0x017b: 0x02b8, + 0x017c: 0x02bc, 0x017d: 0x02c0, 0x017e: 0x02c4, + // Block 0x6, offset 0x180 + 0x01a0: 0x02ca, 0x01a1: 0x02ce, + 0x01af: 0x02d2, + 0x01b0: 0x02d6, + // Block 0x7, offset 0x1c0 + 0x01cd: 0x02fb, 0x01ce: 0x02ff, 0x01cf: 0x0303, 0x01d0: 0x0307, 0x01d1: 0x030b, + 0x01d2: 0x030f, 0x01d3: 0x0313, 0x01d4: 0x0317, 0x01d5: 0x031b, 0x01d6: 0x0321, 0x01d7: 0x0327, + 0x01d8: 0x032d, 0x01d9: 0x0333, 0x01da: 0x0339, 0x01db: 0x033f, 0x01dc: 0x0345, + 0x01de: 0x034b, 0x01df: 0x0351, 0x01e0: 0x0357, 0x01e1: 0x035d, 0x01e2: 0x0363, 0x01e3: 0x0368, + 0x01e6: 0x036d, 0x01e7: 0x0371, 0x01e8: 0x0375, 0x01e9: 0x0379, + 0x01ea: 0x037d, 0x01eb: 0x0381, 0x01ec: 0x0385, 0x01ed: 0x038b, 0x01ee: 0x0391, 0x01ef: 0x0396, + 0x01f0: 0x039b, 0x01f4: 0x03a8, 0x01f5: 0x03ac, + 0x01f8: 0x03b0, 0x01f9: 0x03b4, 0x01fa: 0x03b8, 0x01fb: 0x03be, + 0x01fc: 0x03c4, 0x01fd: 0x03c9, 0x01fe: 0x03ce, 0x01ff: 0x03d3, + // Block 0x8, offset 0x200 + 0x0200: 0x03d8, 0x0201: 0x03dc, 0x0202: 0x03e0, 0x0203: 0x03e4, 0x0204: 0x03e8, 0x0205: 0x03ec, + 0x0206: 0x03f0, 0x0207: 0x03f4, 0x0208: 0x03f8, 0x0209: 0x03fc, 0x020a: 0x0400, 0x020b: 0x0404, + 0x020c: 0x0408, 0x020d: 0x040c, 0x020e: 0x0410, 0x020f: 0x0414, 0x0210: 0x0418, 0x0211: 0x041c, + 0x0212: 0x0420, 0x0213: 0x0424, 0x0214: 0x0428, 0x0215: 0x042c, 0x0216: 0x0430, 0x0217: 0x0434, + 0x0218: 0x0438, 0x0219: 0x043c, 0x021a: 0x0440, 0x021b: 0x0444, + 0x021e: 0x0448, 0x021f: 0x044c, + 0x0226: 0x0450, 0x0227: 0x0454, 0x0228: 0x0458, 0x0229: 0x045c, + 0x022a: 0x0460, 0x022b: 0x0466, 0x022c: 0x046c, 0x022d: 0x0472, 0x022e: 0x0478, 0x022f: 0x047c, + 0x0230: 0x0480, 0x0231: 0x0486, 0x0232: 0x048c, 0x0233: 0x0490, + // Block 0x9, offset 0x240 + 0x0240: 0x04cc, 0x0241: 0x04cf, 0x0243: 0x04d2, 0x0244: 0x04d5, + 0x0274: 0x04da, + 0x027e: 0x04e1, + // Block 0xa, offset 0x280 + 0x0285: 0x04e3, + 0x0286: 0x04ee, 0x0287: 0x04f3, 0x0288: 0x04f6, 0x0289: 0x04fb, 0x028a: 0x0500, + 0x028c: 0x0505, 0x028e: 0x050a, 0x028f: 0x050f, 0x0290: 0x0514, + 0x02aa: 0x051b, 0x02ab: 0x0520, 0x02ac: 0x0525, 0x02ad: 0x052a, 0x02ae: 0x052f, 0x02af: 0x0534, + 0x02b0: 0x0539, + // Block 0xb, offset 0x2c0 + 0x02ca: 0x0540, 0x02cb: 0x0545, + 0x02cc: 0x054a, 0x02cd: 0x054f, 0x02ce: 0x0554, + 0x02d3: 0x0562, 0x02d4: 0x0567, + // Block 0xc, offset 0x300 + 0x0300: 0x0584, 0x0301: 0x0589, 0x0303: 0x058e, + 0x0307: 0x0593, + 0x030c: 0x0598, 0x030d: 0x059d, 0x030e: 0x05a2, + 0x0319: 0x05a7, + 0x0339: 0x05ac, + // Block 0xd, offset 0x340 + 0x0350: 0x05b1, 0x0351: 0x05b6, + 0x0353: 0x05bb, 0x0357: 0x05c0, + 0x035c: 0x05c5, 0x035d: 0x05ca, + 0x035e: 0x05cf, + 0x0376: 0x05d4, 0x0377: 0x05d9, + // Block 0xe, offset 0x380 + 0x0381: 0x05de, 0x0382: 0x05e3, + 0x0390: 0x05e8, 0x0391: 0x05ed, + 0x0392: 0x05f2, 0x0393: 0x05f7, 0x0396: 0x05fc, 0x0397: 0x0601, + 0x039a: 0x0606, 0x039b: 0x060b, 0x039c: 0x0610, 0x039d: 0x0615, + 0x039e: 0x061a, 0x039f: 0x061f, 0x03a2: 0x0624, 0x03a3: 0x0629, + 0x03a4: 0x062e, 0x03a5: 0x0633, 0x03a6: 0x0638, 0x03a7: 0x063d, + 0x03aa: 0x0642, 0x03ab: 0x0647, 0x03ac: 0x064c, 0x03ad: 0x0651, 0x03ae: 0x0656, 0x03af: 0x065b, + 0x03b0: 0x0660, 0x03b1: 0x0665, 0x03b2: 0x066a, 0x03b3: 0x066f, 0x03b4: 0x0674, 0x03b5: 0x0679, + 0x03b8: 0x067e, 0x03b9: 0x0683, + // Block 0xf, offset 0x3c0 + 0x03e2: 0x068d, 0x03e3: 0x0692, + 0x03e4: 0x0697, 0x03e5: 0x069c, 0x03e6: 0x06a1, + // Block 0x10, offset 0x400 + 0x0400: 0x06ba, 0x0402: 0x06bf, + 0x0413: 0x06c4, + // Block 0x11, offset 0x440 + 0x0469: 0x06c9, + 0x0471: 0x06d0, 0x0474: 0x06d7, + // Block 0x12, offset 0x480 + 0x0498: 0x06de, 0x0499: 0x06e5, 0x049a: 0x06ec, 0x049b: 0x06f3, 0x049c: 0x06fa, 0x049d: 0x0701, + 0x049e: 0x0708, 0x049f: 0x070f, + // Block 0x13, offset 0x4c0 + 0x04cb: 0x0716, + 0x04cc: 0x071d, + 0x04dc: 0x0724, 0x04dd: 0x072b, + 0x04df: 0x0732, + // Block 0x14, offset 0x500 + 0x0533: 0x0739, + 0x0536: 0x0740, + // Block 0x15, offset 0x540 + 0x0559: 0x0747, 0x055a: 0x074e, 0x055b: 0x0755, + 0x055e: 0x075c, + // Block 0x16, offset 0x580 + 0x0588: 0x0763, 0x058b: 0x076a, + 0x058c: 0x0771, + 0x059c: 0x0778, 0x059d: 0x077f, + // Block 0x17, offset 0x5c0 + 0x05d4: 0x0786, + // Block 0x18, offset 0x600 + 0x060a: 0x078d, 0x060b: 0x0794, + 0x060c: 0x079b, + // Block 0x19, offset 0x640 + 0x0648: 0x07a2, + // Block 0x1a, offset 0x680 + 0x0680: 0x07a9, + 0x0687: 0x07b0, 0x0688: 0x07b7, 0x068a: 0x07be, 0x068b: 0x07c5, + // Block 0x1b, offset 0x6c0 + 0x06ca: 0x07cf, 0x06cb: 0x07d6, + 0x06cc: 0x07dd, + // Block 0x1c, offset 0x700 + 0x071a: 0x07e4, 0x071c: 0x07eb, 0x071d: 0x07f2, + 0x071e: 0x07fc, + // Block 0x1d, offset 0x740 + 0x0743: 0x0823, + 0x074d: 0x082a, + 0x0752: 0x0831, 0x0757: 0x0838, + 0x075c: 0x083f, + 0x0769: 0x0846, + 0x0773: 0x084d, 0x0775: 0x0854, + 0x0776: 0x085b, 0x0778: 0x086c, + // Block 0x1e, offset 0x780 + 0x0781: 0x087d, + 0x0793: 0x0884, + 0x079d: 0x088b, + 0x07a2: 0x0892, + 0x07a7: 0x0899, + 0x07ac: 0x08a0, + 0x07b9: 0x08a7, + // Block 0x1f, offset 0x7c0 + 0x07e6: 0x08ae, + // Block 0x20, offset 0x800 + 0x0806: 0x08b9, 0x0808: 0x08c0, 0x080a: 0x08c7, + 0x080c: 0x08ce, 0x080e: 0x08d5, + 0x0812: 0x08dc, + 0x083b: 0x08e3, + 0x083d: 0x08ea, + // Block 0x21, offset 0x840 + 0x0840: 0x08f1, 0x0841: 0x08f8, 0x0843: 0x08ff, + // Block 0x22, offset 0x880 + 0x0880: 0x09ea, 0x0881: 0x09ee, 0x0882: 0x09f2, 0x0883: 0x09f6, 0x0884: 0x09fa, 0x0885: 0x09fe, + 0x0886: 0x0a02, 0x0887: 0x0a06, 0x0888: 0x0a0a, 0x0889: 0x0a10, 0x088a: 0x0a16, 0x088b: 0x0a1a, + 0x088c: 0x0a1e, 0x088d: 0x0a22, 0x088e: 0x0a26, 0x088f: 0x0a2a, 0x0890: 0x0a2e, 0x0891: 0x0a32, + 0x0892: 0x0a36, 0x0893: 0x0a3a, 0x0894: 0x0a3e, 0x0895: 0x0a44, 0x0896: 0x0a4a, 0x0897: 0x0a50, + 0x0898: 0x0a56, 0x0899: 0x0a5a, 0x089a: 0x0a5e, 0x089b: 0x0a62, 0x089c: 0x0a66, 0x089d: 0x0a6c, + 0x089e: 0x0a72, 0x089f: 0x0a76, 0x08a0: 0x0a7a, 0x08a1: 0x0a7e, 0x08a2: 0x0a82, 0x08a3: 0x0a86, + 0x08a4: 0x0a8a, 0x08a5: 0x0a8e, 0x08a6: 0x0a92, 0x08a7: 0x0a96, 0x08a8: 0x0a9a, 0x08a9: 0x0a9e, + 0x08aa: 0x0aa2, 0x08ab: 0x0aa6, 0x08ac: 0x0aaa, 0x08ad: 0x0aae, 0x08ae: 0x0ab2, 0x08af: 0x0ab8, + 0x08b0: 0x0abe, 0x08b1: 0x0ac2, 0x08b2: 0x0ac6, 0x08b3: 0x0aca, 0x08b4: 0x0ace, 0x08b5: 0x0ad2, + 0x08b6: 0x0ad6, 0x08b7: 0x0ada, 0x08b8: 0x0ade, 0x08b9: 0x0ae4, 0x08ba: 0x0aea, 0x08bb: 0x0aee, + 0x08bc: 0x0af2, 0x08bd: 0x0af6, 0x08be: 0x0afa, 0x08bf: 0x0afe, + // Block 0x23, offset 0x8c0 + 0x08c0: 0x0b02, 0x08c1: 0x0b06, 0x08c2: 0x0b0a, 0x08c3: 0x0b0e, 0x08c4: 0x0b12, 0x08c5: 0x0b16, + 0x08c6: 0x0b1a, 0x08c7: 0x0b1e, 0x08c8: 0x0b22, 0x08c9: 0x0b26, 0x08ca: 0x0b2a, 0x08cb: 0x0b2e, + 0x08cc: 0x0b32, 0x08cd: 0x0b38, 0x08ce: 0x0b3e, 0x08cf: 0x0b44, 0x08d0: 0x0b4a, 0x08d1: 0x0b50, + 0x08d2: 0x0b56, 0x08d3: 0x0b5c, 0x08d4: 0x0b62, 0x08d5: 0x0b66, 0x08d6: 0x0b6a, 0x08d7: 0x0b6e, + 0x08d8: 0x0b72, 0x08d9: 0x0b76, 0x08da: 0x0b7a, 0x08db: 0x0b7e, 0x08dc: 0x0b82, 0x08dd: 0x0b88, + 0x08de: 0x0b8e, 0x08df: 0x0b92, 0x08e0: 0x0b96, 0x08e1: 0x0b9a, 0x08e2: 0x0b9e, 0x08e3: 0x0ba2, + 0x08e4: 0x0ba6, 0x08e5: 0x0bac, 0x08e6: 0x0bb2, 0x08e7: 0x0bb8, 0x08e8: 0x0bbe, 0x08e9: 0x0bc4, + 0x08ea: 0x0bca, 0x08eb: 0x0bce, 0x08ec: 0x0bd2, 0x08ed: 0x0bd6, 0x08ee: 0x0bda, 0x08ef: 0x0bde, + 0x08f0: 0x0be2, 0x08f1: 0x0be6, 0x08f2: 0x0bea, 0x08f3: 0x0bee, 0x08f4: 0x0bf2, 0x08f5: 0x0bf6, + 0x08f6: 0x0bfa, 0x08f7: 0x0bfe, 0x08f8: 0x0c02, 0x08f9: 0x0c08, 0x08fa: 0x0c0e, 0x08fb: 0x0c14, + 0x08fc: 0x0c1a, 0x08fd: 0x0c1e, 0x08fe: 0x0c22, 0x08ff: 0x0c26, + // Block 0x24, offset 0x900 + 0x0900: 0x0c2a, 0x0901: 0x0c2e, 0x0902: 0x0c32, 0x0903: 0x0c36, 0x0904: 0x0c3a, 0x0905: 0x0c3e, + 0x0906: 0x0c42, 0x0907: 0x0c46, 0x0908: 0x0c4a, 0x0909: 0x0c4e, 0x090a: 0x0c52, 0x090b: 0x0c56, + 0x090c: 0x0c5a, 0x090d: 0x0c5e, 0x090e: 0x0c62, 0x090f: 0x0c66, 0x0910: 0x0c6a, 0x0911: 0x0c6e, + 0x0912: 0x0c72, 0x0913: 0x0c76, 0x0914: 0x0c7a, 0x0915: 0x0c7e, 0x0916: 0x0c82, 0x0917: 0x0c86, + 0x0918: 0x0c8a, 0x0919: 0x0c8e, 0x091b: 0x0c96, + 0x0920: 0x0c9b, 0x0921: 0x0c9f, 0x0922: 0x0ca3, 0x0923: 0x0ca7, + 0x0924: 0x0cab, 0x0925: 0x0cb1, 0x0926: 0x0cb7, 0x0927: 0x0cbd, 0x0928: 0x0cc3, 0x0929: 0x0cc9, + 0x092a: 0x0ccf, 0x092b: 0x0cd5, 0x092c: 0x0cdb, 0x092d: 0x0ce1, 0x092e: 0x0ce7, 0x092f: 0x0ced, + 0x0930: 0x0cf3, 0x0931: 0x0cf9, 0x0932: 0x0cff, 0x0933: 0x0d05, 0x0934: 0x0d0b, 0x0935: 0x0d11, + 0x0936: 0x0d17, 0x0937: 0x0d1d, 0x0938: 0x0d23, 0x0939: 0x0d27, 0x093a: 0x0d2b, 0x093b: 0x0d2f, + 0x093c: 0x0d33, 0x093d: 0x0d37, 0x093e: 0x0d3b, 0x093f: 0x0d41, + // Block 0x25, offset 0x940 + 0x0940: 0x0d47, 0x0941: 0x0d4d, 0x0942: 0x0d53, 0x0943: 0x0d59, 0x0944: 0x0d5f, 0x0945: 0x0d65, + 0x0946: 0x0d6b, 0x0947: 0x0d71, 0x0948: 0x0d77, 0x0949: 0x0d7b, 0x094a: 0x0d7f, 0x094b: 0x0d83, + 0x094c: 0x0d87, 0x094d: 0x0d8b, 0x094e: 0x0d8f, 0x094f: 0x0d93, 0x0950: 0x0d97, 0x0951: 0x0d9d, + 0x0952: 0x0da3, 0x0953: 0x0da9, 0x0954: 0x0daf, 0x0955: 0x0db5, 0x0956: 0x0dbb, 0x0957: 0x0dc1, + 0x0958: 0x0dc7, 0x0959: 0x0dcd, 0x095a: 0x0dd3, 0x095b: 0x0dd9, 0x095c: 0x0ddf, 0x095d: 0x0de5, + 0x095e: 0x0deb, 0x095f: 0x0df1, 0x0960: 0x0df7, 0x0961: 0x0dfd, 0x0962: 0x0e03, 0x0963: 0x0e09, + 0x0964: 0x0e0f, 0x0965: 0x0e13, 0x0966: 0x0e17, 0x0967: 0x0e1b, 0x0968: 0x0e1f, 0x0969: 0x0e25, + 0x096a: 0x0e2b, 0x096b: 0x0e31, 0x096c: 0x0e37, 0x096d: 0x0e3d, 0x096e: 0x0e43, 0x096f: 0x0e49, + 0x0970: 0x0e4f, 0x0971: 0x0e55, 0x0972: 0x0e5b, 0x0973: 0x0e5f, 0x0974: 0x0e63, 0x0975: 0x0e67, + 0x0976: 0x0e6b, 0x0977: 0x0e6f, 0x0978: 0x0e73, 0x0979: 0x0e77, + // Block 0x26, offset 0x980 + 0x0980: 0x0e7b, 0x0981: 0x0e80, 0x0982: 0x0e85, 0x0983: 0x0e8c, 0x0984: 0x0e93, 0x0985: 0x0e9a, + 0x0986: 0x0ea1, 0x0987: 0x0ea8, 0x0988: 0x0eaf, 0x0989: 0x0eb4, 0x098a: 0x0eb9, 0x098b: 0x0ec0, + 0x098c: 0x0ec7, 0x098d: 0x0ece, 0x098e: 0x0ed5, 0x098f: 0x0edc, 0x0990: 0x0ee3, 0x0991: 0x0ee8, + 0x0992: 0x0eed, 0x0993: 0x0ef4, 0x0994: 0x0efb, 0x0995: 0x0f02, + 0x0998: 0x0f09, 0x0999: 0x0f0e, 0x099a: 0x0f13, 0x099b: 0x0f1a, 0x099c: 0x0f21, 0x099d: 0x0f28, + 0x09a0: 0x0f2f, 0x09a1: 0x0f34, 0x09a2: 0x0f39, 0x09a3: 0x0f40, + 0x09a4: 0x0f47, 0x09a5: 0x0f4e, 0x09a6: 0x0f55, 0x09a7: 0x0f5c, 0x09a8: 0x0f63, 0x09a9: 0x0f68, + 0x09aa: 0x0f6d, 0x09ab: 0x0f74, 0x09ac: 0x0f7b, 0x09ad: 0x0f82, 0x09ae: 0x0f89, 0x09af: 0x0f90, + 0x09b0: 0x0f97, 0x09b1: 0x0f9c, 0x09b2: 0x0fa1, 0x09b3: 0x0fa8, 0x09b4: 0x0faf, 0x09b5: 0x0fb6, + 0x09b6: 0x0fbd, 0x09b7: 0x0fc4, 0x09b8: 0x0fcb, 0x09b9: 0x0fd0, 0x09ba: 0x0fd5, 0x09bb: 0x0fdc, + 0x09bc: 0x0fe3, 0x09bd: 0x0fea, 0x09be: 0x0ff1, 0x09bf: 0x0ff8, + // Block 0x27, offset 0x9c0 + 0x09c0: 0x0fff, 0x09c1: 0x1004, 0x09c2: 0x1009, 0x09c3: 0x1010, 0x09c4: 0x1017, 0x09c5: 0x101e, + 0x09c8: 0x1025, 0x09c9: 0x102a, 0x09ca: 0x102f, 0x09cb: 0x1036, + 0x09cc: 0x103d, 0x09cd: 0x1044, 0x09d0: 0x104b, 0x09d1: 0x1050, + 0x09d2: 0x1055, 0x09d3: 0x105c, 0x09d4: 0x1063, 0x09d5: 0x106a, 0x09d6: 0x1071, 0x09d7: 0x1078, + 0x09d9: 0x107f, 0x09db: 0x1084, 0x09dd: 0x108b, + 0x09df: 0x1092, 0x09e0: 0x1099, 0x09e1: 0x109e, 0x09e2: 0x10a3, 0x09e3: 0x10aa, + 0x09e4: 0x10b1, 0x09e5: 0x10b8, 0x09e6: 0x10bf, 0x09e7: 0x10c6, 0x09e8: 0x10cd, 0x09e9: 0x10d2, + 0x09ea: 0x10d7, 0x09eb: 0x10de, 0x09ec: 0x10e5, 0x09ed: 0x10ec, 0x09ee: 0x10f3, 0x09ef: 0x10fa, + 0x09f0: 0x1101, 0x09f1: 0x0525, 0x09f2: 0x1106, 0x09f3: 0x052a, 0x09f4: 0x110b, 0x09f5: 0x052f, + 0x09f6: 0x1110, 0x09f7: 0x0534, 0x09f8: 0x1115, 0x09f9: 0x054a, 0x09fa: 0x111a, 0x09fb: 0x054f, + 0x09fc: 0x111f, 0x09fd: 0x0554, + // Block 0x28, offset 0xa00 + 0x0a00: 0x1124, 0x0a01: 0x112b, 0x0a02: 0x1132, 0x0a03: 0x113b, 0x0a04: 0x1144, 0x0a05: 0x114d, + 0x0a06: 0x1156, 0x0a07: 0x115f, 0x0a08: 0x1168, 0x0a09: 0x116f, 0x0a0a: 0x1176, 0x0a0b: 0x117f, + 0x0a0c: 0x1188, 0x0a0d: 0x1191, 0x0a0e: 0x119a, 0x0a0f: 0x11a3, 0x0a10: 0x11ac, 0x0a11: 0x11b3, + 0x0a12: 0x11ba, 0x0a13: 0x11c3, 0x0a14: 0x11cc, 0x0a15: 0x11d5, 0x0a16: 0x11de, 0x0a17: 0x11e7, + 0x0a18: 0x11f0, 0x0a19: 0x11f7, 0x0a1a: 0x11fe, 0x0a1b: 0x1207, 0x0a1c: 0x1210, 0x0a1d: 0x1219, + 0x0a1e: 0x1222, 0x0a1f: 0x122b, 0x0a20: 0x1234, 0x0a21: 0x123b, 0x0a22: 0x1242, 0x0a23: 0x124b, + 0x0a24: 0x1254, 0x0a25: 0x125d, 0x0a26: 0x1266, 0x0a27: 0x126f, 0x0a28: 0x1278, 0x0a29: 0x127f, + 0x0a2a: 0x1286, 0x0a2b: 0x128f, 0x0a2c: 0x1298, 0x0a2d: 0x12a1, 0x0a2e: 0x12aa, 0x0a2f: 0x12b3, + 0x0a30: 0x12bc, 0x0a31: 0x12c1, 0x0a32: 0x12c6, 0x0a33: 0x12cd, 0x0a34: 0x12d2, + 0x0a36: 0x12d9, 0x0a37: 0x12de, 0x0a38: 0x12e5, 0x0a39: 0x12ea, 0x0a3a: 0x12ef, 0x0a3b: 0x04ee, + 0x0a3c: 0x12f4, 0x0a3e: 0x12fd, + // Block 0x29, offset 0xa40 + 0x0a41: 0x1304, 0x0a42: 0x130f, 0x0a43: 0x1316, 0x0a44: 0x131b, + 0x0a46: 0x1322, 0x0a47: 0x1327, 0x0a48: 0x132e, 0x0a49: 0x04f6, 0x0a4a: 0x1333, 0x0a4b: 0x04fb, + 0x0a4c: 0x1338, 0x0a4d: 0x133d, 0x0a4e: 0x1349, 0x0a4f: 0x1355, 0x0a50: 0x1361, 0x0a51: 0x1366, + 0x0a52: 0x136b, 0x0a53: 0x0514, 0x0a56: 0x1372, 0x0a57: 0x1377, + 0x0a58: 0x137e, 0x0a59: 0x1383, 0x0a5a: 0x1388, 0x0a5b: 0x0500, 0x0a5d: 0x138d, + 0x0a5e: 0x1399, 0x0a5f: 0x13a5, 0x0a60: 0x13b1, 0x0a61: 0x13b6, 0x0a62: 0x13bb, 0x0a63: 0x0539, + 0x0a64: 0x13c2, 0x0a65: 0x13c7, 0x0a66: 0x13cc, 0x0a67: 0x13d1, 0x0a68: 0x13d8, 0x0a69: 0x13dd, + 0x0a6a: 0x13e2, 0x0a6b: 0x050a, 0x0a6c: 0x13e7, 0x0a6d: 0x13ec, 0x0a6e: 0x04e3, 0x0a6f: 0x13f7, + 0x0a72: 0x13f9, 0x0a73: 0x1400, 0x0a74: 0x1405, + 0x0a76: 0x140c, 0x0a77: 0x1411, 0x0a78: 0x1418, 0x0a79: 0x0505, 0x0a7a: 0x141d, 0x0a7b: 0x050f, + 0x0a7c: 0x1422, 0x0a7d: 0x1427, + // Block 0x2a, offset 0xa80 + 0x0a80: 0x142e, 0x0a81: 0x1432, + // Block 0x2b, offset 0xac0 + 0x0ae6: 0x14d6, + 0x0aea: 0x091c, 0x0aeb: 0x0046, + // Block 0x2c, offset 0xb00 + 0x0b1a: 0x159f, 0x0b1b: 0x15a5, + 0x0b2e: 0x15ab, + // Block 0x2d, offset 0xb40 + 0x0b4d: 0x15b1, 0x0b4e: 0x15b7, 0x0b4f: 0x15bd, + // Block 0x2e, offset 0xb80 + 0x0b84: 0x15c3, + 0x0b89: 0x15c9, + 0x0b8c: 0x15cf, + 0x0ba4: 0x15d5, 0x0ba6: 0x15db, + // Block 0x2f, offset 0xbc0 + 0x0bc1: 0x1603, 0x0bc4: 0x1609, + 0x0bc7: 0x160f, 0x0bc9: 0x1615, + 0x0be0: 0x161b, 0x0be2: 0x161f, + 0x0bed: 0x1625, 0x0bee: 0x162b, 0x0bef: 0x162f, + 0x0bf0: 0x1633, 0x0bf1: 0x1639, 0x0bf4: 0x163f, 0x0bf5: 0x1645, + 0x0bf8: 0x164b, 0x0bf9: 0x1651, + // Block 0x30, offset 0xc00 + 0x0c00: 0x1657, 0x0c01: 0x165d, 0x0c04: 0x1663, 0x0c05: 0x1669, + 0x0c08: 0x166f, 0x0c09: 0x1675, + 0x0c2c: 0x167b, 0x0c2d: 0x1681, 0x0c2e: 0x1687, 0x0c2f: 0x168d, + // Block 0x31, offset 0xc40 + 0x0c60: 0x1693, 0x0c61: 0x1699, 0x0c62: 0x169f, 0x0c63: 0x16a5, + 0x0c6a: 0x16ab, 0x0c6b: 0x16b1, 0x0c6c: 0x16b7, 0x0c6d: 0x16bd, + // Block 0x32, offset 0xc80 + 0x0ca9: 0x16c3, + 0x0caa: 0x16c7, + // Block 0x33, offset 0xcc0 + 0x0cdc: 0x1814, + // Block 0x34, offset 0xd00 + 0x0d0c: 0x1b8a, 0x0d0e: 0x1b91, 0x0d10: 0x1b98, + 0x0d12: 0x1b9f, 0x0d14: 0x1ba6, 0x0d16: 0x1bad, + 0x0d18: 0x1bb4, 0x0d1a: 0x1bbb, 0x0d1c: 0x1bc2, + 0x0d1e: 0x1bc9, 0x0d20: 0x1bd0, 0x0d22: 0x1bd7, + 0x0d25: 0x1bde, 0x0d27: 0x1be5, 0x0d29: 0x1bec, + 0x0d30: 0x1bf3, 0x0d31: 0x1bfa, 0x0d33: 0x1c01, 0x0d34: 0x1c08, + 0x0d36: 0x1c0f, 0x0d37: 0x1c16, 0x0d39: 0x1c1d, 0x0d3a: 0x1c24, + 0x0d3c: 0x1c2b, 0x0d3d: 0x1c32, + // Block 0x35, offset 0xd40 + 0x0d54: 0x1c39, + 0x0d5e: 0x1c4a, + 0x0d6c: 0x1c58, 0x0d6e: 0x1c5f, + 0x0d70: 0x1c66, 0x0d72: 0x1c6d, 0x0d74: 0x1c74, + 0x0d76: 0x1c7b, 0x0d78: 0x1c82, 0x0d7a: 0x1c89, + 0x0d7c: 0x1c90, 0x0d7e: 0x1c97, + // Block 0x36, offset 0xd80 + 0x0d80: 0x1c9e, 0x0d82: 0x1ca5, 0x0d85: 0x1cac, + 0x0d87: 0x1cb3, 0x0d89: 0x1cba, + 0x0d90: 0x1cc1, 0x0d91: 0x1cc8, + 0x0d93: 0x1ccf, 0x0d94: 0x1cd6, 0x0d96: 0x1cdd, 0x0d97: 0x1ce4, + 0x0d99: 0x1ceb, 0x0d9a: 0x1cf2, 0x0d9c: 0x1cf9, 0x0d9d: 0x1d00, + 0x0db4: 0x1d07, + 0x0db7: 0x1d0e, 0x0db8: 0x1d15, 0x0db9: 0x1d1c, 0x0dba: 0x1d23, + 0x0dbe: 0x1d2a, + // Block 0x37, offset 0xdc0 + 0x0dc0: 0x2a81, 0x0dc1: 0x2a85, 0x0dc2: 0x1a9e, 0x0dc3: 0x2a89, 0x0dc4: 0x2a8d, 0x0dc5: 0x2a91, + 0x0dc6: 0x2a95, 0x0dc7: 0x1b76, 0x0dc8: 0x1b76, 0x0dc9: 0x2a99, 0x0dca: 0x1abe, 0x0dcb: 0x2a9d, + 0x0dcc: 0x2aa1, 0x0dcd: 0x2aa5, 0x0dce: 0x2aa9, 0x0dcf: 0x2aad, 0x0dd0: 0x2ab1, 0x0dd1: 0x2ab5, + 0x0dd2: 0x2ab9, 0x0dd3: 0x2abd, 0x0dd4: 0x2ac1, 0x0dd5: 0x2ac5, 0x0dd6: 0x2ac9, 0x0dd7: 0x2acd, + 0x0dd8: 0x2ad1, 0x0dd9: 0x2ad5, 0x0dda: 0x2ad9, 0x0ddb: 0x2add, 0x0ddc: 0x2ae1, 0x0ddd: 0x2ae5, + 0x0dde: 0x2ae9, 0x0ddf: 0x2aed, 0x0de0: 0x2af1, 0x0de1: 0x2af5, 0x0de2: 0x2af9, 0x0de3: 0x2afd, + 0x0de4: 0x2b01, 0x0de5: 0x2b05, 0x0de6: 0x2b09, 0x0de7: 0x2b0d, 0x0de8: 0x2b11, 0x0de9: 0x2b15, + 0x0dea: 0x2b19, 0x0deb: 0x2b1d, 0x0dec: 0x2b21, 0x0ded: 0x2b25, 0x0dee: 0x2b29, 0x0def: 0x2b2d, + 0x0df0: 0x2b31, 0x0df1: 0x2b35, 0x0df2: 0x2b39, 0x0df3: 0x2b3d, 0x0df4: 0x1a16, 0x0df5: 0x2b41, + 0x0df6: 0x2b45, 0x0df7: 0x2b49, 0x0df8: 0x2b4d, 0x0df9: 0x2b51, 0x0dfa: 0x2b55, 0x0dfb: 0x2b59, + 0x0dfc: 0x2b5d, 0x0dfd: 0x2b61, 0x0dfe: 0x2b65, 0x0dff: 0x2b69, + // Block 0x38, offset 0xe00 + 0x0e00: 0x1b3a, 0x0e01: 0x2b6d, 0x0e02: 0x2b71, 0x0e03: 0x2b75, 0x0e04: 0x2b79, 0x0e05: 0x2b7d, + 0x0e06: 0x2b81, 0x0e07: 0x2b85, 0x0e08: 0x2b89, 0x0e09: 0x2b8d, 0x0e0a: 0x2b91, 0x0e0b: 0x2b95, + 0x0e0c: 0x2b99, 0x0e0d: 0x2b9d, 0x0e0e: 0x2ba1, 0x0e0f: 0x2ba5, 0x0e10: 0x2ba9, 0x0e11: 0x2bad, + 0x0e12: 0x2bb1, 0x0e13: 0x2bb5, 0x0e14: 0x2bb9, 0x0e15: 0x2bbd, 0x0e16: 0x2bc1, 0x0e17: 0x2bc5, + 0x0e18: 0x2bc9, 0x0e19: 0x2bcd, 0x0e1a: 0x2bd1, 0x0e1b: 0x2bd5, 0x0e1c: 0x2ac1, 0x0e1d: 0x2bd9, + 0x0e1e: 0x2bdd, 0x0e1f: 0x2be1, 0x0e20: 0x2be5, 0x0e21: 0x2be9, 0x0e22: 0x2bed, 0x0e23: 0x2bf1, + 0x0e24: 0x2bf5, 0x0e25: 0x2bf9, 0x0e26: 0x2bfd, 0x0e27: 0x2c01, 0x0e28: 0x2c05, 0x0e29: 0x2c09, + 0x0e2a: 0x2c0d, 0x0e2b: 0x2c11, 0x0e2c: 0x2c15, 0x0e2d: 0x2c19, 0x0e2e: 0x2c1d, 0x0e2f: 0x2c21, + 0x0e30: 0x2c25, 0x0e31: 0x1aa6, 0x0e32: 0x2c29, 0x0e33: 0x2c2d, 0x0e34: 0x2c31, 0x0e35: 0x2c35, + 0x0e36: 0x2c39, 0x0e37: 0x2c3d, 0x0e38: 0x2c41, 0x0e39: 0x2c45, 0x0e3a: 0x2c49, 0x0e3b: 0x2c4d, + 0x0e3c: 0x2c51, 0x0e3d: 0x2c55, 0x0e3e: 0x2c59, 0x0e3f: 0x2c5d, + // Block 0x39, offset 0xe40 + 0x0e40: 0x2c61, 0x0e41: 0x18ba, 0x0e42: 0x2c65, 0x0e43: 0x2c69, 0x0e44: 0x2c6d, 0x0e45: 0x2c71, + 0x0e46: 0x2c75, 0x0e47: 0x2c79, 0x0e48: 0x2c7d, 0x0e49: 0x2c81, 0x0e4a: 0x186e, 0x0e4b: 0x2c85, + 0x0e4c: 0x2c89, 0x0e4d: 0x2c8d, 0x0e4e: 0x2c91, 0x0e4f: 0x2c95, 0x0e50: 0x2c99, 0x0e51: 0x2c9d, + 0x0e52: 0x2ca1, 0x0e53: 0x2ca5, 0x0e54: 0x2ca9, 0x0e55: 0x2cad, 0x0e56: 0x2cb1, 0x0e57: 0x2cb5, + 0x0e58: 0x2cb9, 0x0e59: 0x2cbd, 0x0e5a: 0x2cc1, 0x0e5b: 0x2cc5, 0x0e5c: 0x2cc9, 0x0e5d: 0x2ccd, + 0x0e5e: 0x2cd1, 0x0e5f: 0x2cd5, 0x0e60: 0x2cd9, 0x0e61: 0x2c21, 0x0e62: 0x2cdd, 0x0e63: 0x2ce1, + 0x0e64: 0x2ce5, 0x0e65: 0x2ce9, 0x0e66: 0x2ced, 0x0e67: 0x2cf1, 0x0e68: 0x2cf5, 0x0e69: 0x2cf9, + 0x0e6a: 0x2be1, 0x0e6b: 0x2cfd, 0x0e6c: 0x2d01, 0x0e6d: 0x2d05, 0x0e6e: 0x2d09, 0x0e6f: 0x2d0d, + 0x0e70: 0x2d11, 0x0e71: 0x2d15, 0x0e72: 0x2d19, 0x0e73: 0x2d1d, 0x0e74: 0x2d21, 0x0e75: 0x2d25, + 0x0e76: 0x2d29, 0x0e77: 0x2d2d, 0x0e78: 0x2d31, 0x0e79: 0x2d35, 0x0e7a: 0x2d39, 0x0e7b: 0x2d3d, + 0x0e7c: 0x2d41, 0x0e7d: 0x2d45, 0x0e7e: 0x2d49, 0x0e7f: 0x2ac1, + // Block 0x3a, offset 0xe80 + 0x0e80: 0x2d4d, 0x0e81: 0x2d51, 0x0e82: 0x2d55, 0x0e83: 0x2d59, 0x0e84: 0x1b72, 0x0e85: 0x2d5d, + 0x0e86: 0x2d61, 0x0e87: 0x2d65, 0x0e88: 0x2d69, 0x0e89: 0x2d6d, 0x0e8a: 0x2d71, 0x0e8b: 0x2d75, + 0x0e8c: 0x2d79, 0x0e8d: 0x2d7d, 0x0e8e: 0x2d81, 0x0e8f: 0x2d85, 0x0e90: 0x2d89, 0x0e91: 0x2173, + 0x0e92: 0x2d8d, 0x0e93: 0x2d91, 0x0e94: 0x2d95, 0x0e95: 0x2d99, 0x0e96: 0x2d9d, 0x0e97: 0x2da1, + 0x0e98: 0x2da5, 0x0e99: 0x2da9, 0x0e9a: 0x2dad, 0x0e9b: 0x2be9, 0x0e9c: 0x2db1, 0x0e9d: 0x2db5, + 0x0e9e: 0x2db9, 0x0e9f: 0x2dbd, 0x0ea0: 0x2dc1, 0x0ea1: 0x2dc5, 0x0ea2: 0x2dc9, 0x0ea3: 0x2dcd, + 0x0ea4: 0x2dd1, 0x0ea5: 0x2dd5, 0x0ea6: 0x2dd9, 0x0ea7: 0x2ddd, 0x0ea8: 0x2de1, 0x0ea9: 0x1aba, + 0x0eaa: 0x2de5, 0x0eab: 0x2de9, 0x0eac: 0x2ded, 0x0ead: 0x2df1, 0x0eae: 0x2df5, 0x0eaf: 0x2df9, + 0x0eb0: 0x2dfd, 0x0eb1: 0x2e01, 0x0eb2: 0x2e05, 0x0eb3: 0x2e09, 0x0eb4: 0x2e0d, 0x0eb5: 0x2e11, + 0x0eb6: 0x2e15, 0x0eb7: 0x19f6, 0x0eb8: 0x2e19, 0x0eb9: 0x2e1d, 0x0eba: 0x2e21, 0x0ebb: 0x2e25, + 0x0ebc: 0x2e29, 0x0ebd: 0x2e2d, 0x0ebe: 0x2e31, 0x0ebf: 0x2e35, + // Block 0x3b, offset 0xec0 + 0x0ec0: 0x2e39, 0x0ec1: 0x2e3d, 0x0ec2: 0x2e41, 0x0ec3: 0x2e45, 0x0ec4: 0x2e49, 0x0ec5: 0x2e4d, + 0x0ec6: 0x2e51, 0x0ec7: 0x2e55, 0x0ec8: 0x1a62, 0x0ec9: 0x2e59, 0x0eca: 0x1a6e, 0x0ecb: 0x2e5d, + 0x0ecc: 0x2e61, 0x0ecd: 0x2e65, 0x0ed0: 0x2e69, + 0x0ed2: 0x2e6d, 0x0ed5: 0x2e71, 0x0ed6: 0x2e75, 0x0ed7: 0x2e79, + 0x0ed8: 0x2e7d, 0x0ed9: 0x2e81, 0x0eda: 0x2e85, 0x0edb: 0x2e89, 0x0edc: 0x2e8d, 0x0edd: 0x2e91, + 0x0ede: 0x1a12, 0x0ee0: 0x2e95, 0x0ee2: 0x2e99, + 0x0ee5: 0x2e9d, 0x0ee6: 0x2ea1, + 0x0eea: 0x2ea5, 0x0eeb: 0x2ea9, 0x0eec: 0x2ead, 0x0eed: 0x2eb1, + 0x0ef0: 0x2eb5, 0x0ef1: 0x2eb9, 0x0ef2: 0x2ebd, 0x0ef3: 0x2ec1, 0x0ef4: 0x2ec5, 0x0ef5: 0x2ec9, + 0x0ef6: 0x2ecd, 0x0ef7: 0x2ed1, 0x0ef8: 0x2ed5, 0x0ef9: 0x2ed9, 0x0efa: 0x2edd, 0x0efb: 0x2ee1, + 0x0efc: 0x18d6, 0x0efd: 0x2ee5, 0x0efe: 0x2ee9, 0x0eff: 0x2eed, + // Block 0x3c, offset 0xf00 + 0x0f00: 0x2ef1, 0x0f01: 0x2ef5, 0x0f02: 0x2ef9, 0x0f03: 0x2efd, 0x0f04: 0x2f01, 0x0f05: 0x2f05, + 0x0f06: 0x2f09, 0x0f07: 0x2f0d, 0x0f08: 0x2f11, 0x0f09: 0x2f15, 0x0f0a: 0x2f19, 0x0f0b: 0x2f1d, + 0x0f0c: 0x2187, 0x0f0d: 0x2f21, 0x0f0e: 0x2f25, 0x0f0f: 0x2f29, 0x0f10: 0x2f2d, 0x0f11: 0x2197, + 0x0f12: 0x2f31, 0x0f13: 0x2f35, 0x0f14: 0x2f39, 0x0f15: 0x2f3d, 0x0f16: 0x2f41, 0x0f17: 0x2cb1, + 0x0f18: 0x2f45, 0x0f19: 0x2f49, 0x0f1a: 0x2f4d, 0x0f1b: 0x2f51, 0x0f1c: 0x2f55, 0x0f1d: 0x2f59, + 0x0f1e: 0x2f59, 0x0f1f: 0x2f5d, 0x0f20: 0x2f61, 0x0f21: 0x2f65, 0x0f22: 0x2f69, 0x0f23: 0x2f6d, + 0x0f24: 0x2f71, 0x0f25: 0x2f75, 0x0f26: 0x2f79, 0x0f27: 0x2e9d, 0x0f28: 0x2f7d, 0x0f29: 0x2f81, + 0x0f2a: 0x2f85, 0x0f2b: 0x2f89, 0x0f2c: 0x2f8d, 0x0f2d: 0x2f92, + 0x0f30: 0x2f96, 0x0f31: 0x2f9a, 0x0f32: 0x2f9e, 0x0f33: 0x2fa2, 0x0f34: 0x2fa6, 0x0f35: 0x2faa, + 0x0f36: 0x2fae, 0x0f37: 0x2fb2, 0x0f38: 0x2ecd, 0x0f39: 0x2fb6, 0x0f3a: 0x2fba, 0x0f3b: 0x2fbe, + 0x0f3c: 0x2e69, 0x0f3d: 0x2fc2, 0x0f3e: 0x2fc6, 0x0f3f: 0x2fca, + // Block 0x3d, offset 0xf40 + 0x0f40: 0x2fce, 0x0f41: 0x2fd2, 0x0f42: 0x2fd6, 0x0f43: 0x2fda, 0x0f44: 0x2fde, 0x0f45: 0x2fe2, + 0x0f46: 0x2fe6, 0x0f47: 0x2fea, 0x0f48: 0x2fee, 0x0f49: 0x2eed, 0x0f4a: 0x2ff2, 0x0f4b: 0x2ef1, + 0x0f4c: 0x2ff6, 0x0f4d: 0x2ffa, 0x0f4e: 0x2ffe, 0x0f4f: 0x3002, 0x0f50: 0x3006, 0x0f51: 0x2e6d, + 0x0f52: 0x2b15, 0x0f53: 0x300a, 0x0f54: 0x300e, 0x0f55: 0x195a, 0x0f56: 0x2c25, 0x0f57: 0x2d71, + 0x0f58: 0x3012, 0x0f59: 0x3016, 0x0f5a: 0x2f0d, 0x0f5b: 0x301a, 0x0f5c: 0x2f11, 0x0f5d: 0x301e, + 0x0f5e: 0x3022, 0x0f5f: 0x3026, 0x0f60: 0x2e75, 0x0f61: 0x302a, 0x0f62: 0x302e, 0x0f63: 0x3032, + 0x0f64: 0x3036, 0x0f65: 0x303a, 0x0f66: 0x2e79, 0x0f67: 0x303e, 0x0f68: 0x3042, 0x0f69: 0x3046, + 0x0f6a: 0x304a, 0x0f6b: 0x304e, 0x0f6c: 0x3052, 0x0f6d: 0x2f41, 0x0f6e: 0x3056, 0x0f6f: 0x305a, + 0x0f70: 0x2cb1, 0x0f71: 0x305e, 0x0f72: 0x2f51, 0x0f73: 0x3062, 0x0f74: 0x3066, 0x0f75: 0x306a, + 0x0f76: 0x306e, 0x0f77: 0x3072, 0x0f78: 0x2f65, 0x0f79: 0x3076, 0x0f7a: 0x2e99, 0x0f7b: 0x307a, + 0x0f7c: 0x2f69, 0x0f7d: 0x2bd9, 0x0f7e: 0x307e, 0x0f7f: 0x2f6d, + // Block 0x3e, offset 0xf80 + 0x0f80: 0x3082, 0x0f81: 0x2f75, 0x0f82: 0x3086, 0x0f83: 0x308a, 0x0f84: 0x308e, 0x0f85: 0x3092, + 0x0f86: 0x3096, 0x0f87: 0x2f7d, 0x0f88: 0x2e8d, 0x0f89: 0x309a, 0x0f8a: 0x2f81, 0x0f8b: 0x309e, + 0x0f8c: 0x2f85, 0x0f8d: 0x30a2, 0x0f8e: 0x1b76, 0x0f8f: 0x30a6, 0x0f90: 0x30ab, 0x0f91: 0x30b0, + 0x0f92: 0x30b5, 0x0f93: 0x30b9, 0x0f94: 0x30bd, 0x0f95: 0x30c1, 0x0f96: 0x30c6, 0x0f97: 0x30cb, + 0x0f98: 0x30d0, 0x0f99: 0x30d4, + // Block 0x3f, offset 0xfc0 + 0x0fdd: 0x3105, + 0x0fdf: 0x310a, + 0x0fea: 0x3124, 0x0feb: 0x3129, 0x0fec: 0x312e, 0x0fed: 0x3135, 0x0fee: 0x313c, 0x0fef: 0x3141, + 0x0ff0: 0x3146, 0x0ff1: 0x314b, 0x0ff2: 0x3150, 0x0ff3: 0x3155, 0x0ff4: 0x315a, 0x0ff5: 0x315f, + 0x0ff6: 0x3164, 0x0ff8: 0x3169, 0x0ff9: 0x316e, 0x0ffa: 0x3173, 0x0ffb: 0x3178, + 0x0ffc: 0x317d, 0x0ffe: 0x3182, + // Block 0x40, offset 0x1000 + 0x1000: 0x3187, 0x1001: 0x318c, 0x1003: 0x3191, 0x1004: 0x3196, + 0x1006: 0x319b, 0x1007: 0x31a0, 0x1008: 0x31a5, 0x1009: 0x31aa, 0x100a: 0x31af, 0x100b: 0x31b4, + 0x100c: 0x31b9, 0x100d: 0x31be, 0x100e: 0x31c3, + // Block 0x41, offset 0x1040 + 0x105a: 0x3a73, 0x105c: 0x3a7c, + 0x106b: 0x3a85, + // Block 0x42, offset 0x1080 + 0x109e: 0x3a8e, 0x109f: 0x3a97, 0x10a0: 0x3aa0, 0x10a1: 0x3aad, 0x10a2: 0x3aba, 0x10a3: 0x3ac7, + 0x10a4: 0x3ad4, + // Block 0x43, offset 0x10c0 + 0x10fb: 0x3ae1, + 0x10fc: 0x3aea, 0x10fd: 0x3af3, 0x10fe: 0x3b00, 0x10ff: 0x3b0d, + // Block 0x44, offset 0x1100 + 0x1100: 0x3b1a, + // Block 0x45, offset 0x1140 + 0x1140: 0x3d23, 0x1141: 0x3d27, 0x1142: 0x3d2b, 0x1143: 0x3d2f, 0x1144: 0x3d34, 0x1145: 0x2eb5, + 0x1146: 0x3d38, 0x1147: 0x3d3c, 0x1148: 0x3d40, 0x1149: 0x3d44, 0x114a: 0x2eb9, 0x114b: 0x3d48, + 0x114c: 0x3d4c, 0x114d: 0x3d50, 0x114e: 0x2ebd, 0x114f: 0x3d55, 0x1150: 0x3d59, 0x1151: 0x3d5d, + 0x1152: 0x3d61, 0x1153: 0x3d66, 0x1154: 0x3d6a, 0x1155: 0x3c71, 0x1156: 0x3d6e, 0x1157: 0x3d73, + 0x1158: 0x3d77, 0x1159: 0x3d7b, 0x115a: 0x3d7f, 0x115b: 0x2f9a, 0x115c: 0x3d83, 0x115d: 0x1866, + 0x115e: 0x3d88, 0x115f: 0x3d8c, 0x1160: 0x3d90, 0x1161: 0x3d94, 0x1162: 0x3cb9, 0x1163: 0x3d98, + 0x1164: 0x3d9c, 0x1165: 0x2fae, 0x1166: 0x2ec1, 0x1167: 0x2ec5, 0x1168: 0x2fb2, 0x1169: 0x3da0, + 0x116a: 0x3da4, 0x116b: 0x2bf1, 0x116c: 0x3da8, 0x116d: 0x2ec9, 0x116e: 0x3dac, 0x116f: 0x3db0, + 0x1170: 0x3db4, 0x1171: 0x3db8, 0x1172: 0x3db8, 0x1173: 0x3db8, 0x1174: 0x3dbc, 0x1175: 0x3dc1, + 0x1176: 0x3dc5, 0x1177: 0x3dc9, 0x1178: 0x3dcd, 0x1179: 0x3dd2, 0x117a: 0x3dd6, 0x117b: 0x3dda, + 0x117c: 0x3dde, 0x117d: 0x3de2, 0x117e: 0x3de6, 0x117f: 0x3dea, + // Block 0x46, offset 0x1180 + 0x1180: 0x3dee, 0x1181: 0x3df2, 0x1182: 0x3df6, 0x1183: 0x3dfa, 0x1184: 0x3dfe, 0x1185: 0x3e02, + 0x1186: 0x3e02, 0x1187: 0x2fba, 0x1188: 0x3e06, 0x1189: 0x3e0a, 0x118a: 0x3e0e, 0x118b: 0x3e12, + 0x118c: 0x2ed1, 0x118d: 0x3e16, 0x118e: 0x3e1a, 0x118f: 0x3e1e, 0x1190: 0x2e39, 0x1191: 0x3e22, + 0x1192: 0x3e26, 0x1193: 0x3e2a, 0x1194: 0x3e2e, 0x1195: 0x3e32, 0x1196: 0x3e36, 0x1197: 0x3e3a, + 0x1198: 0x3e3e, 0x1199: 0x3e42, 0x119a: 0x3e47, 0x119b: 0x3e4b, 0x119c: 0x3e4f, 0x119d: 0x3c55, + 0x119e: 0x3e53, 0x119f: 0x3e57, 0x11a0: 0x3e5b, 0x11a1: 0x3e60, 0x11a2: 0x3e65, 0x11a3: 0x3e69, + 0x11a4: 0x3e6d, 0x11a5: 0x3e71, 0x11a6: 0x3e75, 0x11a7: 0x3e79, 0x11a8: 0x3e7d, 0x11a9: 0x3e81, + 0x11aa: 0x3e85, 0x11ab: 0x3e85, 0x11ac: 0x3e89, 0x11ad: 0x3e8e, 0x11ae: 0x3e92, 0x11af: 0x2be1, + 0x11b0: 0x3e96, 0x11b1: 0x3e9a, 0x11b2: 0x3e9f, 0x11b3: 0x3ea3, 0x11b4: 0x3ea7, 0x11b5: 0x18ce, + 0x11b6: 0x3eab, 0x11b7: 0x3eaf, 0x11b8: 0x18d6, 0x11b9: 0x3eb3, 0x11ba: 0x3eb7, 0x11bb: 0x3ebb, + 0x11bc: 0x3ec0, 0x11bd: 0x3ec4, 0x11be: 0x3ec9, 0x11bf: 0x3ecd, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x3ed1, 0x11c1: 0x3ed5, 0x11c2: 0x3ed9, 0x11c3: 0x3edd, 0x11c4: 0x3ee1, 0x11c5: 0x3ee5, + 0x11c6: 0x3ee9, 0x11c7: 0x3eed, 0x11c8: 0x3ef1, 0x11c9: 0x3ef5, 0x11ca: 0x3efa, 0x11cb: 0x3efe, + 0x11cc: 0x3f02, 0x11cd: 0x3f06, 0x11ce: 0x2b11, 0x11cf: 0x3f0a, 0x11d0: 0x18fe, 0x11d1: 0x3f0f, + 0x11d2: 0x3f0f, 0x11d3: 0x3f14, 0x11d4: 0x3f18, 0x11d5: 0x3f18, 0x11d6: 0x3f1c, 0x11d7: 0x3f20, + 0x11d8: 0x3f25, 0x11d9: 0x3f2a, 0x11da: 0x3f2e, 0x11db: 0x3f32, 0x11dc: 0x3f36, 0x11dd: 0x3f3a, + 0x11de: 0x3f3e, 0x11df: 0x3f42, 0x11e0: 0x3f46, 0x11e1: 0x3f4a, 0x11e2: 0x3f4e, 0x11e3: 0x2ee5, + 0x11e4: 0x3f52, 0x11e5: 0x3f57, 0x11e6: 0x3f5b, 0x11e7: 0x3f5f, 0x11e8: 0x2fea, 0x11e9: 0x3f5f, + 0x11ea: 0x3f63, 0x11eb: 0x2eed, 0x11ec: 0x3f67, 0x11ed: 0x3f6b, 0x11ee: 0x3f6f, 0x11ef: 0x3f73, + 0x11f0: 0x2ef1, 0x11f1: 0x2aa5, 0x11f2: 0x3f77, 0x11f3: 0x3f7b, 0x11f4: 0x3f7f, 0x11f5: 0x3f83, + 0x11f6: 0x3f87, 0x11f7: 0x3f8b, 0x11f8: 0x3f8f, 0x11f9: 0x3f94, 0x11fa: 0x3f98, 0x11fb: 0x3f9c, + 0x11fc: 0x3fa0, 0x11fd: 0x3fa4, 0x11fe: 0x3fa8, 0x11ff: 0x3fad, + // Block 0x48, offset 0x1200 + 0x1200: 0x3fb1, 0x1201: 0x3fb5, 0x1202: 0x3fb9, 0x1203: 0x3fbd, 0x1204: 0x3fc1, 0x1205: 0x3fc5, + 0x1206: 0x3fc9, 0x1207: 0x3fcd, 0x1208: 0x2ef5, 0x1209: 0x3fd1, 0x120a: 0x3fd5, 0x120b: 0x3fda, + 0x120c: 0x3fde, 0x120d: 0x3fe2, 0x120e: 0x3fe6, 0x120f: 0x2efd, 0x1210: 0x3fea, 0x1211: 0x3fee, + 0x1212: 0x3ff2, 0x1213: 0x3ff6, 0x1214: 0x3ffa, 0x1215: 0x3ffe, 0x1216: 0x4002, 0x1217: 0x4006, + 0x1218: 0x2b15, 0x1219: 0x300a, 0x121a: 0x400a, 0x121b: 0x400e, 0x121c: 0x4012, 0x121d: 0x4016, + 0x121e: 0x401b, 0x121f: 0x401f, 0x1220: 0x4023, 0x1221: 0x4027, 0x1222: 0x2f01, 0x1223: 0x402b, + 0x1224: 0x4030, 0x1225: 0x4034, 0x1226: 0x4038, 0x1227: 0x30b5, 0x1228: 0x403c, 0x1229: 0x4040, + 0x122a: 0x4044, 0x122b: 0x4048, 0x122c: 0x404c, 0x122d: 0x4051, 0x122e: 0x4055, 0x122f: 0x4059, + 0x1230: 0x405d, 0x1231: 0x4062, 0x1232: 0x4066, 0x1233: 0x406a, 0x1234: 0x406e, 0x1235: 0x2c25, + 0x1236: 0x4072, 0x1237: 0x4076, 0x1238: 0x407b, 0x1239: 0x4080, 0x123a: 0x4085, 0x123b: 0x4089, + 0x123c: 0x408e, 0x123d: 0x4092, 0x123e: 0x4096, 0x123f: 0x409a, + // Block 0x49, offset 0x1240 + 0x1240: 0x409e, 0x1241: 0x2f05, 0x1242: 0x2d71, 0x1243: 0x40a2, 0x1244: 0x40a6, 0x1245: 0x40aa, + 0x1246: 0x40ae, 0x1247: 0x40b3, 0x1248: 0x40b7, 0x1249: 0x40bb, 0x124a: 0x40bf, 0x124b: 0x3016, + 0x124c: 0x40c3, 0x124d: 0x40c7, 0x124e: 0x40cc, 0x124f: 0x40d0, 0x1250: 0x40d4, 0x1251: 0x40d9, + 0x1252: 0x40de, 0x1253: 0x40e2, 0x1254: 0x301a, 0x1255: 0x40e6, 0x1256: 0x40ea, 0x1257: 0x40ee, + 0x1258: 0x40f2, 0x1259: 0x40f6, 0x125a: 0x40fa, 0x125b: 0x40fe, 0x125c: 0x4103, 0x125d: 0x4107, + 0x125e: 0x410c, 0x125f: 0x4110, 0x1260: 0x4115, 0x1261: 0x3022, 0x1262: 0x4119, 0x1263: 0x411d, + 0x1264: 0x4122, 0x1265: 0x4126, 0x1266: 0x412a, 0x1267: 0x412f, 0x1268: 0x4134, 0x1269: 0x4138, + 0x126a: 0x413c, 0x126b: 0x4140, 0x126c: 0x4144, 0x126d: 0x4144, 0x126e: 0x4148, 0x126f: 0x414c, + 0x1270: 0x302a, 0x1271: 0x4150, 0x1272: 0x4154, 0x1273: 0x4158, 0x1274: 0x415c, 0x1275: 0x4160, + 0x1276: 0x4165, 0x1277: 0x4169, 0x1278: 0x2bed, 0x1279: 0x416e, 0x127a: 0x4173, 0x127b: 0x4177, + 0x127c: 0x417c, 0x127d: 0x4181, 0x127e: 0x4186, 0x127f: 0x418a, + // Block 0x4a, offset 0x1280 + 0x1280: 0x3042, 0x1281: 0x418e, 0x1282: 0x4193, 0x1283: 0x4198, 0x1284: 0x419d, 0x1285: 0x41a2, + 0x1286: 0x41a6, 0x1287: 0x41a6, 0x1288: 0x3046, 0x1289: 0x30bd, 0x128a: 0x41aa, 0x128b: 0x41ae, + 0x128c: 0x41b2, 0x128d: 0x41b6, 0x128e: 0x41bb, 0x128f: 0x2b59, 0x1290: 0x304e, 0x1291: 0x41bf, + 0x1292: 0x41c3, 0x1293: 0x2f2d, 0x1294: 0x41c8, 0x1295: 0x41cd, 0x1296: 0x2e89, 0x1297: 0x41d2, + 0x1298: 0x41d6, 0x1299: 0x2f39, 0x129a: 0x41da, 0x129b: 0x41de, 0x129c: 0x41e2, 0x129d: 0x41e7, + 0x129e: 0x41e7, 0x129f: 0x41ec, 0x12a0: 0x41f0, 0x12a1: 0x41f4, 0x12a2: 0x41f9, 0x12a3: 0x41fd, + 0x12a4: 0x4201, 0x12a5: 0x4205, 0x12a6: 0x420a, 0x12a7: 0x420e, 0x12a8: 0x4212, 0x12a9: 0x4216, + 0x12aa: 0x421a, 0x12ab: 0x421e, 0x12ac: 0x4223, 0x12ad: 0x4227, 0x12ae: 0x422b, 0x12af: 0x422f, + 0x12b0: 0x4233, 0x12b1: 0x4237, 0x12b2: 0x423b, 0x12b3: 0x4240, 0x12b4: 0x4245, 0x12b5: 0x4249, + 0x12b6: 0x424e, 0x12b7: 0x4252, 0x12b8: 0x4257, 0x12b9: 0x425b, 0x12ba: 0x2f51, 0x12bb: 0x425f, + 0x12bc: 0x4264, 0x12bd: 0x4269, 0x12be: 0x426d, 0x12bf: 0x4272, + // Block 0x4b, offset 0x12c0 + 0x12c0: 0x4276, 0x12c1: 0x427b, 0x12c2: 0x427f, 0x12c3: 0x4283, 0x12c4: 0x4287, 0x12c5: 0x428b, + 0x12c6: 0x428f, 0x12c7: 0x4293, 0x12c8: 0x4298, 0x12c9: 0x429d, 0x12ca: 0x42a2, 0x12cb: 0x3f14, + 0x12cc: 0x42a7, 0x12cd: 0x42ab, 0x12ce: 0x42af, 0x12cf: 0x42b3, 0x12d0: 0x42b7, 0x12d1: 0x42bb, + 0x12d2: 0x42bf, 0x12d3: 0x42c3, 0x12d4: 0x42c7, 0x12d5: 0x42cb, 0x12d6: 0x42cf, 0x12d7: 0x42d3, + 0x12d8: 0x2c31, 0x12d9: 0x42d8, 0x12da: 0x42dc, 0x12db: 0x42e0, 0x12dc: 0x42e4, 0x12dd: 0x42e8, + 0x12de: 0x42ec, 0x12df: 0x2f5d, 0x12e0: 0x42f0, 0x12e1: 0x42f4, 0x12e2: 0x42f8, 0x12e3: 0x42fc, + 0x12e4: 0x4300, 0x12e5: 0x4305, 0x12e6: 0x430a, 0x12e7: 0x430f, 0x12e8: 0x4313, 0x12e9: 0x4317, + 0x12ea: 0x431b, 0x12eb: 0x431f, 0x12ec: 0x4324, 0x12ed: 0x4328, 0x12ee: 0x432d, 0x12ef: 0x4331, + 0x12f0: 0x4335, 0x12f1: 0x433a, 0x12f2: 0x433f, 0x12f3: 0x4343, 0x12f4: 0x2b45, 0x12f5: 0x4347, + 0x12f6: 0x434b, 0x12f7: 0x434f, 0x12f8: 0x4353, 0x12f9: 0x4357, 0x12fa: 0x435b, 0x12fb: 0x306a, + 0x12fc: 0x435f, 0x12fd: 0x4363, 0x12fe: 0x4367, 0x12ff: 0x436b, + // Block 0x4c, offset 0x1300 + 0x1300: 0x436f, 0x1301: 0x4373, 0x1302: 0x4377, 0x1303: 0x437b, 0x1304: 0x1a66, 0x1305: 0x437f, + 0x1306: 0x4384, 0x1307: 0x4388, 0x1308: 0x438c, 0x1309: 0x4390, 0x130a: 0x4394, 0x130b: 0x4398, + 0x130c: 0x439d, 0x130d: 0x43a2, 0x130e: 0x43a6, 0x130f: 0x43aa, 0x1310: 0x307e, 0x1311: 0x3082, + 0x1312: 0x1a82, 0x1313: 0x43ae, 0x1314: 0x43b3, 0x1315: 0x43b7, 0x1316: 0x43bb, 0x1317: 0x43bf, + 0x1318: 0x43c3, 0x1319: 0x43c8, 0x131a: 0x43cd, 0x131b: 0x43d1, 0x131c: 0x43d5, 0x131d: 0x43d9, + 0x131e: 0x43de, 0x131f: 0x3086, 0x1320: 0x43e2, 0x1321: 0x43e7, 0x1322: 0x43ec, 0x1323: 0x43f0, + 0x1324: 0x43f4, 0x1325: 0x43f8, 0x1326: 0x43fd, 0x1327: 0x4401, 0x1328: 0x4405, 0x1329: 0x4409, + 0x132a: 0x440d, 0x132b: 0x4411, 0x132c: 0x4415, 0x132d: 0x4419, 0x132e: 0x441e, 0x132f: 0x4422, + 0x1330: 0x4426, 0x1331: 0x442a, 0x1332: 0x442f, 0x1333: 0x4433, 0x1334: 0x4437, 0x1335: 0x443b, + 0x1336: 0x443f, 0x1337: 0x4444, 0x1338: 0x4449, 0x1339: 0x444d, 0x133a: 0x4451, 0x133b: 0x4455, + 0x133c: 0x445a, 0x133d: 0x445e, 0x133e: 0x309e, 0x133f: 0x309e, + // Block 0x4d, offset 0x1340 + 0x1340: 0x4463, 0x1341: 0x4467, 0x1342: 0x446c, 0x1343: 0x4470, 0x1344: 0x4474, 0x1345: 0x4478, + 0x1346: 0x447c, 0x1347: 0x4480, 0x1348: 0x4484, 0x1349: 0x4488, 0x134a: 0x30a2, 0x134b: 0x448d, + 0x134c: 0x4491, 0x134d: 0x4495, 0x134e: 0x4499, 0x134f: 0x449d, 0x1350: 0x44a1, 0x1351: 0x44a6, + 0x1352: 0x44aa, 0x1353: 0x44af, 0x1354: 0x44b4, 0x1355: 0x1b42, 0x1356: 0x44b9, 0x1357: 0x1b52, + 0x1358: 0x44bd, 0x1359: 0x44c1, 0x135a: 0x44c5, 0x135b: 0x44c9, 0x135c: 0x1b66, 0x135d: 0x44cd, +} + +// nfcDecompLookup: 832 bytes +// Block 0 is the null block. +var nfcDecompLookup = [832]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c3: 0x03, 0x0c4: 0x04, 0x0c5: 0x05, 0x0c6: 0x06, 0x0c7: 0x07, + 0x0c8: 0x08, 0x0cd: 0x09, 0x0ce: 0x0a, 0x0cf: 0x0b, + 0x0d0: 0x0c, 0x0d1: 0x0d, 0x0d3: 0x0e, + 0x0d8: 0x0f, 0x0db: 0x10, + 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, + 0x0ef: 0x08, + 0x0f0: 0x0c, + // Block 0x4, offset 0x100 + 0x124: 0x11, 0x125: 0x12, 0x127: 0x13, + 0x128: 0x14, 0x129: 0x15, 0x12d: 0x16, 0x12e: 0x17, 0x12f: 0x18, + 0x131: 0x19, 0x133: 0x1a, 0x135: 0x1b, 0x137: 0x1c, + 0x13d: 0x1d, 0x13e: 0x1e, + // Block 0x5, offset 0x140 + 0x140: 0x1f, + 0x16c: 0x20, 0x16d: 0x21, + 0x178: 0x22, 0x179: 0x23, 0x17a: 0x24, 0x17b: 0x25, 0x17c: 0x26, 0x17d: 0x27, 0x17e: 0x28, 0x17f: 0x29, + // Block 0x6, offset 0x180 + 0x180: 0x2a, 0x184: 0x2b, 0x186: 0x2c, 0x187: 0x2d, + 0x188: 0x2e, 0x189: 0x2f, 0x18a: 0x30, 0x18b: 0x31, 0x18c: 0x32, + 0x1ab: 0x33, + // Block 0x7, offset 0x1c0 + 0x1c1: 0x34, 0x1c2: 0x35, 0x1c3: 0x36, + // Block 0x8, offset 0x200 + 0x224: 0x37, 0x225: 0x38, 0x226: 0x39, 0x227: 0x3a, + 0x228: 0x3b, 0x229: 0x3c, 0x22a: 0x3d, 0x22b: 0x3e, 0x22c: 0x3f, 0x22d: 0x40, + // Block 0x9, offset 0x240 + 0x242: 0x41, + // Block 0xa, offset 0x280 + 0x285: 0x42, 0x286: 0x43, 0x287: 0x44, + // Block 0xb, offset 0x2c0 + 0x2e0: 0x45, 0x2e1: 0x46, 0x2e2: 0x47, 0x2e3: 0x48, 0x2e4: 0x49, 0x2e5: 0x4a, 0x2e6: 0x4b, 0x2e7: 0x4c, + 0x2e8: 0x4d, + // Block 0xc, offset 0x300 + 0x311: 0x09, + 0x31d: 0x0a, + 0x32f: 0x0b, +} + +var nfcDecompTrie = trie{nfcDecompLookup[:], nfcDecompValues[:]} + +// nfkcDecompValues: 10176 entries, 20352 bytes +// Block 2 is the null block. +var nfkcDecompValues = [10176]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00e0: 0x0001, + 0x00e8: 0x0003, + 0x00ea: 0x0007, 0x00ef: 0x0009, + 0x00f2: 0x000d, 0x00f3: 0x000f, 0x00f4: 0x0011, 0x00f5: 0x0015, + 0x00f8: 0x0018, 0x00f9: 0x001c, 0x00fa: 0x001e, + 0x00fc: 0x0020, 0x00fd: 0x0026, 0x00fe: 0x002c, + // Block 0x4, offset 0x100 + 0x0100: 0x0032, 0x0101: 0x0036, 0x0102: 0x003a, 0x0103: 0x003e, 0x0104: 0x0042, 0x0105: 0x0046, + 0x0107: 0x004a, 0x0108: 0x004e, 0x0109: 0x0052, 0x010a: 0x0056, 0x010b: 0x005a, + 0x010c: 0x005e, 0x010d: 0x0062, 0x010e: 0x0066, 0x010f: 0x006a, 0x0111: 0x006e, + 0x0112: 0x0072, 0x0113: 0x0076, 0x0114: 0x007a, 0x0115: 0x007e, 0x0116: 0x0082, + 0x0119: 0x0086, 0x011a: 0x008a, 0x011b: 0x008e, 0x011c: 0x0092, 0x011d: 0x0096, + 0x0120: 0x009a, 0x0121: 0x009e, 0x0122: 0x00a2, 0x0123: 0x00a6, + 0x0124: 0x00aa, 0x0125: 0x00ae, 0x0127: 0x00b2, 0x0128: 0x00b6, 0x0129: 0x00ba, + 0x012a: 0x00be, 0x012b: 0x00c2, 0x012c: 0x00c6, 0x012d: 0x00ca, 0x012e: 0x00ce, 0x012f: 0x00d2, + 0x0131: 0x00d6, 0x0132: 0x00da, 0x0133: 0x00de, 0x0134: 0x00e2, 0x0135: 0x00e6, + 0x0136: 0x00ea, 0x0139: 0x00ee, 0x013a: 0x00f2, 0x013b: 0x00f6, + 0x013c: 0x00fa, 0x013d: 0x00fe, 0x013f: 0x0102, + // Block 0x5, offset 0x140 + 0x0140: 0x0106, 0x0141: 0x010a, 0x0142: 0x010e, 0x0143: 0x0112, 0x0144: 0x0116, 0x0145: 0x011a, + 0x0146: 0x011e, 0x0147: 0x0122, 0x0148: 0x0126, 0x0149: 0x012a, 0x014a: 0x012e, 0x014b: 0x0132, + 0x014c: 0x0136, 0x014d: 0x013a, 0x014e: 0x013e, 0x014f: 0x0142, + 0x0152: 0x0146, 0x0153: 0x014a, 0x0154: 0x014e, 0x0155: 0x0152, 0x0156: 0x0156, 0x0157: 0x015a, + 0x0158: 0x015e, 0x0159: 0x0162, 0x015a: 0x0166, 0x015b: 0x016a, 0x015c: 0x016e, 0x015d: 0x0172, + 0x015e: 0x0176, 0x015f: 0x017a, 0x0160: 0x017e, 0x0161: 0x0182, 0x0162: 0x0186, 0x0163: 0x018a, + 0x0164: 0x018e, 0x0165: 0x0192, 0x0168: 0x0196, 0x0169: 0x019a, + 0x016a: 0x019e, 0x016b: 0x01a2, 0x016c: 0x01a6, 0x016d: 0x01aa, 0x016e: 0x01ae, 0x016f: 0x01b2, + 0x0170: 0x01b6, 0x0172: 0x01ba, 0x0173: 0x01bd, 0x0174: 0x01c0, 0x0175: 0x01c4, + 0x0176: 0x01c8, 0x0177: 0x01cc, 0x0179: 0x01d0, 0x017a: 0x01d4, 0x017b: 0x01d8, + 0x017c: 0x01dc, 0x017d: 0x01e0, 0x017e: 0x01e4, 0x017f: 0x01e8, + // Block 0x6, offset 0x180 + 0x0180: 0x01ec, 0x0183: 0x01f0, 0x0184: 0x01f4, 0x0185: 0x01f8, + 0x0186: 0x01fc, 0x0187: 0x0200, 0x0188: 0x0204, 0x0189: 0x0208, + 0x018c: 0x020c, 0x018d: 0x0210, 0x018e: 0x0214, 0x018f: 0x0218, 0x0190: 0x021c, 0x0191: 0x0220, + 0x0194: 0x0224, 0x0195: 0x0228, 0x0196: 0x022c, 0x0197: 0x0230, + 0x0198: 0x0234, 0x0199: 0x0238, 0x019a: 0x023c, 0x019b: 0x0240, 0x019c: 0x0244, 0x019d: 0x0248, + 0x019e: 0x024c, 0x019f: 0x0250, 0x01a0: 0x0254, 0x01a1: 0x0258, 0x01a2: 0x025c, 0x01a3: 0x0260, + 0x01a4: 0x0264, 0x01a5: 0x0268, 0x01a8: 0x026c, 0x01a9: 0x0270, + 0x01aa: 0x0274, 0x01ab: 0x0278, 0x01ac: 0x027c, 0x01ad: 0x0280, 0x01ae: 0x0284, 0x01af: 0x0288, + 0x01b0: 0x028c, 0x01b1: 0x0290, 0x01b2: 0x0294, 0x01b3: 0x0298, 0x01b4: 0x029c, 0x01b5: 0x02a0, + 0x01b6: 0x02a4, 0x01b7: 0x02a8, 0x01b8: 0x02ac, 0x01b9: 0x02b0, 0x01ba: 0x02b4, 0x01bb: 0x02b8, + 0x01bc: 0x02bc, 0x01bd: 0x02c0, 0x01be: 0x02c4, 0x01bf: 0x02c8, + // Block 0x7, offset 0x1c0 + 0x01e0: 0x02ca, 0x01e1: 0x02ce, + 0x01ef: 0x02d2, + 0x01f0: 0x02d6, + // Block 0x8, offset 0x200 + 0x0204: 0x02da, 0x0205: 0x02df, + 0x0206: 0x02e4, 0x0207: 0x02e9, 0x0208: 0x02ec, 0x0209: 0x02ef, 0x020a: 0x02f2, 0x020b: 0x02f5, + 0x020c: 0x02f8, 0x020d: 0x02fb, 0x020e: 0x02ff, 0x020f: 0x0303, 0x0210: 0x0307, 0x0211: 0x030b, + 0x0212: 0x030f, 0x0213: 0x0313, 0x0214: 0x0317, 0x0215: 0x031b, 0x0216: 0x0321, 0x0217: 0x0327, + 0x0218: 0x032d, 0x0219: 0x0333, 0x021a: 0x0339, 0x021b: 0x033f, 0x021c: 0x0345, + 0x021e: 0x034b, 0x021f: 0x0351, 0x0220: 0x0357, 0x0221: 0x035d, 0x0222: 0x0363, 0x0223: 0x0368, + 0x0226: 0x036d, 0x0227: 0x0371, 0x0228: 0x0375, 0x0229: 0x0379, + 0x022a: 0x037d, 0x022b: 0x0381, 0x022c: 0x0385, 0x022d: 0x038b, 0x022e: 0x0391, 0x022f: 0x0396, + 0x0230: 0x039b, 0x0231: 0x039f, 0x0232: 0x03a2, 0x0233: 0x03a5, 0x0234: 0x03a8, 0x0235: 0x03ac, + 0x0238: 0x03b0, 0x0239: 0x03b4, 0x023a: 0x03b8, 0x023b: 0x03be, + 0x023c: 0x03c4, 0x023d: 0x03c9, 0x023e: 0x03ce, 0x023f: 0x03d3, + // Block 0x9, offset 0x240 + 0x0240: 0x03d8, 0x0241: 0x03dc, 0x0242: 0x03e0, 0x0243: 0x03e4, 0x0244: 0x03e8, 0x0245: 0x03ec, + 0x0246: 0x03f0, 0x0247: 0x03f4, 0x0248: 0x03f8, 0x0249: 0x03fc, 0x024a: 0x0400, 0x024b: 0x0404, + 0x024c: 0x0408, 0x024d: 0x040c, 0x024e: 0x0410, 0x024f: 0x0414, 0x0250: 0x0418, 0x0251: 0x041c, + 0x0252: 0x0420, 0x0253: 0x0424, 0x0254: 0x0428, 0x0255: 0x042c, 0x0256: 0x0430, 0x0257: 0x0434, + 0x0258: 0x0438, 0x0259: 0x043c, 0x025a: 0x0440, 0x025b: 0x0444, + 0x025e: 0x0448, 0x025f: 0x044c, + 0x0266: 0x0450, 0x0267: 0x0454, 0x0268: 0x0458, 0x0269: 0x045c, + 0x026a: 0x0460, 0x026b: 0x0466, 0x026c: 0x046c, 0x026d: 0x0472, 0x026e: 0x0478, 0x026f: 0x047c, + 0x0270: 0x0480, 0x0271: 0x0486, 0x0272: 0x048c, 0x0273: 0x0490, + // Block 0xa, offset 0x280 + 0x02b0: 0x0494, 0x02b1: 0x0496, 0x02b2: 0x0499, 0x02b3: 0x049b, 0x02b4: 0x049d, 0x02b5: 0x04a0, + 0x02b6: 0x04a3, 0x02b7: 0x04a6, 0x02b8: 0x04a8, + // Block 0xb, offset 0x2c0 + 0x02d8: 0x04aa, 0x02d9: 0x04ae, 0x02da: 0x04b2, 0x02db: 0x04b6, 0x02dc: 0x04ba, 0x02dd: 0x04be, + 0x02e0: 0x04c2, 0x02e1: 0x04c5, 0x02e2: 0x02c8, 0x02e3: 0x04c7, + 0x02e4: 0x04c9, + // Block 0xc, offset 0x300 + 0x0300: 0x04cc, 0x0301: 0x04cf, 0x0303: 0x04d2, 0x0304: 0x04d5, + 0x0334: 0x04da, + 0x033a: 0x04dd, + 0x033e: 0x04e1, + // Block 0xd, offset 0x340 + 0x0344: 0x0011, 0x0345: 0x04e8, + 0x0346: 0x04ee, 0x0347: 0x04f3, 0x0348: 0x04f6, 0x0349: 0x04fb, 0x034a: 0x0500, + 0x034c: 0x0505, 0x034e: 0x050a, 0x034f: 0x050f, 0x0350: 0x0514, + 0x036a: 0x051b, 0x036b: 0x0520, 0x036c: 0x0525, 0x036d: 0x052a, 0x036e: 0x052f, 0x036f: 0x0534, + 0x0370: 0x0539, + // Block 0xe, offset 0x380 + 0x038a: 0x0540, 0x038b: 0x0545, + 0x038c: 0x054a, 0x038d: 0x054f, 0x038e: 0x0554, 0x0390: 0x0559, 0x0391: 0x055c, + 0x0392: 0x055f, 0x0393: 0x050a, 0x0394: 0x0520, 0x0395: 0x056c, 0x0396: 0x056f, + 0x03b0: 0x0572, 0x03b1: 0x0575, 0x03b2: 0x0578, 0x03b4: 0x057b, 0x03b5: 0x057e, + 0x03b9: 0x0581, + // Block 0xf, offset 0x3c0 + 0x03c0: 0x0584, 0x03c1: 0x0589, 0x03c3: 0x058e, + 0x03c7: 0x0593, + 0x03cc: 0x0598, 0x03cd: 0x059d, 0x03ce: 0x05a2, + 0x03d9: 0x05a7, + 0x03f9: 0x05ac, + // Block 0x10, offset 0x400 + 0x0410: 0x05b1, 0x0411: 0x05b6, + 0x0413: 0x05bb, 0x0417: 0x05c0, + 0x041c: 0x05c5, 0x041d: 0x05ca, + 0x041e: 0x05cf, + 0x0436: 0x05d4, 0x0437: 0x05d9, + // Block 0x11, offset 0x440 + 0x0441: 0x05de, 0x0442: 0x05e3, + 0x0450: 0x05e8, 0x0451: 0x05ed, + 0x0452: 0x05f2, 0x0453: 0x05f7, 0x0456: 0x05fc, 0x0457: 0x0601, + 0x045a: 0x0606, 0x045b: 0x060b, 0x045c: 0x0610, 0x045d: 0x0615, + 0x045e: 0x061a, 0x045f: 0x061f, 0x0462: 0x0624, 0x0463: 0x0629, + 0x0464: 0x062e, 0x0465: 0x0633, 0x0466: 0x0638, 0x0467: 0x063d, + 0x046a: 0x0642, 0x046b: 0x0647, 0x046c: 0x064c, 0x046d: 0x0651, 0x046e: 0x0656, 0x046f: 0x065b, + 0x0470: 0x0660, 0x0471: 0x0665, 0x0472: 0x066a, 0x0473: 0x066f, 0x0474: 0x0674, 0x0475: 0x0679, + 0x0478: 0x067e, 0x0479: 0x0683, + // Block 0x12, offset 0x480 + 0x0487: 0x0688, + // Block 0x13, offset 0x4c0 + 0x04e2: 0x068d, 0x04e3: 0x0692, + 0x04e4: 0x0697, 0x04e5: 0x069c, 0x04e6: 0x06a1, + // Block 0x14, offset 0x500 + 0x0535: 0x06a6, + 0x0536: 0x06ab, 0x0537: 0x06b0, 0x0538: 0x06b5, + // Block 0x15, offset 0x540 + 0x0540: 0x06ba, 0x0542: 0x06bf, + 0x0553: 0x06c4, + // Block 0x16, offset 0x580 + 0x05a9: 0x06c9, + 0x05b1: 0x06d0, 0x05b4: 0x06d7, + // Block 0x17, offset 0x5c0 + 0x05d8: 0x06de, 0x05d9: 0x06e5, 0x05da: 0x06ec, 0x05db: 0x06f3, 0x05dc: 0x06fa, 0x05dd: 0x0701, + 0x05de: 0x0708, 0x05df: 0x070f, + // Block 0x18, offset 0x600 + 0x060b: 0x0716, + 0x060c: 0x071d, + 0x061c: 0x0724, 0x061d: 0x072b, + 0x061f: 0x0732, + // Block 0x19, offset 0x640 + 0x0673: 0x0739, + 0x0676: 0x0740, + // Block 0x1a, offset 0x680 + 0x0699: 0x0747, 0x069a: 0x074e, 0x069b: 0x0755, + 0x069e: 0x075c, + // Block 0x1b, offset 0x6c0 + 0x06c8: 0x0763, 0x06cb: 0x076a, + 0x06cc: 0x0771, + 0x06dc: 0x0778, 0x06dd: 0x077f, + // Block 0x1c, offset 0x700 + 0x0714: 0x0786, + // Block 0x1d, offset 0x740 + 0x074a: 0x078d, 0x074b: 0x0794, + 0x074c: 0x079b, + // Block 0x1e, offset 0x780 + 0x0788: 0x07a2, + // Block 0x1f, offset 0x7c0 + 0x07c0: 0x07a9, + 0x07c7: 0x07b0, 0x07c8: 0x07b7, 0x07ca: 0x07be, 0x07cb: 0x07c5, + // Block 0x20, offset 0x800 + 0x080a: 0x07cf, 0x080b: 0x07d6, + 0x080c: 0x07dd, + // Block 0x21, offset 0x840 + 0x085a: 0x07e4, 0x085c: 0x07eb, 0x085d: 0x07f2, + 0x085e: 0x07fc, + // Block 0x22, offset 0x880 + 0x08b3: 0x0803, + // Block 0x23, offset 0x8c0 + 0x08f3: 0x080a, + // Block 0x24, offset 0x900 + 0x091c: 0x0811, 0x091d: 0x0818, + // Block 0x25, offset 0x940 + 0x094c: 0x081f, + // Block 0x26, offset 0x980 + 0x0983: 0x0823, + 0x098d: 0x082a, + 0x0992: 0x0831, 0x0997: 0x0838, + 0x099c: 0x083f, + 0x09a9: 0x0846, + 0x09b3: 0x084d, 0x09b5: 0x0854, + 0x09b6: 0x085b, 0x09b7: 0x0862, 0x09b8: 0x086c, 0x09b9: 0x0873, + // Block 0x27, offset 0x9c0 + 0x09c1: 0x087d, + 0x09d3: 0x0884, + 0x09dd: 0x088b, + 0x09e2: 0x0892, + 0x09e7: 0x0899, + 0x09ec: 0x08a0, + 0x09f9: 0x08a7, + // Block 0x28, offset 0xa00 + 0x0a26: 0x08ae, + // Block 0x29, offset 0xa40 + 0x0a7c: 0x08b5, + // Block 0x2a, offset 0xa80 + 0x0a86: 0x08b9, 0x0a88: 0x08c0, 0x0a8a: 0x08c7, + 0x0a8c: 0x08ce, 0x0a8e: 0x08d5, + 0x0a92: 0x08dc, + 0x0abb: 0x08e3, + 0x0abd: 0x08ea, + // Block 0x2b, offset 0xac0 + 0x0ac0: 0x08f1, 0x0ac1: 0x08f8, 0x0ac3: 0x08ff, + // Block 0x2c, offset 0xb00 + 0x0b2c: 0x0906, 0x0b2d: 0x0908, 0x0b2e: 0x090b, + 0x0b30: 0x090d, 0x0b31: 0x090f, 0x0b32: 0x0911, 0x0b33: 0x0914, 0x0b34: 0x0916, 0x0b35: 0x0918, + 0x0b36: 0x091a, 0x0b37: 0x091c, 0x0b38: 0x091e, 0x0b39: 0x0920, 0x0b3a: 0x0922, + 0x0b3c: 0x0924, 0x0b3d: 0x0926, 0x0b3e: 0x0929, 0x0b3f: 0x092b, + // Block 0x2d, offset 0xb40 + 0x0b40: 0x092d, 0x0b41: 0x092f, 0x0b42: 0x0931, 0x0b43: 0x0007, 0x0b44: 0x0933, 0x0b45: 0x0936, + 0x0b46: 0x0939, 0x0b47: 0x093d, 0x0b48: 0x093f, 0x0b49: 0x0941, 0x0b4a: 0x0943, 0x0b4b: 0x0946, + 0x0b4c: 0x0949, 0x0b4d: 0x094c, 0x0b4f: 0x094e, 0x0b50: 0x0950, 0x0b51: 0x0952, + 0x0b52: 0x001e, 0x0b53: 0x0955, 0x0b54: 0x0958, 0x0b55: 0x095c, 0x0b56: 0x0960, 0x0b57: 0x0962, + 0x0b58: 0x0964, 0x0b59: 0x0966, 0x0b5a: 0x096a, 0x0b5b: 0x096d, 0x0b5c: 0x096f, 0x0b5d: 0x0559, + 0x0b5e: 0x0973, 0x0b5f: 0x0976, 0x0b60: 0x056c, 0x0b61: 0x0979, 0x0b62: 0x097c, 0x0b63: 0x049b, + 0x0b64: 0x0964, 0x0b65: 0x096d, 0x0b66: 0x0559, 0x0b67: 0x0973, 0x0b68: 0x0575, 0x0b69: 0x056c, + 0x0b6a: 0x0979, + 0x0b78: 0x097e, + // Block 0x2e, offset 0xb80 + 0x0b9b: 0x0981, 0x0b9c: 0x0984, 0x0b9d: 0x0986, + 0x0b9e: 0x0989, 0x0b9f: 0x0949, 0x0ba0: 0x098c, 0x0ba1: 0x098e, 0x0ba2: 0x0991, 0x0ba3: 0x0994, + 0x0ba4: 0x0997, 0x0ba5: 0x099a, 0x0ba6: 0x099d, 0x0ba7: 0x09a0, 0x0ba8: 0x09a4, 0x0ba9: 0x09a7, + 0x0baa: 0x09aa, 0x0bab: 0x09ae, 0x0bac: 0x09b1, 0x0bad: 0x09b4, 0x0bae: 0x09b7, 0x0baf: 0x09ba, + 0x0bb0: 0x09bd, 0x0bb1: 0x09c0, 0x0bb2: 0x09c3, 0x0bb3: 0x09c6, 0x0bb4: 0x09c9, 0x0bb5: 0x09cc, + 0x0bb6: 0x09cf, 0x0bb7: 0x09d2, 0x0bb8: 0x09d5, 0x0bb9: 0x09d9, 0x0bba: 0x09dc, 0x0bbb: 0x09df, + 0x0bbc: 0x09e1, 0x0bbd: 0x09e4, 0x0bbe: 0x09e7, 0x0bbf: 0x055c, + // Block 0x2f, offset 0xbc0 + 0x0bc0: 0x09ea, 0x0bc1: 0x09ee, 0x0bc2: 0x09f2, 0x0bc3: 0x09f6, 0x0bc4: 0x09fa, 0x0bc5: 0x09fe, + 0x0bc6: 0x0a02, 0x0bc7: 0x0a06, 0x0bc8: 0x0a0a, 0x0bc9: 0x0a10, 0x0bca: 0x0a16, 0x0bcb: 0x0a1a, + 0x0bcc: 0x0a1e, 0x0bcd: 0x0a22, 0x0bce: 0x0a26, 0x0bcf: 0x0a2a, 0x0bd0: 0x0a2e, 0x0bd1: 0x0a32, + 0x0bd2: 0x0a36, 0x0bd3: 0x0a3a, 0x0bd4: 0x0a3e, 0x0bd5: 0x0a44, 0x0bd6: 0x0a4a, 0x0bd7: 0x0a50, + 0x0bd8: 0x0a56, 0x0bd9: 0x0a5a, 0x0bda: 0x0a5e, 0x0bdb: 0x0a62, 0x0bdc: 0x0a66, 0x0bdd: 0x0a6c, + 0x0bde: 0x0a72, 0x0bdf: 0x0a76, 0x0be0: 0x0a7a, 0x0be1: 0x0a7e, 0x0be2: 0x0a82, 0x0be3: 0x0a86, + 0x0be4: 0x0a8a, 0x0be5: 0x0a8e, 0x0be6: 0x0a92, 0x0be7: 0x0a96, 0x0be8: 0x0a9a, 0x0be9: 0x0a9e, + 0x0bea: 0x0aa2, 0x0beb: 0x0aa6, 0x0bec: 0x0aaa, 0x0bed: 0x0aae, 0x0bee: 0x0ab2, 0x0bef: 0x0ab8, + 0x0bf0: 0x0abe, 0x0bf1: 0x0ac2, 0x0bf2: 0x0ac6, 0x0bf3: 0x0aca, 0x0bf4: 0x0ace, 0x0bf5: 0x0ad2, + 0x0bf6: 0x0ad6, 0x0bf7: 0x0ada, 0x0bf8: 0x0ade, 0x0bf9: 0x0ae4, 0x0bfa: 0x0aea, 0x0bfb: 0x0aee, + 0x0bfc: 0x0af2, 0x0bfd: 0x0af6, 0x0bfe: 0x0afa, 0x0bff: 0x0afe, + // Block 0x30, offset 0xc00 + 0x0c00: 0x0b02, 0x0c01: 0x0b06, 0x0c02: 0x0b0a, 0x0c03: 0x0b0e, 0x0c04: 0x0b12, 0x0c05: 0x0b16, + 0x0c06: 0x0b1a, 0x0c07: 0x0b1e, 0x0c08: 0x0b22, 0x0c09: 0x0b26, 0x0c0a: 0x0b2a, 0x0c0b: 0x0b2e, + 0x0c0c: 0x0b32, 0x0c0d: 0x0b38, 0x0c0e: 0x0b3e, 0x0c0f: 0x0b44, 0x0c10: 0x0b4a, 0x0c11: 0x0b50, + 0x0c12: 0x0b56, 0x0c13: 0x0b5c, 0x0c14: 0x0b62, 0x0c15: 0x0b66, 0x0c16: 0x0b6a, 0x0c17: 0x0b6e, + 0x0c18: 0x0b72, 0x0c19: 0x0b76, 0x0c1a: 0x0b7a, 0x0c1b: 0x0b7e, 0x0c1c: 0x0b82, 0x0c1d: 0x0b88, + 0x0c1e: 0x0b8e, 0x0c1f: 0x0b92, 0x0c20: 0x0b96, 0x0c21: 0x0b9a, 0x0c22: 0x0b9e, 0x0c23: 0x0ba2, + 0x0c24: 0x0ba6, 0x0c25: 0x0bac, 0x0c26: 0x0bb2, 0x0c27: 0x0bb8, 0x0c28: 0x0bbe, 0x0c29: 0x0bc4, + 0x0c2a: 0x0bca, 0x0c2b: 0x0bce, 0x0c2c: 0x0bd2, 0x0c2d: 0x0bd6, 0x0c2e: 0x0bda, 0x0c2f: 0x0bde, + 0x0c30: 0x0be2, 0x0c31: 0x0be6, 0x0c32: 0x0bea, 0x0c33: 0x0bee, 0x0c34: 0x0bf2, 0x0c35: 0x0bf6, + 0x0c36: 0x0bfa, 0x0c37: 0x0bfe, 0x0c38: 0x0c02, 0x0c39: 0x0c08, 0x0c3a: 0x0c0e, 0x0c3b: 0x0c14, + 0x0c3c: 0x0c1a, 0x0c3d: 0x0c1e, 0x0c3e: 0x0c22, 0x0c3f: 0x0c26, + // Block 0x31, offset 0xc40 + 0x0c40: 0x0c2a, 0x0c41: 0x0c2e, 0x0c42: 0x0c32, 0x0c43: 0x0c36, 0x0c44: 0x0c3a, 0x0c45: 0x0c3e, + 0x0c46: 0x0c42, 0x0c47: 0x0c46, 0x0c48: 0x0c4a, 0x0c49: 0x0c4e, 0x0c4a: 0x0c52, 0x0c4b: 0x0c56, + 0x0c4c: 0x0c5a, 0x0c4d: 0x0c5e, 0x0c4e: 0x0c62, 0x0c4f: 0x0c66, 0x0c50: 0x0c6a, 0x0c51: 0x0c6e, + 0x0c52: 0x0c72, 0x0c53: 0x0c76, 0x0c54: 0x0c7a, 0x0c55: 0x0c7e, 0x0c56: 0x0c82, 0x0c57: 0x0c86, + 0x0c58: 0x0c8a, 0x0c59: 0x0c8e, 0x0c5a: 0x0c92, 0x0c5b: 0x0b9a, + 0x0c60: 0x0c9b, 0x0c61: 0x0c9f, 0x0c62: 0x0ca3, 0x0c63: 0x0ca7, + 0x0c64: 0x0cab, 0x0c65: 0x0cb1, 0x0c66: 0x0cb7, 0x0c67: 0x0cbd, 0x0c68: 0x0cc3, 0x0c69: 0x0cc9, + 0x0c6a: 0x0ccf, 0x0c6b: 0x0cd5, 0x0c6c: 0x0cdb, 0x0c6d: 0x0ce1, 0x0c6e: 0x0ce7, 0x0c6f: 0x0ced, + 0x0c70: 0x0cf3, 0x0c71: 0x0cf9, 0x0c72: 0x0cff, 0x0c73: 0x0d05, 0x0c74: 0x0d0b, 0x0c75: 0x0d11, + 0x0c76: 0x0d17, 0x0c77: 0x0d1d, 0x0c78: 0x0d23, 0x0c79: 0x0d27, 0x0c7a: 0x0d2b, 0x0c7b: 0x0d2f, + 0x0c7c: 0x0d33, 0x0c7d: 0x0d37, 0x0c7e: 0x0d3b, 0x0c7f: 0x0d41, + // Block 0x32, offset 0xc80 + 0x0c80: 0x0d47, 0x0c81: 0x0d4d, 0x0c82: 0x0d53, 0x0c83: 0x0d59, 0x0c84: 0x0d5f, 0x0c85: 0x0d65, + 0x0c86: 0x0d6b, 0x0c87: 0x0d71, 0x0c88: 0x0d77, 0x0c89: 0x0d7b, 0x0c8a: 0x0d7f, 0x0c8b: 0x0d83, + 0x0c8c: 0x0d87, 0x0c8d: 0x0d8b, 0x0c8e: 0x0d8f, 0x0c8f: 0x0d93, 0x0c90: 0x0d97, 0x0c91: 0x0d9d, + 0x0c92: 0x0da3, 0x0c93: 0x0da9, 0x0c94: 0x0daf, 0x0c95: 0x0db5, 0x0c96: 0x0dbb, 0x0c97: 0x0dc1, + 0x0c98: 0x0dc7, 0x0c99: 0x0dcd, 0x0c9a: 0x0dd3, 0x0c9b: 0x0dd9, 0x0c9c: 0x0ddf, 0x0c9d: 0x0de5, + 0x0c9e: 0x0deb, 0x0c9f: 0x0df1, 0x0ca0: 0x0df7, 0x0ca1: 0x0dfd, 0x0ca2: 0x0e03, 0x0ca3: 0x0e09, + 0x0ca4: 0x0e0f, 0x0ca5: 0x0e13, 0x0ca6: 0x0e17, 0x0ca7: 0x0e1b, 0x0ca8: 0x0e1f, 0x0ca9: 0x0e25, + 0x0caa: 0x0e2b, 0x0cab: 0x0e31, 0x0cac: 0x0e37, 0x0cad: 0x0e3d, 0x0cae: 0x0e43, 0x0caf: 0x0e49, + 0x0cb0: 0x0e4f, 0x0cb1: 0x0e55, 0x0cb2: 0x0e5b, 0x0cb3: 0x0e5f, 0x0cb4: 0x0e63, 0x0cb5: 0x0e67, + 0x0cb6: 0x0e6b, 0x0cb7: 0x0e6f, 0x0cb8: 0x0e73, 0x0cb9: 0x0e77, + // Block 0x33, offset 0xcc0 + 0x0cc0: 0x0e7b, 0x0cc1: 0x0e80, 0x0cc2: 0x0e85, 0x0cc3: 0x0e8c, 0x0cc4: 0x0e93, 0x0cc5: 0x0e9a, + 0x0cc6: 0x0ea1, 0x0cc7: 0x0ea8, 0x0cc8: 0x0eaf, 0x0cc9: 0x0eb4, 0x0cca: 0x0eb9, 0x0ccb: 0x0ec0, + 0x0ccc: 0x0ec7, 0x0ccd: 0x0ece, 0x0cce: 0x0ed5, 0x0ccf: 0x0edc, 0x0cd0: 0x0ee3, 0x0cd1: 0x0ee8, + 0x0cd2: 0x0eed, 0x0cd3: 0x0ef4, 0x0cd4: 0x0efb, 0x0cd5: 0x0f02, + 0x0cd8: 0x0f09, 0x0cd9: 0x0f0e, 0x0cda: 0x0f13, 0x0cdb: 0x0f1a, 0x0cdc: 0x0f21, 0x0cdd: 0x0f28, + 0x0ce0: 0x0f2f, 0x0ce1: 0x0f34, 0x0ce2: 0x0f39, 0x0ce3: 0x0f40, + 0x0ce4: 0x0f47, 0x0ce5: 0x0f4e, 0x0ce6: 0x0f55, 0x0ce7: 0x0f5c, 0x0ce8: 0x0f63, 0x0ce9: 0x0f68, + 0x0cea: 0x0f6d, 0x0ceb: 0x0f74, 0x0cec: 0x0f7b, 0x0ced: 0x0f82, 0x0cee: 0x0f89, 0x0cef: 0x0f90, + 0x0cf0: 0x0f97, 0x0cf1: 0x0f9c, 0x0cf2: 0x0fa1, 0x0cf3: 0x0fa8, 0x0cf4: 0x0faf, 0x0cf5: 0x0fb6, + 0x0cf6: 0x0fbd, 0x0cf7: 0x0fc4, 0x0cf8: 0x0fcb, 0x0cf9: 0x0fd0, 0x0cfa: 0x0fd5, 0x0cfb: 0x0fdc, + 0x0cfc: 0x0fe3, 0x0cfd: 0x0fea, 0x0cfe: 0x0ff1, 0x0cff: 0x0ff8, + // Block 0x34, offset 0xd00 + 0x0d00: 0x0fff, 0x0d01: 0x1004, 0x0d02: 0x1009, 0x0d03: 0x1010, 0x0d04: 0x1017, 0x0d05: 0x101e, + 0x0d08: 0x1025, 0x0d09: 0x102a, 0x0d0a: 0x102f, 0x0d0b: 0x1036, + 0x0d0c: 0x103d, 0x0d0d: 0x1044, 0x0d10: 0x104b, 0x0d11: 0x1050, + 0x0d12: 0x1055, 0x0d13: 0x105c, 0x0d14: 0x1063, 0x0d15: 0x106a, 0x0d16: 0x1071, 0x0d17: 0x1078, + 0x0d19: 0x107f, 0x0d1b: 0x1084, 0x0d1d: 0x108b, + 0x0d1f: 0x1092, 0x0d20: 0x1099, 0x0d21: 0x109e, 0x0d22: 0x10a3, 0x0d23: 0x10aa, + 0x0d24: 0x10b1, 0x0d25: 0x10b8, 0x0d26: 0x10bf, 0x0d27: 0x10c6, 0x0d28: 0x10cd, 0x0d29: 0x10d2, + 0x0d2a: 0x10d7, 0x0d2b: 0x10de, 0x0d2c: 0x10e5, 0x0d2d: 0x10ec, 0x0d2e: 0x10f3, 0x0d2f: 0x10fa, + 0x0d30: 0x1101, 0x0d31: 0x0525, 0x0d32: 0x1106, 0x0d33: 0x052a, 0x0d34: 0x110b, 0x0d35: 0x052f, + 0x0d36: 0x1110, 0x0d37: 0x0534, 0x0d38: 0x1115, 0x0d39: 0x054a, 0x0d3a: 0x111a, 0x0d3b: 0x054f, + 0x0d3c: 0x111f, 0x0d3d: 0x0554, + // Block 0x35, offset 0xd40 + 0x0d40: 0x1124, 0x0d41: 0x112b, 0x0d42: 0x1132, 0x0d43: 0x113b, 0x0d44: 0x1144, 0x0d45: 0x114d, + 0x0d46: 0x1156, 0x0d47: 0x115f, 0x0d48: 0x1168, 0x0d49: 0x116f, 0x0d4a: 0x1176, 0x0d4b: 0x117f, + 0x0d4c: 0x1188, 0x0d4d: 0x1191, 0x0d4e: 0x119a, 0x0d4f: 0x11a3, 0x0d50: 0x11ac, 0x0d51: 0x11b3, + 0x0d52: 0x11ba, 0x0d53: 0x11c3, 0x0d54: 0x11cc, 0x0d55: 0x11d5, 0x0d56: 0x11de, 0x0d57: 0x11e7, + 0x0d58: 0x11f0, 0x0d59: 0x11f7, 0x0d5a: 0x11fe, 0x0d5b: 0x1207, 0x0d5c: 0x1210, 0x0d5d: 0x1219, + 0x0d5e: 0x1222, 0x0d5f: 0x122b, 0x0d60: 0x1234, 0x0d61: 0x123b, 0x0d62: 0x1242, 0x0d63: 0x124b, + 0x0d64: 0x1254, 0x0d65: 0x125d, 0x0d66: 0x1266, 0x0d67: 0x126f, 0x0d68: 0x1278, 0x0d69: 0x127f, + 0x0d6a: 0x1286, 0x0d6b: 0x128f, 0x0d6c: 0x1298, 0x0d6d: 0x12a1, 0x0d6e: 0x12aa, 0x0d6f: 0x12b3, + 0x0d70: 0x12bc, 0x0d71: 0x12c1, 0x0d72: 0x12c6, 0x0d73: 0x12cd, 0x0d74: 0x12d2, + 0x0d76: 0x12d9, 0x0d77: 0x12de, 0x0d78: 0x12e5, 0x0d79: 0x12ea, 0x0d7a: 0x12ef, 0x0d7b: 0x04ee, + 0x0d7c: 0x12f4, 0x0d7d: 0x12f9, 0x0d7e: 0x12fd, 0x0d7f: 0x12f9, + // Block 0x36, offset 0xd80 + 0x0d80: 0x1300, 0x0d81: 0x1309, 0x0d82: 0x130f, 0x0d83: 0x1316, 0x0d84: 0x131b, + 0x0d86: 0x1322, 0x0d87: 0x1327, 0x0d88: 0x132e, 0x0d89: 0x04f6, 0x0d8a: 0x1333, 0x0d8b: 0x04fb, + 0x0d8c: 0x1338, 0x0d8d: 0x1343, 0x0d8e: 0x134f, 0x0d8f: 0x135b, 0x0d90: 0x1361, 0x0d91: 0x1366, + 0x0d92: 0x136b, 0x0d93: 0x0514, 0x0d96: 0x1372, 0x0d97: 0x1377, + 0x0d98: 0x137e, 0x0d99: 0x1383, 0x0d9a: 0x1388, 0x0d9b: 0x0500, 0x0d9d: 0x1393, + 0x0d9e: 0x139f, 0x0d9f: 0x13ab, 0x0da0: 0x13b1, 0x0da1: 0x13b6, 0x0da2: 0x13bb, 0x0da3: 0x0539, + 0x0da4: 0x13c2, 0x0da5: 0x13c7, 0x0da6: 0x13cc, 0x0da7: 0x13d1, 0x0da8: 0x13d8, 0x0da9: 0x13dd, + 0x0daa: 0x13e2, 0x0dab: 0x050a, 0x0dac: 0x13e7, 0x0dad: 0x13f1, 0x0dae: 0x04e8, 0x0daf: 0x13f7, + 0x0db2: 0x13f9, 0x0db3: 0x1400, 0x0db4: 0x1405, + 0x0db6: 0x140c, 0x0db7: 0x1411, 0x0db8: 0x1418, 0x0db9: 0x0505, 0x0dba: 0x141d, 0x0dbb: 0x050f, + 0x0dbc: 0x1422, 0x0dbd: 0x0011, 0x0dbe: 0x142a, + // Block 0x37, offset 0xdc0 + 0x0dc0: 0x0001, 0x0dc1: 0x0001, 0x0dc2: 0x0001, 0x0dc3: 0x0001, 0x0dc4: 0x0001, 0x0dc5: 0x0001, + 0x0dc6: 0x0001, 0x0dc7: 0x0001, 0x0dc8: 0x0001, 0x0dc9: 0x0001, 0x0dca: 0x0001, + 0x0dd1: 0x1436, + 0x0dd7: 0x143a, + 0x0de4: 0x143e, 0x0de5: 0x1440, 0x0de6: 0x1443, + 0x0def: 0x0001, + 0x0df3: 0x1447, 0x0df4: 0x144e, + 0x0df6: 0x1458, 0x0df7: 0x145f, + 0x0dfc: 0x1469, 0x0dfe: 0x146c, + // Block 0x38, offset 0xe00 + 0x0e07: 0x1470, 0x0e08: 0x1473, 0x0e09: 0x1476, + 0x0e17: 0x1479, + 0x0e1f: 0x0001, + 0x0e30: 0x1486, 0x0e31: 0x097c, 0x0e34: 0x1488, 0x0e35: 0x148a, + 0x0e36: 0x148c, 0x0e37: 0x148e, 0x0e38: 0x1490, 0x0e39: 0x1492, 0x0e3a: 0x1494, 0x0e3b: 0x1496, + 0x0e3c: 0x149a, 0x0e3d: 0x149c, 0x0e3e: 0x149e, 0x0e3f: 0x14a0, + // Block 0x39, offset 0xe40 + 0x0e40: 0x1486, 0x0e41: 0x001c, 0x0e42: 0x000d, 0x0e43: 0x000f, 0x0e44: 0x1488, 0x0e45: 0x148a, + 0x0e46: 0x148c, 0x0e47: 0x148e, 0x0e48: 0x1490, 0x0e49: 0x1492, 0x0e4a: 0x1494, 0x0e4b: 0x1496, + 0x0e4c: 0x149a, 0x0e4d: 0x149c, 0x0e4e: 0x149e, 0x0e50: 0x0007, 0x0e51: 0x0941, + 0x0e52: 0x001e, 0x0e53: 0x04c7, 0x0e54: 0x0943, 0x0e55: 0x0494, 0x0e56: 0x094e, 0x0e57: 0x04c5, + 0x0e58: 0x0950, 0x0e59: 0x14a0, 0x0e5a: 0x0960, 0x0e5b: 0x02c8, 0x0e5c: 0x0962, + 0x0e68: 0x14a2, + // Block 0x3a, offset 0xe80 + 0x0e80: 0x14a5, 0x0e81: 0x14a9, 0x0e82: 0x14ad, 0x0e83: 0x14af, 0x0e85: 0x14b3, + 0x0e86: 0x14b7, 0x0e87: 0x14bb, 0x0e89: 0x14be, 0x0e8a: 0x094c, 0x0e8b: 0x0916, + 0x0e8c: 0x0916, 0x0e8d: 0x0916, 0x0e8e: 0x0494, 0x0e8f: 0x14c2, 0x0e90: 0x0918, 0x0e91: 0x0918, + 0x0e92: 0x091e, 0x0e93: 0x04c5, 0x0e95: 0x0922, 0x0e96: 0x14c5, + 0x0e99: 0x0929, 0x0e9a: 0x14c8, 0x0e9b: 0x092b, 0x0e9c: 0x092b, 0x0e9d: 0x092b, + 0x0ea0: 0x14ca, 0x0ea1: 0x14cd, 0x0ea2: 0x14d1, + 0x0ea4: 0x14d4, 0x0ea6: 0x14d6, 0x0ea8: 0x14d4, + 0x0eaa: 0x091c, 0x0eab: 0x0046, 0x0eac: 0x090b, 0x0ead: 0x14ad, 0x0eaf: 0x0941, + 0x0eb0: 0x090f, 0x0eb1: 0x14d9, 0x0eb3: 0x0920, 0x0eb4: 0x001e, 0x0eb5: 0x14db, + 0x0eb6: 0x14de, 0x0eb7: 0x14e1, 0x0eb8: 0x14e4, 0x0eb9: 0x097c, 0x0ebb: 0x14e7, + 0x0ebc: 0x056f, 0x0ebd: 0x0973, 0x0ebe: 0x14eb, 0x0ebf: 0x14ee, + // Block 0x3b, offset 0xec0 + 0x0ec0: 0x14f1, 0x0ec5: 0x090d, + 0x0ec6: 0x093f, 0x0ec7: 0x0941, 0x0ec8: 0x097c, 0x0ec9: 0x0499, + 0x0ed0: 0x14f5, 0x0ed1: 0x14fb, + 0x0ed2: 0x1501, 0x0ed3: 0x1508, 0x0ed4: 0x150e, 0x0ed5: 0x1514, 0x0ed6: 0x151a, 0x0ed7: 0x1520, + 0x0ed8: 0x1526, 0x0ed9: 0x152c, 0x0eda: 0x1532, 0x0edb: 0x1538, 0x0edc: 0x153e, 0x0edd: 0x1544, + 0x0ede: 0x154a, 0x0edf: 0x1550, 0x0ee0: 0x0918, 0x0ee1: 0x1555, 0x0ee2: 0x1558, 0x0ee3: 0x155c, + 0x0ee4: 0x155f, 0x0ee5: 0x1561, 0x0ee6: 0x1564, 0x0ee7: 0x1568, 0x0ee8: 0x156d, 0x0ee9: 0x1570, + 0x0eea: 0x1572, 0x0eeb: 0x1575, 0x0eec: 0x091e, 0x0eed: 0x14ad, 0x0eee: 0x090d, 0x0eef: 0x0920, + 0x0ef0: 0x097c, 0x0ef1: 0x1579, 0x0ef2: 0x157c, 0x0ef3: 0x1580, 0x0ef4: 0x096d, 0x0ef5: 0x1583, + 0x0ef6: 0x1586, 0x0ef7: 0x158a, 0x0ef8: 0x158f, 0x0ef9: 0x04c7, 0x0efa: 0x1592, 0x0efb: 0x1595, + 0x0efc: 0x04c5, 0x0efd: 0x0984, 0x0efe: 0x093f, 0x0eff: 0x0950, + // Block 0x3c, offset 0xf00 + 0x0f09: 0x1599, + 0x0f1a: 0x159f, 0x0f1b: 0x15a5, + 0x0f2e: 0x15ab, + // Block 0x3d, offset 0xf40 + 0x0f4d: 0x15b1, 0x0f4e: 0x15b7, 0x0f4f: 0x15bd, + // Block 0x3e, offset 0xf80 + 0x0f84: 0x15c3, + 0x0f89: 0x15c9, + 0x0f8c: 0x15cf, + 0x0fa4: 0x15d5, 0x0fa6: 0x15db, + 0x0fac: 0x15e1, 0x0fad: 0x15e8, 0x0faf: 0x15f2, + 0x0fb0: 0x15f9, + // Block 0x3f, offset 0xfc0 + 0x0fc1: 0x1603, 0x0fc4: 0x1609, + 0x0fc7: 0x160f, 0x0fc9: 0x1615, + 0x0fe0: 0x161b, 0x0fe2: 0x161f, + 0x0fed: 0x1625, 0x0fee: 0x162b, 0x0fef: 0x162f, + 0x0ff0: 0x1633, 0x0ff1: 0x1639, 0x0ff4: 0x163f, 0x0ff5: 0x1645, + 0x0ff8: 0x164b, 0x0ff9: 0x1651, + // Block 0x40, offset 0x1000 + 0x1000: 0x1657, 0x1001: 0x165d, 0x1004: 0x1663, 0x1005: 0x1669, + 0x1008: 0x166f, 0x1009: 0x1675, + 0x102c: 0x167b, 0x102d: 0x1681, 0x102e: 0x1687, 0x102f: 0x168d, + // Block 0x41, offset 0x1040 + 0x1060: 0x1693, 0x1061: 0x1699, 0x1062: 0x169f, 0x1063: 0x16a5, + 0x106a: 0x16ab, 0x106b: 0x16b1, 0x106c: 0x16b7, 0x106d: 0x16bd, + // Block 0x42, offset 0x1080 + 0x10a9: 0x16c3, + 0x10aa: 0x16c7, + // Block 0x43, offset 0x10c0 + 0x10e0: 0x001c, 0x10e1: 0x000d, 0x10e2: 0x000f, 0x10e3: 0x1488, + 0x10e4: 0x148a, 0x10e5: 0x148c, 0x10e6: 0x148e, 0x10e7: 0x1490, 0x10e8: 0x1492, 0x10e9: 0x16cb, + 0x10ea: 0x16ce, 0x10eb: 0x16d1, 0x10ec: 0x16d4, 0x10ed: 0x16d7, 0x10ee: 0x16da, 0x10ef: 0x16dd, + 0x10f0: 0x16e0, 0x10f1: 0x16e3, 0x10f2: 0x16e6, 0x10f3: 0x16e9, 0x10f4: 0x16ec, 0x10f5: 0x16f0, + 0x10f6: 0x16f4, 0x10f7: 0x16f8, 0x10f8: 0x16fc, 0x10f9: 0x1700, 0x10fa: 0x1704, 0x10fb: 0x1708, + 0x10fc: 0x170c, 0x10fd: 0x1710, 0x10fe: 0x1715, 0x10ff: 0x171a, + // Block 0x44, offset 0x1100 + 0x1100: 0x171f, 0x1101: 0x1724, 0x1102: 0x1729, 0x1103: 0x172e, 0x1104: 0x1733, 0x1105: 0x1738, + 0x1106: 0x173d, 0x1107: 0x1742, 0x1108: 0x1747, 0x1109: 0x174a, 0x110a: 0x174d, 0x110b: 0x1750, + 0x110c: 0x1753, 0x110d: 0x1756, 0x110e: 0x1759, 0x110f: 0x175c, 0x1110: 0x175f, 0x1111: 0x1762, + 0x1112: 0x1766, 0x1113: 0x176a, 0x1114: 0x176e, 0x1115: 0x1772, 0x1116: 0x1776, 0x1117: 0x177a, + 0x1118: 0x177e, 0x1119: 0x1782, 0x111a: 0x1786, 0x111b: 0x178a, 0x111c: 0x178e, 0x111d: 0x1792, + 0x111e: 0x1796, 0x111f: 0x179a, 0x1120: 0x179e, 0x1121: 0x17a2, 0x1122: 0x17a6, 0x1123: 0x17aa, + 0x1124: 0x17ae, 0x1125: 0x17b2, 0x1126: 0x17b6, 0x1127: 0x17ba, 0x1128: 0x17be, 0x1129: 0x17c2, + 0x112a: 0x17c6, 0x112b: 0x17ca, 0x112c: 0x17ce, 0x112d: 0x17d2, 0x112e: 0x17d6, 0x112f: 0x17da, + 0x1130: 0x17de, 0x1131: 0x17e2, 0x1132: 0x17e6, 0x1133: 0x17ea, 0x1134: 0x17ee, 0x1135: 0x17f2, + 0x1136: 0x0906, 0x1137: 0x090b, 0x1138: 0x14ad, 0x1139: 0x090d, 0x113a: 0x090f, 0x113b: 0x14d9, + 0x113c: 0x0914, 0x113d: 0x0916, 0x113e: 0x0918, 0x113f: 0x091a, + // Block 0x45, offset 0x1140 + 0x1140: 0x091c, 0x1141: 0x091e, 0x1142: 0x0920, 0x1143: 0x0922, 0x1144: 0x0924, 0x1145: 0x0929, + 0x1146: 0x14c8, 0x1147: 0x092b, 0x1148: 0x17f6, 0x1149: 0x092d, 0x114a: 0x092f, 0x114b: 0x155f, + 0x114c: 0x0931, 0x114d: 0x1570, 0x114e: 0x17f8, 0x114f: 0x14d4, 0x1150: 0x0007, 0x1151: 0x093d, + 0x1152: 0x0984, 0x1153: 0x093f, 0x1154: 0x0941, 0x1155: 0x098c, 0x1156: 0x094c, 0x1157: 0x0494, + 0x1158: 0x097c, 0x1159: 0x0499, 0x115a: 0x094e, 0x115b: 0x04c5, 0x115c: 0x0950, 0x115d: 0x14a0, + 0x115e: 0x001e, 0x115f: 0x0960, 0x1160: 0x17fa, 0x1161: 0x049b, 0x1162: 0x02c8, 0x1163: 0x0962, + 0x1164: 0x0964, 0x1165: 0x096d, 0x1166: 0x04a6, 0x1167: 0x04c7, 0x1168: 0x04a8, 0x1169: 0x09df, + 0x116a: 0x1486, + // Block 0x46, offset 0x1180 + 0x118c: 0x17fc, + // Block 0x47, offset 0x11c0 + 0x11f4: 0x1809, 0x11f5: 0x180d, + 0x11f6: 0x1810, + // Block 0x48, offset 0x1200 + 0x121c: 0x1814, + // Block 0x49, offset 0x1240 + 0x127c: 0x0499, 0x127d: 0x155f, + // Block 0x4a, offset 0x1280 + 0x12af: 0x181a, + // Block 0x4b, offset 0x12c0 + 0x12df: 0x181e, + // Block 0x4c, offset 0x1300 + 0x1333: 0x1822, + // Block 0x4d, offset 0x1340 + 0x1340: 0x1826, 0x1341: 0x182a, 0x1342: 0x182e, 0x1343: 0x1832, 0x1344: 0x1836, 0x1345: 0x183a, + 0x1346: 0x183e, 0x1347: 0x1842, 0x1348: 0x1846, 0x1349: 0x184a, 0x134a: 0x184e, 0x134b: 0x1852, + 0x134c: 0x1856, 0x134d: 0x185a, 0x134e: 0x185e, 0x134f: 0x1862, 0x1350: 0x1866, 0x1351: 0x186a, + 0x1352: 0x186e, 0x1353: 0x1872, 0x1354: 0x1876, 0x1355: 0x187a, 0x1356: 0x187e, 0x1357: 0x1882, + 0x1358: 0x1886, 0x1359: 0x188a, 0x135a: 0x188e, 0x135b: 0x1892, 0x135c: 0x1896, 0x135d: 0x189a, + 0x135e: 0x189e, 0x135f: 0x18a2, 0x1360: 0x18a6, 0x1361: 0x18aa, 0x1362: 0x18ae, 0x1363: 0x18b2, + 0x1364: 0x18b6, 0x1365: 0x18ba, 0x1366: 0x18be, 0x1367: 0x18c2, 0x1368: 0x18c6, 0x1369: 0x18ca, + 0x136a: 0x18ce, 0x136b: 0x18d2, 0x136c: 0x18d6, 0x136d: 0x18da, 0x136e: 0x18de, 0x136f: 0x18e2, + 0x1370: 0x18e6, 0x1371: 0x18ea, 0x1372: 0x18ee, 0x1373: 0x18f2, 0x1374: 0x18f6, 0x1375: 0x18fa, + 0x1376: 0x18fe, 0x1377: 0x1902, 0x1378: 0x1906, 0x1379: 0x190a, 0x137a: 0x190e, 0x137b: 0x1912, + 0x137c: 0x1916, 0x137d: 0x191a, 0x137e: 0x191e, 0x137f: 0x1922, + // Block 0x4e, offset 0x1380 + 0x1380: 0x1926, 0x1381: 0x192a, 0x1382: 0x192e, 0x1383: 0x1932, 0x1384: 0x1936, 0x1385: 0x193a, + 0x1386: 0x193e, 0x1387: 0x1942, 0x1388: 0x1946, 0x1389: 0x194a, 0x138a: 0x194e, 0x138b: 0x1952, + 0x138c: 0x1956, 0x138d: 0x195a, 0x138e: 0x195e, 0x138f: 0x1962, 0x1390: 0x1966, 0x1391: 0x196a, + 0x1392: 0x196e, 0x1393: 0x1972, 0x1394: 0x1976, 0x1395: 0x197a, 0x1396: 0x197e, 0x1397: 0x1982, + 0x1398: 0x1986, 0x1399: 0x198a, 0x139a: 0x198e, 0x139b: 0x1992, 0x139c: 0x1996, 0x139d: 0x199a, + 0x139e: 0x199e, 0x139f: 0x19a2, 0x13a0: 0x19a6, 0x13a1: 0x19aa, 0x13a2: 0x19ae, 0x13a3: 0x19b2, + 0x13a4: 0x19b6, 0x13a5: 0x19ba, 0x13a6: 0x19be, 0x13a7: 0x19c2, 0x13a8: 0x19c6, 0x13a9: 0x19ca, + 0x13aa: 0x19ce, 0x13ab: 0x19d2, 0x13ac: 0x19d6, 0x13ad: 0x19da, 0x13ae: 0x19de, 0x13af: 0x19e2, + 0x13b0: 0x19e6, 0x13b1: 0x19ea, 0x13b2: 0x19ee, 0x13b3: 0x19f2, 0x13b4: 0x19f6, 0x13b5: 0x19fa, + 0x13b6: 0x19fe, 0x13b7: 0x1a02, 0x13b8: 0x1a06, 0x13b9: 0x1a0a, 0x13ba: 0x1a0e, 0x13bb: 0x1a12, + 0x13bc: 0x1a16, 0x13bd: 0x1a1a, 0x13be: 0x1a1e, 0x13bf: 0x1a22, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x1a26, 0x13c1: 0x1a2a, 0x13c2: 0x1a2e, 0x13c3: 0x1a32, 0x13c4: 0x1a36, 0x13c5: 0x1a3a, + 0x13c6: 0x1a3e, 0x13c7: 0x1a42, 0x13c8: 0x1a46, 0x13c9: 0x1a4a, 0x13ca: 0x1a4e, 0x13cb: 0x1a52, + 0x13cc: 0x1a56, 0x13cd: 0x1a5a, 0x13ce: 0x1a5e, 0x13cf: 0x1a62, 0x13d0: 0x1a66, 0x13d1: 0x1a6a, + 0x13d2: 0x1a6e, 0x13d3: 0x1a72, 0x13d4: 0x1a76, 0x13d5: 0x1a7a, 0x13d6: 0x1a7e, 0x13d7: 0x1a82, + 0x13d8: 0x1a86, 0x13d9: 0x1a8a, 0x13da: 0x1a8e, 0x13db: 0x1a92, 0x13dc: 0x1a96, 0x13dd: 0x1a9a, + 0x13de: 0x1a9e, 0x13df: 0x1aa2, 0x13e0: 0x1aa6, 0x13e1: 0x1aaa, 0x13e2: 0x1aae, 0x13e3: 0x1ab2, + 0x13e4: 0x1ab6, 0x13e5: 0x1aba, 0x13e6: 0x1abe, 0x13e7: 0x1ac2, 0x13e8: 0x1ac6, 0x13e9: 0x1aca, + 0x13ea: 0x1ace, 0x13eb: 0x1ad2, 0x13ec: 0x1ad6, 0x13ed: 0x1ada, 0x13ee: 0x1ade, 0x13ef: 0x1ae2, + 0x13f0: 0x1ae6, 0x13f1: 0x1aea, 0x13f2: 0x1aee, 0x13f3: 0x1af2, 0x13f4: 0x1af6, 0x13f5: 0x1afa, + 0x13f6: 0x1afe, 0x13f7: 0x1b02, 0x13f8: 0x1b06, 0x13f9: 0x1b0a, 0x13fa: 0x1b0e, 0x13fb: 0x1b12, + 0x13fc: 0x1b16, 0x13fd: 0x1b1a, 0x13fe: 0x1b1e, 0x13ff: 0x1b22, + // Block 0x50, offset 0x1400 + 0x1400: 0x1b26, 0x1401: 0x1b2a, 0x1402: 0x1b2e, 0x1403: 0x1b32, 0x1404: 0x1b36, 0x1405: 0x1b3a, + 0x1406: 0x1b3e, 0x1407: 0x1b42, 0x1408: 0x1b46, 0x1409: 0x1b4a, 0x140a: 0x1b4e, 0x140b: 0x1b52, + 0x140c: 0x1b56, 0x140d: 0x1b5a, 0x140e: 0x1b5e, 0x140f: 0x1b62, 0x1410: 0x1b66, 0x1411: 0x1b6a, + 0x1412: 0x1b6e, 0x1413: 0x1b72, 0x1414: 0x1b76, 0x1415: 0x1b7a, + // Block 0x51, offset 0x1440 + 0x1440: 0x0001, + 0x1476: 0x1b7e, 0x1478: 0x1882, 0x1479: 0x1b82, 0x147a: 0x1b86, + // Block 0x52, offset 0x1480 + 0x148c: 0x1b8a, 0x148e: 0x1b91, 0x1490: 0x1b98, + 0x1492: 0x1b9f, 0x1494: 0x1ba6, 0x1496: 0x1bad, + 0x1498: 0x1bb4, 0x149a: 0x1bbb, 0x149c: 0x1bc2, + 0x149e: 0x1bc9, 0x14a0: 0x1bd0, 0x14a2: 0x1bd7, + 0x14a5: 0x1bde, 0x14a7: 0x1be5, 0x14a9: 0x1bec, + 0x14b0: 0x1bf3, 0x14b1: 0x1bfa, 0x14b3: 0x1c01, 0x14b4: 0x1c08, + 0x14b6: 0x1c0f, 0x14b7: 0x1c16, 0x14b9: 0x1c1d, 0x14ba: 0x1c24, + 0x14bc: 0x1c2b, 0x14bd: 0x1c32, + // Block 0x53, offset 0x14c0 + 0x14d4: 0x1c39, + 0x14db: 0x1c40, 0x14dc: 0x1c45, + 0x14de: 0x1c4a, 0x14df: 0x1c51, + 0x14ec: 0x1c58, 0x14ee: 0x1c5f, + 0x14f0: 0x1c66, 0x14f2: 0x1c6d, 0x14f4: 0x1c74, + 0x14f6: 0x1c7b, 0x14f8: 0x1c82, 0x14fa: 0x1c89, + 0x14fc: 0x1c90, 0x14fe: 0x1c97, + // Block 0x54, offset 0x1500 + 0x1500: 0x1c9e, 0x1502: 0x1ca5, 0x1505: 0x1cac, + 0x1507: 0x1cb3, 0x1509: 0x1cba, + 0x1510: 0x1cc1, 0x1511: 0x1cc8, + 0x1513: 0x1ccf, 0x1514: 0x1cd6, 0x1516: 0x1cdd, 0x1517: 0x1ce4, + 0x1519: 0x1ceb, 0x151a: 0x1cf2, 0x151c: 0x1cf9, 0x151d: 0x1d00, + 0x1534: 0x1d07, + 0x1537: 0x1d0e, 0x1538: 0x1d15, 0x1539: 0x1d1c, 0x153a: 0x1d23, + 0x153e: 0x1d2a, 0x153f: 0x1d31, + // Block 0x55, offset 0x1540 + 0x1571: 0x1d38, 0x1572: 0x1d3c, 0x1573: 0x1d40, 0x1574: 0x1d44, 0x1575: 0x1d48, + 0x1576: 0x1d4c, 0x1577: 0x1d50, 0x1578: 0x1d54, 0x1579: 0x1d58, 0x157a: 0x1d5c, 0x157b: 0x1d60, + 0x157c: 0x1d64, 0x157d: 0x1d68, 0x157e: 0x1d6c, 0x157f: 0x1d70, + // Block 0x56, offset 0x1580 + 0x1580: 0x1d74, 0x1581: 0x1d78, 0x1582: 0x1d7c, 0x1583: 0x1d80, 0x1584: 0x1d84, 0x1585: 0x1d88, + 0x1586: 0x1d8c, 0x1587: 0x1d90, 0x1588: 0x1d94, 0x1589: 0x1d98, 0x158a: 0x1d9c, 0x158b: 0x1da0, + 0x158c: 0x1da4, 0x158d: 0x1da8, 0x158e: 0x1dac, 0x158f: 0x1db0, 0x1590: 0x1db4, 0x1591: 0x1db8, + 0x1592: 0x1dbc, 0x1593: 0x1dc0, 0x1594: 0x1dc4, 0x1595: 0x1dc8, 0x1596: 0x1dcc, 0x1597: 0x1dd0, + 0x1598: 0x1dd4, 0x1599: 0x1dd8, 0x159a: 0x1ddc, 0x159b: 0x1de0, 0x159c: 0x1de4, 0x159d: 0x1de8, + 0x159e: 0x1dec, 0x159f: 0x1df0, 0x15a0: 0x1df4, 0x15a1: 0x1df8, 0x15a2: 0x1dfc, 0x15a3: 0x1e00, + 0x15a4: 0x1e04, 0x15a5: 0x1e08, 0x15a6: 0x1e0c, 0x15a7: 0x1e10, 0x15a8: 0x1e14, 0x15a9: 0x1e18, + 0x15aa: 0x1e1c, 0x15ab: 0x1e20, 0x15ac: 0x1e24, 0x15ad: 0x1e28, 0x15ae: 0x1e2c, 0x15af: 0x1e30, + 0x15b0: 0x1e34, 0x15b1: 0x1e38, 0x15b2: 0x1e3c, 0x15b3: 0x1e40, 0x15b4: 0x1e44, 0x15b5: 0x1e48, + 0x15b6: 0x1e4c, 0x15b7: 0x1e50, 0x15b8: 0x1e54, 0x15b9: 0x1e58, 0x15ba: 0x1e5c, 0x15bb: 0x1e60, + 0x15bc: 0x1e64, 0x15bd: 0x1e68, 0x15be: 0x1e6c, 0x15bf: 0x1e70, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x1e74, 0x15c1: 0x1e78, 0x15c2: 0x1e7c, 0x15c3: 0x1e80, 0x15c4: 0x1e84, 0x15c5: 0x1e88, + 0x15c6: 0x1e8c, 0x15c7: 0x1e90, 0x15c8: 0x1e94, 0x15c9: 0x1e98, 0x15ca: 0x1e9c, 0x15cb: 0x1ea0, + 0x15cc: 0x1ea4, 0x15cd: 0x1ea8, 0x15ce: 0x1eac, + 0x15d2: 0x1826, 0x15d3: 0x183e, 0x15d4: 0x1eb0, 0x15d5: 0x1eb4, 0x15d6: 0x1eb8, 0x15d7: 0x1ebc, + 0x15d8: 0x1ec0, 0x15d9: 0x1ec4, 0x15da: 0x1836, 0x15db: 0x1ec8, 0x15dc: 0x1ecc, 0x15dd: 0x1ed0, + 0x15de: 0x1ed4, 0x15df: 0x1846, + // Block 0x58, offset 0x1600 + 0x1600: 0x1ed8, 0x1601: 0x1ede, 0x1602: 0x1ee4, 0x1603: 0x1eea, 0x1604: 0x1ef0, 0x1605: 0x1ef6, + 0x1606: 0x1efc, 0x1607: 0x1f02, 0x1608: 0x1f08, 0x1609: 0x1f0e, 0x160a: 0x1f14, 0x160b: 0x1f1a, + 0x160c: 0x1f20, 0x160d: 0x1f26, 0x160e: 0x1f2c, 0x160f: 0x1f35, 0x1610: 0x1f3e, 0x1611: 0x1f47, + 0x1612: 0x1f50, 0x1613: 0x1f59, 0x1614: 0x1f62, 0x1615: 0x1f6b, 0x1616: 0x1f74, 0x1617: 0x1f7d, + 0x1618: 0x1f86, 0x1619: 0x1f8f, 0x161a: 0x1f98, 0x161b: 0x1fa1, 0x161c: 0x1faa, 0x161d: 0x1fb3, + 0x161e: 0x1fc5, 0x1620: 0x1fd4, 0x1621: 0x1fda, 0x1622: 0x1fe0, 0x1623: 0x1fe6, + 0x1624: 0x1fec, 0x1625: 0x1ff2, 0x1626: 0x1ff8, 0x1627: 0x1ffe, 0x1628: 0x2004, 0x1629: 0x200a, + 0x162a: 0x2010, 0x162b: 0x2016, 0x162c: 0x201c, 0x162d: 0x2022, 0x162e: 0x2028, 0x162f: 0x202e, + 0x1630: 0x2034, 0x1631: 0x203a, 0x1632: 0x2040, 0x1633: 0x2046, 0x1634: 0x204c, 0x1635: 0x2052, + 0x1636: 0x2058, 0x1637: 0x205e, 0x1638: 0x2064, 0x1639: 0x206a, 0x163a: 0x2070, 0x163b: 0x2076, + 0x163c: 0x207c, 0x163d: 0x2082, 0x163e: 0x2088, 0x163f: 0x208e, + // Block 0x59, offset 0x1640 + 0x1640: 0x2094, 0x1641: 0x209a, 0x1642: 0x20a0, 0x1643: 0x20a6, 0x1644: 0x20ac, 0x1645: 0x20b0, + 0x1646: 0x192e, 0x1647: 0x20b4, + 0x1650: 0x20b8, 0x1651: 0x20bc, + 0x1652: 0x20bf, 0x1653: 0x20c2, 0x1654: 0x20c5, 0x1655: 0x20c8, 0x1656: 0x20cb, 0x1657: 0x20ce, + 0x1658: 0x20d1, 0x1659: 0x20d4, 0x165a: 0x20d7, 0x165b: 0x20da, 0x165c: 0x20dd, 0x165d: 0x20e0, + 0x165e: 0x20e3, 0x165f: 0x20e6, 0x1660: 0x1d38, 0x1661: 0x1d44, 0x1662: 0x1d50, 0x1663: 0x1d58, + 0x1664: 0x1d78, 0x1665: 0x1d7c, 0x1666: 0x1d88, 0x1667: 0x1d90, 0x1668: 0x1d94, 0x1669: 0x1d9c, + 0x166a: 0x1da0, 0x166b: 0x1da4, 0x166c: 0x1da8, 0x166d: 0x1dac, 0x166e: 0x20e9, 0x166f: 0x20f0, + 0x1670: 0x20f7, 0x1671: 0x20fe, 0x1672: 0x2105, 0x1673: 0x210c, 0x1674: 0x2113, 0x1675: 0x211a, + 0x1676: 0x2121, 0x1677: 0x2128, 0x1678: 0x212f, 0x1679: 0x2136, 0x167a: 0x213d, 0x167b: 0x2144, + 0x167c: 0x214b, 0x167d: 0x215b, 0x167e: 0x2168, + // Block 0x5a, offset 0x1680 + 0x1680: 0x1826, 0x1681: 0x183e, 0x1682: 0x1eb0, 0x1683: 0x1eb4, 0x1684: 0x216f, 0x1685: 0x2173, + 0x1686: 0x2177, 0x1687: 0x1852, 0x1688: 0x217b, 0x1689: 0x1882, 0x168a: 0x194a, 0x168b: 0x197a, + 0x168c: 0x1976, 0x168d: 0x194e, 0x168e: 0x1abe, 0x168f: 0x18a2, 0x1690: 0x1942, 0x1691: 0x217f, + 0x1692: 0x2183, 0x1693: 0x2187, 0x1694: 0x218b, 0x1695: 0x218f, 0x1696: 0x2193, 0x1697: 0x2197, + 0x1698: 0x219b, 0x1699: 0x219f, 0x169a: 0x21a3, 0x169b: 0x18ba, 0x169c: 0x21a7, 0x169d: 0x21ab, + 0x169e: 0x21af, 0x169f: 0x21b3, 0x16a0: 0x21b7, 0x16a1: 0x21bb, 0x16a2: 0x21bf, 0x16a3: 0x21c3, + 0x16a4: 0x1eb8, 0x16a5: 0x1ebc, 0x16a6: 0x1ec0, 0x16a7: 0x21c7, 0x16a8: 0x21cb, 0x16a9: 0x21cf, + 0x16aa: 0x21d3, 0x16ab: 0x21d7, 0x16ac: 0x21db, 0x16ad: 0x21df, 0x16ae: 0x21e3, 0x16af: 0x21e7, + 0x16b0: 0x21eb, 0x16b1: 0x21ef, 0x16b2: 0x21f2, 0x16b3: 0x21f5, 0x16b4: 0x21f8, 0x16b5: 0x21fb, + 0x16b6: 0x21fe, 0x16b7: 0x2201, 0x16b8: 0x2204, 0x16b9: 0x2207, 0x16ba: 0x220a, 0x16bb: 0x220d, + 0x16bc: 0x2210, 0x16bd: 0x2213, 0x16be: 0x2216, 0x16bf: 0x2219, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x221c, 0x16c1: 0x2221, 0x16c2: 0x2226, 0x16c3: 0x222b, 0x16c4: 0x2230, 0x16c5: 0x2235, + 0x16c6: 0x223a, 0x16c7: 0x223f, 0x16c8: 0x2244, 0x16c9: 0x2249, 0x16ca: 0x224f, 0x16cb: 0x2255, + 0x16cc: 0x225b, 0x16cd: 0x225e, 0x16ce: 0x2262, 0x16cf: 0x2265, 0x16d0: 0x2269, 0x16d1: 0x226d, + 0x16d2: 0x2271, 0x16d3: 0x2275, 0x16d4: 0x2279, 0x16d5: 0x227d, 0x16d6: 0x2281, 0x16d7: 0x2285, + 0x16d8: 0x2289, 0x16d9: 0x228d, 0x16da: 0x2291, 0x16db: 0x2295, 0x16dc: 0x2299, 0x16dd: 0x229d, + 0x16de: 0x22a1, 0x16df: 0x22a5, 0x16e0: 0x22a9, 0x16e1: 0x22ad, 0x16e2: 0x22b1, 0x16e3: 0x22b5, + 0x16e4: 0x22b9, 0x16e5: 0x22bd, 0x16e6: 0x22c1, 0x16e7: 0x22c5, 0x16e8: 0x22c9, 0x16e9: 0x22cd, + 0x16ea: 0x22d1, 0x16eb: 0x22d5, 0x16ec: 0x22d9, 0x16ed: 0x22dd, 0x16ee: 0x22e1, 0x16ef: 0x22e5, + 0x16f0: 0x22e9, 0x16f1: 0x22ed, 0x16f2: 0x22f1, 0x16f3: 0x22f5, 0x16f4: 0x22f9, 0x16f5: 0x22fd, + 0x16f6: 0x2301, 0x16f7: 0x2305, 0x16f8: 0x2309, 0x16f9: 0x230d, 0x16fa: 0x2311, 0x16fb: 0x2315, + 0x16fc: 0x2319, 0x16fd: 0x231d, 0x16fe: 0x2321, + // Block 0x5c, offset 0x1700 + 0x1700: 0x2325, 0x1701: 0x2335, 0x1702: 0x2342, 0x1703: 0x2352, 0x1704: 0x235c, 0x1705: 0x236c, + 0x1706: 0x2376, 0x1707: 0x2380, 0x1708: 0x2393, 0x1709: 0x23a0, 0x170a: 0x23aa, 0x170b: 0x23b4, + 0x170c: 0x23be, 0x170d: 0x23cb, 0x170e: 0x23d8, 0x170f: 0x23e5, 0x1710: 0x23f2, 0x1711: 0x23ff, + 0x1712: 0x240c, 0x1713: 0x2419, 0x1714: 0x242c, 0x1715: 0x2433, 0x1716: 0x2446, 0x1717: 0x2459, + 0x1718: 0x2469, 0x1719: 0x2476, 0x171a: 0x2489, 0x171b: 0x249c, 0x171c: 0x24a9, 0x171d: 0x24b3, + 0x171e: 0x24bd, 0x171f: 0x24ca, 0x1720: 0x24d7, 0x1721: 0x24e7, 0x1722: 0x24f7, 0x1723: 0x2501, + 0x1724: 0x250b, 0x1725: 0x2518, 0x1726: 0x2522, 0x1727: 0x252c, 0x1728: 0x2533, 0x1729: 0x253a, + 0x172a: 0x2544, 0x172b: 0x254e, 0x172c: 0x2561, 0x172d: 0x256e, 0x172e: 0x257e, 0x172f: 0x2591, + 0x1730: 0x259e, 0x1731: 0x25a8, 0x1732: 0x25b2, 0x1733: 0x25c5, 0x1734: 0x25d2, 0x1735: 0x25e5, + 0x1736: 0x25ef, 0x1737: 0x25ff, 0x1738: 0x2609, 0x1739: 0x2616, 0x173a: 0x2620, 0x173b: 0x262d, + 0x173c: 0x263d, 0x173d: 0x264a, 0x173e: 0x265a, 0x173f: 0x2667, + // Block 0x5d, offset 0x1740 + 0x1740: 0x266e, 0x1741: 0x267e, 0x1742: 0x2688, 0x1743: 0x2692, 0x1744: 0x269f, 0x1745: 0x26a9, + 0x1746: 0x26b3, 0x1747: 0x26bd, 0x1748: 0x26cd, 0x1749: 0x26da, 0x174a: 0x26e1, 0x174b: 0x26f4, + 0x174c: 0x26fe, 0x174d: 0x270e, 0x174e: 0x271b, 0x174f: 0x2728, 0x1750: 0x2732, 0x1751: 0x273c, + 0x1752: 0x2749, 0x1753: 0x2750, 0x1754: 0x275d, 0x1755: 0x276d, 0x1756: 0x2774, 0x1757: 0x2787, + 0x1758: 0x2791, 0x1759: 0x2796, 0x175a: 0x279b, 0x175b: 0x27a0, 0x175c: 0x27a5, 0x175d: 0x27aa, + 0x175e: 0x27af, 0x175f: 0x27b4, 0x1760: 0x27b9, 0x1761: 0x27be, 0x1762: 0x27c3, 0x1763: 0x27c9, + 0x1764: 0x27cf, 0x1765: 0x27d5, 0x1766: 0x27db, 0x1767: 0x27e1, 0x1768: 0x27e7, 0x1769: 0x27ed, + 0x176a: 0x27f3, 0x176b: 0x27f9, 0x176c: 0x27ff, 0x176d: 0x2805, 0x176e: 0x280b, 0x176f: 0x2811, + 0x1770: 0x2817, 0x1771: 0x281d, 0x1772: 0x2821, 0x1773: 0x2824, 0x1774: 0x2827, 0x1775: 0x282b, + 0x1776: 0x282e, 0x1777: 0x2831, 0x1778: 0x2834, 0x1779: 0x2838, 0x177a: 0x283c, 0x177b: 0x283f, + 0x177c: 0x2846, 0x177d: 0x284d, 0x177e: 0x2854, 0x177f: 0x285b, + // Block 0x5e, offset 0x1780 + 0x1780: 0x2868, 0x1781: 0x286b, 0x1782: 0x286e, 0x1783: 0x2872, 0x1784: 0x2875, 0x1785: 0x2878, + 0x1786: 0x287b, 0x1787: 0x287e, 0x1788: 0x2881, 0x1789: 0x2885, 0x178a: 0x288a, 0x178b: 0x288d, + 0x178c: 0x2890, 0x178d: 0x2894, 0x178e: 0x2898, 0x178f: 0x289b, 0x1790: 0x289e, 0x1791: 0x28a1, + 0x1792: 0x28a5, 0x1793: 0x28a9, 0x1794: 0x28ad, 0x1795: 0x28b1, 0x1796: 0x28b5, 0x1797: 0x28b8, + 0x1798: 0x28bb, 0x1799: 0x28be, 0x179a: 0x28c1, 0x179b: 0x28c4, 0x179c: 0x28c8, 0x179d: 0x28cb, + 0x179e: 0x28ce, 0x179f: 0x28d1, 0x17a0: 0x28d5, 0x17a1: 0x28d9, 0x17a2: 0x28dc, 0x17a3: 0x28e0, + 0x17a4: 0x28e4, 0x17a5: 0x28e8, 0x17a6: 0x28eb, 0x17a7: 0x28ef, 0x17a8: 0x28f5, 0x17a9: 0x28fc, + 0x17aa: 0x28ff, 0x17ab: 0x2903, 0x17ac: 0x2907, 0x17ad: 0x290b, 0x17ae: 0x290f, 0x17af: 0x2917, + 0x17b0: 0x2920, 0x17b1: 0x2923, 0x17b2: 0x2926, 0x17b3: 0x292a, 0x17b4: 0x292d, 0x17b5: 0x2930, + 0x17b6: 0x2933, 0x17b7: 0x2937, 0x17b8: 0x293a, 0x17b9: 0x293d, 0x17ba: 0x2940, 0x17bb: 0x2943, + 0x17bc: 0x2946, 0x17bd: 0x294a, 0x17be: 0x294d, 0x17bf: 0x2950, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x2953, 0x17c1: 0x2957, 0x17c2: 0x295b, 0x17c3: 0x2960, 0x17c4: 0x2963, 0x17c5: 0x2966, + 0x17c6: 0x2969, 0x17c7: 0x2970, 0x17c8: 0x2974, 0x17c9: 0x2977, 0x17ca: 0x297a, 0x17cb: 0x297d, + 0x17cc: 0x2980, 0x17cd: 0x2983, 0x17ce: 0x2986, 0x17cf: 0x2989, 0x17d0: 0x298c, 0x17d1: 0x298f, + 0x17d2: 0x2992, 0x17d3: 0x2996, 0x17d4: 0x2999, 0x17d5: 0x299c, 0x17d6: 0x29a0, 0x17d7: 0x29a4, + 0x17d8: 0x29a7, 0x17d9: 0x29ac, 0x17da: 0x29b0, 0x17db: 0x29b3, 0x17dc: 0x29b6, 0x17dd: 0x29b9, + 0x17de: 0x29bc, 0x17df: 0x29c2, 0x17e0: 0x29c8, 0x17e1: 0x29cd, 0x17e2: 0x29d2, 0x17e3: 0x29d7, + 0x17e4: 0x29dc, 0x17e5: 0x29e1, 0x17e6: 0x29e6, 0x17e7: 0x29eb, 0x17e8: 0x29f0, 0x17e9: 0x29f5, + 0x17ea: 0x29fb, 0x17eb: 0x2a01, 0x17ec: 0x2a07, 0x17ed: 0x2a0d, 0x17ee: 0x2a13, 0x17ef: 0x2a19, + 0x17f0: 0x2a1f, 0x17f1: 0x2a25, 0x17f2: 0x2a2b, 0x17f3: 0x2a31, 0x17f4: 0x2a37, 0x17f5: 0x2a3d, + 0x17f6: 0x2a43, 0x17f7: 0x2a49, 0x17f8: 0x2a4f, 0x17f9: 0x2a55, 0x17fa: 0x2a5b, 0x17fb: 0x2a61, + 0x17fc: 0x2a67, 0x17fd: 0x2a6d, 0x17fe: 0x2a73, 0x17ff: 0x2a79, + // Block 0x60, offset 0x1800 + 0x1830: 0x2a7d, + // Block 0x61, offset 0x1840 + 0x1840: 0x2a81, 0x1841: 0x2a85, 0x1842: 0x1a9e, 0x1843: 0x2a89, 0x1844: 0x2a8d, 0x1845: 0x2a91, + 0x1846: 0x2a95, 0x1847: 0x1b76, 0x1848: 0x1b76, 0x1849: 0x2a99, 0x184a: 0x1abe, 0x184b: 0x2a9d, + 0x184c: 0x2aa1, 0x184d: 0x2aa5, 0x184e: 0x2aa9, 0x184f: 0x2aad, 0x1850: 0x2ab1, 0x1851: 0x2ab5, + 0x1852: 0x2ab9, 0x1853: 0x2abd, 0x1854: 0x2ac1, 0x1855: 0x2ac5, 0x1856: 0x2ac9, 0x1857: 0x2acd, + 0x1858: 0x2ad1, 0x1859: 0x2ad5, 0x185a: 0x2ad9, 0x185b: 0x2add, 0x185c: 0x2ae1, 0x185d: 0x2ae5, + 0x185e: 0x2ae9, 0x185f: 0x2aed, 0x1860: 0x2af1, 0x1861: 0x2af5, 0x1862: 0x2af9, 0x1863: 0x2afd, + 0x1864: 0x2b01, 0x1865: 0x2b05, 0x1866: 0x2b09, 0x1867: 0x2b0d, 0x1868: 0x2b11, 0x1869: 0x2b15, + 0x186a: 0x2b19, 0x186b: 0x2b1d, 0x186c: 0x2b21, 0x186d: 0x2b25, 0x186e: 0x2b29, 0x186f: 0x2b2d, + 0x1870: 0x2b31, 0x1871: 0x2b35, 0x1872: 0x2b39, 0x1873: 0x2b3d, 0x1874: 0x1a16, 0x1875: 0x2b41, + 0x1876: 0x2b45, 0x1877: 0x2b49, 0x1878: 0x2b4d, 0x1879: 0x2b51, 0x187a: 0x2b55, 0x187b: 0x2b59, + 0x187c: 0x2b5d, 0x187d: 0x2b61, 0x187e: 0x2b65, 0x187f: 0x2b69, + // Block 0x62, offset 0x1880 + 0x1880: 0x1b3a, 0x1881: 0x2b6d, 0x1882: 0x2b71, 0x1883: 0x2b75, 0x1884: 0x2b79, 0x1885: 0x2b7d, + 0x1886: 0x2b81, 0x1887: 0x2b85, 0x1888: 0x2b89, 0x1889: 0x2b8d, 0x188a: 0x2b91, 0x188b: 0x2b95, + 0x188c: 0x2b99, 0x188d: 0x2b9d, 0x188e: 0x2ba1, 0x188f: 0x2ba5, 0x1890: 0x2ba9, 0x1891: 0x2bad, + 0x1892: 0x2bb1, 0x1893: 0x2bb5, 0x1894: 0x2bb9, 0x1895: 0x2bbd, 0x1896: 0x2bc1, 0x1897: 0x2bc5, + 0x1898: 0x2bc9, 0x1899: 0x2bcd, 0x189a: 0x2bd1, 0x189b: 0x2bd5, 0x189c: 0x2ac1, 0x189d: 0x2bd9, + 0x189e: 0x2bdd, 0x189f: 0x2be1, 0x18a0: 0x2be5, 0x18a1: 0x2be9, 0x18a2: 0x2bed, 0x18a3: 0x2bf1, + 0x18a4: 0x2bf5, 0x18a5: 0x2bf9, 0x18a6: 0x2bfd, 0x18a7: 0x2c01, 0x18a8: 0x2c05, 0x18a9: 0x2c09, + 0x18aa: 0x2c0d, 0x18ab: 0x2c11, 0x18ac: 0x2c15, 0x18ad: 0x2c19, 0x18ae: 0x2c1d, 0x18af: 0x2c21, + 0x18b0: 0x2c25, 0x18b1: 0x1aa6, 0x18b2: 0x2c29, 0x18b3: 0x2c2d, 0x18b4: 0x2c31, 0x18b5: 0x2c35, + 0x18b6: 0x2c39, 0x18b7: 0x2c3d, 0x18b8: 0x2c41, 0x18b9: 0x2c45, 0x18ba: 0x2c49, 0x18bb: 0x2c4d, + 0x18bc: 0x2c51, 0x18bd: 0x2c55, 0x18be: 0x2c59, 0x18bf: 0x2c5d, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x2c61, 0x18c1: 0x18ba, 0x18c2: 0x2c65, 0x18c3: 0x2c69, 0x18c4: 0x2c6d, 0x18c5: 0x2c71, + 0x18c6: 0x2c75, 0x18c7: 0x2c79, 0x18c8: 0x2c7d, 0x18c9: 0x2c81, 0x18ca: 0x186e, 0x18cb: 0x2c85, + 0x18cc: 0x2c89, 0x18cd: 0x2c8d, 0x18ce: 0x2c91, 0x18cf: 0x2c95, 0x18d0: 0x2c99, 0x18d1: 0x2c9d, + 0x18d2: 0x2ca1, 0x18d3: 0x2ca5, 0x18d4: 0x2ca9, 0x18d5: 0x2cad, 0x18d6: 0x2cb1, 0x18d7: 0x2cb5, + 0x18d8: 0x2cb9, 0x18d9: 0x2cbd, 0x18da: 0x2cc1, 0x18db: 0x2cc5, 0x18dc: 0x2cc9, 0x18dd: 0x2ccd, + 0x18de: 0x2cd1, 0x18df: 0x2cd5, 0x18e0: 0x2cd9, 0x18e1: 0x2c21, 0x18e2: 0x2cdd, 0x18e3: 0x2ce1, + 0x18e4: 0x2ce5, 0x18e5: 0x2ce9, 0x18e6: 0x2ced, 0x18e7: 0x2cf1, 0x18e8: 0x2cf5, 0x18e9: 0x2cf9, + 0x18ea: 0x2be1, 0x18eb: 0x2cfd, 0x18ec: 0x2d01, 0x18ed: 0x2d05, 0x18ee: 0x2d09, 0x18ef: 0x2d0d, + 0x18f0: 0x2d11, 0x18f1: 0x2d15, 0x18f2: 0x2d19, 0x18f3: 0x2d1d, 0x18f4: 0x2d21, 0x18f5: 0x2d25, + 0x18f6: 0x2d29, 0x18f7: 0x2d2d, 0x18f8: 0x2d31, 0x18f9: 0x2d35, 0x18fa: 0x2d39, 0x18fb: 0x2d3d, + 0x18fc: 0x2d41, 0x18fd: 0x2d45, 0x18fe: 0x2d49, 0x18ff: 0x2ac1, + // Block 0x64, offset 0x1900 + 0x1900: 0x2d4d, 0x1901: 0x2d51, 0x1902: 0x2d55, 0x1903: 0x2d59, 0x1904: 0x1b72, 0x1905: 0x2d5d, + 0x1906: 0x2d61, 0x1907: 0x2d65, 0x1908: 0x2d69, 0x1909: 0x2d6d, 0x190a: 0x2d71, 0x190b: 0x2d75, + 0x190c: 0x2d79, 0x190d: 0x2d7d, 0x190e: 0x2d81, 0x190f: 0x2d85, 0x1910: 0x2d89, 0x1911: 0x2173, + 0x1912: 0x2d8d, 0x1913: 0x2d91, 0x1914: 0x2d95, 0x1915: 0x2d99, 0x1916: 0x2d9d, 0x1917: 0x2da1, + 0x1918: 0x2da5, 0x1919: 0x2da9, 0x191a: 0x2dad, 0x191b: 0x2be9, 0x191c: 0x2db1, 0x191d: 0x2db5, + 0x191e: 0x2db9, 0x191f: 0x2dbd, 0x1920: 0x2dc1, 0x1921: 0x2dc5, 0x1922: 0x2dc9, 0x1923: 0x2dcd, + 0x1924: 0x2dd1, 0x1925: 0x2dd5, 0x1926: 0x2dd9, 0x1927: 0x2ddd, 0x1928: 0x2de1, 0x1929: 0x1aba, + 0x192a: 0x2de5, 0x192b: 0x2de9, 0x192c: 0x2ded, 0x192d: 0x2df1, 0x192e: 0x2df5, 0x192f: 0x2df9, + 0x1930: 0x2dfd, 0x1931: 0x2e01, 0x1932: 0x2e05, 0x1933: 0x2e09, 0x1934: 0x2e0d, 0x1935: 0x2e11, + 0x1936: 0x2e15, 0x1937: 0x19f6, 0x1938: 0x2e19, 0x1939: 0x2e1d, 0x193a: 0x2e21, 0x193b: 0x2e25, + 0x193c: 0x2e29, 0x193d: 0x2e2d, 0x193e: 0x2e31, 0x193f: 0x2e35, + // Block 0x65, offset 0x1940 + 0x1940: 0x2e39, 0x1941: 0x2e3d, 0x1942: 0x2e41, 0x1943: 0x2e45, 0x1944: 0x2e49, 0x1945: 0x2e4d, + 0x1946: 0x2e51, 0x1947: 0x2e55, 0x1948: 0x1a62, 0x1949: 0x2e59, 0x194a: 0x1a6e, 0x194b: 0x2e5d, + 0x194c: 0x2e61, 0x194d: 0x2e65, 0x1950: 0x2e69, + 0x1952: 0x2e6d, 0x1955: 0x2e71, 0x1956: 0x2e75, 0x1957: 0x2e79, + 0x1958: 0x2e7d, 0x1959: 0x2e81, 0x195a: 0x2e85, 0x195b: 0x2e89, 0x195c: 0x2e8d, 0x195d: 0x2e91, + 0x195e: 0x1a12, 0x1960: 0x2e95, 0x1962: 0x2e99, + 0x1965: 0x2e9d, 0x1966: 0x2ea1, + 0x196a: 0x2ea5, 0x196b: 0x2ea9, 0x196c: 0x2ead, 0x196d: 0x2eb1, + 0x1970: 0x2eb5, 0x1971: 0x2eb9, 0x1972: 0x2ebd, 0x1973: 0x2ec1, 0x1974: 0x2ec5, 0x1975: 0x2ec9, + 0x1976: 0x2ecd, 0x1977: 0x2ed1, 0x1978: 0x2ed5, 0x1979: 0x2ed9, 0x197a: 0x2edd, 0x197b: 0x2ee1, + 0x197c: 0x18d6, 0x197d: 0x2ee5, 0x197e: 0x2ee9, 0x197f: 0x2eed, + // Block 0x66, offset 0x1980 + 0x1980: 0x2ef1, 0x1981: 0x2ef5, 0x1982: 0x2ef9, 0x1983: 0x2efd, 0x1984: 0x2f01, 0x1985: 0x2f05, + 0x1986: 0x2f09, 0x1987: 0x2f0d, 0x1988: 0x2f11, 0x1989: 0x2f15, 0x198a: 0x2f19, 0x198b: 0x2f1d, + 0x198c: 0x2187, 0x198d: 0x2f21, 0x198e: 0x2f25, 0x198f: 0x2f29, 0x1990: 0x2f2d, 0x1991: 0x2197, + 0x1992: 0x2f31, 0x1993: 0x2f35, 0x1994: 0x2f39, 0x1995: 0x2f3d, 0x1996: 0x2f41, 0x1997: 0x2cb1, + 0x1998: 0x2f45, 0x1999: 0x2f49, 0x199a: 0x2f4d, 0x199b: 0x2f51, 0x199c: 0x2f55, 0x199d: 0x2f59, + 0x199e: 0x2f59, 0x199f: 0x2f5d, 0x19a0: 0x2f61, 0x19a1: 0x2f65, 0x19a2: 0x2f69, 0x19a3: 0x2f6d, + 0x19a4: 0x2f71, 0x19a5: 0x2f75, 0x19a6: 0x2f79, 0x19a7: 0x2e9d, 0x19a8: 0x2f7d, 0x19a9: 0x2f81, + 0x19aa: 0x2f85, 0x19ab: 0x2f89, 0x19ac: 0x2f8d, 0x19ad: 0x2f92, + 0x19b0: 0x2f96, 0x19b1: 0x2f9a, 0x19b2: 0x2f9e, 0x19b3: 0x2fa2, 0x19b4: 0x2fa6, 0x19b5: 0x2faa, + 0x19b6: 0x2fae, 0x19b7: 0x2fb2, 0x19b8: 0x2ecd, 0x19b9: 0x2fb6, 0x19ba: 0x2fba, 0x19bb: 0x2fbe, + 0x19bc: 0x2e69, 0x19bd: 0x2fc2, 0x19be: 0x2fc6, 0x19bf: 0x2fca, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x2fce, 0x19c1: 0x2fd2, 0x19c2: 0x2fd6, 0x19c3: 0x2fda, 0x19c4: 0x2fde, 0x19c5: 0x2fe2, + 0x19c6: 0x2fe6, 0x19c7: 0x2fea, 0x19c8: 0x2fee, 0x19c9: 0x2eed, 0x19ca: 0x2ff2, 0x19cb: 0x2ef1, + 0x19cc: 0x2ff6, 0x19cd: 0x2ffa, 0x19ce: 0x2ffe, 0x19cf: 0x3002, 0x19d0: 0x3006, 0x19d1: 0x2e6d, + 0x19d2: 0x2b15, 0x19d3: 0x300a, 0x19d4: 0x300e, 0x19d5: 0x195a, 0x19d6: 0x2c25, 0x19d7: 0x2d71, + 0x19d8: 0x3012, 0x19d9: 0x3016, 0x19da: 0x2f0d, 0x19db: 0x301a, 0x19dc: 0x2f11, 0x19dd: 0x301e, + 0x19de: 0x3022, 0x19df: 0x3026, 0x19e0: 0x2e75, 0x19e1: 0x302a, 0x19e2: 0x302e, 0x19e3: 0x3032, + 0x19e4: 0x3036, 0x19e5: 0x303a, 0x19e6: 0x2e79, 0x19e7: 0x303e, 0x19e8: 0x3042, 0x19e9: 0x3046, + 0x19ea: 0x304a, 0x19eb: 0x304e, 0x19ec: 0x3052, 0x19ed: 0x2f41, 0x19ee: 0x3056, 0x19ef: 0x305a, + 0x19f0: 0x2cb1, 0x19f1: 0x305e, 0x19f2: 0x2f51, 0x19f3: 0x3062, 0x19f4: 0x3066, 0x19f5: 0x306a, + 0x19f6: 0x306e, 0x19f7: 0x3072, 0x19f8: 0x2f65, 0x19f9: 0x3076, 0x19fa: 0x2e99, 0x19fb: 0x307a, + 0x19fc: 0x2f69, 0x19fd: 0x2bd9, 0x19fe: 0x307e, 0x19ff: 0x2f6d, + // Block 0x68, offset 0x1a00 + 0x1a00: 0x3082, 0x1a01: 0x2f75, 0x1a02: 0x3086, 0x1a03: 0x308a, 0x1a04: 0x308e, 0x1a05: 0x3092, + 0x1a06: 0x3096, 0x1a07: 0x2f7d, 0x1a08: 0x2e8d, 0x1a09: 0x309a, 0x1a0a: 0x2f81, 0x1a0b: 0x309e, + 0x1a0c: 0x2f85, 0x1a0d: 0x30a2, 0x1a0e: 0x1b76, 0x1a0f: 0x30a6, 0x1a10: 0x30ab, 0x1a11: 0x30b0, + 0x1a12: 0x30b5, 0x1a13: 0x30b9, 0x1a14: 0x30bd, 0x1a15: 0x30c1, 0x1a16: 0x30c6, 0x1a17: 0x30cb, + 0x1a18: 0x30d0, 0x1a19: 0x30d4, + // Block 0x69, offset 0x1a40 + 0x1a40: 0x30d8, 0x1a41: 0x30db, 0x1a42: 0x30de, 0x1a43: 0x30e1, 0x1a44: 0x30e5, 0x1a45: 0x30e9, + 0x1a46: 0x30e9, + 0x1a53: 0x30ec, 0x1a54: 0x30f1, 0x1a55: 0x30f6, 0x1a56: 0x30fb, 0x1a57: 0x3100, + 0x1a5d: 0x3105, + 0x1a5f: 0x310a, 0x1a60: 0x310f, 0x1a61: 0x14db, 0x1a62: 0x14e4, 0x1a63: 0x3112, + 0x1a64: 0x3115, 0x1a65: 0x3118, 0x1a66: 0x311b, 0x1a67: 0x311e, 0x1a68: 0x3121, 0x1a69: 0x1494, + 0x1a6a: 0x3124, 0x1a6b: 0x3129, 0x1a6c: 0x312e, 0x1a6d: 0x3135, 0x1a6e: 0x313c, 0x1a6f: 0x3141, + 0x1a70: 0x3146, 0x1a71: 0x314b, 0x1a72: 0x3150, 0x1a73: 0x3155, 0x1a74: 0x315a, 0x1a75: 0x315f, + 0x1a76: 0x3164, 0x1a78: 0x3169, 0x1a79: 0x316e, 0x1a7a: 0x3173, 0x1a7b: 0x3178, + 0x1a7c: 0x317d, 0x1a7e: 0x3182, + // Block 0x6a, offset 0x1a80 + 0x1a80: 0x3187, 0x1a81: 0x318c, 0x1a83: 0x3191, 0x1a84: 0x3196, + 0x1a86: 0x319b, 0x1a87: 0x31a0, 0x1a88: 0x31a5, 0x1a89: 0x31aa, 0x1a8a: 0x31af, 0x1a8b: 0x31b4, + 0x1a8c: 0x31b9, 0x1a8d: 0x31be, 0x1a8e: 0x31c3, 0x1a8f: 0x31c8, 0x1a90: 0x31cd, 0x1a91: 0x31cd, + 0x1a92: 0x31d0, 0x1a93: 0x31d0, 0x1a94: 0x31d0, 0x1a95: 0x31d0, 0x1a96: 0x31d3, 0x1a97: 0x31d3, + 0x1a98: 0x31d3, 0x1a99: 0x31d3, 0x1a9a: 0x31d6, 0x1a9b: 0x31d6, 0x1a9c: 0x31d6, 0x1a9d: 0x31d6, + 0x1a9e: 0x31d9, 0x1a9f: 0x31d9, 0x1aa0: 0x31d9, 0x1aa1: 0x31d9, 0x1aa2: 0x31dc, 0x1aa3: 0x31dc, + 0x1aa4: 0x31dc, 0x1aa5: 0x31dc, 0x1aa6: 0x31df, 0x1aa7: 0x31df, 0x1aa8: 0x31df, 0x1aa9: 0x31df, + 0x1aaa: 0x31e2, 0x1aab: 0x31e2, 0x1aac: 0x31e2, 0x1aad: 0x31e2, 0x1aae: 0x31e5, 0x1aaf: 0x31e5, + 0x1ab0: 0x31e5, 0x1ab1: 0x31e5, 0x1ab2: 0x31e8, 0x1ab3: 0x31e8, 0x1ab4: 0x31e8, 0x1ab5: 0x31e8, + 0x1ab6: 0x31eb, 0x1ab7: 0x31eb, 0x1ab8: 0x31eb, 0x1ab9: 0x31eb, 0x1aba: 0x31ee, 0x1abb: 0x31ee, + 0x1abc: 0x31ee, 0x1abd: 0x31ee, 0x1abe: 0x31f1, 0x1abf: 0x31f1, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0x31f1, 0x1ac1: 0x31f1, 0x1ac2: 0x31f4, 0x1ac3: 0x31f4, 0x1ac4: 0x31f7, 0x1ac5: 0x31f7, + 0x1ac6: 0x31fa, 0x1ac7: 0x31fa, 0x1ac8: 0x31fd, 0x1ac9: 0x31fd, 0x1aca: 0x3200, 0x1acb: 0x3200, + 0x1acc: 0x3203, 0x1acd: 0x3203, 0x1ace: 0x3206, 0x1acf: 0x3206, 0x1ad0: 0x3206, 0x1ad1: 0x3206, + 0x1ad2: 0x3209, 0x1ad3: 0x3209, 0x1ad4: 0x3209, 0x1ad5: 0x3209, 0x1ad6: 0x320c, 0x1ad7: 0x320c, + 0x1ad8: 0x320c, 0x1ad9: 0x320c, 0x1ada: 0x320f, 0x1adb: 0x320f, 0x1adc: 0x320f, 0x1add: 0x320f, + 0x1ade: 0x3212, 0x1adf: 0x3212, 0x1ae0: 0x3215, 0x1ae1: 0x3215, 0x1ae2: 0x3215, 0x1ae3: 0x3215, + 0x1ae4: 0x06ba, 0x1ae5: 0x06ba, 0x1ae6: 0x3218, 0x1ae7: 0x3218, 0x1ae8: 0x3218, 0x1ae9: 0x3218, + 0x1aea: 0x321b, 0x1aeb: 0x321b, 0x1aec: 0x321b, 0x1aed: 0x321b, 0x1aee: 0x321e, 0x1aef: 0x321e, + 0x1af0: 0x06c4, 0x1af1: 0x06c4, + // Block 0x6c, offset 0x1b00 + 0x1b13: 0x3221, 0x1b14: 0x3221, 0x1b15: 0x3221, 0x1b16: 0x3221, 0x1b17: 0x3224, + 0x1b18: 0x3224, 0x1b19: 0x3227, 0x1b1a: 0x3227, 0x1b1b: 0x322a, 0x1b1c: 0x322a, 0x1b1d: 0x06b0, + 0x1b1e: 0x322d, 0x1b1f: 0x322d, 0x1b20: 0x3230, 0x1b21: 0x3230, 0x1b22: 0x3233, 0x1b23: 0x3233, + 0x1b24: 0x3236, 0x1b25: 0x3236, 0x1b26: 0x3236, 0x1b27: 0x3236, 0x1b28: 0x3239, 0x1b29: 0x3239, + 0x1b2a: 0x323c, 0x1b2b: 0x323c, 0x1b2c: 0x3243, 0x1b2d: 0x3243, 0x1b2e: 0x324a, 0x1b2f: 0x324a, + 0x1b30: 0x3251, 0x1b31: 0x3251, 0x1b32: 0x3258, 0x1b33: 0x3258, 0x1b34: 0x325f, 0x1b35: 0x325f, + 0x1b36: 0x3266, 0x1b37: 0x3266, 0x1b38: 0x3266, 0x1b39: 0x326d, 0x1b3a: 0x326d, 0x1b3b: 0x326d, + 0x1b3c: 0x3274, 0x1b3d: 0x3274, 0x1b3e: 0x3274, 0x1b3f: 0x3274, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x3277, 0x1b41: 0x327e, 0x1b42: 0x3285, 0x1b43: 0x326d, 0x1b44: 0x328c, 0x1b45: 0x3293, + 0x1b46: 0x3298, 0x1b47: 0x329d, 0x1b48: 0x32a2, 0x1b49: 0x32a7, 0x1b4a: 0x32ac, 0x1b4b: 0x32b1, + 0x1b4c: 0x32b6, 0x1b4d: 0x32bb, 0x1b4e: 0x32c0, 0x1b4f: 0x32c5, 0x1b50: 0x32ca, 0x1b51: 0x32cf, + 0x1b52: 0x32d4, 0x1b53: 0x32d9, 0x1b54: 0x32de, 0x1b55: 0x32e3, 0x1b56: 0x32e8, 0x1b57: 0x32ed, + 0x1b58: 0x32f2, 0x1b59: 0x32f7, 0x1b5a: 0x32fc, 0x1b5b: 0x3301, 0x1b5c: 0x3306, 0x1b5d: 0x330b, + 0x1b5e: 0x3310, 0x1b5f: 0x3315, 0x1b60: 0x331a, 0x1b61: 0x331f, 0x1b62: 0x3324, 0x1b63: 0x3329, + 0x1b64: 0x332e, 0x1b65: 0x3333, 0x1b66: 0x3338, 0x1b67: 0x333d, 0x1b68: 0x3342, 0x1b69: 0x3347, + 0x1b6a: 0x334c, 0x1b6b: 0x3351, 0x1b6c: 0x3356, 0x1b6d: 0x335b, 0x1b6e: 0x3360, 0x1b6f: 0x3365, + 0x1b70: 0x336a, 0x1b71: 0x336f, 0x1b72: 0x3374, 0x1b73: 0x3379, 0x1b74: 0x337e, 0x1b75: 0x3383, + 0x1b76: 0x3388, 0x1b77: 0x338d, 0x1b78: 0x3392, 0x1b79: 0x3397, 0x1b7a: 0x339c, 0x1b7b: 0x33a1, + 0x1b7c: 0x33a6, 0x1b7d: 0x33ab, 0x1b7e: 0x33b0, 0x1b7f: 0x33b5, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x33ba, 0x1b81: 0x33bf, 0x1b82: 0x33c4, 0x1b83: 0x33c9, 0x1b84: 0x33ce, 0x1b85: 0x33d3, + 0x1b86: 0x33d8, 0x1b87: 0x33dd, 0x1b88: 0x33e2, 0x1b89: 0x33e7, 0x1b8a: 0x33ec, 0x1b8b: 0x33f1, + 0x1b8c: 0x33f6, 0x1b8d: 0x33fb, 0x1b8e: 0x3400, 0x1b8f: 0x3405, 0x1b90: 0x340a, 0x1b91: 0x340f, + 0x1b92: 0x3414, 0x1b93: 0x3419, 0x1b94: 0x341e, 0x1b95: 0x3423, 0x1b96: 0x3428, 0x1b97: 0x342d, + 0x1b98: 0x3432, 0x1b99: 0x3437, 0x1b9a: 0x343c, 0x1b9b: 0x3441, 0x1b9c: 0x3446, 0x1b9d: 0x344b, + 0x1b9e: 0x3450, 0x1b9f: 0x3456, 0x1ba0: 0x345c, 0x1ba1: 0x3462, 0x1ba2: 0x3468, 0x1ba3: 0x346e, + 0x1ba4: 0x3474, 0x1ba5: 0x347b, 0x1ba6: 0x3285, 0x1ba7: 0x3482, 0x1ba8: 0x326d, 0x1ba9: 0x328c, + 0x1baa: 0x3489, 0x1bab: 0x348e, 0x1bac: 0x32a2, 0x1bad: 0x3493, 0x1bae: 0x32a7, 0x1baf: 0x32ac, + 0x1bb0: 0x3498, 0x1bb1: 0x349d, 0x1bb2: 0x32c0, 0x1bb3: 0x34a2, 0x1bb4: 0x32c5, 0x1bb5: 0x32ca, + 0x1bb6: 0x34a7, 0x1bb7: 0x34ac, 0x1bb8: 0x32d4, 0x1bb9: 0x34b1, 0x1bba: 0x32d9, 0x1bbb: 0x32de, + 0x1bbc: 0x336f, 0x1bbd: 0x3374, 0x1bbe: 0x3383, 0x1bbf: 0x3388, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0x338d, 0x1bc1: 0x33a1, 0x1bc2: 0x33a6, 0x1bc3: 0x33ab, 0x1bc4: 0x33b0, 0x1bc5: 0x33c4, + 0x1bc6: 0x33c9, 0x1bc7: 0x33ce, 0x1bc8: 0x34b6, 0x1bc9: 0x33e2, 0x1bca: 0x34bb, 0x1bcb: 0x34c0, + 0x1bcc: 0x3400, 0x1bcd: 0x34c5, 0x1bce: 0x3405, 0x1bcf: 0x340a, 0x1bd0: 0x344b, 0x1bd1: 0x34ca, + 0x1bd2: 0x34cf, 0x1bd3: 0x3432, 0x1bd4: 0x34d4, 0x1bd5: 0x3437, 0x1bd6: 0x343c, 0x1bd7: 0x3277, + 0x1bd8: 0x327e, 0x1bd9: 0x34d9, 0x1bda: 0x3285, 0x1bdb: 0x34e0, 0x1bdc: 0x3293, 0x1bdd: 0x3298, + 0x1bde: 0x329d, 0x1bdf: 0x32a2, 0x1be0: 0x34e7, 0x1be1: 0x32b1, 0x1be2: 0x32b6, 0x1be3: 0x32bb, + 0x1be4: 0x32c0, 0x1be5: 0x34ec, 0x1be6: 0x32d4, 0x1be7: 0x32e3, 0x1be8: 0x32e8, 0x1be9: 0x32ed, + 0x1bea: 0x32f2, 0x1beb: 0x32f7, 0x1bec: 0x3301, 0x1bed: 0x3306, 0x1bee: 0x330b, 0x1bef: 0x3310, + 0x1bf0: 0x3315, 0x1bf1: 0x331a, 0x1bf2: 0x34f1, 0x1bf3: 0x331f, 0x1bf4: 0x3324, 0x1bf5: 0x3329, + 0x1bf6: 0x332e, 0x1bf7: 0x3333, 0x1bf8: 0x3338, 0x1bf9: 0x3342, 0x1bfa: 0x3347, 0x1bfb: 0x334c, + 0x1bfc: 0x3351, 0x1bfd: 0x3356, 0x1bfe: 0x335b, 0x1bff: 0x3360, + // Block 0x70, offset 0x1c00 + 0x1c00: 0x3365, 0x1c01: 0x336a, 0x1c02: 0x3379, 0x1c03: 0x337e, 0x1c04: 0x3392, 0x1c05: 0x3397, + 0x1c06: 0x339c, 0x1c07: 0x33a1, 0x1c08: 0x33a6, 0x1c09: 0x33b5, 0x1c0a: 0x33ba, 0x1c0b: 0x33bf, + 0x1c0c: 0x33c4, 0x1c0d: 0x34f6, 0x1c0e: 0x33d3, 0x1c0f: 0x33d8, 0x1c10: 0x33dd, 0x1c11: 0x33e2, + 0x1c12: 0x33f1, 0x1c13: 0x33f6, 0x1c14: 0x33fb, 0x1c15: 0x3400, 0x1c16: 0x34fb, 0x1c17: 0x340f, + 0x1c18: 0x3414, 0x1c19: 0x3500, 0x1c1a: 0x3423, 0x1c1b: 0x3428, 0x1c1c: 0x342d, 0x1c1d: 0x3432, + 0x1c1e: 0x3505, 0x1c1f: 0x3285, 0x1c20: 0x34e0, 0x1c21: 0x32a2, 0x1c22: 0x34e7, 0x1c23: 0x32c0, + 0x1c24: 0x34ec, 0x1c25: 0x32d4, 0x1c26: 0x350a, 0x1c27: 0x3315, 0x1c28: 0x350f, 0x1c29: 0x3514, + 0x1c2a: 0x3519, 0x1c2b: 0x33a1, 0x1c2c: 0x33a6, 0x1c2d: 0x33c4, 0x1c2e: 0x3400, 0x1c2f: 0x34fb, + 0x1c30: 0x3432, 0x1c31: 0x3505, 0x1c32: 0x351e, 0x1c33: 0x3525, 0x1c34: 0x352c, 0x1c35: 0x3533, + 0x1c36: 0x3538, 0x1c37: 0x353d, 0x1c38: 0x3542, 0x1c39: 0x3547, 0x1c3a: 0x354c, 0x1c3b: 0x3551, + 0x1c3c: 0x3556, 0x1c3d: 0x355b, 0x1c3e: 0x3560, 0x1c3f: 0x3565, + // Block 0x71, offset 0x1c40 + 0x1c40: 0x356a, 0x1c41: 0x356f, 0x1c42: 0x3574, 0x1c43: 0x3579, 0x1c44: 0x357e, 0x1c45: 0x3583, + 0x1c46: 0x3588, 0x1c47: 0x358d, 0x1c48: 0x3592, 0x1c49: 0x3597, 0x1c4a: 0x359c, 0x1c4b: 0x35a1, + 0x1c4c: 0x3514, 0x1c4d: 0x35a6, 0x1c4e: 0x35ab, 0x1c4f: 0x35b0, 0x1c50: 0x35b5, 0x1c51: 0x3533, + 0x1c52: 0x3538, 0x1c53: 0x353d, 0x1c54: 0x3542, 0x1c55: 0x3547, 0x1c56: 0x354c, 0x1c57: 0x3551, + 0x1c58: 0x3556, 0x1c59: 0x355b, 0x1c5a: 0x3560, 0x1c5b: 0x3565, 0x1c5c: 0x356a, 0x1c5d: 0x356f, + 0x1c5e: 0x3574, 0x1c5f: 0x3579, 0x1c60: 0x357e, 0x1c61: 0x3583, 0x1c62: 0x3588, 0x1c63: 0x358d, + 0x1c64: 0x3592, 0x1c65: 0x3597, 0x1c66: 0x359c, 0x1c67: 0x35a1, 0x1c68: 0x3514, 0x1c69: 0x35a6, + 0x1c6a: 0x35ab, 0x1c6b: 0x35b0, 0x1c6c: 0x35b5, 0x1c6d: 0x3597, 0x1c6e: 0x359c, 0x1c6f: 0x35a1, + 0x1c70: 0x3514, 0x1c71: 0x350f, 0x1c72: 0x3519, 0x1c73: 0x333d, 0x1c74: 0x3306, 0x1c75: 0x330b, + 0x1c76: 0x3310, 0x1c77: 0x3597, 0x1c78: 0x359c, 0x1c79: 0x35a1, 0x1c7a: 0x333d, 0x1c7b: 0x3342, + 0x1c7c: 0x35ba, 0x1c7d: 0x35ba, + // Block 0x72, offset 0x1c80 + 0x1c90: 0x35bf, 0x1c91: 0x35c6, + 0x1c92: 0x35c6, 0x1c93: 0x35cd, 0x1c94: 0x35d4, 0x1c95: 0x35db, 0x1c96: 0x35e2, 0x1c97: 0x35e9, + 0x1c98: 0x35f0, 0x1c99: 0x35f0, 0x1c9a: 0x35f7, 0x1c9b: 0x35fe, 0x1c9c: 0x3605, 0x1c9d: 0x360c, + 0x1c9e: 0x3613, 0x1c9f: 0x361a, 0x1ca0: 0x361a, 0x1ca1: 0x3621, 0x1ca2: 0x3628, 0x1ca3: 0x3628, + 0x1ca4: 0x362f, 0x1ca5: 0x362f, 0x1ca6: 0x3636, 0x1ca7: 0x363d, 0x1ca8: 0x363d, 0x1ca9: 0x3644, + 0x1caa: 0x364b, 0x1cab: 0x364b, 0x1cac: 0x3652, 0x1cad: 0x3652, 0x1cae: 0x3659, 0x1caf: 0x3660, + 0x1cb0: 0x3660, 0x1cb1: 0x3667, 0x1cb2: 0x3667, 0x1cb3: 0x366e, 0x1cb4: 0x3675, 0x1cb5: 0x367c, + 0x1cb6: 0x3683, 0x1cb7: 0x3683, 0x1cb8: 0x368a, 0x1cb9: 0x3691, 0x1cba: 0x3698, 0x1cbb: 0x369f, + 0x1cbc: 0x36a6, 0x1cbd: 0x36a6, 0x1cbe: 0x36ad, 0x1cbf: 0x36b4, + // Block 0x73, offset 0x1cc0 + 0x1cc0: 0x36bb, 0x1cc1: 0x36c2, 0x1cc2: 0x36c9, 0x1cc3: 0x36d0, 0x1cc4: 0x36d0, 0x1cc5: 0x36d7, + 0x1cc6: 0x36d7, 0x1cc7: 0x36de, 0x1cc8: 0x36de, 0x1cc9: 0x36e5, 0x1cca: 0x36ec, 0x1ccb: 0x36f3, + 0x1ccc: 0x36fa, 0x1ccd: 0x3701, 0x1cce: 0x3708, 0x1ccf: 0x370f, + 0x1cd2: 0x3716, 0x1cd3: 0x371d, 0x1cd4: 0x3724, 0x1cd5: 0x372b, 0x1cd6: 0x3732, 0x1cd7: 0x3739, + 0x1cd8: 0x3739, 0x1cd9: 0x3740, 0x1cda: 0x3747, 0x1cdb: 0x374e, 0x1cdc: 0x3755, 0x1cdd: 0x3755, + 0x1cde: 0x375c, 0x1cdf: 0x3763, 0x1ce0: 0x376a, 0x1ce1: 0x3771, 0x1ce2: 0x3778, 0x1ce3: 0x377f, + 0x1ce4: 0x3786, 0x1ce5: 0x378d, 0x1ce6: 0x3794, 0x1ce7: 0x379b, 0x1ce8: 0x37a2, 0x1ce9: 0x37a9, + 0x1cea: 0x37b0, 0x1ceb: 0x37b7, 0x1cec: 0x37be, 0x1ced: 0x37c5, 0x1cee: 0x37cc, 0x1cef: 0x37d3, + 0x1cf0: 0x37da, 0x1cf1: 0x37e1, 0x1cf2: 0x37e8, 0x1cf3: 0x37ef, 0x1cf4: 0x36ad, 0x1cf5: 0x36bb, + 0x1cf6: 0x37f6, 0x1cf7: 0x37fd, 0x1cf8: 0x3804, 0x1cf9: 0x380b, 0x1cfa: 0x3812, 0x1cfb: 0x3819, + 0x1cfc: 0x3812, 0x1cfd: 0x3804, 0x1cfe: 0x3820, 0x1cff: 0x3827, + // Block 0x74, offset 0x1d00 + 0x1d00: 0x382e, 0x1d01: 0x3835, 0x1d02: 0x383c, 0x1d03: 0x3819, 0x1d04: 0x367c, 0x1d05: 0x3636, + 0x1d06: 0x3843, 0x1d07: 0x384a, + 0x1d30: 0x3851, 0x1d31: 0x3858, 0x1d32: 0x385f, 0x1d33: 0x3868, 0x1d34: 0x3871, 0x1d35: 0x387a, + 0x1d36: 0x3883, 0x1d37: 0x388c, 0x1d38: 0x3895, 0x1d39: 0x389e, 0x1d3a: 0x38a5, 0x1d3b: 0x38c7, + 0x1d3c: 0x38d7, + // Block 0x75, offset 0x1d40 + 0x1d50: 0x38e0, 0x1d51: 0x38e2, + 0x1d52: 0x38e6, 0x1d53: 0x38ea, 0x1d54: 0x04e1, 0x1d55: 0x38ec, 0x1d56: 0x38ee, 0x1d57: 0x38f0, + 0x1d58: 0x38f4, 0x1d59: 0x1443, + 0x1d70: 0x1440, 0x1d71: 0x38f8, 0x1d72: 0x38fc, 0x1d73: 0x3900, 0x1d74: 0x3900, 0x1d75: 0x149c, + 0x1d76: 0x149e, 0x1d77: 0x3902, 0x1d78: 0x3904, 0x1d79: 0x3906, 0x1d7a: 0x390a, 0x1d7b: 0x390e, + 0x1d7c: 0x3912, 0x1d7d: 0x3916, 0x1d7e: 0x391a, 0x1d7f: 0x16c3, + // Block 0x76, offset 0x1d80 + 0x1d80: 0x16c7, 0x1d81: 0x391e, 0x1d82: 0x3922, 0x1d83: 0x3926, 0x1d84: 0x392a, + 0x1d87: 0x392e, 0x1d88: 0x3930, 0x1d89: 0x146c, 0x1d8a: 0x146c, 0x1d8b: 0x146c, + 0x1d8c: 0x146c, 0x1d8d: 0x3900, 0x1d8e: 0x3900, 0x1d8f: 0x3900, 0x1d90: 0x38e0, 0x1d91: 0x38e2, + 0x1d92: 0x143e, 0x1d94: 0x04e1, 0x1d95: 0x38ea, 0x1d96: 0x38ee, 0x1d97: 0x38ec, + 0x1d98: 0x38f8, 0x1d99: 0x149c, 0x1d9a: 0x149e, 0x1d9b: 0x3902, 0x1d9c: 0x3904, 0x1d9d: 0x3906, + 0x1d9e: 0x390a, 0x1d9f: 0x3932, 0x1da0: 0x3934, 0x1da1: 0x3936, 0x1da2: 0x1494, 0x1da3: 0x3938, + 0x1da4: 0x393a, 0x1da5: 0x393c, 0x1da6: 0x149a, 0x1da8: 0x393e, 0x1da9: 0x3940, + 0x1daa: 0x3942, 0x1dab: 0x3944, + 0x1db0: 0x3946, 0x1db1: 0x394a, 0x1db2: 0x394f, 0x1db4: 0x3953, + 0x1db6: 0x3957, 0x1db7: 0x395b, 0x1db8: 0x3960, 0x1db9: 0x3964, 0x1dba: 0x3969, 0x1dbb: 0x396d, + 0x1dbc: 0x3972, 0x1dbd: 0x3976, 0x1dbe: 0x397b, 0x1dbf: 0x397f, + // Block 0x77, offset 0x1dc0 + 0x1dc0: 0x3984, 0x1dc1: 0x068d, 0x1dc2: 0x068d, 0x1dc3: 0x0692, 0x1dc4: 0x0692, 0x1dc5: 0x0697, + 0x1dc6: 0x0697, 0x1dc7: 0x069c, 0x1dc8: 0x069c, 0x1dc9: 0x06a1, 0x1dca: 0x06a1, 0x1dcb: 0x06a1, + 0x1dcc: 0x06a1, 0x1dcd: 0x3987, 0x1dce: 0x3987, 0x1dcf: 0x398a, 0x1dd0: 0x398a, 0x1dd1: 0x398a, + 0x1dd2: 0x398a, 0x1dd3: 0x398d, 0x1dd4: 0x398d, 0x1dd5: 0x3990, 0x1dd6: 0x3990, 0x1dd7: 0x3990, + 0x1dd8: 0x3990, 0x1dd9: 0x3993, 0x1dda: 0x3993, 0x1ddb: 0x3993, 0x1ddc: 0x3993, 0x1ddd: 0x3996, + 0x1dde: 0x3996, 0x1ddf: 0x3996, 0x1de0: 0x3996, 0x1de1: 0x3999, 0x1de2: 0x3999, 0x1de3: 0x3999, + 0x1de4: 0x3999, 0x1de5: 0x399c, 0x1de6: 0x399c, 0x1de7: 0x399c, 0x1de8: 0x399c, 0x1de9: 0x399f, + 0x1dea: 0x399f, 0x1deb: 0x39a2, 0x1dec: 0x39a2, 0x1ded: 0x39a5, 0x1dee: 0x39a5, 0x1def: 0x39a8, + 0x1df0: 0x39a8, 0x1df1: 0x39ab, 0x1df2: 0x39ab, 0x1df3: 0x39ab, 0x1df4: 0x39ab, 0x1df5: 0x39ae, + 0x1df6: 0x39ae, 0x1df7: 0x39ae, 0x1df8: 0x39ae, 0x1df9: 0x39b1, 0x1dfa: 0x39b1, 0x1dfb: 0x39b1, + 0x1dfc: 0x39b1, 0x1dfd: 0x39b4, 0x1dfe: 0x39b4, 0x1dff: 0x39b4, + // Block 0x78, offset 0x1e00 + 0x1e00: 0x39b4, 0x1e01: 0x39b7, 0x1e02: 0x39b7, 0x1e03: 0x39b7, 0x1e04: 0x39b7, 0x1e05: 0x39ba, + 0x1e06: 0x39ba, 0x1e07: 0x39ba, 0x1e08: 0x39ba, 0x1e09: 0x39bd, 0x1e0a: 0x39bd, 0x1e0b: 0x39bd, + 0x1e0c: 0x39bd, 0x1e0d: 0x39c0, 0x1e0e: 0x39c0, 0x1e0f: 0x39c0, 0x1e10: 0x39c0, 0x1e11: 0x39c3, + 0x1e12: 0x39c3, 0x1e13: 0x39c3, 0x1e14: 0x39c3, 0x1e15: 0x39c6, 0x1e16: 0x39c6, 0x1e17: 0x39c6, + 0x1e18: 0x39c6, 0x1e19: 0x39c9, 0x1e1a: 0x39c9, 0x1e1b: 0x39c9, 0x1e1c: 0x39c9, 0x1e1d: 0x39cc, + 0x1e1e: 0x39cc, 0x1e1f: 0x39cc, 0x1e20: 0x39cc, 0x1e21: 0x39cf, 0x1e22: 0x39cf, 0x1e23: 0x39cf, + 0x1e24: 0x39cf, 0x1e25: 0x39d2, 0x1e26: 0x39d2, 0x1e27: 0x39d2, 0x1e28: 0x39d2, 0x1e29: 0x39d5, + 0x1e2a: 0x39d5, 0x1e2b: 0x39d5, 0x1e2c: 0x39d5, 0x1e2d: 0x39d8, 0x1e2e: 0x39d8, 0x1e2f: 0x3239, + 0x1e30: 0x3239, 0x1e31: 0x39db, 0x1e32: 0x39db, 0x1e33: 0x39db, 0x1e34: 0x39db, 0x1e35: 0x39de, + 0x1e36: 0x39de, 0x1e37: 0x39e5, 0x1e38: 0x39e5, 0x1e39: 0x39ec, 0x1e3a: 0x39ec, 0x1e3b: 0x39f3, + 0x1e3c: 0x39f3, + // Block 0x79, offset 0x1e40 + 0x1e41: 0x38ec, 0x1e42: 0x39f8, 0x1e43: 0x3932, 0x1e44: 0x3940, 0x1e45: 0x3942, + 0x1e46: 0x3934, 0x1e47: 0x39fa, 0x1e48: 0x149c, 0x1e49: 0x149e, 0x1e4a: 0x3936, 0x1e4b: 0x1494, + 0x1e4c: 0x38e0, 0x1e4d: 0x3938, 0x1e4e: 0x143e, 0x1e4f: 0x39fc, 0x1e50: 0x1486, 0x1e51: 0x001c, + 0x1e52: 0x000d, 0x1e53: 0x000f, 0x1e54: 0x1488, 0x1e55: 0x148a, 0x1e56: 0x148c, 0x1e57: 0x148e, + 0x1e58: 0x1490, 0x1e59: 0x1492, 0x1e5a: 0x38ea, 0x1e5b: 0x04e1, 0x1e5c: 0x393a, 0x1e5d: 0x149a, + 0x1e5e: 0x393c, 0x1e5f: 0x38ee, 0x1e60: 0x3944, 0x1e61: 0x0906, 0x1e62: 0x090b, 0x1e63: 0x14ad, + 0x1e64: 0x090d, 0x1e65: 0x090f, 0x1e66: 0x14d9, 0x1e67: 0x0914, 0x1e68: 0x0916, 0x1e69: 0x0918, + 0x1e6a: 0x091a, 0x1e6b: 0x091c, 0x1e6c: 0x091e, 0x1e6d: 0x0920, 0x1e6e: 0x0922, 0x1e6f: 0x0924, + 0x1e70: 0x0929, 0x1e71: 0x14c8, 0x1e72: 0x092b, 0x1e73: 0x17f6, 0x1e74: 0x092d, 0x1e75: 0x092f, + 0x1e76: 0x155f, 0x1e77: 0x0931, 0x1e78: 0x1570, 0x1e79: 0x17f8, 0x1e7a: 0x14d4, 0x1e7b: 0x392e, + 0x1e7c: 0x393e, 0x1e7d: 0x3930, 0x1e7e: 0x39fe, 0x1e7f: 0x3900, + // Block 0x7a, offset 0x1e80 + 0x1e80: 0x13f7, 0x1e81: 0x0007, 0x1e82: 0x093d, 0x1e83: 0x0984, 0x1e84: 0x093f, 0x1e85: 0x0941, + 0x1e86: 0x098c, 0x1e87: 0x094c, 0x1e88: 0x0494, 0x1e89: 0x097c, 0x1e8a: 0x0499, 0x1e8b: 0x094e, + 0x1e8c: 0x04c5, 0x1e8d: 0x0950, 0x1e8e: 0x14a0, 0x1e8f: 0x001e, 0x1e90: 0x0960, 0x1e91: 0x17fa, + 0x1e92: 0x049b, 0x1e93: 0x02c8, 0x1e94: 0x0962, 0x1e95: 0x0964, 0x1e96: 0x096d, 0x1e97: 0x04a6, + 0x1e98: 0x04c7, 0x1e99: 0x04a8, 0x1e9a: 0x09df, 0x1e9b: 0x3902, 0x1e9c: 0x3a00, 0x1e9d: 0x3904, + 0x1e9e: 0x3a02, 0x1e9f: 0x3a04, 0x1ea0: 0x3a08, 0x1ea1: 0x38e6, 0x1ea2: 0x391e, 0x1ea3: 0x3922, + 0x1ea4: 0x38e2, 0x1ea5: 0x3a0c, 0x1ea6: 0x2321, 0x1ea7: 0x3a10, 0x1ea8: 0x3a14, 0x1ea9: 0x3a18, + 0x1eaa: 0x3a1c, 0x1eab: 0x3a20, 0x1eac: 0x3a24, 0x1ead: 0x3a28, 0x1eae: 0x3a2c, 0x1eaf: 0x3a30, + 0x1eb0: 0x3a34, 0x1eb1: 0x2269, 0x1eb2: 0x226d, 0x1eb3: 0x2271, 0x1eb4: 0x2275, 0x1eb5: 0x2279, + 0x1eb6: 0x227d, 0x1eb7: 0x2281, 0x1eb8: 0x2285, 0x1eb9: 0x2289, 0x1eba: 0x228d, 0x1ebb: 0x2291, + 0x1ebc: 0x2295, 0x1ebd: 0x2299, 0x1ebe: 0x229d, 0x1ebf: 0x22a1, + // Block 0x7b, offset 0x1ec0 + 0x1ec0: 0x22a5, 0x1ec1: 0x22a9, 0x1ec2: 0x22ad, 0x1ec3: 0x22b1, 0x1ec4: 0x22b5, 0x1ec5: 0x22b9, + 0x1ec6: 0x22bd, 0x1ec7: 0x22c1, 0x1ec8: 0x22c5, 0x1ec9: 0x22c9, 0x1eca: 0x22cd, 0x1ecb: 0x22d1, + 0x1ecc: 0x22d5, 0x1ecd: 0x22d9, 0x1ece: 0x22dd, 0x1ecf: 0x22e1, 0x1ed0: 0x22e5, 0x1ed1: 0x22e9, + 0x1ed2: 0x22ed, 0x1ed3: 0x22f1, 0x1ed4: 0x22f5, 0x1ed5: 0x22f9, 0x1ed6: 0x22fd, 0x1ed7: 0x2301, + 0x1ed8: 0x2305, 0x1ed9: 0x2309, 0x1eda: 0x230d, 0x1edb: 0x2311, 0x1edc: 0x2315, 0x1edd: 0x3a38, + 0x1ede: 0x3a3c, 0x1edf: 0x3a40, 0x1ee0: 0x1e04, 0x1ee1: 0x1d38, 0x1ee2: 0x1d3c, 0x1ee3: 0x1d40, + 0x1ee4: 0x1d44, 0x1ee5: 0x1d48, 0x1ee6: 0x1d4c, 0x1ee7: 0x1d50, 0x1ee8: 0x1d54, 0x1ee9: 0x1d58, + 0x1eea: 0x1d5c, 0x1eeb: 0x1d60, 0x1eec: 0x1d64, 0x1eed: 0x1d68, 0x1eee: 0x1d6c, 0x1eef: 0x1d70, + 0x1ef0: 0x1d74, 0x1ef1: 0x1d78, 0x1ef2: 0x1d7c, 0x1ef3: 0x1d80, 0x1ef4: 0x1d84, 0x1ef5: 0x1d88, + 0x1ef6: 0x1d8c, 0x1ef7: 0x1d90, 0x1ef8: 0x1d94, 0x1ef9: 0x1d98, 0x1efa: 0x1d9c, 0x1efb: 0x1da0, + 0x1efc: 0x1da4, 0x1efd: 0x1da8, 0x1efe: 0x1dac, + // Block 0x7c, offset 0x1f00 + 0x1f02: 0x1db0, 0x1f03: 0x1db4, 0x1f04: 0x1db8, 0x1f05: 0x1dbc, + 0x1f06: 0x1dc0, 0x1f07: 0x1dc4, 0x1f0a: 0x1dc8, 0x1f0b: 0x1dcc, + 0x1f0c: 0x1dd0, 0x1f0d: 0x1dd4, 0x1f0e: 0x1dd8, 0x1f0f: 0x1ddc, + 0x1f12: 0x1de0, 0x1f13: 0x1de4, 0x1f14: 0x1de8, 0x1f15: 0x1dec, 0x1f16: 0x1df0, 0x1f17: 0x1df4, + 0x1f1a: 0x1df8, 0x1f1b: 0x1dfc, 0x1f1c: 0x1e00, + 0x1f20: 0x3a44, 0x1f21: 0x3a47, 0x1f22: 0x3a4a, 0x1f23: 0x0009, + 0x1f24: 0x3a4d, 0x1f25: 0x3a50, 0x1f26: 0x3a53, 0x1f28: 0x3a57, 0x1f29: 0x3a5b, + 0x1f2a: 0x3a5f, 0x1f2b: 0x3a63, 0x1f2c: 0x3a67, 0x1f2d: 0x3a6b, 0x1f2e: 0x3a6f, + // Block 0x7d, offset 0x1f40 + 0x1f5a: 0x3a73, 0x1f5c: 0x3a7c, + 0x1f6b: 0x3a85, + // Block 0x7e, offset 0x1f80 + 0x1f9e: 0x3a8e, 0x1f9f: 0x3a97, 0x1fa0: 0x3aa0, 0x1fa1: 0x3aad, 0x1fa2: 0x3aba, 0x1fa3: 0x3ac7, + 0x1fa4: 0x3ad4, + // Block 0x7f, offset 0x1fc0 + 0x1ffb: 0x3ae1, + 0x1ffc: 0x3aea, 0x1ffd: 0x3af3, 0x1ffe: 0x3b00, 0x1fff: 0x3b0d, + // Block 0x80, offset 0x2000 + 0x2000: 0x3b1a, + // Block 0x81, offset 0x2040 + 0x2040: 0x0906, 0x2041: 0x090b, 0x2042: 0x14ad, 0x2043: 0x090d, 0x2044: 0x090f, 0x2045: 0x14d9, + 0x2046: 0x0914, 0x2047: 0x0916, 0x2048: 0x0918, 0x2049: 0x091a, 0x204a: 0x091c, 0x204b: 0x091e, + 0x204c: 0x0920, 0x204d: 0x0922, 0x204e: 0x0924, 0x204f: 0x0929, 0x2050: 0x14c8, 0x2051: 0x092b, + 0x2052: 0x17f6, 0x2053: 0x092d, 0x2054: 0x092f, 0x2055: 0x155f, 0x2056: 0x0931, 0x2057: 0x1570, + 0x2058: 0x17f8, 0x2059: 0x14d4, 0x205a: 0x0007, 0x205b: 0x093d, 0x205c: 0x0984, 0x205d: 0x093f, + 0x205e: 0x0941, 0x205f: 0x098c, 0x2060: 0x094c, 0x2061: 0x0494, 0x2062: 0x097c, 0x2063: 0x0499, + 0x2064: 0x094e, 0x2065: 0x04c5, 0x2066: 0x0950, 0x2067: 0x14a0, 0x2068: 0x001e, 0x2069: 0x0960, + 0x206a: 0x17fa, 0x206b: 0x049b, 0x206c: 0x02c8, 0x206d: 0x0962, 0x206e: 0x0964, 0x206f: 0x096d, + 0x2070: 0x04a6, 0x2071: 0x04c7, 0x2072: 0x04a8, 0x2073: 0x09df, 0x2074: 0x0906, 0x2075: 0x090b, + 0x2076: 0x14ad, 0x2077: 0x090d, 0x2078: 0x090f, 0x2079: 0x14d9, 0x207a: 0x0914, 0x207b: 0x0916, + 0x207c: 0x0918, 0x207d: 0x091a, 0x207e: 0x091c, 0x207f: 0x091e, + // Block 0x82, offset 0x2080 + 0x2080: 0x0920, 0x2081: 0x0922, 0x2082: 0x0924, 0x2083: 0x0929, 0x2084: 0x14c8, 0x2085: 0x092b, + 0x2086: 0x17f6, 0x2087: 0x092d, 0x2088: 0x092f, 0x2089: 0x155f, 0x208a: 0x0931, 0x208b: 0x1570, + 0x208c: 0x17f8, 0x208d: 0x14d4, 0x208e: 0x0007, 0x208f: 0x093d, 0x2090: 0x0984, 0x2091: 0x093f, + 0x2092: 0x0941, 0x2093: 0x098c, 0x2094: 0x094c, 0x2096: 0x097c, 0x2097: 0x0499, + 0x2098: 0x094e, 0x2099: 0x04c5, 0x209a: 0x0950, 0x209b: 0x14a0, 0x209c: 0x001e, 0x209d: 0x0960, + 0x209e: 0x17fa, 0x209f: 0x049b, 0x20a0: 0x02c8, 0x20a1: 0x0962, 0x20a2: 0x0964, 0x20a3: 0x096d, + 0x20a4: 0x04a6, 0x20a5: 0x04c7, 0x20a6: 0x04a8, 0x20a7: 0x09df, 0x20a8: 0x0906, 0x20a9: 0x090b, + 0x20aa: 0x14ad, 0x20ab: 0x090d, 0x20ac: 0x090f, 0x20ad: 0x14d9, 0x20ae: 0x0914, 0x20af: 0x0916, + 0x20b0: 0x0918, 0x20b1: 0x091a, 0x20b2: 0x091c, 0x20b3: 0x091e, 0x20b4: 0x0920, 0x20b5: 0x0922, + 0x20b6: 0x0924, 0x20b7: 0x0929, 0x20b8: 0x14c8, 0x20b9: 0x092b, 0x20ba: 0x17f6, 0x20bb: 0x092d, + 0x20bc: 0x092f, 0x20bd: 0x155f, 0x20be: 0x0931, 0x20bf: 0x1570, + // Block 0x83, offset 0x20c0 + 0x20c0: 0x17f8, 0x20c1: 0x14d4, 0x20c2: 0x0007, 0x20c3: 0x093d, 0x20c4: 0x0984, 0x20c5: 0x093f, + 0x20c6: 0x0941, 0x20c7: 0x098c, 0x20c8: 0x094c, 0x20c9: 0x0494, 0x20ca: 0x097c, 0x20cb: 0x0499, + 0x20cc: 0x094e, 0x20cd: 0x04c5, 0x20ce: 0x0950, 0x20cf: 0x14a0, 0x20d0: 0x001e, 0x20d1: 0x0960, + 0x20d2: 0x17fa, 0x20d3: 0x049b, 0x20d4: 0x02c8, 0x20d5: 0x0962, 0x20d6: 0x0964, 0x20d7: 0x096d, + 0x20d8: 0x04a6, 0x20d9: 0x04c7, 0x20da: 0x04a8, 0x20db: 0x09df, 0x20dc: 0x0906, + 0x20de: 0x14ad, 0x20df: 0x090d, 0x20e2: 0x0914, + 0x20e5: 0x091a, 0x20e6: 0x091c, 0x20e9: 0x0922, + 0x20ea: 0x0924, 0x20eb: 0x0929, 0x20ec: 0x14c8, 0x20ee: 0x17f6, 0x20ef: 0x092d, + 0x20f0: 0x092f, 0x20f1: 0x155f, 0x20f2: 0x0931, 0x20f3: 0x1570, 0x20f4: 0x17f8, 0x20f5: 0x14d4, + 0x20f6: 0x0007, 0x20f7: 0x093d, 0x20f8: 0x0984, 0x20f9: 0x093f, 0x20fb: 0x098c, + 0x20fd: 0x0494, 0x20fe: 0x097c, 0x20ff: 0x0499, + // Block 0x84, offset 0x2100 + 0x2100: 0x094e, 0x2101: 0x04c5, 0x2102: 0x0950, 0x2103: 0x14a0, 0x2105: 0x0960, + 0x2106: 0x17fa, 0x2107: 0x049b, 0x2108: 0x02c8, 0x2109: 0x0962, 0x210a: 0x0964, 0x210b: 0x096d, + 0x210c: 0x04a6, 0x210d: 0x04c7, 0x210e: 0x04a8, 0x210f: 0x09df, 0x2110: 0x0906, 0x2111: 0x090b, + 0x2112: 0x14ad, 0x2113: 0x090d, 0x2114: 0x090f, 0x2115: 0x14d9, 0x2116: 0x0914, 0x2117: 0x0916, + 0x2118: 0x0918, 0x2119: 0x091a, 0x211a: 0x091c, 0x211b: 0x091e, 0x211c: 0x0920, 0x211d: 0x0922, + 0x211e: 0x0924, 0x211f: 0x0929, 0x2120: 0x14c8, 0x2121: 0x092b, 0x2122: 0x17f6, 0x2123: 0x092d, + 0x2124: 0x092f, 0x2125: 0x155f, 0x2126: 0x0931, 0x2127: 0x1570, 0x2128: 0x17f8, 0x2129: 0x14d4, + 0x212a: 0x0007, 0x212b: 0x093d, 0x212c: 0x0984, 0x212d: 0x093f, 0x212e: 0x0941, 0x212f: 0x098c, + 0x2130: 0x094c, 0x2131: 0x0494, 0x2132: 0x097c, 0x2133: 0x0499, 0x2134: 0x094e, 0x2135: 0x04c5, + 0x2136: 0x0950, 0x2137: 0x14a0, 0x2138: 0x001e, 0x2139: 0x0960, 0x213a: 0x17fa, 0x213b: 0x049b, + 0x213c: 0x02c8, 0x213d: 0x0962, 0x213e: 0x0964, 0x213f: 0x096d, + // Block 0x85, offset 0x2140 + 0x2140: 0x04a6, 0x2141: 0x04c7, 0x2142: 0x04a8, 0x2143: 0x09df, 0x2144: 0x0906, 0x2145: 0x090b, + 0x2147: 0x090d, 0x2148: 0x090f, 0x2149: 0x14d9, 0x214a: 0x0914, + 0x214d: 0x091a, 0x214e: 0x091c, 0x214f: 0x091e, 0x2150: 0x0920, 0x2151: 0x0922, + 0x2152: 0x0924, 0x2153: 0x0929, 0x2154: 0x14c8, 0x2156: 0x17f6, 0x2157: 0x092d, + 0x2158: 0x092f, 0x2159: 0x155f, 0x215a: 0x0931, 0x215b: 0x1570, 0x215c: 0x17f8, + 0x215e: 0x0007, 0x215f: 0x093d, 0x2160: 0x0984, 0x2161: 0x093f, 0x2162: 0x0941, 0x2163: 0x098c, + 0x2164: 0x094c, 0x2165: 0x0494, 0x2166: 0x097c, 0x2167: 0x0499, 0x2168: 0x094e, 0x2169: 0x04c5, + 0x216a: 0x0950, 0x216b: 0x14a0, 0x216c: 0x001e, 0x216d: 0x0960, 0x216e: 0x17fa, 0x216f: 0x049b, + 0x2170: 0x02c8, 0x2171: 0x0962, 0x2172: 0x0964, 0x2173: 0x096d, 0x2174: 0x04a6, 0x2175: 0x04c7, + 0x2176: 0x04a8, 0x2177: 0x09df, 0x2178: 0x0906, 0x2179: 0x090b, 0x217b: 0x090d, + 0x217c: 0x090f, 0x217d: 0x14d9, 0x217e: 0x0914, + // Block 0x86, offset 0x2180 + 0x2180: 0x0918, 0x2181: 0x091a, 0x2182: 0x091c, 0x2183: 0x091e, 0x2184: 0x0920, + 0x2186: 0x0924, 0x218a: 0x17f6, 0x218b: 0x092d, + 0x218c: 0x092f, 0x218d: 0x155f, 0x218e: 0x0931, 0x218f: 0x1570, 0x2190: 0x17f8, + 0x2192: 0x0007, 0x2193: 0x093d, 0x2194: 0x0984, 0x2195: 0x093f, 0x2196: 0x0941, 0x2197: 0x098c, + 0x2198: 0x094c, 0x2199: 0x0494, 0x219a: 0x097c, 0x219b: 0x0499, 0x219c: 0x094e, 0x219d: 0x04c5, + 0x219e: 0x0950, 0x219f: 0x14a0, 0x21a0: 0x001e, 0x21a1: 0x0960, 0x21a2: 0x17fa, 0x21a3: 0x049b, + 0x21a4: 0x02c8, 0x21a5: 0x0962, 0x21a6: 0x0964, 0x21a7: 0x096d, 0x21a8: 0x04a6, 0x21a9: 0x04c7, + 0x21aa: 0x04a8, 0x21ab: 0x09df, 0x21ac: 0x0906, 0x21ad: 0x090b, 0x21ae: 0x14ad, 0x21af: 0x090d, + 0x21b0: 0x090f, 0x21b1: 0x14d9, 0x21b2: 0x0914, 0x21b3: 0x0916, 0x21b4: 0x0918, 0x21b5: 0x091a, + 0x21b6: 0x091c, 0x21b7: 0x091e, 0x21b8: 0x0920, 0x21b9: 0x0922, 0x21ba: 0x0924, 0x21bb: 0x0929, + 0x21bc: 0x14c8, 0x21bd: 0x092b, 0x21be: 0x17f6, 0x21bf: 0x092d, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x092f, 0x21c1: 0x155f, 0x21c2: 0x0931, 0x21c3: 0x1570, 0x21c4: 0x17f8, 0x21c5: 0x14d4, + 0x21c6: 0x0007, 0x21c7: 0x093d, 0x21c8: 0x0984, 0x21c9: 0x093f, 0x21ca: 0x0941, 0x21cb: 0x098c, + 0x21cc: 0x094c, 0x21cd: 0x0494, 0x21ce: 0x097c, 0x21cf: 0x0499, 0x21d0: 0x094e, 0x21d1: 0x04c5, + 0x21d2: 0x0950, 0x21d3: 0x14a0, 0x21d4: 0x001e, 0x21d5: 0x0960, 0x21d6: 0x17fa, 0x21d7: 0x049b, + 0x21d8: 0x02c8, 0x21d9: 0x0962, 0x21da: 0x0964, 0x21db: 0x096d, 0x21dc: 0x04a6, 0x21dd: 0x04c7, + 0x21de: 0x04a8, 0x21df: 0x09df, 0x21e0: 0x0906, 0x21e1: 0x090b, 0x21e2: 0x14ad, 0x21e3: 0x090d, + 0x21e4: 0x090f, 0x21e5: 0x14d9, 0x21e6: 0x0914, 0x21e7: 0x0916, 0x21e8: 0x0918, 0x21e9: 0x091a, + 0x21ea: 0x091c, 0x21eb: 0x091e, 0x21ec: 0x0920, 0x21ed: 0x0922, 0x21ee: 0x0924, 0x21ef: 0x0929, + 0x21f0: 0x14c8, 0x21f1: 0x092b, 0x21f2: 0x17f6, 0x21f3: 0x092d, 0x21f4: 0x092f, 0x21f5: 0x155f, + 0x21f6: 0x0931, 0x21f7: 0x1570, 0x21f8: 0x17f8, 0x21f9: 0x14d4, 0x21fa: 0x0007, 0x21fb: 0x093d, + 0x21fc: 0x0984, 0x21fd: 0x093f, 0x21fe: 0x0941, 0x21ff: 0x098c, + // Block 0x88, offset 0x2200 + 0x2200: 0x094c, 0x2201: 0x0494, 0x2202: 0x097c, 0x2203: 0x0499, 0x2204: 0x094e, 0x2205: 0x04c5, + 0x2206: 0x0950, 0x2207: 0x14a0, 0x2208: 0x001e, 0x2209: 0x0960, 0x220a: 0x17fa, 0x220b: 0x049b, + 0x220c: 0x02c8, 0x220d: 0x0962, 0x220e: 0x0964, 0x220f: 0x096d, 0x2210: 0x04a6, 0x2211: 0x04c7, + 0x2212: 0x04a8, 0x2213: 0x09df, 0x2214: 0x0906, 0x2215: 0x090b, 0x2216: 0x14ad, 0x2217: 0x090d, + 0x2218: 0x090f, 0x2219: 0x14d9, 0x221a: 0x0914, 0x221b: 0x0916, 0x221c: 0x0918, 0x221d: 0x091a, + 0x221e: 0x091c, 0x221f: 0x091e, 0x2220: 0x0920, 0x2221: 0x0922, 0x2222: 0x0924, 0x2223: 0x0929, + 0x2224: 0x14c8, 0x2225: 0x092b, 0x2226: 0x17f6, 0x2227: 0x092d, 0x2228: 0x092f, 0x2229: 0x155f, + 0x222a: 0x0931, 0x222b: 0x1570, 0x222c: 0x17f8, 0x222d: 0x14d4, 0x222e: 0x0007, 0x222f: 0x093d, + 0x2230: 0x0984, 0x2231: 0x093f, 0x2232: 0x0941, 0x2233: 0x098c, 0x2234: 0x094c, 0x2235: 0x0494, + 0x2236: 0x097c, 0x2237: 0x0499, 0x2238: 0x094e, 0x2239: 0x04c5, 0x223a: 0x0950, 0x223b: 0x14a0, + 0x223c: 0x001e, 0x223d: 0x0960, 0x223e: 0x17fa, 0x223f: 0x049b, + // Block 0x89, offset 0x2240 + 0x2240: 0x02c8, 0x2241: 0x0962, 0x2242: 0x0964, 0x2243: 0x096d, 0x2244: 0x04a6, 0x2245: 0x04c7, + 0x2246: 0x04a8, 0x2247: 0x09df, 0x2248: 0x0906, 0x2249: 0x090b, 0x224a: 0x14ad, 0x224b: 0x090d, + 0x224c: 0x090f, 0x224d: 0x14d9, 0x224e: 0x0914, 0x224f: 0x0916, 0x2250: 0x0918, 0x2251: 0x091a, + 0x2252: 0x091c, 0x2253: 0x091e, 0x2254: 0x0920, 0x2255: 0x0922, 0x2256: 0x0924, 0x2257: 0x0929, + 0x2258: 0x14c8, 0x2259: 0x092b, 0x225a: 0x17f6, 0x225b: 0x092d, 0x225c: 0x092f, 0x225d: 0x155f, + 0x225e: 0x0931, 0x225f: 0x1570, 0x2260: 0x17f8, 0x2261: 0x14d4, 0x2262: 0x0007, 0x2263: 0x093d, + 0x2264: 0x0984, 0x2265: 0x093f, 0x2266: 0x0941, 0x2267: 0x098c, 0x2268: 0x094c, 0x2269: 0x0494, + 0x226a: 0x097c, 0x226b: 0x0499, 0x226c: 0x094e, 0x226d: 0x04c5, 0x226e: 0x0950, 0x226f: 0x14a0, + 0x2270: 0x001e, 0x2271: 0x0960, 0x2272: 0x17fa, 0x2273: 0x049b, 0x2274: 0x02c8, 0x2275: 0x0962, + 0x2276: 0x0964, 0x2277: 0x096d, 0x2278: 0x04a6, 0x2279: 0x04c7, 0x227a: 0x04a8, 0x227b: 0x09df, + 0x227c: 0x0906, 0x227d: 0x090b, 0x227e: 0x14ad, 0x227f: 0x090d, + // Block 0x8a, offset 0x2280 + 0x2280: 0x090f, 0x2281: 0x14d9, 0x2282: 0x0914, 0x2283: 0x0916, 0x2284: 0x0918, 0x2285: 0x091a, + 0x2286: 0x091c, 0x2287: 0x091e, 0x2288: 0x0920, 0x2289: 0x0922, 0x228a: 0x0924, 0x228b: 0x0929, + 0x228c: 0x14c8, 0x228d: 0x092b, 0x228e: 0x17f6, 0x228f: 0x092d, 0x2290: 0x092f, 0x2291: 0x155f, + 0x2292: 0x0931, 0x2293: 0x1570, 0x2294: 0x17f8, 0x2295: 0x14d4, 0x2296: 0x0007, 0x2297: 0x093d, + 0x2298: 0x0984, 0x2299: 0x093f, 0x229a: 0x0941, 0x229b: 0x098c, 0x229c: 0x094c, 0x229d: 0x0494, + 0x229e: 0x097c, 0x229f: 0x0499, 0x22a0: 0x094e, 0x22a1: 0x04c5, 0x22a2: 0x0950, 0x22a3: 0x14a0, + 0x22a4: 0x001e, 0x22a5: 0x0960, 0x22a6: 0x17fa, 0x22a7: 0x049b, 0x22a8: 0x02c8, 0x22a9: 0x0962, + 0x22aa: 0x0964, 0x22ab: 0x096d, 0x22ac: 0x04a6, 0x22ad: 0x04c7, 0x22ae: 0x04a8, 0x22af: 0x09df, + 0x22b0: 0x0906, 0x22b1: 0x090b, 0x22b2: 0x14ad, 0x22b3: 0x090d, 0x22b4: 0x090f, 0x22b5: 0x14d9, + 0x22b6: 0x0914, 0x22b7: 0x0916, 0x22b8: 0x0918, 0x22b9: 0x091a, 0x22ba: 0x091c, 0x22bb: 0x091e, + 0x22bc: 0x0920, 0x22bd: 0x0922, 0x22be: 0x0924, 0x22bf: 0x0929, + // Block 0x8b, offset 0x22c0 + 0x22c0: 0x14c8, 0x22c1: 0x092b, 0x22c2: 0x17f6, 0x22c3: 0x092d, 0x22c4: 0x092f, 0x22c5: 0x155f, + 0x22c6: 0x0931, 0x22c7: 0x1570, 0x22c8: 0x17f8, 0x22c9: 0x14d4, 0x22ca: 0x0007, 0x22cb: 0x093d, + 0x22cc: 0x0984, 0x22cd: 0x093f, 0x22ce: 0x0941, 0x22cf: 0x098c, 0x22d0: 0x094c, 0x22d1: 0x0494, + 0x22d2: 0x097c, 0x22d3: 0x0499, 0x22d4: 0x094e, 0x22d5: 0x04c5, 0x22d6: 0x0950, 0x22d7: 0x14a0, + 0x22d8: 0x001e, 0x22d9: 0x0960, 0x22da: 0x17fa, 0x22db: 0x049b, 0x22dc: 0x02c8, 0x22dd: 0x0962, + 0x22de: 0x0964, 0x22df: 0x096d, 0x22e0: 0x04a6, 0x22e1: 0x04c7, 0x22e2: 0x04a8, 0x22e3: 0x09df, + 0x22e4: 0x3b27, 0x22e5: 0x3b2a, 0x22e8: 0x3b2d, 0x22e9: 0x3b30, + 0x22ea: 0x14eb, 0x22eb: 0x3b33, 0x22ec: 0x3b36, 0x22ed: 0x3b39, 0x22ee: 0x3b3c, 0x22ef: 0x057b, + 0x22f0: 0x3b3f, 0x22f1: 0x3b42, 0x22f2: 0x3b45, 0x22f3: 0x3b48, 0x22f4: 0x3b4b, 0x22f5: 0x3b4e, + 0x22f6: 0x3b51, 0x22f7: 0x14ee, 0x22f8: 0x3b54, 0x22f9: 0x057b, 0x22fa: 0x0581, 0x22fb: 0x3b57, + 0x22fc: 0x055f, 0x22fd: 0x3b5a, 0x22fe: 0x3b5d, 0x22ff: 0x3b60, + // Block 0x8c, offset 0x2300 + 0x2300: 0x14d6, 0x2301: 0x3b63, 0x2302: 0x3b67, 0x2303: 0x0559, 0x2304: 0x0973, 0x2305: 0x0976, + 0x2306: 0x057e, 0x2307: 0x3b6a, 0x2308: 0x3b6d, 0x2309: 0x055c, 0x230a: 0x12fd, 0x230b: 0x0572, + 0x230c: 0x3b70, 0x230d: 0x0015, 0x230e: 0x3b73, 0x230f: 0x3b76, 0x2310: 0x3b79, 0x2311: 0x056f, + 0x2312: 0x0575, 0x2313: 0x0578, 0x2314: 0x3b7c, 0x2315: 0x3b7f, 0x2316: 0x3b82, 0x2317: 0x056c, + 0x2318: 0x0979, 0x2319: 0x3b85, 0x231a: 0x3b88, 0x231b: 0x3b8b, 0x231c: 0x057e, 0x231d: 0x055c, + 0x231e: 0x0572, 0x231f: 0x056c, 0x2320: 0x0575, 0x2321: 0x056f, 0x2322: 0x3b2d, 0x2323: 0x3b30, + 0x2324: 0x14eb, 0x2325: 0x3b33, 0x2326: 0x3b36, 0x2327: 0x3b39, 0x2328: 0x3b3c, 0x2329: 0x057b, + 0x232a: 0x3b3f, 0x232b: 0x3b42, 0x232c: 0x3b45, 0x232d: 0x3b48, 0x232e: 0x3b4b, 0x232f: 0x3b4e, + 0x2330: 0x3b51, 0x2331: 0x14ee, 0x2332: 0x3b54, 0x2333: 0x057b, 0x2334: 0x0581, 0x2335: 0x3b57, + 0x2336: 0x055f, 0x2337: 0x3b5a, 0x2338: 0x3b5d, 0x2339: 0x3b60, 0x233a: 0x14d6, 0x233b: 0x3b63, + 0x233c: 0x3b67, 0x233d: 0x0559, 0x233e: 0x0973, 0x233f: 0x0976, + // Block 0x8d, offset 0x2340 + 0x2340: 0x057e, 0x2341: 0x3b6a, 0x2342: 0x3b6d, 0x2343: 0x055c, 0x2344: 0x12fd, 0x2345: 0x0572, + 0x2346: 0x3b70, 0x2347: 0x0015, 0x2348: 0x3b73, 0x2349: 0x3b76, 0x234a: 0x3b79, 0x234b: 0x056f, + 0x234c: 0x0575, 0x234d: 0x0578, 0x234e: 0x3b7c, 0x234f: 0x3b7f, 0x2350: 0x3b82, 0x2351: 0x056c, + 0x2352: 0x0979, 0x2353: 0x3b85, 0x2354: 0x3b88, 0x2355: 0x3b8b, 0x2356: 0x057e, 0x2357: 0x055c, + 0x2358: 0x0572, 0x2359: 0x056c, 0x235a: 0x0575, 0x235b: 0x056f, 0x235c: 0x3b2d, 0x235d: 0x3b30, + 0x235e: 0x14eb, 0x235f: 0x3b33, 0x2360: 0x3b36, 0x2361: 0x3b39, 0x2362: 0x3b3c, 0x2363: 0x057b, + 0x2364: 0x3b3f, 0x2365: 0x3b42, 0x2366: 0x3b45, 0x2367: 0x3b48, 0x2368: 0x3b4b, 0x2369: 0x3b4e, + 0x236a: 0x3b51, 0x236b: 0x14ee, 0x236c: 0x3b54, 0x236d: 0x057b, 0x236e: 0x0581, 0x236f: 0x3b57, + 0x2370: 0x055f, 0x2371: 0x3b5a, 0x2372: 0x3b5d, 0x2373: 0x3b60, 0x2374: 0x14d6, 0x2375: 0x3b63, + 0x2376: 0x3b67, 0x2377: 0x0559, 0x2378: 0x0973, 0x2379: 0x0976, 0x237a: 0x057e, 0x237b: 0x3b6a, + 0x237c: 0x3b6d, 0x237d: 0x055c, 0x237e: 0x12fd, 0x237f: 0x0572, + // Block 0x8e, offset 0x2380 + 0x2380: 0x3b70, 0x2381: 0x0015, 0x2382: 0x3b73, 0x2383: 0x3b76, 0x2384: 0x3b79, 0x2385: 0x056f, + 0x2386: 0x0575, 0x2387: 0x0578, 0x2388: 0x3b7c, 0x2389: 0x3b7f, 0x238a: 0x3b82, 0x238b: 0x056c, + 0x238c: 0x0979, 0x238d: 0x3b85, 0x238e: 0x3b88, 0x238f: 0x3b8b, 0x2390: 0x057e, 0x2391: 0x055c, + 0x2392: 0x0572, 0x2393: 0x056c, 0x2394: 0x0575, 0x2395: 0x056f, 0x2396: 0x3b2d, 0x2397: 0x3b30, + 0x2398: 0x14eb, 0x2399: 0x3b33, 0x239a: 0x3b36, 0x239b: 0x3b39, 0x239c: 0x3b3c, 0x239d: 0x057b, + 0x239e: 0x3b3f, 0x239f: 0x3b42, 0x23a0: 0x3b45, 0x23a1: 0x3b48, 0x23a2: 0x3b4b, 0x23a3: 0x3b4e, + 0x23a4: 0x3b51, 0x23a5: 0x14ee, 0x23a6: 0x3b54, 0x23a7: 0x057b, 0x23a8: 0x0581, 0x23a9: 0x3b57, + 0x23aa: 0x055f, 0x23ab: 0x3b5a, 0x23ac: 0x3b5d, 0x23ad: 0x3b60, 0x23ae: 0x14d6, 0x23af: 0x3b63, + 0x23b0: 0x3b67, 0x23b1: 0x0559, 0x23b2: 0x0973, 0x23b3: 0x0976, 0x23b4: 0x057e, 0x23b5: 0x3b6a, + 0x23b6: 0x3b6d, 0x23b7: 0x055c, 0x23b8: 0x12fd, 0x23b9: 0x0572, 0x23ba: 0x3b70, 0x23bb: 0x0015, + 0x23bc: 0x3b73, 0x23bd: 0x3b76, 0x23be: 0x3b79, 0x23bf: 0x056f, + // Block 0x8f, offset 0x23c0 + 0x23c0: 0x0575, 0x23c1: 0x0578, 0x23c2: 0x3b7c, 0x23c3: 0x3b7f, 0x23c4: 0x3b82, 0x23c5: 0x056c, + 0x23c6: 0x0979, 0x23c7: 0x3b85, 0x23c8: 0x3b88, 0x23c9: 0x3b8b, 0x23ca: 0x057e, 0x23cb: 0x055c, + 0x23cc: 0x0572, 0x23cd: 0x056c, 0x23ce: 0x0575, 0x23cf: 0x056f, 0x23d0: 0x3b2d, 0x23d1: 0x3b30, + 0x23d2: 0x14eb, 0x23d3: 0x3b33, 0x23d4: 0x3b36, 0x23d5: 0x3b39, 0x23d6: 0x3b3c, 0x23d7: 0x057b, + 0x23d8: 0x3b3f, 0x23d9: 0x3b42, 0x23da: 0x3b45, 0x23db: 0x3b48, 0x23dc: 0x3b4b, 0x23dd: 0x3b4e, + 0x23de: 0x3b51, 0x23df: 0x14ee, 0x23e0: 0x3b54, 0x23e1: 0x057b, 0x23e2: 0x0581, 0x23e3: 0x3b57, + 0x23e4: 0x055f, 0x23e5: 0x3b5a, 0x23e6: 0x3b5d, 0x23e7: 0x3b60, 0x23e8: 0x14d6, 0x23e9: 0x3b63, + 0x23ea: 0x3b67, 0x23eb: 0x0559, 0x23ec: 0x0973, 0x23ed: 0x0976, 0x23ee: 0x057e, 0x23ef: 0x3b6a, + 0x23f0: 0x3b6d, 0x23f1: 0x055c, 0x23f2: 0x12fd, 0x23f3: 0x0572, 0x23f4: 0x3b70, 0x23f5: 0x0015, + 0x23f6: 0x3b73, 0x23f7: 0x3b76, 0x23f8: 0x3b79, 0x23f9: 0x056f, 0x23fa: 0x0575, 0x23fb: 0x0578, + 0x23fc: 0x3b7c, 0x23fd: 0x3b7f, 0x23fe: 0x3b82, 0x23ff: 0x056c, + // Block 0x90, offset 0x2400 + 0x2400: 0x0979, 0x2401: 0x3b85, 0x2402: 0x3b88, 0x2403: 0x3b8b, 0x2404: 0x057e, 0x2405: 0x055c, + 0x2406: 0x0572, 0x2407: 0x056c, 0x2408: 0x0575, 0x2409: 0x056f, 0x240a: 0x3b8f, 0x240b: 0x3b92, + 0x240e: 0x1486, 0x240f: 0x001c, 0x2410: 0x000d, 0x2411: 0x000f, + 0x2412: 0x1488, 0x2413: 0x148a, 0x2414: 0x148c, 0x2415: 0x148e, 0x2416: 0x1490, 0x2417: 0x1492, + 0x2418: 0x1486, 0x2419: 0x001c, 0x241a: 0x000d, 0x241b: 0x000f, 0x241c: 0x1488, 0x241d: 0x148a, + 0x241e: 0x148c, 0x241f: 0x148e, 0x2420: 0x1490, 0x2421: 0x1492, 0x2422: 0x1486, 0x2423: 0x001c, + 0x2424: 0x000d, 0x2425: 0x000f, 0x2426: 0x1488, 0x2427: 0x148a, 0x2428: 0x148c, 0x2429: 0x148e, + 0x242a: 0x1490, 0x242b: 0x1492, 0x242c: 0x1486, 0x242d: 0x001c, 0x242e: 0x000d, 0x242f: 0x000f, + 0x2430: 0x1488, 0x2431: 0x148a, 0x2432: 0x148c, 0x2433: 0x148e, 0x2434: 0x1490, 0x2435: 0x1492, + 0x2436: 0x1486, 0x2437: 0x001c, 0x2438: 0x000d, 0x2439: 0x000f, 0x243a: 0x1488, 0x243b: 0x148a, + 0x243c: 0x148c, 0x243d: 0x148e, 0x243e: 0x1490, 0x243f: 0x1492, + // Block 0x91, offset 0x2440 + 0x2440: 0x3b95, 0x2441: 0x3b98, 0x2442: 0x3b9b, 0x2443: 0x3b9e, 0x2444: 0x3ba1, 0x2445: 0x3ba4, + 0x2446: 0x3ba7, 0x2447: 0x3baa, 0x2448: 0x3bad, 0x2449: 0x3bb0, 0x244a: 0x3bb3, + 0x2450: 0x3bb6, 0x2451: 0x3bba, + 0x2452: 0x3bbe, 0x2453: 0x3bc2, 0x2454: 0x3bc6, 0x2455: 0x3bca, 0x2456: 0x3bce, 0x2457: 0x3bd2, + 0x2458: 0x3bd6, 0x2459: 0x3bda, 0x245a: 0x3bde, 0x245b: 0x3be2, 0x245c: 0x3be6, 0x245d: 0x3bea, + 0x245e: 0x3bee, 0x245f: 0x3bf2, 0x2460: 0x3bf6, 0x2461: 0x3bfa, 0x2462: 0x3bfe, 0x2463: 0x3c02, + 0x2464: 0x3c06, 0x2465: 0x3c0a, 0x2466: 0x3c0e, 0x2467: 0x3c12, 0x2468: 0x3c16, 0x2469: 0x3c1a, + 0x246a: 0x3c1e, 0x246b: 0x14ad, 0x246c: 0x092b, 0x246d: 0x3c26, 0x246e: 0x3c29, + 0x2470: 0x0906, 0x2471: 0x090b, 0x2472: 0x14ad, 0x2473: 0x090d, 0x2474: 0x090f, 0x2475: 0x14d9, + 0x2476: 0x0914, 0x2477: 0x0916, 0x2478: 0x0918, 0x2479: 0x091a, 0x247a: 0x091c, 0x247b: 0x091e, + 0x247c: 0x0920, 0x247d: 0x0922, 0x247e: 0x0924, 0x247f: 0x0929, + // Block 0x92, offset 0x2480 + 0x2480: 0x14c8, 0x2481: 0x092b, 0x2482: 0x17f6, 0x2483: 0x092d, 0x2484: 0x092f, 0x2485: 0x155f, + 0x2486: 0x0931, 0x2487: 0x1570, 0x2488: 0x17f8, 0x2489: 0x14d4, 0x248a: 0x3c2c, 0x248b: 0x293d, + 0x248c: 0x3c2f, 0x248d: 0x3c32, 0x248e: 0x3c35, 0x248f: 0x3c39, + // Block 0x93, offset 0x24c0 + 0x24d0: 0x3c3c, + // Block 0x94, offset 0x2500 + 0x2500: 0x3c3f, 0x2501: 0x3c46, 0x2502: 0x2291, + 0x2510: 0x1922, 0x2511: 0x3c4d, + 0x2512: 0x3c51, 0x2513: 0x1cb3, 0x2514: 0x183e, 0x2515: 0x3c55, 0x2516: 0x3c59, 0x2517: 0x1ed0, + 0x2518: 0x3c5d, 0x2519: 0x3c61, 0x251a: 0x3c65, 0x251b: 0x2d49, 0x251c: 0x3c69, 0x251d: 0x3c6d, + 0x251e: 0x3c71, 0x251f: 0x3c75, 0x2520: 0x3c79, 0x2521: 0x3c7d, 0x2522: 0x19b2, 0x2523: 0x3c81, + 0x2524: 0x3c85, 0x2525: 0x3c89, 0x2526: 0x3c8d, 0x2527: 0x3c91, 0x2528: 0x3c95, 0x2529: 0x1826, + 0x252a: 0x1eb0, 0x252b: 0x3c99, 0x252c: 0x21c7, 0x252d: 0x1ebc, 0x252e: 0x21cb, 0x252f: 0x3c9d, + 0x2530: 0x1a92, 0x2531: 0x3ca1, 0x2532: 0x3ca5, 0x2533: 0x3ca9, 0x2534: 0x3cad, 0x2535: 0x3cb1, + 0x2536: 0x2183, 0x2537: 0x194a, 0x2538: 0x3cb5, 0x2539: 0x3cb9, 0x253a: 0x3cbd, + // Block 0x95, offset 0x2540 + 0x2540: 0x3cc1, 0x2541: 0x3ccb, 0x2542: 0x3cd5, 0x2543: 0x3cdf, 0x2544: 0x3ce9, 0x2545: 0x3cf3, + 0x2546: 0x3cfd, 0x2547: 0x3d07, 0x2548: 0x3d11, + 0x2550: 0x3d1b, 0x2551: 0x3d1f, + // Block 0x96, offset 0x2580 + 0x2580: 0x3d23, 0x2581: 0x3d27, 0x2582: 0x3d2b, 0x2583: 0x3d2f, 0x2584: 0x3d34, 0x2585: 0x2eb5, + 0x2586: 0x3d38, 0x2587: 0x3d3c, 0x2588: 0x3d40, 0x2589: 0x3d44, 0x258a: 0x2eb9, 0x258b: 0x3d48, + 0x258c: 0x3d4c, 0x258d: 0x3d50, 0x258e: 0x2ebd, 0x258f: 0x3d55, 0x2590: 0x3d59, 0x2591: 0x3d5d, + 0x2592: 0x3d61, 0x2593: 0x3d66, 0x2594: 0x3d6a, 0x2595: 0x3c71, 0x2596: 0x3d6e, 0x2597: 0x3d73, + 0x2598: 0x3d77, 0x2599: 0x3d7b, 0x259a: 0x3d7f, 0x259b: 0x2f9a, 0x259c: 0x3d83, 0x259d: 0x1866, + 0x259e: 0x3d88, 0x259f: 0x3d8c, 0x25a0: 0x3d90, 0x25a1: 0x3d94, 0x25a2: 0x3cb9, 0x25a3: 0x3d98, + 0x25a4: 0x3d9c, 0x25a5: 0x2fae, 0x25a6: 0x2ec1, 0x25a7: 0x2ec5, 0x25a8: 0x2fb2, 0x25a9: 0x3da0, + 0x25aa: 0x3da4, 0x25ab: 0x2bf1, 0x25ac: 0x3da8, 0x25ad: 0x2ec9, 0x25ae: 0x3dac, 0x25af: 0x3db0, + 0x25b0: 0x3db4, 0x25b1: 0x3db8, 0x25b2: 0x3db8, 0x25b3: 0x3db8, 0x25b4: 0x3dbc, 0x25b5: 0x3dc1, + 0x25b6: 0x3dc5, 0x25b7: 0x3dc9, 0x25b8: 0x3dcd, 0x25b9: 0x3dd2, 0x25ba: 0x3dd6, 0x25bb: 0x3dda, + 0x25bc: 0x3dde, 0x25bd: 0x3de2, 0x25be: 0x3de6, 0x25bf: 0x3dea, + // Block 0x97, offset 0x25c0 + 0x25c0: 0x3dee, 0x25c1: 0x3df2, 0x25c2: 0x3df6, 0x25c3: 0x3dfa, 0x25c4: 0x3dfe, 0x25c5: 0x3e02, + 0x25c6: 0x3e02, 0x25c7: 0x2fba, 0x25c8: 0x3e06, 0x25c9: 0x3e0a, 0x25ca: 0x3e0e, 0x25cb: 0x3e12, + 0x25cc: 0x2ed1, 0x25cd: 0x3e16, 0x25ce: 0x3e1a, 0x25cf: 0x3e1e, 0x25d0: 0x2e39, 0x25d1: 0x3e22, + 0x25d2: 0x3e26, 0x25d3: 0x3e2a, 0x25d4: 0x3e2e, 0x25d5: 0x3e32, 0x25d6: 0x3e36, 0x25d7: 0x3e3a, + 0x25d8: 0x3e3e, 0x25d9: 0x3e42, 0x25da: 0x3e47, 0x25db: 0x3e4b, 0x25dc: 0x3e4f, 0x25dd: 0x3c55, + 0x25de: 0x3e53, 0x25df: 0x3e57, 0x25e0: 0x3e5b, 0x25e1: 0x3e60, 0x25e2: 0x3e65, 0x25e3: 0x3e69, + 0x25e4: 0x3e6d, 0x25e5: 0x3e71, 0x25e6: 0x3e75, 0x25e7: 0x3e79, 0x25e8: 0x3e7d, 0x25e9: 0x3e81, + 0x25ea: 0x3e85, 0x25eb: 0x3e85, 0x25ec: 0x3e89, 0x25ed: 0x3e8e, 0x25ee: 0x3e92, 0x25ef: 0x2be1, + 0x25f0: 0x3e96, 0x25f1: 0x3e9a, 0x25f2: 0x3e9f, 0x25f3: 0x3ea3, 0x25f4: 0x3ea7, 0x25f5: 0x18ce, + 0x25f6: 0x3eab, 0x25f7: 0x3eaf, 0x25f8: 0x18d6, 0x25f9: 0x3eb3, 0x25fa: 0x3eb7, 0x25fb: 0x3ebb, + 0x25fc: 0x3ec0, 0x25fd: 0x3ec4, 0x25fe: 0x3ec9, 0x25ff: 0x3ecd, + // Block 0x98, offset 0x2600 + 0x2600: 0x3ed1, 0x2601: 0x3ed5, 0x2602: 0x3ed9, 0x2603: 0x3edd, 0x2604: 0x3ee1, 0x2605: 0x3ee5, + 0x2606: 0x3ee9, 0x2607: 0x3eed, 0x2608: 0x3ef1, 0x2609: 0x3ef5, 0x260a: 0x3efa, 0x260b: 0x3efe, + 0x260c: 0x3f02, 0x260d: 0x3f06, 0x260e: 0x2b11, 0x260f: 0x3f0a, 0x2610: 0x18fe, 0x2611: 0x3f0f, + 0x2612: 0x3f0f, 0x2613: 0x3f14, 0x2614: 0x3f18, 0x2615: 0x3f18, 0x2616: 0x3f1c, 0x2617: 0x3f20, + 0x2618: 0x3f25, 0x2619: 0x3f2a, 0x261a: 0x3f2e, 0x261b: 0x3f32, 0x261c: 0x3f36, 0x261d: 0x3f3a, + 0x261e: 0x3f3e, 0x261f: 0x3f42, 0x2620: 0x3f46, 0x2621: 0x3f4a, 0x2622: 0x3f4e, 0x2623: 0x2ee5, + 0x2624: 0x3f52, 0x2625: 0x3f57, 0x2626: 0x3f5b, 0x2627: 0x3f5f, 0x2628: 0x2fea, 0x2629: 0x3f5f, + 0x262a: 0x3f63, 0x262b: 0x2eed, 0x262c: 0x3f67, 0x262d: 0x3f6b, 0x262e: 0x3f6f, 0x262f: 0x3f73, + 0x2630: 0x2ef1, 0x2631: 0x2aa5, 0x2632: 0x3f77, 0x2633: 0x3f7b, 0x2634: 0x3f7f, 0x2635: 0x3f83, + 0x2636: 0x3f87, 0x2637: 0x3f8b, 0x2638: 0x3f8f, 0x2639: 0x3f94, 0x263a: 0x3f98, 0x263b: 0x3f9c, + 0x263c: 0x3fa0, 0x263d: 0x3fa4, 0x263e: 0x3fa8, 0x263f: 0x3fad, + // Block 0x99, offset 0x2640 + 0x2640: 0x3fb1, 0x2641: 0x3fb5, 0x2642: 0x3fb9, 0x2643: 0x3fbd, 0x2644: 0x3fc1, 0x2645: 0x3fc5, + 0x2646: 0x3fc9, 0x2647: 0x3fcd, 0x2648: 0x2ef5, 0x2649: 0x3fd1, 0x264a: 0x3fd5, 0x264b: 0x3fda, + 0x264c: 0x3fde, 0x264d: 0x3fe2, 0x264e: 0x3fe6, 0x264f: 0x2efd, 0x2650: 0x3fea, 0x2651: 0x3fee, + 0x2652: 0x3ff2, 0x2653: 0x3ff6, 0x2654: 0x3ffa, 0x2655: 0x3ffe, 0x2656: 0x4002, 0x2657: 0x4006, + 0x2658: 0x2b15, 0x2659: 0x300a, 0x265a: 0x400a, 0x265b: 0x400e, 0x265c: 0x4012, 0x265d: 0x4016, + 0x265e: 0x401b, 0x265f: 0x401f, 0x2660: 0x4023, 0x2661: 0x4027, 0x2662: 0x2f01, 0x2663: 0x402b, + 0x2664: 0x4030, 0x2665: 0x4034, 0x2666: 0x4038, 0x2667: 0x30b5, 0x2668: 0x403c, 0x2669: 0x4040, + 0x266a: 0x4044, 0x266b: 0x4048, 0x266c: 0x404c, 0x266d: 0x4051, 0x266e: 0x4055, 0x266f: 0x4059, + 0x2670: 0x405d, 0x2671: 0x4062, 0x2672: 0x4066, 0x2673: 0x406a, 0x2674: 0x406e, 0x2675: 0x2c25, + 0x2676: 0x4072, 0x2677: 0x4076, 0x2678: 0x407b, 0x2679: 0x4080, 0x267a: 0x4085, 0x267b: 0x4089, + 0x267c: 0x408e, 0x267d: 0x4092, 0x267e: 0x4096, 0x267f: 0x409a, + // Block 0x9a, offset 0x2680 + 0x2680: 0x409e, 0x2681: 0x2f05, 0x2682: 0x2d71, 0x2683: 0x40a2, 0x2684: 0x40a6, 0x2685: 0x40aa, + 0x2686: 0x40ae, 0x2687: 0x40b3, 0x2688: 0x40b7, 0x2689: 0x40bb, 0x268a: 0x40bf, 0x268b: 0x3016, + 0x268c: 0x40c3, 0x268d: 0x40c7, 0x268e: 0x40cc, 0x268f: 0x40d0, 0x2690: 0x40d4, 0x2691: 0x40d9, + 0x2692: 0x40de, 0x2693: 0x40e2, 0x2694: 0x301a, 0x2695: 0x40e6, 0x2696: 0x40ea, 0x2697: 0x40ee, + 0x2698: 0x40f2, 0x2699: 0x40f6, 0x269a: 0x40fa, 0x269b: 0x40fe, 0x269c: 0x4103, 0x269d: 0x4107, + 0x269e: 0x410c, 0x269f: 0x4110, 0x26a0: 0x4115, 0x26a1: 0x3022, 0x26a2: 0x4119, 0x26a3: 0x411d, + 0x26a4: 0x4122, 0x26a5: 0x4126, 0x26a6: 0x412a, 0x26a7: 0x412f, 0x26a8: 0x4134, 0x26a9: 0x4138, + 0x26aa: 0x413c, 0x26ab: 0x4140, 0x26ac: 0x4144, 0x26ad: 0x4144, 0x26ae: 0x4148, 0x26af: 0x414c, + 0x26b0: 0x302a, 0x26b1: 0x4150, 0x26b2: 0x4154, 0x26b3: 0x4158, 0x26b4: 0x415c, 0x26b5: 0x4160, + 0x26b6: 0x4165, 0x26b7: 0x4169, 0x26b8: 0x2bed, 0x26b9: 0x416e, 0x26ba: 0x4173, 0x26bb: 0x4177, + 0x26bc: 0x417c, 0x26bd: 0x4181, 0x26be: 0x4186, 0x26bf: 0x418a, + // Block 0x9b, offset 0x26c0 + 0x26c0: 0x3042, 0x26c1: 0x418e, 0x26c2: 0x4193, 0x26c3: 0x4198, 0x26c4: 0x419d, 0x26c5: 0x41a2, + 0x26c6: 0x41a6, 0x26c7: 0x41a6, 0x26c8: 0x3046, 0x26c9: 0x30bd, 0x26ca: 0x41aa, 0x26cb: 0x41ae, + 0x26cc: 0x41b2, 0x26cd: 0x41b6, 0x26ce: 0x41bb, 0x26cf: 0x2b59, 0x26d0: 0x304e, 0x26d1: 0x41bf, + 0x26d2: 0x41c3, 0x26d3: 0x2f2d, 0x26d4: 0x41c8, 0x26d5: 0x41cd, 0x26d6: 0x2e89, 0x26d7: 0x41d2, + 0x26d8: 0x41d6, 0x26d9: 0x2f39, 0x26da: 0x41da, 0x26db: 0x41de, 0x26dc: 0x41e2, 0x26dd: 0x41e7, + 0x26de: 0x41e7, 0x26df: 0x41ec, 0x26e0: 0x41f0, 0x26e1: 0x41f4, 0x26e2: 0x41f9, 0x26e3: 0x41fd, + 0x26e4: 0x4201, 0x26e5: 0x4205, 0x26e6: 0x420a, 0x26e7: 0x420e, 0x26e8: 0x4212, 0x26e9: 0x4216, + 0x26ea: 0x421a, 0x26eb: 0x421e, 0x26ec: 0x4223, 0x26ed: 0x4227, 0x26ee: 0x422b, 0x26ef: 0x422f, + 0x26f0: 0x4233, 0x26f1: 0x4237, 0x26f2: 0x423b, 0x26f3: 0x4240, 0x26f4: 0x4245, 0x26f5: 0x4249, + 0x26f6: 0x424e, 0x26f7: 0x4252, 0x26f8: 0x4257, 0x26f9: 0x425b, 0x26fa: 0x2f51, 0x26fb: 0x425f, + 0x26fc: 0x4264, 0x26fd: 0x4269, 0x26fe: 0x426d, 0x26ff: 0x4272, + // Block 0x9c, offset 0x2700 + 0x2700: 0x4276, 0x2701: 0x427b, 0x2702: 0x427f, 0x2703: 0x4283, 0x2704: 0x4287, 0x2705: 0x428b, + 0x2706: 0x428f, 0x2707: 0x4293, 0x2708: 0x4298, 0x2709: 0x429d, 0x270a: 0x42a2, 0x270b: 0x3f14, + 0x270c: 0x42a7, 0x270d: 0x42ab, 0x270e: 0x42af, 0x270f: 0x42b3, 0x2710: 0x42b7, 0x2711: 0x42bb, + 0x2712: 0x42bf, 0x2713: 0x42c3, 0x2714: 0x42c7, 0x2715: 0x42cb, 0x2716: 0x42cf, 0x2717: 0x42d3, + 0x2718: 0x2c31, 0x2719: 0x42d8, 0x271a: 0x42dc, 0x271b: 0x42e0, 0x271c: 0x42e4, 0x271d: 0x42e8, + 0x271e: 0x42ec, 0x271f: 0x2f5d, 0x2720: 0x42f0, 0x2721: 0x42f4, 0x2722: 0x42f8, 0x2723: 0x42fc, + 0x2724: 0x4300, 0x2725: 0x4305, 0x2726: 0x430a, 0x2727: 0x430f, 0x2728: 0x4313, 0x2729: 0x4317, + 0x272a: 0x431b, 0x272b: 0x431f, 0x272c: 0x4324, 0x272d: 0x4328, 0x272e: 0x432d, 0x272f: 0x4331, + 0x2730: 0x4335, 0x2731: 0x433a, 0x2732: 0x433f, 0x2733: 0x4343, 0x2734: 0x2b45, 0x2735: 0x4347, + 0x2736: 0x434b, 0x2737: 0x434f, 0x2738: 0x4353, 0x2739: 0x4357, 0x273a: 0x435b, 0x273b: 0x306a, + 0x273c: 0x435f, 0x273d: 0x4363, 0x273e: 0x4367, 0x273f: 0x436b, + // Block 0x9d, offset 0x2740 + 0x2740: 0x436f, 0x2741: 0x4373, 0x2742: 0x4377, 0x2743: 0x437b, 0x2744: 0x1a66, 0x2745: 0x437f, + 0x2746: 0x4384, 0x2747: 0x4388, 0x2748: 0x438c, 0x2749: 0x4390, 0x274a: 0x4394, 0x274b: 0x4398, + 0x274c: 0x439d, 0x274d: 0x43a2, 0x274e: 0x43a6, 0x274f: 0x43aa, 0x2750: 0x307e, 0x2751: 0x3082, + 0x2752: 0x1a82, 0x2753: 0x43ae, 0x2754: 0x43b3, 0x2755: 0x43b7, 0x2756: 0x43bb, 0x2757: 0x43bf, + 0x2758: 0x43c3, 0x2759: 0x43c8, 0x275a: 0x43cd, 0x275b: 0x43d1, 0x275c: 0x43d5, 0x275d: 0x43d9, + 0x275e: 0x43de, 0x275f: 0x3086, 0x2760: 0x43e2, 0x2761: 0x43e7, 0x2762: 0x43ec, 0x2763: 0x43f0, + 0x2764: 0x43f4, 0x2765: 0x43f8, 0x2766: 0x43fd, 0x2767: 0x4401, 0x2768: 0x4405, 0x2769: 0x4409, + 0x276a: 0x440d, 0x276b: 0x4411, 0x276c: 0x4415, 0x276d: 0x4419, 0x276e: 0x441e, 0x276f: 0x4422, + 0x2770: 0x4426, 0x2771: 0x442a, 0x2772: 0x442f, 0x2773: 0x4433, 0x2774: 0x4437, 0x2775: 0x443b, + 0x2776: 0x443f, 0x2777: 0x4444, 0x2778: 0x4449, 0x2779: 0x444d, 0x277a: 0x4451, 0x277b: 0x4455, + 0x277c: 0x445a, 0x277d: 0x445e, 0x277e: 0x309e, 0x277f: 0x309e, + // Block 0x9e, offset 0x2780 + 0x2780: 0x4463, 0x2781: 0x4467, 0x2782: 0x446c, 0x2783: 0x4470, 0x2784: 0x4474, 0x2785: 0x4478, + 0x2786: 0x447c, 0x2787: 0x4480, 0x2788: 0x4484, 0x2789: 0x4488, 0x278a: 0x30a2, 0x278b: 0x448d, + 0x278c: 0x4491, 0x278d: 0x4495, 0x278e: 0x4499, 0x278f: 0x449d, 0x2790: 0x44a1, 0x2791: 0x44a6, + 0x2792: 0x44aa, 0x2793: 0x44af, 0x2794: 0x44b4, 0x2795: 0x1b42, 0x2796: 0x44b9, 0x2797: 0x1b52, + 0x2798: 0x44bd, 0x2799: 0x44c1, 0x279a: 0x44c5, 0x279b: 0x44c9, 0x279c: 0x1b66, 0x279d: 0x44cd, +} + +// nfkcDecompLookup: 960 bytes +// Block 0 is the null block. +var nfkcDecompLookup = [960]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c2: 0x03, 0x0c3: 0x04, 0x0c4: 0x05, 0x0c5: 0x06, 0x0c6: 0x07, 0x0c7: 0x08, + 0x0c8: 0x09, 0x0ca: 0x0a, 0x0cb: 0x0b, 0x0cd: 0x0c, 0x0ce: 0x0d, 0x0cf: 0x0e, + 0x0d0: 0x0f, 0x0d1: 0x10, 0x0d3: 0x11, 0x0d6: 0x12, + 0x0d8: 0x13, 0x0d9: 0x14, 0x0db: 0x15, + 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, + 0x0ea: 0x08, 0x0ef: 0x09, + 0x0f0: 0x0e, + // Block 0x4, offset 0x100 + 0x124: 0x16, 0x125: 0x17, 0x127: 0x18, + 0x128: 0x19, 0x129: 0x1a, 0x12d: 0x1b, 0x12e: 0x1c, 0x12f: 0x1d, + 0x131: 0x1e, 0x133: 0x1f, 0x135: 0x20, 0x137: 0x21, + 0x138: 0x22, 0x13a: 0x23, 0x13b: 0x24, 0x13c: 0x25, 0x13d: 0x26, 0x13e: 0x27, + // Block 0x5, offset 0x140 + 0x140: 0x28, 0x143: 0x29, + 0x16c: 0x2a, 0x16d: 0x2b, + 0x174: 0x2c, 0x175: 0x2d, 0x176: 0x2e, + 0x178: 0x2f, 0x179: 0x30, 0x17a: 0x31, 0x17b: 0x32, 0x17c: 0x33, 0x17d: 0x34, 0x17e: 0x35, 0x17f: 0x36, + // Block 0x6, offset 0x180 + 0x180: 0x37, 0x181: 0x38, 0x182: 0x39, 0x184: 0x3a, 0x185: 0x3b, 0x186: 0x3c, 0x187: 0x3d, + 0x188: 0x3e, 0x189: 0x3f, 0x18a: 0x40, 0x18b: 0x41, 0x18c: 0x42, + 0x191: 0x43, 0x192: 0x44, 0x193: 0x45, + 0x1a8: 0x46, 0x1a9: 0x47, 0x1ab: 0x48, + 0x1b1: 0x49, 0x1b5: 0x4a, + 0x1ba: 0x4b, 0x1bb: 0x4c, 0x1bc: 0x4d, 0x1bd: 0x4e, 0x1be: 0x4f, 0x1bf: 0x50, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x51, 0x1c1: 0x52, 0x1c2: 0x53, 0x1c3: 0x54, 0x1c4: 0x55, 0x1c5: 0x56, 0x1c6: 0x57, + 0x1c8: 0x58, 0x1c9: 0x59, 0x1ca: 0x5a, 0x1cb: 0x5b, 0x1cc: 0x5c, 0x1cd: 0x5d, 0x1ce: 0x5e, 0x1cf: 0x5f, + // Block 0x8, offset 0x200 + 0x21d: 0x60, + // Block 0x9, offset 0x240 + 0x264: 0x61, 0x265: 0x62, 0x266: 0x63, 0x267: 0x64, + 0x268: 0x65, 0x269: 0x66, 0x26a: 0x67, 0x26b: 0x68, 0x26c: 0x69, 0x26d: 0x6a, 0x26e: 0x6b, 0x26f: 0x6c, + 0x270: 0x6d, 0x271: 0x6e, 0x272: 0x6f, 0x273: 0x70, 0x274: 0x71, 0x275: 0x72, 0x276: 0x73, 0x277: 0x74, + 0x278: 0x75, 0x279: 0x76, 0x27a: 0x77, 0x27b: 0x78, 0x27c: 0x79, 0x27d: 0x7a, 0x27e: 0x7b, 0x27f: 0x7c, + // Block 0xa, offset 0x280 + 0x282: 0x7d, + // Block 0xb, offset 0x2c0 + 0x2c5: 0x7e, 0x2c6: 0x7f, 0x2c7: 0x80, + 0x2d0: 0x81, 0x2d1: 0x82, 0x2d2: 0x83, 0x2d3: 0x84, 0x2d4: 0x85, 0x2d5: 0x86, 0x2d6: 0x87, 0x2d7: 0x88, + 0x2d8: 0x89, 0x2d9: 0x8a, 0x2da: 0x8b, 0x2db: 0x8c, 0x2dc: 0x8d, 0x2dd: 0x8e, 0x2de: 0x8f, 0x2df: 0x90, + // Block 0xc, offset 0x300 + 0x304: 0x91, 0x305: 0x92, 0x306: 0x93, + 0x308: 0x94, 0x309: 0x95, + // Block 0xd, offset 0x340 + 0x360: 0x96, 0x361: 0x97, 0x362: 0x98, 0x363: 0x99, 0x364: 0x9a, 0x365: 0x9b, 0x366: 0x9c, 0x367: 0x9d, + 0x368: 0x9e, + // Block 0xe, offset 0x380 + 0x391: 0x0a, + 0x39d: 0x0b, 0x39f: 0x0c, + 0x3af: 0x0d, +} + +var nfkcDecompTrie = trie{nfkcDecompLookup[:], nfkcDecompValues[:]} + +// recompMap: 7448 bytes (entries only) +var recompMap = map[uint32]uint32{ + 0x00410300: 0x00C0, + 0x00410301: 0x00C1, + 0x00410302: 0x00C2, + 0x00410303: 0x00C3, + 0x00410308: 0x00C4, + 0x0041030A: 0x00C5, + 0x00430327: 0x00C7, + 0x00450300: 0x00C8, + 0x00450301: 0x00C9, + 0x00450302: 0x00CA, + 0x00450308: 0x00CB, + 0x00490300: 0x00CC, + 0x00490301: 0x00CD, + 0x00490302: 0x00CE, + 0x00490308: 0x00CF, + 0x004E0303: 0x00D1, + 0x004F0300: 0x00D2, + 0x004F0301: 0x00D3, + 0x004F0302: 0x00D4, + 0x004F0303: 0x00D5, + 0x004F0308: 0x00D6, + 0x00550300: 0x00D9, + 0x00550301: 0x00DA, + 0x00550302: 0x00DB, + 0x00550308: 0x00DC, + 0x00590301: 0x00DD, + 0x00610300: 0x00E0, + 0x00610301: 0x00E1, + 0x00610302: 0x00E2, + 0x00610303: 0x00E3, + 0x00610308: 0x00E4, + 0x0061030A: 0x00E5, + 0x00630327: 0x00E7, + 0x00650300: 0x00E8, + 0x00650301: 0x00E9, + 0x00650302: 0x00EA, + 0x00650308: 0x00EB, + 0x00690300: 0x00EC, + 0x00690301: 0x00ED, + 0x00690302: 0x00EE, + 0x00690308: 0x00EF, + 0x006E0303: 0x00F1, + 0x006F0300: 0x00F2, + 0x006F0301: 0x00F3, + 0x006F0302: 0x00F4, + 0x006F0303: 0x00F5, + 0x006F0308: 0x00F6, + 0x00750300: 0x00F9, + 0x00750301: 0x00FA, + 0x00750302: 0x00FB, + 0x00750308: 0x00FC, + 0x00790301: 0x00FD, + 0x00790308: 0x00FF, + 0x00410304: 0x0100, + 0x00610304: 0x0101, + 0x00410306: 0x0102, + 0x00610306: 0x0103, + 0x00410328: 0x0104, + 0x00610328: 0x0105, + 0x00430301: 0x0106, + 0x00630301: 0x0107, + 0x00430302: 0x0108, + 0x00630302: 0x0109, + 0x00430307: 0x010A, + 0x00630307: 0x010B, + 0x0043030C: 0x010C, + 0x0063030C: 0x010D, + 0x0044030C: 0x010E, + 0x0064030C: 0x010F, + 0x00450304: 0x0112, + 0x00650304: 0x0113, + 0x00450306: 0x0114, + 0x00650306: 0x0115, + 0x00450307: 0x0116, + 0x00650307: 0x0117, + 0x00450328: 0x0118, + 0x00650328: 0x0119, + 0x0045030C: 0x011A, + 0x0065030C: 0x011B, + 0x00470302: 0x011C, + 0x00670302: 0x011D, + 0x00470306: 0x011E, + 0x00670306: 0x011F, + 0x00470307: 0x0120, + 0x00670307: 0x0121, + 0x00470327: 0x0122, + 0x00670327: 0x0123, + 0x00480302: 0x0124, + 0x00680302: 0x0125, + 0x00490303: 0x0128, + 0x00690303: 0x0129, + 0x00490304: 0x012A, + 0x00690304: 0x012B, + 0x00490306: 0x012C, + 0x00690306: 0x012D, + 0x00490328: 0x012E, + 0x00690328: 0x012F, + 0x00490307: 0x0130, + 0x004A0302: 0x0134, + 0x006A0302: 0x0135, + 0x004B0327: 0x0136, + 0x006B0327: 0x0137, + 0x004C0301: 0x0139, + 0x006C0301: 0x013A, + 0x004C0327: 0x013B, + 0x006C0327: 0x013C, + 0x004C030C: 0x013D, + 0x006C030C: 0x013E, + 0x004E0301: 0x0143, + 0x006E0301: 0x0144, + 0x004E0327: 0x0145, + 0x006E0327: 0x0146, + 0x004E030C: 0x0147, + 0x006E030C: 0x0148, + 0x004F0304: 0x014C, + 0x006F0304: 0x014D, + 0x004F0306: 0x014E, + 0x006F0306: 0x014F, + 0x004F030B: 0x0150, + 0x006F030B: 0x0151, + 0x00520301: 0x0154, + 0x00720301: 0x0155, + 0x00520327: 0x0156, + 0x00720327: 0x0157, + 0x0052030C: 0x0158, + 0x0072030C: 0x0159, + 0x00530301: 0x015A, + 0x00730301: 0x015B, + 0x00530302: 0x015C, + 0x00730302: 0x015D, + 0x00530327: 0x015E, + 0x00730327: 0x015F, + 0x0053030C: 0x0160, + 0x0073030C: 0x0161, + 0x00540327: 0x0162, + 0x00740327: 0x0163, + 0x0054030C: 0x0164, + 0x0074030C: 0x0165, + 0x00550303: 0x0168, + 0x00750303: 0x0169, + 0x00550304: 0x016A, + 0x00750304: 0x016B, + 0x00550306: 0x016C, + 0x00750306: 0x016D, + 0x0055030A: 0x016E, + 0x0075030A: 0x016F, + 0x0055030B: 0x0170, + 0x0075030B: 0x0171, + 0x00550328: 0x0172, + 0x00750328: 0x0173, + 0x00570302: 0x0174, + 0x00770302: 0x0175, + 0x00590302: 0x0176, + 0x00790302: 0x0177, + 0x00590308: 0x0178, + 0x005A0301: 0x0179, + 0x007A0301: 0x017A, + 0x005A0307: 0x017B, + 0x007A0307: 0x017C, + 0x005A030C: 0x017D, + 0x007A030C: 0x017E, + 0x004F031B: 0x01A0, + 0x006F031B: 0x01A1, + 0x0055031B: 0x01AF, + 0x0075031B: 0x01B0, + 0x0041030C: 0x01CD, + 0x0061030C: 0x01CE, + 0x0049030C: 0x01CF, + 0x0069030C: 0x01D0, + 0x004F030C: 0x01D1, + 0x006F030C: 0x01D2, + 0x0055030C: 0x01D3, + 0x0075030C: 0x01D4, + 0x00DC0304: 0x01D5, + 0x00FC0304: 0x01D6, + 0x00DC0301: 0x01D7, + 0x00FC0301: 0x01D8, + 0x00DC030C: 0x01D9, + 0x00FC030C: 0x01DA, + 0x00DC0300: 0x01DB, + 0x00FC0300: 0x01DC, + 0x00C40304: 0x01DE, + 0x00E40304: 0x01DF, + 0x02260304: 0x01E0, + 0x02270304: 0x01E1, + 0x00C60304: 0x01E2, + 0x00E60304: 0x01E3, + 0x0047030C: 0x01E6, + 0x0067030C: 0x01E7, + 0x004B030C: 0x01E8, + 0x006B030C: 0x01E9, + 0x004F0328: 0x01EA, + 0x006F0328: 0x01EB, + 0x01EA0304: 0x01EC, + 0x01EB0304: 0x01ED, + 0x01B7030C: 0x01EE, + 0x0292030C: 0x01EF, + 0x006A030C: 0x01F0, + 0x00470301: 0x01F4, + 0x00670301: 0x01F5, + 0x004E0300: 0x01F8, + 0x006E0300: 0x01F9, + 0x00C50301: 0x01FA, + 0x00E50301: 0x01FB, + 0x00C60301: 0x01FC, + 0x00E60301: 0x01FD, + 0x00D80301: 0x01FE, + 0x00F80301: 0x01FF, + 0x0041030F: 0x0200, + 0x0061030F: 0x0201, + 0x00410311: 0x0202, + 0x00610311: 0x0203, + 0x0045030F: 0x0204, + 0x0065030F: 0x0205, + 0x00450311: 0x0206, + 0x00650311: 0x0207, + 0x0049030F: 0x0208, + 0x0069030F: 0x0209, + 0x00490311: 0x020A, + 0x00690311: 0x020B, + 0x004F030F: 0x020C, + 0x006F030F: 0x020D, + 0x004F0311: 0x020E, + 0x006F0311: 0x020F, + 0x0052030F: 0x0210, + 0x0072030F: 0x0211, + 0x00520311: 0x0212, + 0x00720311: 0x0213, + 0x0055030F: 0x0214, + 0x0075030F: 0x0215, + 0x00550311: 0x0216, + 0x00750311: 0x0217, + 0x00530326: 0x0218, + 0x00730326: 0x0219, + 0x00540326: 0x021A, + 0x00740326: 0x021B, + 0x0048030C: 0x021E, + 0x0068030C: 0x021F, + 0x00410307: 0x0226, + 0x00610307: 0x0227, + 0x00450327: 0x0228, + 0x00650327: 0x0229, + 0x00D60304: 0x022A, + 0x00F60304: 0x022B, + 0x00D50304: 0x022C, + 0x00F50304: 0x022D, + 0x004F0307: 0x022E, + 0x006F0307: 0x022F, + 0x022E0304: 0x0230, + 0x022F0304: 0x0231, + 0x00590304: 0x0232, + 0x00790304: 0x0233, + 0x00A80301: 0x0385, + 0x03910301: 0x0386, + 0x03950301: 0x0388, + 0x03970301: 0x0389, + 0x03990301: 0x038A, + 0x039F0301: 0x038C, + 0x03A50301: 0x038E, + 0x03A90301: 0x038F, + 0x03CA0301: 0x0390, + 0x03990308: 0x03AA, + 0x03A50308: 0x03AB, + 0x03B10301: 0x03AC, + 0x03B50301: 0x03AD, + 0x03B70301: 0x03AE, + 0x03B90301: 0x03AF, + 0x03CB0301: 0x03B0, + 0x03B90308: 0x03CA, + 0x03C50308: 0x03CB, + 0x03BF0301: 0x03CC, + 0x03C50301: 0x03CD, + 0x03C90301: 0x03CE, + 0x03D20301: 0x03D3, + 0x03D20308: 0x03D4, + 0x04150300: 0x0400, + 0x04150308: 0x0401, + 0x04130301: 0x0403, + 0x04060308: 0x0407, + 0x041A0301: 0x040C, + 0x04180300: 0x040D, + 0x04230306: 0x040E, + 0x04180306: 0x0419, + 0x04380306: 0x0439, + 0x04350300: 0x0450, + 0x04350308: 0x0451, + 0x04330301: 0x0453, + 0x04560308: 0x0457, + 0x043A0301: 0x045C, + 0x04380300: 0x045D, + 0x04430306: 0x045E, + 0x0474030F: 0x0476, + 0x0475030F: 0x0477, + 0x04160306: 0x04C1, + 0x04360306: 0x04C2, + 0x04100306: 0x04D0, + 0x04300306: 0x04D1, + 0x04100308: 0x04D2, + 0x04300308: 0x04D3, + 0x04150306: 0x04D6, + 0x04350306: 0x04D7, + 0x04D80308: 0x04DA, + 0x04D90308: 0x04DB, + 0x04160308: 0x04DC, + 0x04360308: 0x04DD, + 0x04170308: 0x04DE, + 0x04370308: 0x04DF, + 0x04180304: 0x04E2, + 0x04380304: 0x04E3, + 0x04180308: 0x04E4, + 0x04380308: 0x04E5, + 0x041E0308: 0x04E6, + 0x043E0308: 0x04E7, + 0x04E80308: 0x04EA, + 0x04E90308: 0x04EB, + 0x042D0308: 0x04EC, + 0x044D0308: 0x04ED, + 0x04230304: 0x04EE, + 0x04430304: 0x04EF, + 0x04230308: 0x04F0, + 0x04430308: 0x04F1, + 0x0423030B: 0x04F2, + 0x0443030B: 0x04F3, + 0x04270308: 0x04F4, + 0x04470308: 0x04F5, + 0x042B0308: 0x04F8, + 0x044B0308: 0x04F9, + 0x06270653: 0x0622, + 0x06270654: 0x0623, + 0x06480654: 0x0624, + 0x06270655: 0x0625, + 0x064A0654: 0x0626, + 0x06D50654: 0x06C0, + 0x06C10654: 0x06C2, + 0x06D20654: 0x06D3, + 0x0928093C: 0x0929, + 0x0930093C: 0x0931, + 0x0933093C: 0x0934, + 0x09C709BE: 0x09CB, + 0x09C709D7: 0x09CC, + 0x0B470B56: 0x0B48, + 0x0B470B3E: 0x0B4B, + 0x0B470B57: 0x0B4C, + 0x0B920BD7: 0x0B94, + 0x0BC60BBE: 0x0BCA, + 0x0BC70BBE: 0x0BCB, + 0x0BC60BD7: 0x0BCC, + 0x0C460C56: 0x0C48, + 0x0CBF0CD5: 0x0CC0, + 0x0CC60CD5: 0x0CC7, + 0x0CC60CD6: 0x0CC8, + 0x0CC60CC2: 0x0CCA, + 0x0CCA0CD5: 0x0CCB, + 0x0D460D3E: 0x0D4A, + 0x0D470D3E: 0x0D4B, + 0x0D460D57: 0x0D4C, + 0x0DD90DCA: 0x0DDA, + 0x0DD90DCF: 0x0DDC, + 0x0DDC0DCA: 0x0DDD, + 0x0DD90DDF: 0x0DDE, + 0x1025102E: 0x1026, + 0x1B051B35: 0x1B06, + 0x1B071B35: 0x1B08, + 0x1B091B35: 0x1B0A, + 0x1B0B1B35: 0x1B0C, + 0x1B0D1B35: 0x1B0E, + 0x1B111B35: 0x1B12, + 0x1B3A1B35: 0x1B3B, + 0x1B3C1B35: 0x1B3D, + 0x1B3E1B35: 0x1B40, + 0x1B3F1B35: 0x1B41, + 0x1B421B35: 0x1B43, + 0x00410325: 0x1E00, + 0x00610325: 0x1E01, + 0x00420307: 0x1E02, + 0x00620307: 0x1E03, + 0x00420323: 0x1E04, + 0x00620323: 0x1E05, + 0x00420331: 0x1E06, + 0x00620331: 0x1E07, + 0x00C70301: 0x1E08, + 0x00E70301: 0x1E09, + 0x00440307: 0x1E0A, + 0x00640307: 0x1E0B, + 0x00440323: 0x1E0C, + 0x00640323: 0x1E0D, + 0x00440331: 0x1E0E, + 0x00640331: 0x1E0F, + 0x00440327: 0x1E10, + 0x00640327: 0x1E11, + 0x0044032D: 0x1E12, + 0x0064032D: 0x1E13, + 0x01120300: 0x1E14, + 0x01130300: 0x1E15, + 0x01120301: 0x1E16, + 0x01130301: 0x1E17, + 0x0045032D: 0x1E18, + 0x0065032D: 0x1E19, + 0x00450330: 0x1E1A, + 0x00650330: 0x1E1B, + 0x02280306: 0x1E1C, + 0x02290306: 0x1E1D, + 0x00460307: 0x1E1E, + 0x00660307: 0x1E1F, + 0x00470304: 0x1E20, + 0x00670304: 0x1E21, + 0x00480307: 0x1E22, + 0x00680307: 0x1E23, + 0x00480323: 0x1E24, + 0x00680323: 0x1E25, + 0x00480308: 0x1E26, + 0x00680308: 0x1E27, + 0x00480327: 0x1E28, + 0x00680327: 0x1E29, + 0x0048032E: 0x1E2A, + 0x0068032E: 0x1E2B, + 0x00490330: 0x1E2C, + 0x00690330: 0x1E2D, + 0x00CF0301: 0x1E2E, + 0x00EF0301: 0x1E2F, + 0x004B0301: 0x1E30, + 0x006B0301: 0x1E31, + 0x004B0323: 0x1E32, + 0x006B0323: 0x1E33, + 0x004B0331: 0x1E34, + 0x006B0331: 0x1E35, + 0x004C0323: 0x1E36, + 0x006C0323: 0x1E37, + 0x1E360304: 0x1E38, + 0x1E370304: 0x1E39, + 0x004C0331: 0x1E3A, + 0x006C0331: 0x1E3B, + 0x004C032D: 0x1E3C, + 0x006C032D: 0x1E3D, + 0x004D0301: 0x1E3E, + 0x006D0301: 0x1E3F, + 0x004D0307: 0x1E40, + 0x006D0307: 0x1E41, + 0x004D0323: 0x1E42, + 0x006D0323: 0x1E43, + 0x004E0307: 0x1E44, + 0x006E0307: 0x1E45, + 0x004E0323: 0x1E46, + 0x006E0323: 0x1E47, + 0x004E0331: 0x1E48, + 0x006E0331: 0x1E49, + 0x004E032D: 0x1E4A, + 0x006E032D: 0x1E4B, + 0x00D50301: 0x1E4C, + 0x00F50301: 0x1E4D, + 0x00D50308: 0x1E4E, + 0x00F50308: 0x1E4F, + 0x014C0300: 0x1E50, + 0x014D0300: 0x1E51, + 0x014C0301: 0x1E52, + 0x014D0301: 0x1E53, + 0x00500301: 0x1E54, + 0x00700301: 0x1E55, + 0x00500307: 0x1E56, + 0x00700307: 0x1E57, + 0x00520307: 0x1E58, + 0x00720307: 0x1E59, + 0x00520323: 0x1E5A, + 0x00720323: 0x1E5B, + 0x1E5A0304: 0x1E5C, + 0x1E5B0304: 0x1E5D, + 0x00520331: 0x1E5E, + 0x00720331: 0x1E5F, + 0x00530307: 0x1E60, + 0x00730307: 0x1E61, + 0x00530323: 0x1E62, + 0x00730323: 0x1E63, + 0x015A0307: 0x1E64, + 0x015B0307: 0x1E65, + 0x01600307: 0x1E66, + 0x01610307: 0x1E67, + 0x1E620307: 0x1E68, + 0x1E630307: 0x1E69, + 0x00540307: 0x1E6A, + 0x00740307: 0x1E6B, + 0x00540323: 0x1E6C, + 0x00740323: 0x1E6D, + 0x00540331: 0x1E6E, + 0x00740331: 0x1E6F, + 0x0054032D: 0x1E70, + 0x0074032D: 0x1E71, + 0x00550324: 0x1E72, + 0x00750324: 0x1E73, + 0x00550330: 0x1E74, + 0x00750330: 0x1E75, + 0x0055032D: 0x1E76, + 0x0075032D: 0x1E77, + 0x01680301: 0x1E78, + 0x01690301: 0x1E79, + 0x016A0308: 0x1E7A, + 0x016B0308: 0x1E7B, + 0x00560303: 0x1E7C, + 0x00760303: 0x1E7D, + 0x00560323: 0x1E7E, + 0x00760323: 0x1E7F, + 0x00570300: 0x1E80, + 0x00770300: 0x1E81, + 0x00570301: 0x1E82, + 0x00770301: 0x1E83, + 0x00570308: 0x1E84, + 0x00770308: 0x1E85, + 0x00570307: 0x1E86, + 0x00770307: 0x1E87, + 0x00570323: 0x1E88, + 0x00770323: 0x1E89, + 0x00580307: 0x1E8A, + 0x00780307: 0x1E8B, + 0x00580308: 0x1E8C, + 0x00780308: 0x1E8D, + 0x00590307: 0x1E8E, + 0x00790307: 0x1E8F, + 0x005A0302: 0x1E90, + 0x007A0302: 0x1E91, + 0x005A0323: 0x1E92, + 0x007A0323: 0x1E93, + 0x005A0331: 0x1E94, + 0x007A0331: 0x1E95, + 0x00680331: 0x1E96, + 0x00740308: 0x1E97, + 0x0077030A: 0x1E98, + 0x0079030A: 0x1E99, + 0x017F0307: 0x1E9B, + 0x00410323: 0x1EA0, + 0x00610323: 0x1EA1, + 0x00410309: 0x1EA2, + 0x00610309: 0x1EA3, + 0x00C20301: 0x1EA4, + 0x00E20301: 0x1EA5, + 0x00C20300: 0x1EA6, + 0x00E20300: 0x1EA7, + 0x00C20309: 0x1EA8, + 0x00E20309: 0x1EA9, + 0x00C20303: 0x1EAA, + 0x00E20303: 0x1EAB, + 0x1EA00302: 0x1EAC, + 0x1EA10302: 0x1EAD, + 0x01020301: 0x1EAE, + 0x01030301: 0x1EAF, + 0x01020300: 0x1EB0, + 0x01030300: 0x1EB1, + 0x01020309: 0x1EB2, + 0x01030309: 0x1EB3, + 0x01020303: 0x1EB4, + 0x01030303: 0x1EB5, + 0x1EA00306: 0x1EB6, + 0x1EA10306: 0x1EB7, + 0x00450323: 0x1EB8, + 0x00650323: 0x1EB9, + 0x00450309: 0x1EBA, + 0x00650309: 0x1EBB, + 0x00450303: 0x1EBC, + 0x00650303: 0x1EBD, + 0x00CA0301: 0x1EBE, + 0x00EA0301: 0x1EBF, + 0x00CA0300: 0x1EC0, + 0x00EA0300: 0x1EC1, + 0x00CA0309: 0x1EC2, + 0x00EA0309: 0x1EC3, + 0x00CA0303: 0x1EC4, + 0x00EA0303: 0x1EC5, + 0x1EB80302: 0x1EC6, + 0x1EB90302: 0x1EC7, + 0x00490309: 0x1EC8, + 0x00690309: 0x1EC9, + 0x00490323: 0x1ECA, + 0x00690323: 0x1ECB, + 0x004F0323: 0x1ECC, + 0x006F0323: 0x1ECD, + 0x004F0309: 0x1ECE, + 0x006F0309: 0x1ECF, + 0x00D40301: 0x1ED0, + 0x00F40301: 0x1ED1, + 0x00D40300: 0x1ED2, + 0x00F40300: 0x1ED3, + 0x00D40309: 0x1ED4, + 0x00F40309: 0x1ED5, + 0x00D40303: 0x1ED6, + 0x00F40303: 0x1ED7, + 0x1ECC0302: 0x1ED8, + 0x1ECD0302: 0x1ED9, + 0x01A00301: 0x1EDA, + 0x01A10301: 0x1EDB, + 0x01A00300: 0x1EDC, + 0x01A10300: 0x1EDD, + 0x01A00309: 0x1EDE, + 0x01A10309: 0x1EDF, + 0x01A00303: 0x1EE0, + 0x01A10303: 0x1EE1, + 0x01A00323: 0x1EE2, + 0x01A10323: 0x1EE3, + 0x00550323: 0x1EE4, + 0x00750323: 0x1EE5, + 0x00550309: 0x1EE6, + 0x00750309: 0x1EE7, + 0x01AF0301: 0x1EE8, + 0x01B00301: 0x1EE9, + 0x01AF0300: 0x1EEA, + 0x01B00300: 0x1EEB, + 0x01AF0309: 0x1EEC, + 0x01B00309: 0x1EED, + 0x01AF0303: 0x1EEE, + 0x01B00303: 0x1EEF, + 0x01AF0323: 0x1EF0, + 0x01B00323: 0x1EF1, + 0x00590300: 0x1EF2, + 0x00790300: 0x1EF3, + 0x00590323: 0x1EF4, + 0x00790323: 0x1EF5, + 0x00590309: 0x1EF6, + 0x00790309: 0x1EF7, + 0x00590303: 0x1EF8, + 0x00790303: 0x1EF9, + 0x03B10313: 0x1F00, + 0x03B10314: 0x1F01, + 0x1F000300: 0x1F02, + 0x1F010300: 0x1F03, + 0x1F000301: 0x1F04, + 0x1F010301: 0x1F05, + 0x1F000342: 0x1F06, + 0x1F010342: 0x1F07, + 0x03910313: 0x1F08, + 0x03910314: 0x1F09, + 0x1F080300: 0x1F0A, + 0x1F090300: 0x1F0B, + 0x1F080301: 0x1F0C, + 0x1F090301: 0x1F0D, + 0x1F080342: 0x1F0E, + 0x1F090342: 0x1F0F, + 0x03B50313: 0x1F10, + 0x03B50314: 0x1F11, + 0x1F100300: 0x1F12, + 0x1F110300: 0x1F13, + 0x1F100301: 0x1F14, + 0x1F110301: 0x1F15, + 0x03950313: 0x1F18, + 0x03950314: 0x1F19, + 0x1F180300: 0x1F1A, + 0x1F190300: 0x1F1B, + 0x1F180301: 0x1F1C, + 0x1F190301: 0x1F1D, + 0x03B70313: 0x1F20, + 0x03B70314: 0x1F21, + 0x1F200300: 0x1F22, + 0x1F210300: 0x1F23, + 0x1F200301: 0x1F24, + 0x1F210301: 0x1F25, + 0x1F200342: 0x1F26, + 0x1F210342: 0x1F27, + 0x03970313: 0x1F28, + 0x03970314: 0x1F29, + 0x1F280300: 0x1F2A, + 0x1F290300: 0x1F2B, + 0x1F280301: 0x1F2C, + 0x1F290301: 0x1F2D, + 0x1F280342: 0x1F2E, + 0x1F290342: 0x1F2F, + 0x03B90313: 0x1F30, + 0x03B90314: 0x1F31, + 0x1F300300: 0x1F32, + 0x1F310300: 0x1F33, + 0x1F300301: 0x1F34, + 0x1F310301: 0x1F35, + 0x1F300342: 0x1F36, + 0x1F310342: 0x1F37, + 0x03990313: 0x1F38, + 0x03990314: 0x1F39, + 0x1F380300: 0x1F3A, + 0x1F390300: 0x1F3B, + 0x1F380301: 0x1F3C, + 0x1F390301: 0x1F3D, + 0x1F380342: 0x1F3E, + 0x1F390342: 0x1F3F, + 0x03BF0313: 0x1F40, + 0x03BF0314: 0x1F41, + 0x1F400300: 0x1F42, + 0x1F410300: 0x1F43, + 0x1F400301: 0x1F44, + 0x1F410301: 0x1F45, + 0x039F0313: 0x1F48, + 0x039F0314: 0x1F49, + 0x1F480300: 0x1F4A, + 0x1F490300: 0x1F4B, + 0x1F480301: 0x1F4C, + 0x1F490301: 0x1F4D, + 0x03C50313: 0x1F50, + 0x03C50314: 0x1F51, + 0x1F500300: 0x1F52, + 0x1F510300: 0x1F53, + 0x1F500301: 0x1F54, + 0x1F510301: 0x1F55, + 0x1F500342: 0x1F56, + 0x1F510342: 0x1F57, + 0x03A50314: 0x1F59, + 0x1F590300: 0x1F5B, + 0x1F590301: 0x1F5D, + 0x1F590342: 0x1F5F, + 0x03C90313: 0x1F60, + 0x03C90314: 0x1F61, + 0x1F600300: 0x1F62, + 0x1F610300: 0x1F63, + 0x1F600301: 0x1F64, + 0x1F610301: 0x1F65, + 0x1F600342: 0x1F66, + 0x1F610342: 0x1F67, + 0x03A90313: 0x1F68, + 0x03A90314: 0x1F69, + 0x1F680300: 0x1F6A, + 0x1F690300: 0x1F6B, + 0x1F680301: 0x1F6C, + 0x1F690301: 0x1F6D, + 0x1F680342: 0x1F6E, + 0x1F690342: 0x1F6F, + 0x03B10300: 0x1F70, + 0x03B50300: 0x1F72, + 0x03B70300: 0x1F74, + 0x03B90300: 0x1F76, + 0x03BF0300: 0x1F78, + 0x03C50300: 0x1F7A, + 0x03C90300: 0x1F7C, + 0x1F000345: 0x1F80, + 0x1F010345: 0x1F81, + 0x1F020345: 0x1F82, + 0x1F030345: 0x1F83, + 0x1F040345: 0x1F84, + 0x1F050345: 0x1F85, + 0x1F060345: 0x1F86, + 0x1F070345: 0x1F87, + 0x1F080345: 0x1F88, + 0x1F090345: 0x1F89, + 0x1F0A0345: 0x1F8A, + 0x1F0B0345: 0x1F8B, + 0x1F0C0345: 0x1F8C, + 0x1F0D0345: 0x1F8D, + 0x1F0E0345: 0x1F8E, + 0x1F0F0345: 0x1F8F, + 0x1F200345: 0x1F90, + 0x1F210345: 0x1F91, + 0x1F220345: 0x1F92, + 0x1F230345: 0x1F93, + 0x1F240345: 0x1F94, + 0x1F250345: 0x1F95, + 0x1F260345: 0x1F96, + 0x1F270345: 0x1F97, + 0x1F280345: 0x1F98, + 0x1F290345: 0x1F99, + 0x1F2A0345: 0x1F9A, + 0x1F2B0345: 0x1F9B, + 0x1F2C0345: 0x1F9C, + 0x1F2D0345: 0x1F9D, + 0x1F2E0345: 0x1F9E, + 0x1F2F0345: 0x1F9F, + 0x1F600345: 0x1FA0, + 0x1F610345: 0x1FA1, + 0x1F620345: 0x1FA2, + 0x1F630345: 0x1FA3, + 0x1F640345: 0x1FA4, + 0x1F650345: 0x1FA5, + 0x1F660345: 0x1FA6, + 0x1F670345: 0x1FA7, + 0x1F680345: 0x1FA8, + 0x1F690345: 0x1FA9, + 0x1F6A0345: 0x1FAA, + 0x1F6B0345: 0x1FAB, + 0x1F6C0345: 0x1FAC, + 0x1F6D0345: 0x1FAD, + 0x1F6E0345: 0x1FAE, + 0x1F6F0345: 0x1FAF, + 0x03B10306: 0x1FB0, + 0x03B10304: 0x1FB1, + 0x1F700345: 0x1FB2, + 0x03B10345: 0x1FB3, + 0x03AC0345: 0x1FB4, + 0x03B10342: 0x1FB6, + 0x1FB60345: 0x1FB7, + 0x03910306: 0x1FB8, + 0x03910304: 0x1FB9, + 0x03910300: 0x1FBA, + 0x03910345: 0x1FBC, + 0x00A80342: 0x1FC1, + 0x1F740345: 0x1FC2, + 0x03B70345: 0x1FC3, + 0x03AE0345: 0x1FC4, + 0x03B70342: 0x1FC6, + 0x1FC60345: 0x1FC7, + 0x03950300: 0x1FC8, + 0x03970300: 0x1FCA, + 0x03970345: 0x1FCC, + 0x1FBF0300: 0x1FCD, + 0x1FBF0301: 0x1FCE, + 0x1FBF0342: 0x1FCF, + 0x03B90306: 0x1FD0, + 0x03B90304: 0x1FD1, + 0x03CA0300: 0x1FD2, + 0x03B90342: 0x1FD6, + 0x03CA0342: 0x1FD7, + 0x03990306: 0x1FD8, + 0x03990304: 0x1FD9, + 0x03990300: 0x1FDA, + 0x1FFE0300: 0x1FDD, + 0x1FFE0301: 0x1FDE, + 0x1FFE0342: 0x1FDF, + 0x03C50306: 0x1FE0, + 0x03C50304: 0x1FE1, + 0x03CB0300: 0x1FE2, + 0x03C10313: 0x1FE4, + 0x03C10314: 0x1FE5, + 0x03C50342: 0x1FE6, + 0x03CB0342: 0x1FE7, + 0x03A50306: 0x1FE8, + 0x03A50304: 0x1FE9, + 0x03A50300: 0x1FEA, + 0x03A10314: 0x1FEC, + 0x00A80300: 0x1FED, + 0x1F7C0345: 0x1FF2, + 0x03C90345: 0x1FF3, + 0x03CE0345: 0x1FF4, + 0x03C90342: 0x1FF6, + 0x1FF60345: 0x1FF7, + 0x039F0300: 0x1FF8, + 0x03A90300: 0x1FFA, + 0x03A90345: 0x1FFC, + 0x21900338: 0x219A, + 0x21920338: 0x219B, + 0x21940338: 0x21AE, + 0x21D00338: 0x21CD, + 0x21D40338: 0x21CE, + 0x21D20338: 0x21CF, + 0x22030338: 0x2204, + 0x22080338: 0x2209, + 0x220B0338: 0x220C, + 0x22230338: 0x2224, + 0x22250338: 0x2226, + 0x223C0338: 0x2241, + 0x22430338: 0x2244, + 0x22450338: 0x2247, + 0x22480338: 0x2249, + 0x003D0338: 0x2260, + 0x22610338: 0x2262, + 0x224D0338: 0x226D, + 0x003C0338: 0x226E, + 0x003E0338: 0x226F, + 0x22640338: 0x2270, + 0x22650338: 0x2271, + 0x22720338: 0x2274, + 0x22730338: 0x2275, + 0x22760338: 0x2278, + 0x22770338: 0x2279, + 0x227A0338: 0x2280, + 0x227B0338: 0x2281, + 0x22820338: 0x2284, + 0x22830338: 0x2285, + 0x22860338: 0x2288, + 0x22870338: 0x2289, + 0x22A20338: 0x22AC, + 0x22A80338: 0x22AD, + 0x22A90338: 0x22AE, + 0x22AB0338: 0x22AF, + 0x227C0338: 0x22E0, + 0x227D0338: 0x22E1, + 0x22910338: 0x22E2, + 0x22920338: 0x22E3, + 0x22B20338: 0x22EA, + 0x22B30338: 0x22EB, + 0x22B40338: 0x22EC, + 0x22B50338: 0x22ED, + 0x304B3099: 0x304C, + 0x304D3099: 0x304E, + 0x304F3099: 0x3050, + 0x30513099: 0x3052, + 0x30533099: 0x3054, + 0x30553099: 0x3056, + 0x30573099: 0x3058, + 0x30593099: 0x305A, + 0x305B3099: 0x305C, + 0x305D3099: 0x305E, + 0x305F3099: 0x3060, + 0x30613099: 0x3062, + 0x30643099: 0x3065, + 0x30663099: 0x3067, + 0x30683099: 0x3069, + 0x306F3099: 0x3070, + 0x306F309A: 0x3071, + 0x30723099: 0x3073, + 0x3072309A: 0x3074, + 0x30753099: 0x3076, + 0x3075309A: 0x3077, + 0x30783099: 0x3079, + 0x3078309A: 0x307A, + 0x307B3099: 0x307C, + 0x307B309A: 0x307D, + 0x30463099: 0x3094, + 0x309D3099: 0x309E, + 0x30AB3099: 0x30AC, + 0x30AD3099: 0x30AE, + 0x30AF3099: 0x30B0, + 0x30B13099: 0x30B2, + 0x30B33099: 0x30B4, + 0x30B53099: 0x30B6, + 0x30B73099: 0x30B8, + 0x30B93099: 0x30BA, + 0x30BB3099: 0x30BC, + 0x30BD3099: 0x30BE, + 0x30BF3099: 0x30C0, + 0x30C13099: 0x30C2, + 0x30C43099: 0x30C5, + 0x30C63099: 0x30C7, + 0x30C83099: 0x30C9, + 0x30CF3099: 0x30D0, + 0x30CF309A: 0x30D1, + 0x30D23099: 0x30D3, + 0x30D2309A: 0x30D4, + 0x30D53099: 0x30D6, + 0x30D5309A: 0x30D7, + 0x30D83099: 0x30D9, + 0x30D8309A: 0x30DA, + 0x30DB3099: 0x30DC, + 0x30DB309A: 0x30DD, + 0x30A63099: 0x30F4, + 0x30EF3099: 0x30F7, + 0x30F03099: 0x30F8, + 0x30F13099: 0x30F9, + 0x30F23099: 0x30FA, + 0x30FD3099: 0x30FE, + 0x109910BA: 0x1109A, + 0x109B10BA: 0x1109C, + 0x10A510BA: 0x110AB, +} + +// charInfoValues: 10944 entries, 21888 bytes +// Block 2 is the null block. +var charInfoValues = [10944]uint16{ + // Block 0x0, offset 0x0 + 0x003c: 0x8800, 0x003d: 0x8800, 0x003e: 0x8800, + // Block 0x1, offset 0x40 + 0x0041: 0x8800, 0x0042: 0x8800, 0x0043: 0x8800, 0x0044: 0x8800, 0x0045: 0x8800, + 0x0046: 0x8800, 0x0047: 0x8800, 0x0048: 0x8800, 0x0049: 0x8800, 0x004a: 0x8800, 0x004b: 0x8800, + 0x004c: 0x8800, 0x004d: 0x8800, 0x004e: 0x8800, 0x004f: 0x8800, 0x0050: 0x8800, + 0x0052: 0x8800, 0x0053: 0x8800, 0x0054: 0x8800, 0x0055: 0x8800, 0x0056: 0x8800, 0x0057: 0x8800, + 0x0058: 0x8800, 0x0059: 0x8800, 0x005a: 0x8800, + 0x0061: 0x8800, 0x0062: 0x8800, 0x0063: 0x8800, + 0x0064: 0x8800, 0x0065: 0x8800, 0x0066: 0x8800, 0x0067: 0x8800, 0x0068: 0x8800, 0x0069: 0x8800, + 0x006a: 0x8800, 0x006b: 0x8800, 0x006c: 0x8800, 0x006d: 0x8800, 0x006e: 0x8800, 0x006f: 0x8800, + 0x0070: 0x8800, 0x0072: 0x8800, 0x0073: 0x8800, 0x0074: 0x8800, 0x0075: 0x8800, + 0x0076: 0x8800, 0x0077: 0x8800, 0x0078: 0x8800, 0x0079: 0x8800, 0x007a: 0x8800, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00e0: 0x3000, + 0x00e8: 0x3800, + 0x00ea: 0x3000, 0x00ef: 0x3000, + 0x00f2: 0x3000, 0x00f3: 0x3000, 0x00f4: 0x3000, 0x00f5: 0x3000, + 0x00f8: 0x3000, 0x00f9: 0x3000, 0x00fa: 0x3000, + 0x00fc: 0x3000, 0x00fd: 0x3000, 0x00fe: 0x3000, + // Block 0x4, offset 0x100 + 0x0100: 0x1100, 0x0101: 0x1100, 0x0102: 0x9900, 0x0103: 0x1100, 0x0104: 0x9900, 0x0105: 0x9900, + 0x0106: 0x8800, 0x0107: 0x9900, 0x0108: 0x1100, 0x0109: 0x1100, 0x010a: 0x9900, 0x010b: 0x1100, + 0x010c: 0x1100, 0x010d: 0x1100, 0x010e: 0x1100, 0x010f: 0x9900, 0x0111: 0x1100, + 0x0112: 0x1100, 0x0113: 0x1100, 0x0114: 0x9900, 0x0115: 0x9900, 0x0116: 0x9900, + 0x0118: 0x8800, 0x0119: 0x1100, 0x011a: 0x1100, 0x011b: 0x1100, 0x011c: 0x9900, 0x011d: 0x1100, + 0x0120: 0x1100, 0x0121: 0x1100, 0x0122: 0x9900, 0x0123: 0x1100, + 0x0124: 0x9900, 0x0125: 0x9900, 0x0126: 0x8800, 0x0127: 0x9900, 0x0128: 0x1100, 0x0129: 0x1100, + 0x012a: 0x9900, 0x012b: 0x1100, 0x012c: 0x1100, 0x012d: 0x1100, 0x012e: 0x1100, 0x012f: 0x9900, + 0x0131: 0x1100, 0x0132: 0x1100, 0x0133: 0x1100, 0x0134: 0x9900, 0x0135: 0x9900, + 0x0136: 0x9900, 0x0138: 0x8800, 0x0139: 0x1100, 0x013a: 0x1100, 0x013b: 0x1100, + 0x013c: 0x9900, 0x013d: 0x1100, 0x013f: 0x1100, + // Block 0x5, offset 0x140 + 0x0140: 0x1100, 0x0141: 0x1100, 0x0142: 0x9900, 0x0143: 0x9900, 0x0144: 0x1100, 0x0145: 0x1100, + 0x0146: 0x1100, 0x0147: 0x1100, 0x0148: 0x1100, 0x0149: 0x1100, 0x014a: 0x1100, 0x014b: 0x1100, + 0x014c: 0x1100, 0x014d: 0x1100, 0x014e: 0x1100, 0x014f: 0x1100, + 0x0152: 0x9900, 0x0153: 0x9900, 0x0154: 0x1100, 0x0155: 0x1100, 0x0156: 0x1100, 0x0157: 0x1100, + 0x0158: 0x1100, 0x0159: 0x1100, 0x015a: 0x1100, 0x015b: 0x1100, 0x015c: 0x1100, 0x015d: 0x1100, + 0x015e: 0x1100, 0x015f: 0x1100, 0x0160: 0x1100, 0x0161: 0x1100, 0x0162: 0x1100, 0x0163: 0x1100, + 0x0164: 0x1100, 0x0165: 0x1100, 0x0168: 0x1100, 0x0169: 0x1100, + 0x016a: 0x1100, 0x016b: 0x1100, 0x016c: 0x1100, 0x016d: 0x1100, 0x016e: 0x1100, 0x016f: 0x1100, + 0x0170: 0x1100, 0x0172: 0x3000, 0x0173: 0x3000, 0x0174: 0x1100, 0x0175: 0x1100, + 0x0176: 0x1100, 0x0177: 0x1100, 0x0179: 0x1100, 0x017a: 0x1100, 0x017b: 0x1100, + 0x017c: 0x1100, 0x017d: 0x1100, 0x017e: 0x1100, 0x017f: 0x3000, + // Block 0x6, offset 0x180 + 0x0180: 0x3000, 0x0183: 0x1100, 0x0184: 0x1100, 0x0185: 0x1100, + 0x0186: 0x1100, 0x0187: 0x1100, 0x0188: 0x1100, 0x0189: 0x3000, + 0x018c: 0x9900, 0x018d: 0x9900, 0x018e: 0x1100, 0x018f: 0x1100, 0x0190: 0x1100, 0x0191: 0x1100, + 0x0194: 0x1100, 0x0195: 0x1100, 0x0196: 0x1100, 0x0197: 0x1100, + 0x0198: 0x1100, 0x0199: 0x1100, 0x019a: 0x9900, 0x019b: 0x9900, 0x019c: 0x1100, 0x019d: 0x1100, + 0x019e: 0x1100, 0x019f: 0x1100, 0x01a0: 0x9900, 0x01a1: 0x9900, 0x01a2: 0x1100, 0x01a3: 0x1100, + 0x01a4: 0x1100, 0x01a5: 0x1100, 0x01a8: 0x9900, 0x01a9: 0x9900, + 0x01aa: 0x9900, 0x01ab: 0x9900, 0x01ac: 0x1100, 0x01ad: 0x1100, 0x01ae: 0x1100, 0x01af: 0x1100, + 0x01b0: 0x1100, 0x01b1: 0x1100, 0x01b2: 0x1100, 0x01b3: 0x1100, 0x01b4: 0x1100, 0x01b5: 0x1100, + 0x01b6: 0x1100, 0x01b7: 0x1100, 0x01b8: 0x1100, 0x01b9: 0x1100, 0x01ba: 0x1100, 0x01bb: 0x1100, + 0x01bc: 0x1100, 0x01bd: 0x1100, 0x01be: 0x1100, 0x01bf: 0x3800, + // Block 0x7, offset 0x1c0 + 0x01e0: 0x9900, 0x01e1: 0x9900, + 0x01ef: 0x9900, + 0x01f0: 0x9900, + 0x01f7: 0x8800, + // Block 0x8, offset 0x200 + 0x0204: 0x3000, 0x0205: 0x3000, + 0x0206: 0x3000, 0x0207: 0x3000, 0x0208: 0x3000, 0x0209: 0x3000, 0x020a: 0x3000, 0x020b: 0x3000, + 0x020c: 0x3000, 0x020d: 0x1100, 0x020e: 0x1100, 0x020f: 0x1100, 0x0210: 0x1100, 0x0211: 0x1100, + 0x0212: 0x1100, 0x0213: 0x1100, 0x0214: 0x1100, 0x0215: 0x1100, 0x0216: 0x1100, 0x0217: 0x1100, + 0x0218: 0x1100, 0x0219: 0x1100, 0x021a: 0x1100, 0x021b: 0x1100, 0x021c: 0x1100, + 0x021e: 0x1100, 0x021f: 0x1100, 0x0220: 0x1100, 0x0221: 0x1100, 0x0222: 0x1100, 0x0223: 0x1100, + 0x0226: 0x1100, 0x0227: 0x1100, 0x0228: 0x1100, 0x0229: 0x1100, + 0x022a: 0x9900, 0x022b: 0x9900, 0x022c: 0x1100, 0x022d: 0x1100, 0x022e: 0x1100, 0x022f: 0x1100, + 0x0230: 0x1100, 0x0231: 0x3000, 0x0232: 0x3000, 0x0233: 0x3000, 0x0234: 0x1100, 0x0235: 0x1100, + 0x0238: 0x1100, 0x0239: 0x1100, 0x023a: 0x1100, 0x023b: 0x1100, + 0x023c: 0x1100, 0x023d: 0x1100, 0x023e: 0x1100, 0x023f: 0x1100, + // Block 0x9, offset 0x240 + 0x0240: 0x1100, 0x0241: 0x1100, 0x0242: 0x1100, 0x0243: 0x1100, 0x0244: 0x1100, 0x0245: 0x1100, + 0x0246: 0x1100, 0x0247: 0x1100, 0x0248: 0x1100, 0x0249: 0x1100, 0x024a: 0x1100, 0x024b: 0x1100, + 0x024c: 0x1100, 0x024d: 0x1100, 0x024e: 0x1100, 0x024f: 0x1100, 0x0250: 0x1100, 0x0251: 0x1100, + 0x0252: 0x1100, 0x0253: 0x1100, 0x0254: 0x1100, 0x0255: 0x1100, 0x0256: 0x1100, 0x0257: 0x1100, + 0x0258: 0x1100, 0x0259: 0x1100, 0x025a: 0x1100, 0x025b: 0x1100, + 0x025e: 0x1100, 0x025f: 0x1100, + 0x0266: 0x9900, 0x0267: 0x9900, 0x0268: 0x9900, 0x0269: 0x9900, + 0x026a: 0x1100, 0x026b: 0x1100, 0x026c: 0x1100, 0x026d: 0x1100, 0x026e: 0x9900, 0x026f: 0x9900, + 0x0270: 0x1100, 0x0271: 0x1100, 0x0272: 0x1100, 0x0273: 0x1100, + // Block 0xa, offset 0x280 + 0x0292: 0x8800, + 0x02b0: 0x3000, 0x02b1: 0x3000, 0x02b2: 0x3000, 0x02b3: 0x3000, 0x02b4: 0x3000, 0x02b5: 0x3000, + 0x02b6: 0x3000, 0x02b7: 0x3000, 0x02b8: 0x3000, + // Block 0xb, offset 0x2c0 + 0x02d8: 0x3000, 0x02d9: 0x3000, 0x02da: 0x3000, 0x02db: 0x3000, 0x02dc: 0x3000, 0x02dd: 0x3000, + 0x02e0: 0x3000, 0x02e1: 0x3000, 0x02e2: 0x3000, 0x02e3: 0x3000, + 0x02e4: 0x3000, + // Block 0xc, offset 0x300 + 0x0300: 0x66e6, 0x0301: 0x66e6, 0x0302: 0x66e6, 0x0303: 0x66e6, 0x0304: 0x66e6, 0x0305: 0x00e6, + 0x0306: 0x66e6, 0x0307: 0x66e6, 0x0308: 0x66e6, 0x0309: 0x66e6, 0x030a: 0x66e6, 0x030b: 0x66e6, + 0x030c: 0x66e6, 0x030d: 0x00e6, 0x030e: 0x00e6, 0x030f: 0x66e6, 0x0310: 0x00e6, 0x0311: 0x66e6, + 0x0312: 0x00e6, 0x0313: 0x66e6, 0x0314: 0x66e6, 0x0315: 0x00e8, 0x0316: 0x00dc, 0x0317: 0x00dc, + 0x0318: 0x00dc, 0x0319: 0x00dc, 0x031a: 0x00e8, 0x031b: 0x66d8, 0x031c: 0x00dc, 0x031d: 0x00dc, + 0x031e: 0x00dc, 0x031f: 0x00dc, 0x0320: 0x00dc, 0x0321: 0x00ca, 0x0322: 0x00ca, 0x0323: 0x66dc, + 0x0324: 0x66dc, 0x0325: 0x66dc, 0x0326: 0x66dc, 0x0327: 0x66ca, 0x0328: 0x66ca, 0x0329: 0x00dc, + 0x032a: 0x00dc, 0x032b: 0x00dc, 0x032c: 0x00dc, 0x032d: 0x66dc, 0x032e: 0x66dc, 0x032f: 0x00dc, + 0x0330: 0x66dc, 0x0331: 0x66dc, 0x0332: 0x00dc, 0x0333: 0x00dc, 0x0334: 0x0001, 0x0335: 0x0001, + 0x0336: 0x0001, 0x0337: 0x0001, 0x0338: 0x6601, 0x0339: 0x00dc, 0x033a: 0x00dc, 0x033b: 0x00dc, + 0x033c: 0x00dc, 0x033d: 0x00e6, 0x033e: 0x00e6, 0x033f: 0x00e6, + // Block 0xd, offset 0x340 + 0x0340: 0x33e6, 0x0341: 0x33e6, 0x0342: 0x66e6, 0x0343: 0x33e6, 0x0344: 0x33e6, 0x0345: 0x66f0, + 0x0346: 0x00e6, 0x0347: 0x00dc, 0x0348: 0x00dc, 0x0349: 0x00dc, 0x034a: 0x00e6, 0x034b: 0x00e6, + 0x034c: 0x00e6, 0x034d: 0x00dc, 0x034e: 0x00dc, 0x0350: 0x00e6, 0x0351: 0x00e6, + 0x0352: 0x00e6, 0x0353: 0x00dc, 0x0354: 0x00dc, 0x0355: 0x00dc, 0x0356: 0x00dc, 0x0357: 0x00e6, + 0x0358: 0x00e8, 0x0359: 0x00dc, 0x035a: 0x00dc, 0x035b: 0x00e6, 0x035c: 0x00e9, 0x035d: 0x00ea, + 0x035e: 0x00ea, 0x035f: 0x00e9, 0x0360: 0x00ea, 0x0361: 0x00ea, 0x0362: 0x00e9, 0x0363: 0x00e6, + 0x0364: 0x00e6, 0x0365: 0x00e6, 0x0366: 0x00e6, 0x0367: 0x00e6, 0x0368: 0x00e6, 0x0369: 0x00e6, + 0x036a: 0x00e6, 0x036b: 0x00e6, 0x036c: 0x00e6, 0x036d: 0x00e6, 0x036e: 0x00e6, 0x036f: 0x00e6, + 0x0374: 0x3300, + 0x037a: 0x3000, + 0x037e: 0x3300, + // Block 0xe, offset 0x380 + 0x0384: 0x3000, 0x0385: 0x3100, + 0x0386: 0x1100, 0x0387: 0x3300, 0x0388: 0x1100, 0x0389: 0x1100, 0x038a: 0x1100, + 0x038c: 0x1100, 0x038e: 0x1100, 0x038f: 0x1100, 0x0390: 0x1100, 0x0391: 0x8800, + 0x0395: 0x8800, 0x0397: 0x8800, + 0x0399: 0x8800, + 0x039f: 0x8800, 0x03a1: 0x8800, + 0x03a5: 0x8800, 0x03a9: 0x8800, + 0x03aa: 0x1100, 0x03ab: 0x1100, 0x03ac: 0x9900, 0x03ad: 0x1100, 0x03ae: 0x9900, 0x03af: 0x1100, + 0x03b0: 0x1100, 0x03b1: 0x8800, 0x03b5: 0x8800, + 0x03b7: 0x8800, 0x03b9: 0x8800, + 0x03bf: 0x8800, + // Block 0xf, offset 0x3c0 + 0x03c1: 0x8800, 0x03c5: 0x8800, + 0x03c9: 0x8800, 0x03ca: 0x9900, 0x03cb: 0x9900, + 0x03cc: 0x1100, 0x03cd: 0x1100, 0x03ce: 0x9900, 0x03d0: 0x3000, 0x03d1: 0x3000, + 0x03d2: 0x3800, 0x03d3: 0x3100, 0x03d4: 0x3100, 0x03d5: 0x3000, 0x03d6: 0x3000, + 0x03f0: 0x3000, 0x03f1: 0x3000, 0x03f2: 0x3000, 0x03f4: 0x3000, 0x03f5: 0x3000, + 0x03f9: 0x3000, + // Block 0x10, offset 0x400 + 0x0400: 0x1100, 0x0401: 0x1100, 0x0403: 0x1100, + 0x0406: 0x8800, 0x0407: 0x1100, + 0x040c: 0x1100, 0x040d: 0x1100, 0x040e: 0x1100, 0x0410: 0x8800, + 0x0413: 0x8800, 0x0415: 0x8800, 0x0416: 0x8800, 0x0417: 0x8800, + 0x0418: 0x8800, 0x0419: 0x1100, 0x041a: 0x8800, + 0x041e: 0x8800, 0x0423: 0x8800, + 0x0427: 0x8800, + 0x042b: 0x8800, 0x042d: 0x8800, + 0x0430: 0x8800, 0x0433: 0x8800, 0x0435: 0x8800, + 0x0436: 0x8800, 0x0437: 0x8800, 0x0438: 0x8800, 0x0439: 0x1100, 0x043a: 0x8800, + 0x043e: 0x8800, + // Block 0x11, offset 0x440 + 0x0443: 0x8800, + 0x0447: 0x8800, 0x044b: 0x8800, + 0x044d: 0x8800, 0x0450: 0x1100, 0x0451: 0x1100, + 0x0453: 0x1100, 0x0456: 0x8800, 0x0457: 0x1100, + 0x045c: 0x1100, 0x045d: 0x1100, + 0x045e: 0x1100, + 0x0474: 0x8800, 0x0475: 0x8800, + 0x0476: 0x1100, 0x0477: 0x1100, + // Block 0x12, offset 0x480 + 0x0483: 0x00e6, 0x0484: 0x00e6, 0x0485: 0x00e6, + 0x0486: 0x00e6, 0x0487: 0x00e6, + // Block 0x13, offset 0x4c0 + 0x04c1: 0x1100, 0x04c2: 0x1100, + 0x04d0: 0x1100, 0x04d1: 0x1100, + 0x04d2: 0x1100, 0x04d3: 0x1100, 0x04d6: 0x1100, 0x04d7: 0x1100, + 0x04d8: 0x8800, 0x04d9: 0x8800, 0x04da: 0x1100, 0x04db: 0x1100, 0x04dc: 0x1100, 0x04dd: 0x1100, + 0x04de: 0x1100, 0x04df: 0x1100, 0x04e2: 0x1100, 0x04e3: 0x1100, + 0x04e4: 0x1100, 0x04e5: 0x1100, 0x04e6: 0x1100, 0x04e7: 0x1100, 0x04e8: 0x8800, 0x04e9: 0x8800, + 0x04ea: 0x1100, 0x04eb: 0x1100, 0x04ec: 0x1100, 0x04ed: 0x1100, 0x04ee: 0x1100, 0x04ef: 0x1100, + 0x04f0: 0x1100, 0x04f1: 0x1100, 0x04f2: 0x1100, 0x04f3: 0x1100, 0x04f4: 0x1100, 0x04f5: 0x1100, + 0x04f8: 0x1100, 0x04f9: 0x1100, + // Block 0x14, offset 0x500 + 0x0507: 0x3000, + 0x0511: 0x00dc, + 0x0512: 0x00e6, 0x0513: 0x00e6, 0x0514: 0x00e6, 0x0515: 0x00e6, 0x0516: 0x00dc, 0x0517: 0x00e6, + 0x0518: 0x00e6, 0x0519: 0x00e6, 0x051a: 0x00de, 0x051b: 0x00dc, 0x051c: 0x00e6, 0x051d: 0x00e6, + 0x051e: 0x00e6, 0x051f: 0x00e6, 0x0520: 0x00e6, 0x0521: 0x00e6, 0x0522: 0x00dc, 0x0523: 0x00dc, + 0x0524: 0x00dc, 0x0525: 0x00dc, 0x0526: 0x00dc, 0x0527: 0x00dc, 0x0528: 0x00e6, 0x0529: 0x00e6, + 0x052a: 0x00dc, 0x052b: 0x00e6, 0x052c: 0x00e6, 0x052d: 0x00de, 0x052e: 0x00e4, 0x052f: 0x00e6, + 0x0530: 0x000a, 0x0531: 0x000b, 0x0532: 0x000c, 0x0533: 0x000d, 0x0534: 0x000e, 0x0535: 0x000f, + 0x0536: 0x0010, 0x0537: 0x0011, 0x0538: 0x0012, 0x0539: 0x0013, 0x053a: 0x0013, 0x053b: 0x0014, + 0x053c: 0x0015, 0x053d: 0x0016, 0x053f: 0x0017, + // Block 0x15, offset 0x540 + 0x0541: 0x0018, 0x0542: 0x0019, 0x0544: 0x00e6, 0x0545: 0x00dc, + 0x0547: 0x0012, + // Block 0x16, offset 0x580 + 0x0590: 0x00e6, 0x0591: 0x00e6, + 0x0592: 0x00e6, 0x0593: 0x00e6, 0x0594: 0x00e6, 0x0595: 0x00e6, 0x0596: 0x00e6, 0x0597: 0x00e6, + 0x0598: 0x001e, 0x0599: 0x001f, 0x059a: 0x0020, + 0x05a2: 0x1100, 0x05a3: 0x1100, + 0x05a4: 0x1100, 0x05a5: 0x1100, 0x05a6: 0x1100, 0x05a7: 0x8800, + // Block 0x17, offset 0x5c0 + 0x05c8: 0x8800, 0x05ca: 0x8800, 0x05cb: 0x001b, + 0x05cc: 0x001c, 0x05cd: 0x001d, 0x05ce: 0x001e, 0x05cf: 0x001f, 0x05d0: 0x0020, 0x05d1: 0x0021, + 0x05d2: 0x0022, 0x05d3: 0x66e6, 0x05d4: 0x66e6, 0x05d5: 0x66dc, 0x05d6: 0x00dc, 0x05d7: 0x00e6, + 0x05d8: 0x00e6, 0x05d9: 0x00e6, 0x05da: 0x00e6, 0x05db: 0x00e6, 0x05dc: 0x00dc, 0x05dd: 0x00e6, + 0x05de: 0x00e6, 0x05df: 0x00dc, + 0x05f0: 0x0023, 0x05f5: 0x3000, + 0x05f6: 0x3000, 0x05f7: 0x3000, 0x05f8: 0x3000, + // Block 0x18, offset 0x600 + 0x0600: 0x1100, 0x0601: 0x8800, 0x0602: 0x1100, + 0x0612: 0x8800, 0x0613: 0x1100, 0x0615: 0x8800, 0x0616: 0x00e6, 0x0617: 0x00e6, + 0x0618: 0x00e6, 0x0619: 0x00e6, 0x061a: 0x00e6, 0x061b: 0x00e6, 0x061c: 0x00e6, + 0x061f: 0x00e6, 0x0620: 0x00e6, 0x0621: 0x00e6, 0x0622: 0x00e6, 0x0623: 0x00dc, + 0x0624: 0x00e6, 0x0627: 0x00e6, 0x0628: 0x00e6, + 0x062a: 0x00dc, 0x062b: 0x00e6, 0x062c: 0x00e6, 0x062d: 0x00dc, + // Block 0x19, offset 0x640 + 0x0651: 0x0024, + 0x0670: 0x00e6, 0x0671: 0x00dc, 0x0672: 0x00e6, 0x0673: 0x00e6, 0x0674: 0x00dc, 0x0675: 0x00e6, + 0x0676: 0x00e6, 0x0677: 0x00dc, 0x0678: 0x00dc, 0x0679: 0x00dc, 0x067a: 0x00e6, 0x067b: 0x00dc, + 0x067c: 0x00dc, 0x067d: 0x00e6, 0x067e: 0x00dc, 0x067f: 0x00e6, + // Block 0x1a, offset 0x680 + 0x0680: 0x00e6, 0x0681: 0x00e6, 0x0682: 0x00dc, 0x0683: 0x00e6, 0x0684: 0x00dc, 0x0685: 0x00e6, + 0x0686: 0x00dc, 0x0687: 0x00e6, 0x0688: 0x00dc, 0x0689: 0x00e6, 0x068a: 0x00e6, + // Block 0x1b, offset 0x6c0 + 0x06eb: 0x00e6, 0x06ec: 0x00e6, 0x06ed: 0x00e6, 0x06ee: 0x00e6, 0x06ef: 0x00e6, + 0x06f0: 0x00e6, 0x06f1: 0x00e6, 0x06f2: 0x00dc, 0x06f3: 0x00e6, + // Block 0x1c, offset 0x700 + 0x0716: 0x00e6, 0x0717: 0x00e6, + 0x0718: 0x00e6, 0x0719: 0x00e6, 0x071b: 0x00e6, 0x071c: 0x00e6, 0x071d: 0x00e6, + 0x071e: 0x00e6, 0x071f: 0x00e6, 0x0720: 0x00e6, 0x0721: 0x00e6, 0x0722: 0x00e6, 0x0723: 0x00e6, + 0x0725: 0x00e6, 0x0726: 0x00e6, 0x0727: 0x00e6, 0x0729: 0x00e6, + 0x072a: 0x00e6, 0x072b: 0x00e6, 0x072c: 0x00e6, 0x072d: 0x00e6, + // Block 0x1d, offset 0x740 + 0x0759: 0x00dc, 0x075a: 0x00dc, 0x075b: 0x00dc, + // Block 0x1e, offset 0x780 + 0x07a8: 0x8800, 0x07a9: 0x1100, + 0x07b0: 0x8800, 0x07b1: 0x1100, 0x07b3: 0x8800, 0x07b4: 0x1100, + 0x07bc: 0x6607, + // Block 0x1f, offset 0x7c0 + 0x07cd: 0x0009, 0x07d1: 0x00e6, + 0x07d2: 0x00dc, 0x07d3: 0x00e6, 0x07d4: 0x00e6, + 0x07d8: 0x3300, 0x07d9: 0x3300, 0x07da: 0x3300, 0x07db: 0x3300, 0x07dc: 0x3300, 0x07dd: 0x3300, + 0x07de: 0x3300, 0x07df: 0x3300, + // Block 0x20, offset 0x800 + 0x083c: 0x0007, 0x083e: 0x6600, + // Block 0x21, offset 0x840 + 0x0847: 0x8800, 0x084b: 0x1100, + 0x084c: 0x1100, 0x084d: 0x0009, + 0x0857: 0x6600, + 0x085c: 0x3300, 0x085d: 0x3300, + 0x085f: 0x3300, + // Block 0x22, offset 0x880 + 0x08b3: 0x3300, + 0x08b6: 0x3300, + 0x08bc: 0x0007, + // Block 0x23, offset 0x8c0 + 0x08cd: 0x0009, + 0x08d9: 0x3300, 0x08da: 0x3300, 0x08db: 0x3300, + 0x08de: 0x3300, + // Block 0x24, offset 0x900 + 0x093c: 0x0007, + // Block 0x25, offset 0x940 + 0x094d: 0x0009, + // Block 0x26, offset 0x980 + 0x0987: 0x8800, 0x0988: 0x1100, 0x098b: 0x1100, + 0x098c: 0x1100, 0x098d: 0x0009, + 0x0996: 0x6600, 0x0997: 0x6600, + 0x099c: 0x3300, 0x099d: 0x3300, + // Block 0x27, offset 0x9c0 + 0x09d2: 0x8800, 0x09d4: 0x1100, + 0x09fe: 0x6600, + // Block 0x28, offset 0xa00 + 0x0a06: 0x8800, 0x0a07: 0x8800, 0x0a0a: 0x1100, 0x0a0b: 0x1100, + 0x0a0c: 0x1100, 0x0a0d: 0x0009, + 0x0a17: 0x6600, + // Block 0x29, offset 0xa40 + 0x0a46: 0x8800, 0x0a48: 0x1100, + 0x0a4d: 0x0009, + 0x0a55: 0x0054, 0x0a56: 0x665b, + // Block 0x2a, offset 0xa80 + 0x0abc: 0x0007, 0x0abf: 0x8800, + // Block 0x2b, offset 0xac0 + 0x0ac0: 0x1100, 0x0ac2: 0x6600, + 0x0ac6: 0x8800, 0x0ac7: 0x1100, 0x0ac8: 0x1100, 0x0aca: 0x9900, 0x0acb: 0x1100, + 0x0acd: 0x0009, + 0x0ad5: 0x6600, 0x0ad6: 0x6600, + // Block 0x2c, offset 0xb00 + 0x0b3e: 0x6600, + // Block 0x2d, offset 0xb40 + 0x0b4a: 0x6609, + 0x0b4f: 0x6600, + 0x0b59: 0x8800, 0x0b5a: 0x1100, 0x0b5c: 0x9900, 0x0b5d: 0x1100, + 0x0b5e: 0x1100, 0x0b5f: 0x6600, + // Block 0x2e, offset 0xb80 + 0x0bb3: 0x3000, + 0x0bb8: 0x0067, 0x0bb9: 0x0067, 0x0bba: 0x0009, + // Block 0x2f, offset 0xbc0 + 0x0bc8: 0x006b, 0x0bc9: 0x006b, 0x0bca: 0x006b, 0x0bcb: 0x006b, + // Block 0x30, offset 0xc00 + 0x0c33: 0x3000, + 0x0c38: 0x0076, 0x0c39: 0x0076, + // Block 0x31, offset 0xc40 + 0x0c48: 0x007a, 0x0c49: 0x007a, 0x0c4a: 0x007a, 0x0c4b: 0x007a, + 0x0c5c: 0x3000, 0x0c5d: 0x3000, + // Block 0x32, offset 0xc80 + 0x0c8c: 0x3000, + 0x0c98: 0x00dc, 0x0c99: 0x00dc, + 0x0cb5: 0x00dc, + 0x0cb7: 0x00dc, 0x0cb9: 0x00d8, + // Block 0x33, offset 0xcc0 + 0x0cc3: 0x3300, + 0x0ccd: 0x3300, + 0x0cd2: 0x3300, 0x0cd7: 0x3300, + 0x0cdc: 0x3300, + 0x0ce9: 0x3300, + 0x0cf1: 0x0081, 0x0cf2: 0x0082, 0x0cf3: 0x3300, 0x0cf4: 0x0084, 0x0cf5: 0x3300, + 0x0cf6: 0x3300, 0x0cf7: 0x3000, 0x0cf8: 0x3300, 0x0cf9: 0x3000, 0x0cfa: 0x0082, 0x0cfb: 0x0082, + 0x0cfc: 0x0082, 0x0cfd: 0x0082, + // Block 0x34, offset 0xd00 + 0x0d00: 0x0082, 0x0d01: 0x3300, 0x0d02: 0x00e6, 0x0d03: 0x00e6, 0x0d04: 0x0009, + 0x0d06: 0x00e6, 0x0d07: 0x00e6, + 0x0d13: 0x3300, + 0x0d1d: 0x3300, + 0x0d22: 0x3300, + 0x0d27: 0x3300, + 0x0d2c: 0x3300, + 0x0d39: 0x3300, + // Block 0x35, offset 0xd40 + 0x0d46: 0x00dc, + // Block 0x36, offset 0xd80 + 0x0da5: 0x8800, 0x0da6: 0x1100, + 0x0dae: 0x6600, + 0x0db7: 0x0007, 0x0db9: 0x0009, 0x0dba: 0x0009, + // Block 0x37, offset 0xdc0 + 0x0dcd: 0x00dc, + // Block 0x38, offset 0xe00 + 0x0e3c: 0x3000, + // Block 0x39, offset 0xe40 + 0x0e61: 0x6600, 0x0e62: 0x6600, 0x0e63: 0x6600, + 0x0e64: 0x6600, 0x0e65: 0x6600, 0x0e66: 0x6600, 0x0e67: 0x6600, 0x0e68: 0x6600, 0x0e69: 0x6600, + 0x0e6a: 0x6600, 0x0e6b: 0x6600, 0x0e6c: 0x6600, 0x0e6d: 0x6600, 0x0e6e: 0x6600, 0x0e6f: 0x6600, + 0x0e70: 0x6600, 0x0e71: 0x6600, 0x0e72: 0x6600, 0x0e73: 0x6600, 0x0e74: 0x6600, 0x0e75: 0x6600, + // Block 0x3a, offset 0xe80 + 0x0ea8: 0x6600, 0x0ea9: 0x6600, + 0x0eaa: 0x6600, 0x0eab: 0x6600, 0x0eac: 0x6600, 0x0ead: 0x6600, 0x0eae: 0x6600, 0x0eaf: 0x6600, + 0x0eb0: 0x6600, 0x0eb1: 0x6600, 0x0eb2: 0x6600, 0x0eb3: 0x6600, 0x0eb4: 0x6600, 0x0eb5: 0x6600, + 0x0eb6: 0x6600, 0x0eb7: 0x6600, 0x0eb8: 0x6600, 0x0eb9: 0x6600, 0x0eba: 0x6600, 0x0ebb: 0x6600, + 0x0ebc: 0x6600, 0x0ebd: 0x6600, 0x0ebe: 0x6600, 0x0ebf: 0x6600, + // Block 0x3b, offset 0xec0 + 0x0ec0: 0x6600, 0x0ec1: 0x6600, 0x0ec2: 0x6600, + // Block 0x3c, offset 0xf00 + 0x0f1d: 0x00e6, + 0x0f1e: 0x00e6, 0x0f1f: 0x00e6, + // Block 0x3d, offset 0xf40 + 0x0f54: 0x0009, + 0x0f74: 0x0009, + // Block 0x3e, offset 0xf80 + 0x0f92: 0x0009, + 0x0f9d: 0x00e6, + // Block 0x3f, offset 0xfc0 + 0x0fe9: 0x00e4, + // Block 0x40, offset 0x1000 + 0x1039: 0x00de, 0x103a: 0x00e6, 0x103b: 0x00dc, + // Block 0x41, offset 0x1040 + 0x1057: 0x00e6, + 0x1058: 0x00dc, + // Block 0x42, offset 0x1080 + 0x10a0: 0x0009, + 0x10b5: 0x00e6, + 0x10b6: 0x00e6, 0x10b7: 0x00e6, 0x10b8: 0x00e6, 0x10b9: 0x00e6, 0x10ba: 0x00e6, 0x10bb: 0x00e6, + 0x10bc: 0x00e6, 0x10bf: 0x00dc, + // Block 0x43, offset 0x10c0 + 0x10c5: 0x8800, + 0x10c6: 0x1100, 0x10c7: 0x8800, 0x10c8: 0x1100, 0x10c9: 0x8800, 0x10ca: 0x1100, 0x10cb: 0x8800, + 0x10cc: 0x1100, 0x10cd: 0x8800, 0x10ce: 0x1100, 0x10d1: 0x8800, + 0x10d2: 0x1100, + 0x10f4: 0x0007, 0x10f5: 0x6600, + 0x10fa: 0x8800, 0x10fb: 0x1100, + 0x10fc: 0x8800, 0x10fd: 0x1100, 0x10fe: 0x8800, 0x10ff: 0x8800, + // Block 0x44, offset 0x1100 + 0x1100: 0x1100, 0x1101: 0x1100, 0x1102: 0x8800, 0x1103: 0x1100, 0x1104: 0x0009, + 0x112b: 0x00e6, 0x112c: 0x00dc, 0x112d: 0x00e6, 0x112e: 0x00e6, 0x112f: 0x00e6, + 0x1130: 0x00e6, 0x1131: 0x00e6, 0x1132: 0x00e6, 0x1133: 0x00e6, + // Block 0x45, offset 0x1140 + 0x116a: 0x0009, + // Block 0x46, offset 0x1180 + 0x11a6: 0x0007, + 0x11b2: 0x0009, 0x11b3: 0x0009, + // Block 0x47, offset 0x11c0 + 0x11f7: 0x0007, + // Block 0x48, offset 0x1200 + 0x1210: 0x00e6, 0x1211: 0x00e6, + 0x1212: 0x00e6, 0x1214: 0x0001, 0x1215: 0x00dc, 0x1216: 0x00dc, 0x1217: 0x00dc, + 0x1218: 0x00dc, 0x1219: 0x00dc, 0x121a: 0x00e6, 0x121b: 0x00e6, 0x121c: 0x00dc, 0x121d: 0x00dc, + 0x121e: 0x00dc, 0x121f: 0x00dc, 0x1220: 0x00e6, 0x1222: 0x0001, 0x1223: 0x0001, + 0x1224: 0x0001, 0x1225: 0x0001, 0x1226: 0x0001, 0x1227: 0x0001, 0x1228: 0x0001, + 0x122d: 0x00dc, + // Block 0x49, offset 0x1240 + 0x126c: 0x3000, 0x126d: 0x3000, 0x126e: 0x3000, + 0x1270: 0x3000, 0x1271: 0x3000, 0x1272: 0x3000, 0x1273: 0x3000, 0x1274: 0x3000, 0x1275: 0x3000, + 0x1276: 0x3000, 0x1277: 0x3000, 0x1278: 0x3000, 0x1279: 0x3000, 0x127a: 0x3000, + 0x127c: 0x3000, 0x127d: 0x3000, 0x127e: 0x3000, 0x127f: 0x3000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x3000, 0x1281: 0x3000, 0x1282: 0x3000, 0x1283: 0x3000, 0x1284: 0x3000, 0x1285: 0x3000, + 0x1286: 0x3000, 0x1287: 0x3000, 0x1288: 0x3000, 0x1289: 0x3000, 0x128a: 0x3000, 0x128b: 0x3000, + 0x128c: 0x3000, 0x128d: 0x3000, 0x128f: 0x3000, 0x1290: 0x3000, 0x1291: 0x3000, + 0x1292: 0x3000, 0x1293: 0x3000, 0x1294: 0x3000, 0x1295: 0x3000, 0x1296: 0x3000, 0x1297: 0x3000, + 0x1298: 0x3000, 0x1299: 0x3000, 0x129a: 0x3000, 0x129b: 0x3000, 0x129c: 0x3000, 0x129d: 0x3000, + 0x129e: 0x3000, 0x129f: 0x3000, 0x12a0: 0x3000, 0x12a1: 0x3000, 0x12a2: 0x3000, 0x12a3: 0x3000, + 0x12a4: 0x3000, 0x12a5: 0x3000, 0x12a6: 0x3000, 0x12a7: 0x3000, 0x12a8: 0x3000, 0x12a9: 0x3000, + 0x12aa: 0x3000, + 0x12b8: 0x3000, + // Block 0x4b, offset 0x12c0 + 0x12db: 0x3000, 0x12dc: 0x3000, 0x12dd: 0x3000, + 0x12de: 0x3000, 0x12df: 0x3000, 0x12e0: 0x3000, 0x12e1: 0x3000, 0x12e2: 0x3000, 0x12e3: 0x3000, + 0x12e4: 0x3000, 0x12e5: 0x3000, 0x12e6: 0x3000, 0x12e7: 0x3000, 0x12e8: 0x3000, 0x12e9: 0x3000, + 0x12ea: 0x3000, 0x12eb: 0x3000, 0x12ec: 0x3000, 0x12ed: 0x3000, 0x12ee: 0x3000, 0x12ef: 0x3000, + 0x12f0: 0x3000, 0x12f1: 0x3000, 0x12f2: 0x3000, 0x12f3: 0x3000, 0x12f4: 0x3000, 0x12f5: 0x3000, + 0x12f6: 0x3000, 0x12f7: 0x3000, 0x12f8: 0x3000, 0x12f9: 0x3000, 0x12fa: 0x3000, 0x12fb: 0x3000, + 0x12fc: 0x3000, 0x12fd: 0x3000, 0x12fe: 0x3000, 0x12ff: 0x3000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x00e6, 0x1301: 0x00e6, 0x1302: 0x00dc, 0x1303: 0x00e6, 0x1304: 0x00e6, 0x1305: 0x00e6, + 0x1306: 0x00e6, 0x1307: 0x00e6, 0x1308: 0x00e6, 0x1309: 0x00e6, 0x130a: 0x00dc, 0x130b: 0x00e6, + 0x130c: 0x00e6, 0x130d: 0x00ea, 0x130e: 0x00d6, 0x130f: 0x00dc, 0x1310: 0x00ca, 0x1311: 0x00e6, + 0x1312: 0x00e6, 0x1313: 0x00e6, 0x1314: 0x00e6, 0x1315: 0x00e6, 0x1316: 0x00e6, 0x1317: 0x00e6, + 0x1318: 0x00e6, 0x1319: 0x00e6, 0x131a: 0x00e6, 0x131b: 0x00e6, 0x131c: 0x00e6, 0x131d: 0x00e6, + 0x131e: 0x00e6, 0x131f: 0x00e6, 0x1320: 0x00e6, 0x1321: 0x00e6, 0x1322: 0x00e6, 0x1323: 0x00e6, + 0x1324: 0x00e6, 0x1325: 0x00e6, 0x1326: 0x00e6, + 0x133c: 0x00e9, 0x133d: 0x00dc, 0x133e: 0x00e6, 0x133f: 0x00dc, + // Block 0x4d, offset 0x1340 + 0x1340: 0x1100, 0x1341: 0x1100, 0x1342: 0x1100, 0x1343: 0x1100, 0x1344: 0x1100, 0x1345: 0x1100, + 0x1346: 0x1100, 0x1347: 0x1100, 0x1348: 0x1100, 0x1349: 0x1100, 0x134a: 0x1100, 0x134b: 0x1100, + 0x134c: 0x1100, 0x134d: 0x1100, 0x134e: 0x1100, 0x134f: 0x1100, 0x1350: 0x1100, 0x1351: 0x1100, + 0x1352: 0x1100, 0x1353: 0x1100, 0x1354: 0x1100, 0x1355: 0x1100, 0x1356: 0x1100, 0x1357: 0x1100, + 0x1358: 0x1100, 0x1359: 0x1100, 0x135a: 0x1100, 0x135b: 0x1100, 0x135c: 0x1100, 0x135d: 0x1100, + 0x135e: 0x1100, 0x135f: 0x1100, 0x1360: 0x1100, 0x1361: 0x1100, 0x1362: 0x1100, 0x1363: 0x1100, + 0x1364: 0x1100, 0x1365: 0x1100, 0x1366: 0x1100, 0x1367: 0x1100, 0x1368: 0x1100, 0x1369: 0x1100, + 0x136a: 0x1100, 0x136b: 0x1100, 0x136c: 0x1100, 0x136d: 0x1100, 0x136e: 0x1100, 0x136f: 0x1100, + 0x1370: 0x1100, 0x1371: 0x1100, 0x1372: 0x1100, 0x1373: 0x1100, 0x1374: 0x1100, 0x1375: 0x1100, + 0x1376: 0x9900, 0x1377: 0x9900, 0x1378: 0x1100, 0x1379: 0x1100, 0x137a: 0x1100, 0x137b: 0x1100, + 0x137c: 0x1100, 0x137d: 0x1100, 0x137e: 0x1100, 0x137f: 0x1100, + // Block 0x4e, offset 0x1380 + 0x1380: 0x1100, 0x1381: 0x1100, 0x1382: 0x1100, 0x1383: 0x1100, 0x1384: 0x1100, 0x1385: 0x1100, + 0x1386: 0x1100, 0x1387: 0x1100, 0x1388: 0x1100, 0x1389: 0x1100, 0x138a: 0x1100, 0x138b: 0x1100, + 0x138c: 0x1100, 0x138d: 0x1100, 0x138e: 0x1100, 0x138f: 0x1100, 0x1390: 0x1100, 0x1391: 0x1100, + 0x1392: 0x1100, 0x1393: 0x1100, 0x1394: 0x1100, 0x1395: 0x1100, 0x1396: 0x1100, 0x1397: 0x1100, + 0x1398: 0x1100, 0x1399: 0x1100, 0x139a: 0x9900, 0x139b: 0x9900, 0x139c: 0x1100, 0x139d: 0x1100, + 0x139e: 0x1100, 0x139f: 0x1100, 0x13a0: 0x1100, 0x13a1: 0x1100, 0x13a2: 0x9900, 0x13a3: 0x9900, + 0x13a4: 0x1100, 0x13a5: 0x1100, 0x13a6: 0x1100, 0x13a7: 0x1100, 0x13a8: 0x1100, 0x13a9: 0x1100, + 0x13aa: 0x1100, 0x13ab: 0x1100, 0x13ac: 0x1100, 0x13ad: 0x1100, 0x13ae: 0x1100, 0x13af: 0x1100, + 0x13b0: 0x1100, 0x13b1: 0x1100, 0x13b2: 0x1100, 0x13b3: 0x1100, 0x13b4: 0x1100, 0x13b5: 0x1100, + 0x13b6: 0x1100, 0x13b7: 0x1100, 0x13b8: 0x1100, 0x13b9: 0x1100, 0x13ba: 0x1100, 0x13bb: 0x1100, + 0x13bc: 0x1100, 0x13bd: 0x1100, 0x13be: 0x1100, 0x13bf: 0x1100, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x1100, 0x13c1: 0x1100, 0x13c2: 0x1100, 0x13c3: 0x1100, 0x13c4: 0x1100, 0x13c5: 0x1100, + 0x13c6: 0x1100, 0x13c7: 0x1100, 0x13c8: 0x1100, 0x13c9: 0x1100, 0x13ca: 0x1100, 0x13cb: 0x1100, + 0x13cc: 0x1100, 0x13cd: 0x1100, 0x13ce: 0x1100, 0x13cf: 0x1100, 0x13d0: 0x1100, 0x13d1: 0x1100, + 0x13d2: 0x1100, 0x13d3: 0x1100, 0x13d4: 0x1100, 0x13d5: 0x1100, 0x13d6: 0x1100, 0x13d7: 0x1100, + 0x13d8: 0x1100, 0x13d9: 0x1100, 0x13da: 0x3000, 0x13db: 0x3100, + 0x13e0: 0x9900, 0x13e1: 0x9900, 0x13e2: 0x1100, 0x13e3: 0x1100, + 0x13e4: 0x1100, 0x13e5: 0x1100, 0x13e6: 0x1100, 0x13e7: 0x1100, 0x13e8: 0x1100, 0x13e9: 0x1100, + 0x13ea: 0x1100, 0x13eb: 0x1100, 0x13ec: 0x1100, 0x13ed: 0x1100, 0x13ee: 0x1100, 0x13ef: 0x1100, + 0x13f0: 0x1100, 0x13f1: 0x1100, 0x13f2: 0x1100, 0x13f3: 0x1100, 0x13f4: 0x1100, 0x13f5: 0x1100, + 0x13f6: 0x1100, 0x13f7: 0x1100, 0x13f8: 0x9900, 0x13f9: 0x9900, 0x13fa: 0x1100, 0x13fb: 0x1100, + 0x13fc: 0x1100, 0x13fd: 0x1100, 0x13fe: 0x1100, 0x13ff: 0x1100, + // Block 0x50, offset 0x1400 + 0x1400: 0x1100, 0x1401: 0x1100, 0x1402: 0x1100, 0x1403: 0x1100, 0x1404: 0x1100, 0x1405: 0x1100, + 0x1406: 0x1100, 0x1407: 0x1100, 0x1408: 0x1100, 0x1409: 0x1100, 0x140a: 0x1100, 0x140b: 0x1100, + 0x140c: 0x9900, 0x140d: 0x9900, 0x140e: 0x1100, 0x140f: 0x1100, 0x1410: 0x1100, 0x1411: 0x1100, + 0x1412: 0x1100, 0x1413: 0x1100, 0x1414: 0x1100, 0x1415: 0x1100, 0x1416: 0x1100, 0x1417: 0x1100, + 0x1418: 0x1100, 0x1419: 0x1100, 0x141a: 0x1100, 0x141b: 0x1100, 0x141c: 0x1100, 0x141d: 0x1100, + 0x141e: 0x1100, 0x141f: 0x1100, 0x1420: 0x1100, 0x1421: 0x1100, 0x1422: 0x1100, 0x1423: 0x1100, + 0x1424: 0x1100, 0x1425: 0x1100, 0x1426: 0x1100, 0x1427: 0x1100, 0x1428: 0x1100, 0x1429: 0x1100, + 0x142a: 0x1100, 0x142b: 0x1100, 0x142c: 0x1100, 0x142d: 0x1100, 0x142e: 0x1100, 0x142f: 0x1100, + 0x1430: 0x1100, 0x1431: 0x1100, 0x1432: 0x1100, 0x1433: 0x1100, 0x1434: 0x1100, 0x1435: 0x1100, + 0x1436: 0x1100, 0x1437: 0x1100, 0x1438: 0x1100, 0x1439: 0x1100, + // Block 0x51, offset 0x1440 + 0x1440: 0x9900, 0x1441: 0x9900, 0x1442: 0x9900, 0x1443: 0x9900, 0x1444: 0x9900, 0x1445: 0x9900, + 0x1446: 0x9900, 0x1447: 0x9900, 0x1448: 0x9900, 0x1449: 0x9900, 0x144a: 0x9900, 0x144b: 0x9900, + 0x144c: 0x9900, 0x144d: 0x9900, 0x144e: 0x9900, 0x144f: 0x9900, 0x1450: 0x9900, 0x1451: 0x9900, + 0x1452: 0x1100, 0x1453: 0x1100, 0x1454: 0x1100, 0x1455: 0x1100, + 0x1458: 0x9900, 0x1459: 0x9900, 0x145a: 0x1100, 0x145b: 0x1100, 0x145c: 0x1100, 0x145d: 0x1100, + 0x1460: 0x9900, 0x1461: 0x9900, 0x1462: 0x9900, 0x1463: 0x9900, + 0x1464: 0x9900, 0x1465: 0x9900, 0x1466: 0x9900, 0x1467: 0x9900, 0x1468: 0x9900, 0x1469: 0x9900, + 0x146a: 0x9900, 0x146b: 0x9900, 0x146c: 0x9900, 0x146d: 0x9900, 0x146e: 0x9900, 0x146f: 0x9900, + 0x1470: 0x9900, 0x1471: 0x9900, 0x1472: 0x1100, 0x1473: 0x1100, 0x1474: 0x1100, 0x1475: 0x1100, + 0x1476: 0x1100, 0x1477: 0x1100, 0x1478: 0x9900, 0x1479: 0x9900, 0x147a: 0x1100, 0x147b: 0x1100, + 0x147c: 0x1100, 0x147d: 0x1100, 0x147e: 0x1100, 0x147f: 0x1100, + // Block 0x52, offset 0x1480 + 0x1480: 0x9900, 0x1481: 0x9900, 0x1482: 0x1100, 0x1483: 0x1100, 0x1484: 0x1100, 0x1485: 0x1100, + 0x1488: 0x9900, 0x1489: 0x9900, 0x148a: 0x1100, 0x148b: 0x1100, + 0x148c: 0x1100, 0x148d: 0x1100, 0x1490: 0x9900, 0x1491: 0x9900, + 0x1492: 0x1100, 0x1493: 0x1100, 0x1494: 0x1100, 0x1495: 0x1100, 0x1496: 0x1100, 0x1497: 0x1100, + 0x1499: 0x9900, 0x149b: 0x1100, 0x149d: 0x1100, + 0x149f: 0x1100, 0x14a0: 0x9900, 0x14a1: 0x9900, 0x14a2: 0x9900, 0x14a3: 0x9900, + 0x14a4: 0x9900, 0x14a5: 0x9900, 0x14a6: 0x9900, 0x14a7: 0x9900, 0x14a8: 0x9900, 0x14a9: 0x9900, + 0x14aa: 0x9900, 0x14ab: 0x9900, 0x14ac: 0x9900, 0x14ad: 0x9900, 0x14ae: 0x9900, 0x14af: 0x9900, + 0x14b0: 0x9900, 0x14b1: 0x3300, 0x14b2: 0x1100, 0x14b3: 0x3300, 0x14b4: 0x9900, 0x14b5: 0x3300, + 0x14b6: 0x1100, 0x14b7: 0x3300, 0x14b8: 0x1100, 0x14b9: 0x3300, 0x14ba: 0x1100, 0x14bb: 0x3300, + 0x14bc: 0x9900, 0x14bd: 0x3300, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x1100, 0x14c1: 0x1100, 0x14c2: 0x1100, 0x14c3: 0x1100, 0x14c4: 0x1100, 0x14c5: 0x1100, + 0x14c6: 0x1100, 0x14c7: 0x1100, 0x14c8: 0x1100, 0x14c9: 0x1100, 0x14ca: 0x1100, 0x14cb: 0x1100, + 0x14cc: 0x1100, 0x14cd: 0x1100, 0x14ce: 0x1100, 0x14cf: 0x1100, 0x14d0: 0x1100, 0x14d1: 0x1100, + 0x14d2: 0x1100, 0x14d3: 0x1100, 0x14d4: 0x1100, 0x14d5: 0x1100, 0x14d6: 0x1100, 0x14d7: 0x1100, + 0x14d8: 0x1100, 0x14d9: 0x1100, 0x14da: 0x1100, 0x14db: 0x1100, 0x14dc: 0x1100, 0x14dd: 0x1100, + 0x14de: 0x1100, 0x14df: 0x1100, 0x14e0: 0x1100, 0x14e1: 0x1100, 0x14e2: 0x1100, 0x14e3: 0x1100, + 0x14e4: 0x1100, 0x14e5: 0x1100, 0x14e6: 0x1100, 0x14e7: 0x1100, 0x14e8: 0x1100, 0x14e9: 0x1100, + 0x14ea: 0x1100, 0x14eb: 0x1100, 0x14ec: 0x1100, 0x14ed: 0x1100, 0x14ee: 0x1100, 0x14ef: 0x1100, + 0x14f0: 0x1100, 0x14f1: 0x1100, 0x14f2: 0x1100, 0x14f3: 0x1100, 0x14f4: 0x1100, + 0x14f6: 0x9900, 0x14f7: 0x1100, 0x14f8: 0x1100, 0x14f9: 0x1100, 0x14fa: 0x1100, 0x14fb: 0x3300, + 0x14fc: 0x1100, 0x14fd: 0x3000, 0x14fe: 0x3300, 0x14ff: 0x3800, + // Block 0x54, offset 0x1500 + 0x1500: 0x3000, 0x1501: 0x3100, 0x1502: 0x1100, 0x1503: 0x1100, 0x1504: 0x1100, + 0x1506: 0x9900, 0x1507: 0x1100, 0x1508: 0x1100, 0x1509: 0x3300, 0x150a: 0x1100, 0x150b: 0x3300, + 0x150c: 0x1100, 0x150d: 0x3100, 0x150e: 0x3100, 0x150f: 0x3100, 0x1510: 0x1100, 0x1511: 0x1100, + 0x1512: 0x1100, 0x1513: 0x3300, 0x1516: 0x1100, 0x1517: 0x1100, + 0x1518: 0x1100, 0x1519: 0x1100, 0x151a: 0x1100, 0x151b: 0x3300, 0x151d: 0x3100, + 0x151e: 0x3100, 0x151f: 0x3100, 0x1520: 0x1100, 0x1521: 0x1100, 0x1522: 0x1100, 0x1523: 0x3300, + 0x1524: 0x1100, 0x1525: 0x1100, 0x1526: 0x1100, 0x1527: 0x1100, 0x1528: 0x1100, 0x1529: 0x1100, + 0x152a: 0x1100, 0x152b: 0x3300, 0x152c: 0x1100, 0x152d: 0x3100, 0x152e: 0x3300, 0x152f: 0x3300, + 0x1532: 0x1100, 0x1533: 0x1100, 0x1534: 0x1100, + 0x1536: 0x9900, 0x1537: 0x1100, 0x1538: 0x1100, 0x1539: 0x3300, 0x153a: 0x1100, 0x153b: 0x3300, + 0x153c: 0x1100, 0x153d: 0x3300, 0x153e: 0x3800, + // Block 0x55, offset 0x1540 + 0x1540: 0x3300, 0x1541: 0x3300, 0x1542: 0x3000, 0x1543: 0x3000, 0x1544: 0x3000, 0x1545: 0x3000, + 0x1546: 0x3000, 0x1547: 0x3000, 0x1548: 0x3000, 0x1549: 0x3000, 0x154a: 0x3000, + 0x1551: 0x3000, + 0x1557: 0x3000, + 0x1564: 0x3000, 0x1565: 0x3000, 0x1566: 0x3000, + 0x156f: 0x3000, + 0x1573: 0x3000, 0x1574: 0x3000, + 0x1576: 0x3000, 0x1577: 0x3000, + 0x157c: 0x3000, 0x157e: 0x3000, + // Block 0x56, offset 0x1580 + 0x1587: 0x3000, 0x1588: 0x3000, 0x1589: 0x3000, + 0x1597: 0x3000, + 0x159f: 0x3000, + 0x15b0: 0x3000, 0x15b1: 0x3000, 0x15b4: 0x3000, 0x15b5: 0x3000, + 0x15b6: 0x3000, 0x15b7: 0x3000, 0x15b8: 0x3000, 0x15b9: 0x3000, 0x15ba: 0x3000, 0x15bb: 0x3000, + 0x15bc: 0x3000, 0x15bd: 0x3000, 0x15be: 0x3000, 0x15bf: 0x3000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x3000, 0x15c1: 0x3000, 0x15c2: 0x3000, 0x15c3: 0x3000, 0x15c4: 0x3000, 0x15c5: 0x3000, + 0x15c6: 0x3000, 0x15c7: 0x3000, 0x15c8: 0x3000, 0x15c9: 0x3000, 0x15ca: 0x3000, 0x15cb: 0x3000, + 0x15cc: 0x3000, 0x15cd: 0x3000, 0x15ce: 0x3000, 0x15d0: 0x3000, 0x15d1: 0x3000, + 0x15d2: 0x3000, 0x15d3: 0x3000, 0x15d4: 0x3000, 0x15d5: 0x3000, 0x15d6: 0x3000, 0x15d7: 0x3000, + 0x15d8: 0x3000, 0x15d9: 0x3000, 0x15da: 0x3000, 0x15db: 0x3000, 0x15dc: 0x3000, + 0x15e8: 0x3000, + // Block 0x58, offset 0x1600 + 0x1610: 0x00e6, 0x1611: 0x00e6, + 0x1612: 0x0001, 0x1613: 0x0001, 0x1614: 0x00e6, 0x1615: 0x00e6, 0x1616: 0x00e6, 0x1617: 0x00e6, + 0x1618: 0x0001, 0x1619: 0x0001, 0x161a: 0x0001, 0x161b: 0x00e6, 0x161c: 0x00e6, + 0x1621: 0x00e6, + 0x1625: 0x0001, 0x1626: 0x0001, 0x1627: 0x00e6, 0x1628: 0x00dc, 0x1629: 0x00e6, + 0x162a: 0x0001, 0x162b: 0x0001, 0x162c: 0x00dc, 0x162d: 0x00dc, 0x162e: 0x00dc, 0x162f: 0x00dc, + 0x1630: 0x00e6, + // Block 0x59, offset 0x1640 + 0x1640: 0x3000, 0x1641: 0x3000, 0x1642: 0x3000, 0x1643: 0x3000, 0x1645: 0x3000, + 0x1646: 0x3000, 0x1647: 0x3000, 0x1649: 0x3000, 0x164a: 0x3000, 0x164b: 0x3000, + 0x164c: 0x3000, 0x164d: 0x3000, 0x164e: 0x3000, 0x164f: 0x3000, 0x1650: 0x3000, 0x1651: 0x3000, + 0x1652: 0x3000, 0x1653: 0x3000, 0x1655: 0x3000, 0x1656: 0x3000, + 0x1659: 0x3000, 0x165a: 0x3000, 0x165b: 0x3000, 0x165c: 0x3000, 0x165d: 0x3000, + 0x1660: 0x3000, 0x1661: 0x3000, 0x1662: 0x3000, + 0x1664: 0x3000, 0x1666: 0x3300, 0x1668: 0x3000, + 0x166a: 0x3300, 0x166b: 0x3300, 0x166c: 0x3000, 0x166d: 0x3000, 0x166f: 0x3000, + 0x1670: 0x3000, 0x1671: 0x3000, 0x1673: 0x3000, 0x1674: 0x3000, 0x1675: 0x3000, + 0x1676: 0x3000, 0x1677: 0x3000, 0x1678: 0x3000, 0x1679: 0x3000, 0x167b: 0x3000, + 0x167c: 0x3000, 0x167d: 0x3000, 0x167e: 0x3000, 0x167f: 0x3000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x3000, 0x1685: 0x3000, + 0x1686: 0x3000, 0x1687: 0x3000, 0x1688: 0x3000, 0x1689: 0x3000, + 0x1690: 0x3000, 0x1691: 0x3000, + 0x1692: 0x3000, 0x1693: 0x3000, 0x1694: 0x3000, 0x1695: 0x3000, 0x1696: 0x3000, 0x1697: 0x3000, + 0x1698: 0x3000, 0x1699: 0x3000, 0x169a: 0x3000, 0x169b: 0x3000, 0x169c: 0x3000, 0x169d: 0x3000, + 0x169e: 0x3000, 0x169f: 0x3000, 0x16a0: 0x3000, 0x16a1: 0x3000, 0x16a2: 0x3000, 0x16a3: 0x3000, + 0x16a4: 0x3000, 0x16a5: 0x3000, 0x16a6: 0x3000, 0x16a7: 0x3000, 0x16a8: 0x3000, 0x16a9: 0x3000, + 0x16aa: 0x3000, 0x16ab: 0x3000, 0x16ac: 0x3000, 0x16ad: 0x3000, 0x16ae: 0x3000, 0x16af: 0x3000, + 0x16b0: 0x3000, 0x16b1: 0x3000, 0x16b2: 0x3000, 0x16b3: 0x3000, 0x16b4: 0x3000, 0x16b5: 0x3000, + 0x16b6: 0x3000, 0x16b7: 0x3000, 0x16b8: 0x3000, 0x16b9: 0x3000, 0x16ba: 0x3000, 0x16bb: 0x3000, + 0x16bc: 0x3000, 0x16bd: 0x3000, 0x16be: 0x3000, 0x16bf: 0x3000, + // Block 0x5b, offset 0x16c0 + 0x16c9: 0x3000, + 0x16d0: 0x8800, + 0x16d2: 0x8800, 0x16d4: 0x8800, + 0x16da: 0x1100, 0x16db: 0x1100, + 0x16ee: 0x1100, + // Block 0x5c, offset 0x1700 + 0x170d: 0x1100, 0x170e: 0x1100, 0x170f: 0x1100, 0x1710: 0x8800, + 0x1712: 0x8800, 0x1714: 0x8800, + // Block 0x5d, offset 0x1740 + 0x1743: 0x8800, 0x1744: 0x1100, + 0x1748: 0x8800, 0x1749: 0x1100, 0x174b: 0x8800, + 0x174c: 0x1100, + 0x1763: 0x8800, + 0x1764: 0x1100, 0x1765: 0x8800, 0x1766: 0x1100, + 0x176c: 0x3000, 0x176d: 0x3000, 0x176f: 0x3000, + 0x1770: 0x3000, + 0x177c: 0x8800, + // Block 0x5e, offset 0x1780 + 0x1781: 0x1100, 0x1783: 0x8800, 0x1784: 0x1100, 0x1785: 0x8800, + 0x1787: 0x1100, 0x1788: 0x8800, 0x1789: 0x1100, + 0x178d: 0x8800, + 0x17a0: 0x1100, 0x17a1: 0x8800, 0x17a2: 0x1100, + 0x17a4: 0x8800, 0x17a5: 0x8800, + 0x17ad: 0x1100, 0x17ae: 0x1100, 0x17af: 0x1100, + 0x17b0: 0x1100, 0x17b1: 0x1100, 0x17b2: 0x8800, 0x17b3: 0x8800, 0x17b4: 0x1100, 0x17b5: 0x1100, + 0x17b6: 0x8800, 0x17b7: 0x8800, 0x17b8: 0x1100, 0x17b9: 0x1100, 0x17ba: 0x8800, 0x17bb: 0x8800, + 0x17bc: 0x8800, 0x17bd: 0x8800, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x1100, 0x17c1: 0x1100, 0x17c2: 0x8800, 0x17c3: 0x8800, 0x17c4: 0x1100, 0x17c5: 0x1100, + 0x17c6: 0x8800, 0x17c7: 0x8800, 0x17c8: 0x1100, 0x17c9: 0x1100, + 0x17d1: 0x8800, + 0x17d2: 0x8800, + 0x17e2: 0x8800, + 0x17e8: 0x8800, 0x17e9: 0x8800, + 0x17eb: 0x8800, 0x17ec: 0x1100, 0x17ed: 0x1100, 0x17ee: 0x1100, 0x17ef: 0x1100, + 0x17f2: 0x8800, 0x17f3: 0x8800, 0x17f4: 0x8800, 0x17f5: 0x8800, + // Block 0x60, offset 0x1800 + 0x1820: 0x1100, 0x1821: 0x1100, 0x1822: 0x1100, 0x1823: 0x1100, + 0x182a: 0x1100, 0x182b: 0x1100, 0x182c: 0x1100, 0x182d: 0x1100, + // Block 0x61, offset 0x1840 + 0x1869: 0x3300, + 0x186a: 0x3300, + // Block 0x62, offset 0x1880 + 0x18a0: 0x3000, 0x18a1: 0x3000, 0x18a2: 0x3000, 0x18a3: 0x3000, + 0x18a4: 0x3000, 0x18a5: 0x3000, 0x18a6: 0x3000, 0x18a7: 0x3000, 0x18a8: 0x3000, 0x18a9: 0x3000, + 0x18aa: 0x3000, 0x18ab: 0x3000, 0x18ac: 0x3000, 0x18ad: 0x3000, 0x18ae: 0x3000, 0x18af: 0x3000, + 0x18b0: 0x3000, 0x18b1: 0x3000, 0x18b2: 0x3000, 0x18b3: 0x3000, 0x18b4: 0x3000, 0x18b5: 0x3000, + 0x18b6: 0x3000, 0x18b7: 0x3000, 0x18b8: 0x3000, 0x18b9: 0x3000, 0x18ba: 0x3000, 0x18bb: 0x3000, + 0x18bc: 0x3000, 0x18bd: 0x3000, 0x18be: 0x3000, 0x18bf: 0x3000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x3000, 0x18c1: 0x3000, 0x18c2: 0x3000, 0x18c3: 0x3000, 0x18c4: 0x3000, 0x18c5: 0x3000, + 0x18c6: 0x3000, 0x18c7: 0x3000, 0x18c8: 0x3000, 0x18c9: 0x3000, 0x18ca: 0x3000, 0x18cb: 0x3000, + 0x18cc: 0x3000, 0x18cd: 0x3000, 0x18ce: 0x3000, 0x18cf: 0x3000, 0x18d0: 0x3000, 0x18d1: 0x3000, + 0x18d2: 0x3000, 0x18d3: 0x3000, 0x18d4: 0x3000, 0x18d5: 0x3000, 0x18d6: 0x3000, 0x18d7: 0x3000, + 0x18d8: 0x3000, 0x18d9: 0x3000, 0x18da: 0x3000, 0x18db: 0x3000, 0x18dc: 0x3000, 0x18dd: 0x3000, + 0x18de: 0x3000, 0x18df: 0x3000, 0x18e0: 0x3000, 0x18e1: 0x3000, 0x18e2: 0x3000, 0x18e3: 0x3000, + 0x18e4: 0x3000, 0x18e5: 0x3000, 0x18e6: 0x3000, 0x18e7: 0x3000, 0x18e8: 0x3000, 0x18e9: 0x3000, + 0x18ea: 0x3000, 0x18eb: 0x3000, 0x18ec: 0x3000, 0x18ed: 0x3000, 0x18ee: 0x3000, 0x18ef: 0x3000, + 0x18f0: 0x3000, 0x18f1: 0x3000, 0x18f2: 0x3000, 0x18f3: 0x3000, 0x18f4: 0x3000, 0x18f5: 0x3000, + 0x18f6: 0x3000, 0x18f7: 0x3000, 0x18f8: 0x3000, 0x18f9: 0x3000, 0x18fa: 0x3000, 0x18fb: 0x3000, + 0x18fc: 0x3000, 0x18fd: 0x3000, 0x18fe: 0x3000, 0x18ff: 0x3000, + // Block 0x64, offset 0x1900 + 0x1900: 0x3000, 0x1901: 0x3000, 0x1902: 0x3000, 0x1903: 0x3000, 0x1904: 0x3000, 0x1905: 0x3000, + 0x1906: 0x3000, 0x1907: 0x3000, 0x1908: 0x3000, 0x1909: 0x3000, 0x190a: 0x3000, 0x190b: 0x3000, + 0x190c: 0x3000, 0x190d: 0x3000, 0x190e: 0x3000, 0x190f: 0x3000, 0x1910: 0x3000, 0x1911: 0x3000, + 0x1912: 0x3000, 0x1913: 0x3000, 0x1914: 0x3000, 0x1915: 0x3000, 0x1916: 0x3000, 0x1917: 0x3000, + 0x1918: 0x3000, 0x1919: 0x3000, 0x191a: 0x3000, 0x191b: 0x3000, 0x191c: 0x3000, 0x191d: 0x3000, + 0x191e: 0x3000, 0x191f: 0x3000, 0x1920: 0x3000, 0x1921: 0x3000, 0x1922: 0x3000, 0x1923: 0x3000, + 0x1924: 0x3000, 0x1925: 0x3000, 0x1926: 0x3000, 0x1927: 0x3000, 0x1928: 0x3000, 0x1929: 0x3000, + 0x192a: 0x3000, + // Block 0x65, offset 0x1940 + 0x194c: 0x3000, + // Block 0x66, offset 0x1980 + 0x19b4: 0x3000, 0x19b5: 0x3000, + 0x19b6: 0x3000, + // Block 0x67, offset 0x19c0 + 0x19dc: 0x3300, + // Block 0x68, offset 0x1a00 + 0x1a3c: 0x3000, 0x1a3d: 0x3000, + // Block 0x69, offset 0x1a40 + 0x1a6f: 0x00e6, + 0x1a70: 0x00e6, 0x1a71: 0x00e6, + // Block 0x6a, offset 0x1a80 + 0x1aaf: 0x3000, + 0x1abf: 0x0009, + // Block 0x6b, offset 0x1ac0 + 0x1ae0: 0x00e6, 0x1ae1: 0x00e6, 0x1ae2: 0x00e6, 0x1ae3: 0x00e6, + 0x1ae4: 0x00e6, 0x1ae5: 0x00e6, 0x1ae6: 0x00e6, 0x1ae7: 0x00e6, 0x1ae8: 0x00e6, 0x1ae9: 0x00e6, + 0x1aea: 0x00e6, 0x1aeb: 0x00e6, 0x1aec: 0x00e6, 0x1aed: 0x00e6, 0x1aee: 0x00e6, 0x1aef: 0x00e6, + 0x1af0: 0x00e6, 0x1af1: 0x00e6, 0x1af2: 0x00e6, 0x1af3: 0x00e6, 0x1af4: 0x00e6, 0x1af5: 0x00e6, + 0x1af6: 0x00e6, 0x1af7: 0x00e6, 0x1af8: 0x00e6, 0x1af9: 0x00e6, 0x1afa: 0x00e6, 0x1afb: 0x00e6, + 0x1afc: 0x00e6, 0x1afd: 0x00e6, 0x1afe: 0x00e6, 0x1aff: 0x00e6, + // Block 0x6c, offset 0x1b00 + 0x1b1f: 0x3000, + // Block 0x6d, offset 0x1b40 + 0x1b73: 0x3000, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x3000, 0x1b81: 0x3000, 0x1b82: 0x3000, 0x1b83: 0x3000, 0x1b84: 0x3000, 0x1b85: 0x3000, + 0x1b86: 0x3000, 0x1b87: 0x3000, 0x1b88: 0x3000, 0x1b89: 0x3000, 0x1b8a: 0x3000, 0x1b8b: 0x3000, + 0x1b8c: 0x3000, 0x1b8d: 0x3000, 0x1b8e: 0x3000, 0x1b8f: 0x3000, 0x1b90: 0x3000, 0x1b91: 0x3000, + 0x1b92: 0x3000, 0x1b93: 0x3000, 0x1b94: 0x3000, 0x1b95: 0x3000, + // Block 0x6f, offset 0x1bc0 + 0x1bc0: 0x3000, + 0x1bea: 0x00da, 0x1beb: 0x00e4, 0x1bec: 0x00e8, 0x1bed: 0x00de, 0x1bee: 0x00e0, 0x1bef: 0x00e0, + 0x1bf6: 0x3000, 0x1bf8: 0x3000, 0x1bf9: 0x3000, 0x1bfa: 0x3000, + // Block 0x70, offset 0x1c00 + 0x1c06: 0x8800, 0x1c0b: 0x8800, + 0x1c0c: 0x1100, 0x1c0d: 0x8800, 0x1c0e: 0x1100, 0x1c0f: 0x8800, 0x1c10: 0x1100, 0x1c11: 0x8800, + 0x1c12: 0x1100, 0x1c13: 0x8800, 0x1c14: 0x1100, 0x1c15: 0x8800, 0x1c16: 0x1100, 0x1c17: 0x8800, + 0x1c18: 0x1100, 0x1c19: 0x8800, 0x1c1a: 0x1100, 0x1c1b: 0x8800, 0x1c1c: 0x1100, 0x1c1d: 0x8800, + 0x1c1e: 0x1100, 0x1c1f: 0x8800, 0x1c20: 0x1100, 0x1c21: 0x8800, 0x1c22: 0x1100, + 0x1c24: 0x8800, 0x1c25: 0x1100, 0x1c26: 0x8800, 0x1c27: 0x1100, 0x1c28: 0x8800, 0x1c29: 0x1100, + 0x1c2f: 0x8800, + 0x1c30: 0x1100, 0x1c31: 0x1100, 0x1c32: 0x8800, 0x1c33: 0x1100, 0x1c34: 0x1100, 0x1c35: 0x8800, + 0x1c36: 0x1100, 0x1c37: 0x1100, 0x1c38: 0x8800, 0x1c39: 0x1100, 0x1c3a: 0x1100, 0x1c3b: 0x8800, + 0x1c3c: 0x1100, 0x1c3d: 0x1100, + // Block 0x71, offset 0x1c40 + 0x1c54: 0x1100, + 0x1c59: 0x6608, 0x1c5a: 0x6608, 0x1c5b: 0x3000, 0x1c5c: 0x3000, 0x1c5d: 0x8800, + 0x1c5e: 0x1100, 0x1c5f: 0x3000, + 0x1c66: 0x8800, + 0x1c6b: 0x8800, 0x1c6c: 0x1100, 0x1c6d: 0x8800, 0x1c6e: 0x1100, 0x1c6f: 0x8800, + 0x1c70: 0x1100, 0x1c71: 0x8800, 0x1c72: 0x1100, 0x1c73: 0x8800, 0x1c74: 0x1100, 0x1c75: 0x8800, + 0x1c76: 0x1100, 0x1c77: 0x8800, 0x1c78: 0x1100, 0x1c79: 0x8800, 0x1c7a: 0x1100, 0x1c7b: 0x8800, + 0x1c7c: 0x1100, 0x1c7d: 0x8800, 0x1c7e: 0x1100, 0x1c7f: 0x8800, + // Block 0x72, offset 0x1c80 + 0x1c80: 0x1100, 0x1c81: 0x8800, 0x1c82: 0x1100, 0x1c84: 0x8800, 0x1c85: 0x1100, + 0x1c86: 0x8800, 0x1c87: 0x1100, 0x1c88: 0x8800, 0x1c89: 0x1100, + 0x1c8f: 0x8800, 0x1c90: 0x1100, 0x1c91: 0x1100, + 0x1c92: 0x8800, 0x1c93: 0x1100, 0x1c94: 0x1100, 0x1c95: 0x8800, 0x1c96: 0x1100, 0x1c97: 0x1100, + 0x1c98: 0x8800, 0x1c99: 0x1100, 0x1c9a: 0x1100, 0x1c9b: 0x8800, 0x1c9c: 0x1100, 0x1c9d: 0x1100, + 0x1caf: 0x8800, + 0x1cb0: 0x8800, 0x1cb1: 0x8800, 0x1cb2: 0x8800, 0x1cb4: 0x1100, + 0x1cb7: 0x1100, 0x1cb8: 0x1100, 0x1cb9: 0x1100, 0x1cba: 0x1100, + 0x1cbd: 0x8800, 0x1cbe: 0x1100, 0x1cbf: 0x3000, + // Block 0x73, offset 0x1cc0 + 0x1cf1: 0x3000, 0x1cf2: 0x3000, 0x1cf3: 0x3000, 0x1cf4: 0x3000, 0x1cf5: 0x3000, + 0x1cf6: 0x3000, 0x1cf7: 0x3000, 0x1cf8: 0x3000, 0x1cf9: 0x3000, 0x1cfa: 0x3000, 0x1cfb: 0x3000, + 0x1cfc: 0x3000, 0x1cfd: 0x3000, 0x1cfe: 0x3000, 0x1cff: 0x3000, + // Block 0x74, offset 0x1d00 + 0x1d00: 0x3000, 0x1d01: 0x3000, 0x1d02: 0x3000, 0x1d03: 0x3000, 0x1d04: 0x3000, 0x1d05: 0x3000, + 0x1d06: 0x3000, 0x1d07: 0x3000, 0x1d08: 0x3000, 0x1d09: 0x3000, 0x1d0a: 0x3000, 0x1d0b: 0x3000, + 0x1d0c: 0x3000, 0x1d0d: 0x3000, 0x1d0e: 0x3000, + 0x1d12: 0x3000, 0x1d13: 0x3000, 0x1d14: 0x3000, 0x1d15: 0x3000, 0x1d16: 0x3000, 0x1d17: 0x3000, + 0x1d18: 0x3000, 0x1d19: 0x3000, 0x1d1a: 0x3000, 0x1d1b: 0x3000, 0x1d1c: 0x3000, 0x1d1d: 0x3000, + 0x1d1e: 0x3000, 0x1d1f: 0x3000, + // Block 0x75, offset 0x1d40 + 0x1d40: 0x3000, 0x1d41: 0x3000, 0x1d42: 0x3000, 0x1d43: 0x3000, 0x1d44: 0x3000, 0x1d45: 0x3000, + 0x1d46: 0x3000, 0x1d47: 0x3000, 0x1d48: 0x3000, 0x1d49: 0x3000, 0x1d4a: 0x3000, 0x1d4b: 0x3000, + 0x1d4c: 0x3000, 0x1d4d: 0x3000, 0x1d4e: 0x3000, 0x1d4f: 0x3000, 0x1d50: 0x3000, 0x1d51: 0x3000, + 0x1d52: 0x3000, 0x1d53: 0x3000, 0x1d54: 0x3000, 0x1d55: 0x3000, 0x1d56: 0x3000, 0x1d57: 0x3000, + 0x1d58: 0x3000, 0x1d59: 0x3000, 0x1d5a: 0x3000, 0x1d5b: 0x3000, 0x1d5c: 0x3000, 0x1d5d: 0x3000, + 0x1d5e: 0x3000, 0x1d60: 0x3000, 0x1d61: 0x3000, 0x1d62: 0x3000, 0x1d63: 0x3000, + 0x1d64: 0x3000, 0x1d65: 0x3000, 0x1d66: 0x3000, 0x1d67: 0x3000, 0x1d68: 0x3000, 0x1d69: 0x3000, + 0x1d6a: 0x3000, 0x1d6b: 0x3000, 0x1d6c: 0x3000, 0x1d6d: 0x3000, 0x1d6e: 0x3000, 0x1d6f: 0x3000, + 0x1d70: 0x3000, 0x1d71: 0x3000, 0x1d72: 0x3000, 0x1d73: 0x3000, 0x1d74: 0x3000, 0x1d75: 0x3000, + 0x1d76: 0x3000, 0x1d77: 0x3000, 0x1d78: 0x3000, 0x1d79: 0x3000, 0x1d7a: 0x3000, 0x1d7b: 0x3000, + 0x1d7c: 0x3000, 0x1d7d: 0x3000, 0x1d7e: 0x3000, 0x1d7f: 0x3000, + // Block 0x76, offset 0x1d80 + 0x1d80: 0x3000, 0x1d81: 0x3000, 0x1d82: 0x3000, 0x1d83: 0x3000, 0x1d84: 0x3000, 0x1d85: 0x3000, + 0x1d86: 0x3000, 0x1d87: 0x3000, + 0x1d90: 0x3000, 0x1d91: 0x3000, + 0x1d92: 0x3000, 0x1d93: 0x3000, 0x1d94: 0x3000, 0x1d95: 0x3000, 0x1d96: 0x3000, 0x1d97: 0x3000, + 0x1d98: 0x3000, 0x1d99: 0x3000, 0x1d9a: 0x3000, 0x1d9b: 0x3000, 0x1d9c: 0x3000, 0x1d9d: 0x3000, + 0x1d9e: 0x3000, 0x1d9f: 0x3000, 0x1da0: 0x3000, 0x1da1: 0x3000, 0x1da2: 0x3000, 0x1da3: 0x3000, + 0x1da4: 0x3000, 0x1da5: 0x3000, 0x1da6: 0x3000, 0x1da7: 0x3000, 0x1da8: 0x3000, 0x1da9: 0x3000, + 0x1daa: 0x3000, 0x1dab: 0x3000, 0x1dac: 0x3000, 0x1dad: 0x3000, 0x1dae: 0x3000, 0x1daf: 0x3000, + 0x1db0: 0x3000, 0x1db1: 0x3000, 0x1db2: 0x3000, 0x1db3: 0x3000, 0x1db4: 0x3000, 0x1db5: 0x3000, + 0x1db6: 0x3000, 0x1db7: 0x3000, 0x1db8: 0x3000, 0x1db9: 0x3000, 0x1dba: 0x3000, 0x1dbb: 0x3000, + 0x1dbc: 0x3000, 0x1dbd: 0x3000, 0x1dbe: 0x3000, + // Block 0x77, offset 0x1dc0 + 0x1dc0: 0x3000, 0x1dc1: 0x3000, 0x1dc2: 0x3000, 0x1dc3: 0x3000, 0x1dc4: 0x3000, 0x1dc5: 0x3000, + 0x1dc6: 0x3000, 0x1dc7: 0x3000, 0x1dc8: 0x3000, 0x1dc9: 0x3000, 0x1dca: 0x3000, 0x1dcb: 0x3000, + 0x1dcc: 0x3000, 0x1dcd: 0x3000, 0x1dce: 0x3000, 0x1dcf: 0x3000, 0x1dd0: 0x3000, 0x1dd1: 0x3000, + 0x1dd2: 0x3000, 0x1dd3: 0x3000, 0x1dd4: 0x3000, 0x1dd5: 0x3000, 0x1dd6: 0x3000, 0x1dd7: 0x3000, + 0x1dd8: 0x3000, 0x1dd9: 0x3000, 0x1dda: 0x3000, 0x1ddb: 0x3000, 0x1ddc: 0x3000, 0x1ddd: 0x3000, + 0x1dde: 0x3000, 0x1ddf: 0x3000, 0x1de0: 0x3000, 0x1de1: 0x3000, 0x1de2: 0x3000, 0x1de3: 0x3000, + 0x1de4: 0x3000, 0x1de5: 0x3000, 0x1de6: 0x3000, 0x1de7: 0x3000, 0x1de8: 0x3000, 0x1de9: 0x3000, + 0x1dea: 0x3000, 0x1deb: 0x3000, 0x1dec: 0x3000, 0x1ded: 0x3000, 0x1dee: 0x3000, 0x1def: 0x3000, + 0x1df0: 0x3000, 0x1df1: 0x3000, 0x1df2: 0x3000, 0x1df3: 0x3000, 0x1df4: 0x3000, 0x1df5: 0x3000, + 0x1df6: 0x3000, 0x1df7: 0x3000, 0x1df8: 0x3000, 0x1df9: 0x3000, 0x1dfa: 0x3000, 0x1dfb: 0x3000, + 0x1dfc: 0x3000, 0x1dfd: 0x3000, 0x1dfe: 0x3000, + // Block 0x78, offset 0x1e00 + 0x1e2f: 0x00e6, + 0x1e3c: 0x00e6, 0x1e3d: 0x00e6, + // Block 0x79, offset 0x1e40 + 0x1e70: 0x00e6, 0x1e71: 0x00e6, + // Block 0x7a, offset 0x1e80 + 0x1eb0: 0x3000, + // Block 0x7b, offset 0x1ec0 + 0x1ec6: 0x0009, + // Block 0x7c, offset 0x1f00 + 0x1f04: 0x0009, + 0x1f20: 0x00e6, 0x1f21: 0x00e6, 0x1f22: 0x00e6, 0x1f23: 0x00e6, + 0x1f24: 0x00e6, 0x1f25: 0x00e6, 0x1f26: 0x00e6, 0x1f27: 0x00e6, 0x1f28: 0x00e6, 0x1f29: 0x00e6, + 0x1f2a: 0x00e6, 0x1f2b: 0x00e6, 0x1f2c: 0x00e6, 0x1f2d: 0x00e6, 0x1f2e: 0x00e6, 0x1f2f: 0x00e6, + 0x1f30: 0x00e6, 0x1f31: 0x00e6, + // Block 0x7d, offset 0x1f40 + 0x1f6b: 0x00dc, 0x1f6c: 0x00dc, 0x1f6d: 0x00dc, + // Block 0x7e, offset 0x1f80 + 0x1f93: 0x0009, + // Block 0x7f, offset 0x1fc0 + 0x1ff3: 0x0007, + // Block 0x80, offset 0x2000 + 0x2000: 0x0009, + // Block 0x81, offset 0x2040 + 0x2070: 0x00e6, 0x2072: 0x00e6, 0x2073: 0x00e6, 0x2074: 0x00dc, + 0x2077: 0x00e6, 0x2078: 0x00e6, + 0x207e: 0x00e6, 0x207f: 0x00e6, + // Block 0x82, offset 0x2080 + 0x2081: 0x00e6, + // Block 0x83, offset 0x20c0 + 0x20ed: 0x0009, + // Block 0x84, offset 0x2100 + 0x2100: 0x1100, 0x2101: 0x1100, 0x2102: 0x1100, 0x2103: 0x1100, 0x2104: 0x1100, 0x2105: 0x1100, + 0x2106: 0x1100, 0x2107: 0x1100, 0x2108: 0x1100, 0x2109: 0x1100, 0x210a: 0x1100, 0x210b: 0x1100, + 0x210c: 0x1100, 0x210d: 0x1100, 0x210e: 0x1100, 0x210f: 0x1100, 0x2110: 0x1100, 0x2111: 0x1100, + 0x2112: 0x1100, 0x2113: 0x1100, 0x2114: 0x1100, 0x2115: 0x1100, 0x2116: 0x1100, 0x2117: 0x1100, + 0x2118: 0x1100, 0x2119: 0x1100, 0x211a: 0x1100, 0x211b: 0x1100, 0x211c: 0x1100, 0x211d: 0x1100, + 0x211e: 0x1100, 0x211f: 0x1100, 0x2120: 0x1100, 0x2121: 0x1100, 0x2122: 0x1100, 0x2123: 0x1100, + 0x2124: 0x1100, 0x2125: 0x1100, 0x2126: 0x1100, 0x2127: 0x1100, 0x2128: 0x1100, 0x2129: 0x1100, + 0x212a: 0x1100, 0x212b: 0x1100, 0x212c: 0x1100, 0x212d: 0x1100, 0x212e: 0x1100, 0x212f: 0x1100, + 0x2130: 0x1100, 0x2131: 0x1100, 0x2132: 0x1100, 0x2133: 0x1100, 0x2134: 0x1100, 0x2135: 0x1100, + 0x2136: 0x1100, 0x2137: 0x1100, 0x2138: 0x1100, 0x2139: 0x1100, 0x213a: 0x1100, 0x213b: 0x1100, + 0x213c: 0x1100, 0x213d: 0x1100, 0x213e: 0x1100, 0x213f: 0x1100, + // Block 0x85, offset 0x2140 + 0x2140: 0x1100, 0x2141: 0x1100, 0x2142: 0x1100, 0x2143: 0x1100, 0x2144: 0x1100, 0x2145: 0x1100, + 0x2146: 0x1100, 0x2147: 0x1100, 0x2148: 0x1100, 0x2149: 0x1100, 0x214a: 0x1100, 0x214b: 0x1100, + 0x214c: 0x1100, 0x214d: 0x1100, 0x214e: 0x1100, 0x214f: 0x1100, 0x2150: 0x1100, 0x2151: 0x1100, + 0x2152: 0x1100, 0x2153: 0x1100, 0x2154: 0x1100, 0x2155: 0x1100, 0x2156: 0x1100, 0x2157: 0x1100, + 0x2158: 0x1100, 0x2159: 0x1100, 0x215a: 0x1100, 0x215b: 0x1100, 0x215c: 0x1100, 0x215d: 0x1100, + 0x215e: 0x1100, 0x215f: 0x1100, 0x2160: 0x1100, 0x2161: 0x1100, 0x2162: 0x1100, 0x2163: 0x1100, + // Block 0x86, offset 0x2180 + 0x2180: 0x3300, 0x2181: 0x3300, 0x2182: 0x3300, 0x2183: 0x3300, 0x2184: 0x3300, 0x2185: 0x3300, + 0x2186: 0x3300, 0x2187: 0x3300, 0x2188: 0x3300, 0x2189: 0x3300, 0x218a: 0x3300, 0x218b: 0x3300, + 0x218c: 0x3300, 0x218d: 0x3300, 0x218e: 0x3300, 0x218f: 0x3300, 0x2190: 0x3300, 0x2191: 0x3300, + 0x2192: 0x3300, 0x2193: 0x3300, 0x2194: 0x3300, 0x2195: 0x3300, 0x2196: 0x3300, 0x2197: 0x3300, + 0x2198: 0x3300, 0x2199: 0x3300, 0x219a: 0x3300, 0x219b: 0x3300, 0x219c: 0x3300, 0x219d: 0x3300, + 0x219e: 0x3300, 0x219f: 0x3300, 0x21a0: 0x3300, 0x21a1: 0x3300, 0x21a2: 0x3300, 0x21a3: 0x3300, + 0x21a4: 0x3300, 0x21a5: 0x3300, 0x21a6: 0x3300, 0x21a7: 0x3300, 0x21a8: 0x3300, 0x21a9: 0x3300, + 0x21aa: 0x3300, 0x21ab: 0x3300, 0x21ac: 0x3300, 0x21ad: 0x3300, 0x21ae: 0x3300, 0x21af: 0x3300, + 0x21b0: 0x3300, 0x21b1: 0x3300, 0x21b2: 0x3300, 0x21b3: 0x3300, 0x21b4: 0x3300, 0x21b5: 0x3300, + 0x21b6: 0x3300, 0x21b7: 0x3300, 0x21b8: 0x3300, 0x21b9: 0x3300, 0x21ba: 0x3300, 0x21bb: 0x3300, + 0x21bc: 0x3300, 0x21bd: 0x3300, 0x21be: 0x3300, 0x21bf: 0x3300, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x3300, 0x21c1: 0x3300, 0x21c2: 0x3300, 0x21c3: 0x3300, 0x21c4: 0x3300, 0x21c5: 0x3300, + 0x21c6: 0x3300, 0x21c7: 0x3300, 0x21c8: 0x3300, 0x21c9: 0x3300, 0x21ca: 0x3300, 0x21cb: 0x3300, + 0x21cc: 0x3300, 0x21cd: 0x3300, 0x21d0: 0x3300, + 0x21d2: 0x3300, 0x21d5: 0x3300, 0x21d6: 0x3300, 0x21d7: 0x3300, + 0x21d8: 0x3300, 0x21d9: 0x3300, 0x21da: 0x3300, 0x21db: 0x3300, 0x21dc: 0x3300, 0x21dd: 0x3300, + 0x21de: 0x3300, 0x21e0: 0x3300, 0x21e2: 0x3300, + 0x21e5: 0x3300, 0x21e6: 0x3300, + 0x21ea: 0x3300, 0x21eb: 0x3300, 0x21ec: 0x3300, 0x21ed: 0x3300, + 0x21f0: 0x3300, 0x21f1: 0x3300, 0x21f2: 0x3300, 0x21f3: 0x3300, 0x21f4: 0x3300, 0x21f5: 0x3300, + 0x21f6: 0x3300, 0x21f7: 0x3300, 0x21f8: 0x3300, 0x21f9: 0x3300, 0x21fa: 0x3300, 0x21fb: 0x3300, + 0x21fc: 0x3300, 0x21fd: 0x3300, 0x21fe: 0x3300, 0x21ff: 0x3300, + // Block 0x88, offset 0x2200 + 0x2200: 0x3300, 0x2201: 0x3300, 0x2202: 0x3300, 0x2203: 0x3300, 0x2204: 0x3300, 0x2205: 0x3300, + 0x2206: 0x3300, 0x2207: 0x3300, 0x2208: 0x3300, 0x2209: 0x3300, 0x220a: 0x3300, 0x220b: 0x3300, + 0x220c: 0x3300, 0x220d: 0x3300, 0x220e: 0x3300, 0x220f: 0x3300, 0x2210: 0x3300, 0x2211: 0x3300, + 0x2212: 0x3300, 0x2213: 0x3300, 0x2214: 0x3300, 0x2215: 0x3300, 0x2216: 0x3300, 0x2217: 0x3300, + 0x2218: 0x3300, 0x2219: 0x3300, 0x221a: 0x3300, 0x221b: 0x3300, 0x221c: 0x3300, 0x221d: 0x3300, + 0x221e: 0x3300, 0x221f: 0x3300, 0x2220: 0x3300, 0x2221: 0x3300, 0x2222: 0x3300, 0x2223: 0x3300, + 0x2224: 0x3300, 0x2225: 0x3300, 0x2226: 0x3300, 0x2227: 0x3300, 0x2228: 0x3300, 0x2229: 0x3300, + 0x222a: 0x3300, 0x222b: 0x3300, 0x222c: 0x3300, 0x222d: 0x3300, + 0x2230: 0x3300, 0x2231: 0x3300, 0x2232: 0x3300, 0x2233: 0x3300, 0x2234: 0x3300, 0x2235: 0x3300, + 0x2236: 0x3300, 0x2237: 0x3300, 0x2238: 0x3300, 0x2239: 0x3300, 0x223a: 0x3300, 0x223b: 0x3300, + 0x223c: 0x3300, 0x223d: 0x3300, 0x223e: 0x3300, 0x223f: 0x3300, + // Block 0x89, offset 0x2240 + 0x2240: 0x3300, 0x2241: 0x3300, 0x2242: 0x3300, 0x2243: 0x3300, 0x2244: 0x3300, 0x2245: 0x3300, + 0x2246: 0x3300, 0x2247: 0x3300, 0x2248: 0x3300, 0x2249: 0x3300, 0x224a: 0x3300, 0x224b: 0x3300, + 0x224c: 0x3300, 0x224d: 0x3300, 0x224e: 0x3300, 0x224f: 0x3300, 0x2250: 0x3300, 0x2251: 0x3300, + 0x2252: 0x3300, 0x2253: 0x3300, 0x2254: 0x3300, 0x2255: 0x3300, 0x2256: 0x3300, 0x2257: 0x3300, + 0x2258: 0x3300, 0x2259: 0x3300, + // Block 0x8a, offset 0x2280 + 0x2280: 0x3000, 0x2281: 0x3000, 0x2282: 0x3000, 0x2283: 0x3000, 0x2284: 0x3000, 0x2285: 0x3000, + 0x2286: 0x3000, + 0x2293: 0x3000, 0x2294: 0x3000, 0x2295: 0x3000, 0x2296: 0x3000, 0x2297: 0x3000, + 0x229d: 0x3300, + 0x229e: 0x001a, 0x229f: 0x3300, 0x22a0: 0x3000, 0x22a1: 0x3000, 0x22a2: 0x3000, 0x22a3: 0x3000, + 0x22a4: 0x3000, 0x22a5: 0x3000, 0x22a6: 0x3000, 0x22a7: 0x3000, 0x22a8: 0x3000, 0x22a9: 0x3000, + 0x22aa: 0x3300, 0x22ab: 0x3300, 0x22ac: 0x3300, 0x22ad: 0x3300, 0x22ae: 0x3300, 0x22af: 0x3300, + 0x22b0: 0x3300, 0x22b1: 0x3300, 0x22b2: 0x3300, 0x22b3: 0x3300, 0x22b4: 0x3300, 0x22b5: 0x3300, + 0x22b6: 0x3300, 0x22b8: 0x3300, 0x22b9: 0x3300, 0x22ba: 0x3300, 0x22bb: 0x3300, + 0x22bc: 0x3300, 0x22be: 0x3300, + // Block 0x8b, offset 0x22c0 + 0x22c0: 0x3300, 0x22c1: 0x3300, 0x22c3: 0x3300, 0x22c4: 0x3300, + 0x22c6: 0x3300, 0x22c7: 0x3300, 0x22c8: 0x3300, 0x22c9: 0x3300, 0x22ca: 0x3300, 0x22cb: 0x3300, + 0x22cc: 0x3300, 0x22cd: 0x3300, 0x22ce: 0x3300, 0x22cf: 0x3000, 0x22d0: 0x3000, 0x22d1: 0x3000, + 0x22d2: 0x3000, 0x22d3: 0x3000, 0x22d4: 0x3000, 0x22d5: 0x3000, 0x22d6: 0x3000, 0x22d7: 0x3000, + 0x22d8: 0x3000, 0x22d9: 0x3000, 0x22da: 0x3000, 0x22db: 0x3000, 0x22dc: 0x3000, 0x22dd: 0x3000, + 0x22de: 0x3000, 0x22df: 0x3000, 0x22e0: 0x3000, 0x22e1: 0x3000, 0x22e2: 0x3000, 0x22e3: 0x3000, + 0x22e4: 0x3000, 0x22e5: 0x3000, 0x22e6: 0x3000, 0x22e7: 0x3000, 0x22e8: 0x3000, 0x22e9: 0x3000, + 0x22ea: 0x3000, 0x22eb: 0x3000, 0x22ec: 0x3000, 0x22ed: 0x3000, 0x22ee: 0x3000, 0x22ef: 0x3000, + 0x22f0: 0x3000, 0x22f1: 0x3000, 0x22f2: 0x3000, 0x22f3: 0x3000, 0x22f4: 0x3000, 0x22f5: 0x3000, + 0x22f6: 0x3000, 0x22f7: 0x3000, 0x22f8: 0x3000, 0x22f9: 0x3000, 0x22fa: 0x3000, 0x22fb: 0x3000, + 0x22fc: 0x3000, 0x22fd: 0x3000, 0x22fe: 0x3000, 0x22ff: 0x3000, + // Block 0x8c, offset 0x2300 + 0x2300: 0x3000, 0x2301: 0x3000, 0x2302: 0x3000, 0x2303: 0x3000, 0x2304: 0x3000, 0x2305: 0x3000, + 0x2306: 0x3000, 0x2307: 0x3000, 0x2308: 0x3000, 0x2309: 0x3000, 0x230a: 0x3000, 0x230b: 0x3000, + 0x230c: 0x3000, 0x230d: 0x3000, 0x230e: 0x3000, 0x230f: 0x3000, 0x2310: 0x3000, 0x2311: 0x3000, + 0x2312: 0x3000, 0x2313: 0x3000, 0x2314: 0x3000, 0x2315: 0x3000, 0x2316: 0x3000, 0x2317: 0x3000, + 0x2318: 0x3000, 0x2319: 0x3000, 0x231a: 0x3000, 0x231b: 0x3000, 0x231c: 0x3000, 0x231d: 0x3000, + 0x231e: 0x3000, 0x231f: 0x3000, 0x2320: 0x3000, 0x2321: 0x3000, 0x2322: 0x3000, 0x2323: 0x3000, + 0x2324: 0x3000, 0x2325: 0x3000, 0x2326: 0x3000, 0x2327: 0x3000, 0x2328: 0x3000, 0x2329: 0x3000, + 0x232a: 0x3000, 0x232b: 0x3000, 0x232c: 0x3000, 0x232d: 0x3000, 0x232e: 0x3000, 0x232f: 0x3000, + 0x2330: 0x3000, 0x2331: 0x3000, + // Block 0x8d, offset 0x2340 + 0x2353: 0x3000, 0x2354: 0x3000, 0x2355: 0x3000, 0x2356: 0x3000, 0x2357: 0x3000, + 0x2358: 0x3000, 0x2359: 0x3000, 0x235a: 0x3000, 0x235b: 0x3000, 0x235c: 0x3000, 0x235d: 0x3000, + 0x235e: 0x3000, 0x235f: 0x3000, 0x2360: 0x3000, 0x2361: 0x3000, 0x2362: 0x3000, 0x2363: 0x3000, + 0x2364: 0x3000, 0x2365: 0x3000, 0x2366: 0x3000, 0x2367: 0x3000, 0x2368: 0x3000, 0x2369: 0x3000, + 0x236a: 0x3000, 0x236b: 0x3000, 0x236c: 0x3000, 0x236d: 0x3000, 0x236e: 0x3000, 0x236f: 0x3000, + 0x2370: 0x3000, 0x2371: 0x3000, 0x2372: 0x3000, 0x2373: 0x3000, 0x2374: 0x3000, 0x2375: 0x3000, + 0x2376: 0x3000, 0x2377: 0x3000, 0x2378: 0x3000, 0x2379: 0x3000, 0x237a: 0x3000, 0x237b: 0x3000, + 0x237c: 0x3000, 0x237d: 0x3000, 0x237e: 0x3000, 0x237f: 0x3000, + // Block 0x8e, offset 0x2380 + 0x2380: 0x3000, 0x2381: 0x3000, 0x2382: 0x3000, 0x2383: 0x3000, 0x2384: 0x3000, 0x2385: 0x3000, + 0x2386: 0x3000, 0x2387: 0x3000, 0x2388: 0x3000, 0x2389: 0x3000, 0x238a: 0x3000, 0x238b: 0x3000, + 0x238c: 0x3000, 0x238d: 0x3000, 0x238e: 0x3000, 0x238f: 0x3000, 0x2390: 0x3000, 0x2391: 0x3000, + 0x2392: 0x3000, 0x2393: 0x3000, 0x2394: 0x3000, 0x2395: 0x3000, 0x2396: 0x3000, 0x2397: 0x3000, + 0x2398: 0x3000, 0x2399: 0x3000, 0x239a: 0x3000, 0x239b: 0x3000, 0x239c: 0x3000, 0x239d: 0x3000, + 0x239e: 0x3000, 0x239f: 0x3000, 0x23a0: 0x3000, 0x23a1: 0x3000, 0x23a2: 0x3000, 0x23a3: 0x3000, + 0x23a4: 0x3000, 0x23a5: 0x3000, 0x23a6: 0x3000, 0x23a7: 0x3000, 0x23a8: 0x3000, 0x23a9: 0x3000, + 0x23aa: 0x3000, 0x23ab: 0x3000, 0x23ac: 0x3000, 0x23ad: 0x3000, 0x23ae: 0x3000, 0x23af: 0x3000, + 0x23b0: 0x3000, 0x23b1: 0x3000, 0x23b2: 0x3000, 0x23b3: 0x3000, 0x23b4: 0x3000, 0x23b5: 0x3000, + 0x23b6: 0x3000, 0x23b7: 0x3000, 0x23b8: 0x3000, 0x23b9: 0x3000, 0x23ba: 0x3000, 0x23bb: 0x3000, + 0x23bc: 0x3000, 0x23bd: 0x3000, + // Block 0x8f, offset 0x23c0 + 0x23d0: 0x3000, 0x23d1: 0x3000, + 0x23d2: 0x3000, 0x23d3: 0x3000, 0x23d4: 0x3000, 0x23d5: 0x3000, 0x23d6: 0x3000, 0x23d7: 0x3000, + 0x23d8: 0x3000, 0x23d9: 0x3000, 0x23da: 0x3000, 0x23db: 0x3000, 0x23dc: 0x3000, 0x23dd: 0x3000, + 0x23de: 0x3000, 0x23df: 0x3000, 0x23e0: 0x3000, 0x23e1: 0x3000, 0x23e2: 0x3000, 0x23e3: 0x3000, + 0x23e4: 0x3000, 0x23e5: 0x3000, 0x23e6: 0x3000, 0x23e7: 0x3000, 0x23e8: 0x3000, 0x23e9: 0x3000, + 0x23ea: 0x3000, 0x23eb: 0x3000, 0x23ec: 0x3000, 0x23ed: 0x3000, 0x23ee: 0x3000, 0x23ef: 0x3000, + 0x23f0: 0x3000, 0x23f1: 0x3000, 0x23f2: 0x3000, 0x23f3: 0x3000, 0x23f4: 0x3000, 0x23f5: 0x3000, + 0x23f6: 0x3000, 0x23f7: 0x3000, 0x23f8: 0x3000, 0x23f9: 0x3000, 0x23fa: 0x3000, 0x23fb: 0x3000, + 0x23fc: 0x3000, 0x23fd: 0x3000, 0x23fe: 0x3000, 0x23ff: 0x3000, + // Block 0x90, offset 0x2400 + 0x2400: 0x3000, 0x2401: 0x3000, 0x2402: 0x3000, 0x2403: 0x3000, 0x2404: 0x3000, 0x2405: 0x3000, + 0x2406: 0x3000, 0x2407: 0x3000, 0x2408: 0x3000, 0x2409: 0x3000, 0x240a: 0x3000, 0x240b: 0x3000, + 0x240c: 0x3000, 0x240d: 0x3000, 0x240e: 0x3000, 0x240f: 0x3000, + 0x2412: 0x3000, 0x2413: 0x3000, 0x2414: 0x3000, 0x2415: 0x3000, 0x2416: 0x3000, 0x2417: 0x3000, + 0x2418: 0x3000, 0x2419: 0x3000, 0x241a: 0x3000, 0x241b: 0x3000, 0x241c: 0x3000, 0x241d: 0x3000, + 0x241e: 0x3000, 0x241f: 0x3000, 0x2420: 0x3000, 0x2421: 0x3000, 0x2422: 0x3000, 0x2423: 0x3000, + 0x2424: 0x3000, 0x2425: 0x3000, 0x2426: 0x3000, 0x2427: 0x3000, 0x2428: 0x3000, 0x2429: 0x3000, + 0x242a: 0x3000, 0x242b: 0x3000, 0x242c: 0x3000, 0x242d: 0x3000, 0x242e: 0x3000, 0x242f: 0x3000, + 0x2430: 0x3000, 0x2431: 0x3000, 0x2432: 0x3000, 0x2433: 0x3000, 0x2434: 0x3000, 0x2435: 0x3000, + 0x2436: 0x3000, 0x2437: 0x3000, 0x2438: 0x3000, 0x2439: 0x3000, 0x243a: 0x3000, 0x243b: 0x3000, + 0x243c: 0x3000, 0x243d: 0x3000, 0x243e: 0x3000, 0x243f: 0x3000, + // Block 0x91, offset 0x2440 + 0x2440: 0x3000, 0x2441: 0x3000, 0x2442: 0x3000, 0x2443: 0x3000, 0x2444: 0x3000, 0x2445: 0x3000, + 0x2446: 0x3000, 0x2447: 0x3000, + 0x2470: 0x3000, 0x2471: 0x3000, 0x2472: 0x3000, 0x2473: 0x3000, 0x2474: 0x3000, 0x2475: 0x3000, + 0x2476: 0x3000, 0x2477: 0x3000, 0x2478: 0x3000, 0x2479: 0x3000, 0x247a: 0x3000, 0x247b: 0x3000, + 0x247c: 0x3000, + // Block 0x92, offset 0x2480 + 0x2490: 0x3000, 0x2491: 0x3000, + 0x2492: 0x3000, 0x2493: 0x3000, 0x2494: 0x3000, 0x2495: 0x3000, 0x2496: 0x3000, 0x2497: 0x3000, + 0x2498: 0x3000, 0x2499: 0x3000, + 0x24a0: 0x00e6, 0x24a1: 0x00e6, 0x24a2: 0x00e6, 0x24a3: 0x00e6, + 0x24a4: 0x00e6, 0x24a5: 0x00e6, 0x24a6: 0x00e6, + 0x24b0: 0x3000, 0x24b1: 0x3000, 0x24b2: 0x3000, 0x24b3: 0x3000, 0x24b4: 0x3000, 0x24b5: 0x3000, + 0x24b6: 0x3000, 0x24b7: 0x3000, 0x24b8: 0x3000, 0x24b9: 0x3000, 0x24ba: 0x3000, 0x24bb: 0x3000, + 0x24bc: 0x3000, 0x24bd: 0x3000, 0x24be: 0x3000, 0x24bf: 0x3000, + // Block 0x93, offset 0x24c0 + 0x24c0: 0x3000, 0x24c1: 0x3000, 0x24c2: 0x3000, 0x24c3: 0x3000, 0x24c4: 0x3000, + 0x24c7: 0x3000, 0x24c8: 0x3000, 0x24c9: 0x3000, 0x24ca: 0x3000, 0x24cb: 0x3000, + 0x24cc: 0x3000, 0x24cd: 0x3000, 0x24ce: 0x3000, 0x24cf: 0x3000, 0x24d0: 0x3000, 0x24d1: 0x3000, + 0x24d2: 0x3000, 0x24d4: 0x3000, 0x24d5: 0x3000, 0x24d6: 0x3000, 0x24d7: 0x3000, + 0x24d8: 0x3000, 0x24d9: 0x3000, 0x24da: 0x3000, 0x24db: 0x3000, 0x24dc: 0x3000, 0x24dd: 0x3000, + 0x24de: 0x3000, 0x24df: 0x3000, 0x24e0: 0x3000, 0x24e1: 0x3000, 0x24e2: 0x3000, 0x24e3: 0x3000, + 0x24e4: 0x3000, 0x24e5: 0x3000, 0x24e6: 0x3000, 0x24e8: 0x3000, 0x24e9: 0x3000, + 0x24ea: 0x3000, 0x24eb: 0x3000, + 0x24f0: 0x3000, 0x24f1: 0x3000, 0x24f2: 0x3000, 0x24f4: 0x3000, + 0x24f6: 0x3000, 0x24f7: 0x3000, 0x24f8: 0x3000, 0x24f9: 0x3000, 0x24fa: 0x3000, 0x24fb: 0x3000, + 0x24fc: 0x3000, 0x24fd: 0x3000, 0x24fe: 0x3000, 0x24ff: 0x3000, + // Block 0x94, offset 0x2500 + 0x2500: 0x3000, 0x2501: 0x3000, 0x2502: 0x3000, 0x2503: 0x3000, 0x2504: 0x3000, 0x2505: 0x3000, + 0x2506: 0x3000, 0x2507: 0x3000, 0x2508: 0x3000, 0x2509: 0x3000, 0x250a: 0x3000, 0x250b: 0x3000, + 0x250c: 0x3000, 0x250d: 0x3000, 0x250e: 0x3000, 0x250f: 0x3000, 0x2510: 0x3000, 0x2511: 0x3000, + 0x2512: 0x3000, 0x2513: 0x3000, 0x2514: 0x3000, 0x2515: 0x3000, 0x2516: 0x3000, 0x2517: 0x3000, + 0x2518: 0x3000, 0x2519: 0x3000, 0x251a: 0x3000, 0x251b: 0x3000, 0x251c: 0x3000, 0x251d: 0x3000, + 0x251e: 0x3000, 0x251f: 0x3000, 0x2520: 0x3000, 0x2521: 0x3000, 0x2522: 0x3000, 0x2523: 0x3000, + 0x2524: 0x3000, 0x2525: 0x3000, 0x2526: 0x3000, 0x2527: 0x3000, 0x2528: 0x3000, 0x2529: 0x3000, + 0x252a: 0x3000, 0x252b: 0x3000, 0x252c: 0x3000, 0x252d: 0x3000, 0x252e: 0x3000, 0x252f: 0x3000, + 0x2530: 0x3000, 0x2531: 0x3000, 0x2532: 0x3000, 0x2533: 0x3000, 0x2534: 0x3000, 0x2535: 0x3000, + 0x2536: 0x3000, 0x2537: 0x3000, 0x2538: 0x3000, 0x2539: 0x3000, 0x253a: 0x3000, 0x253b: 0x3000, + 0x253c: 0x3000, + // Block 0x95, offset 0x2540 + 0x2541: 0x3000, 0x2542: 0x3000, 0x2543: 0x3000, 0x2544: 0x3000, 0x2545: 0x3000, + 0x2546: 0x3000, 0x2547: 0x3000, 0x2548: 0x3000, 0x2549: 0x3000, 0x254a: 0x3000, 0x254b: 0x3000, + 0x254c: 0x3000, 0x254d: 0x3000, 0x254e: 0x3000, 0x254f: 0x3000, 0x2550: 0x3000, 0x2551: 0x3000, + 0x2552: 0x3000, 0x2553: 0x3000, 0x2554: 0x3000, 0x2555: 0x3000, 0x2556: 0x3000, 0x2557: 0x3000, + 0x2558: 0x3000, 0x2559: 0x3000, 0x255a: 0x3000, 0x255b: 0x3000, 0x255c: 0x3000, 0x255d: 0x3000, + 0x255e: 0x3000, 0x255f: 0x3000, 0x2560: 0x3000, 0x2561: 0x3000, 0x2562: 0x3000, 0x2563: 0x3000, + 0x2564: 0x3000, 0x2565: 0x3000, 0x2566: 0x3000, 0x2567: 0x3000, 0x2568: 0x3000, 0x2569: 0x3000, + 0x256a: 0x3000, 0x256b: 0x3000, 0x256c: 0x3000, 0x256d: 0x3000, 0x256e: 0x3000, 0x256f: 0x3000, + 0x2570: 0x3000, 0x2571: 0x3000, 0x2572: 0x3000, 0x2573: 0x3000, 0x2574: 0x3000, 0x2575: 0x3000, + 0x2576: 0x3000, 0x2577: 0x3000, 0x2578: 0x3000, 0x2579: 0x3000, 0x257a: 0x3000, 0x257b: 0x3000, + 0x257c: 0x3000, 0x257d: 0x3000, 0x257e: 0x3000, 0x257f: 0x3000, + // Block 0x96, offset 0x2580 + 0x2582: 0x3000, 0x2583: 0x3000, 0x2584: 0x3000, 0x2585: 0x3000, + 0x2586: 0x3000, 0x2587: 0x3000, 0x258a: 0x3000, 0x258b: 0x3000, + 0x258c: 0x3000, 0x258d: 0x3000, 0x258e: 0x3000, 0x258f: 0x3000, + 0x2592: 0x3000, 0x2593: 0x3000, 0x2594: 0x3000, 0x2595: 0x3000, 0x2596: 0x3000, 0x2597: 0x3000, + 0x259a: 0x3000, 0x259b: 0x3000, 0x259c: 0x3000, + 0x25a0: 0x3000, 0x25a1: 0x3000, 0x25a2: 0x3000, 0x25a3: 0x3000, + 0x25a4: 0x3000, 0x25a5: 0x3000, 0x25a6: 0x3000, 0x25a8: 0x3000, 0x25a9: 0x3000, + 0x25aa: 0x3000, 0x25ab: 0x3000, 0x25ac: 0x3000, 0x25ad: 0x3000, 0x25ae: 0x3000, + // Block 0x97, offset 0x25c0 + 0x25fd: 0x00dc, + // Block 0x98, offset 0x2600 + 0x260d: 0x00dc, 0x260f: 0x00e6, + 0x2638: 0x00e6, 0x2639: 0x0001, 0x263a: 0x00dc, + 0x263f: 0x0009, + // Block 0x99, offset 0x2640 + 0x2659: 0x8800, 0x265a: 0x1100, 0x265b: 0x8800, 0x265c: 0x1100, + 0x2665: 0x8800, + 0x266b: 0x1100, + 0x2679: 0x0009, 0x267a: 0x6607, + // Block 0x9a, offset 0x2680 + 0x269e: 0x3300, 0x269f: 0x3300, 0x26a0: 0x3300, 0x26a1: 0x3300, 0x26a2: 0x3300, 0x26a3: 0x3300, + 0x26a4: 0x3300, 0x26a5: 0x00d8, 0x26a6: 0x00d8, 0x26a7: 0x0001, 0x26a8: 0x0001, 0x26a9: 0x0001, + 0x26ad: 0x00e2, 0x26ae: 0x00d8, 0x26af: 0x00d8, + 0x26b0: 0x00d8, 0x26b1: 0x00d8, 0x26b2: 0x00d8, + 0x26bb: 0x00dc, + 0x26bc: 0x00dc, 0x26bd: 0x00dc, 0x26be: 0x00dc, 0x26bf: 0x00dc, + // Block 0x9b, offset 0x26c0 + 0x26c0: 0x00dc, 0x26c1: 0x00dc, 0x26c2: 0x00dc, 0x26c5: 0x00e6, + 0x26c6: 0x00e6, 0x26c7: 0x00e6, 0x26c8: 0x00e6, 0x26c9: 0x00e6, 0x26ca: 0x00dc, 0x26cb: 0x00dc, + 0x26ea: 0x00e6, 0x26eb: 0x00e6, 0x26ec: 0x00e6, 0x26ed: 0x00e6, + 0x26fb: 0x3300, + 0x26fc: 0x3300, 0x26fd: 0x3300, 0x26fe: 0x3300, 0x26ff: 0x3300, + // Block 0x9c, offset 0x2700 + 0x2700: 0x3300, + // Block 0x9d, offset 0x2740 + 0x2742: 0x00e6, 0x2743: 0x00e6, 0x2744: 0x00e6, + // Block 0x9e, offset 0x2780 + 0x2780: 0x3000, 0x2781: 0x3000, 0x2782: 0x3000, 0x2783: 0x3000, 0x2784: 0x3000, 0x2785: 0x3000, + 0x2786: 0x3000, 0x2787: 0x3000, 0x2788: 0x3000, 0x2789: 0x3000, 0x278a: 0x3000, 0x278b: 0x3000, + 0x278c: 0x3000, 0x278d: 0x3000, 0x278e: 0x3000, 0x278f: 0x3000, 0x2790: 0x3000, 0x2791: 0x3000, + 0x2792: 0x3000, 0x2793: 0x3000, 0x2794: 0x3000, 0x2796: 0x3000, 0x2797: 0x3000, + 0x2798: 0x3000, 0x2799: 0x3000, 0x279a: 0x3000, 0x279b: 0x3000, 0x279c: 0x3000, 0x279d: 0x3000, + 0x279e: 0x3000, 0x279f: 0x3000, 0x27a0: 0x3000, 0x27a1: 0x3000, 0x27a2: 0x3000, 0x27a3: 0x3000, + 0x27a4: 0x3000, 0x27a5: 0x3000, 0x27a6: 0x3000, 0x27a7: 0x3000, 0x27a8: 0x3000, 0x27a9: 0x3000, + 0x27aa: 0x3000, 0x27ab: 0x3000, 0x27ac: 0x3000, 0x27ad: 0x3000, 0x27ae: 0x3000, 0x27af: 0x3000, + 0x27b0: 0x3000, 0x27b1: 0x3000, 0x27b2: 0x3000, 0x27b3: 0x3000, 0x27b4: 0x3000, 0x27b5: 0x3000, + 0x27b6: 0x3000, 0x27b7: 0x3000, 0x27b8: 0x3000, 0x27b9: 0x3000, 0x27ba: 0x3000, 0x27bb: 0x3000, + 0x27bc: 0x3000, 0x27bd: 0x3000, 0x27be: 0x3000, 0x27bf: 0x3000, + // Block 0x9f, offset 0x27c0 + 0x27c0: 0x3000, 0x27c1: 0x3000, 0x27c2: 0x3000, 0x27c3: 0x3000, 0x27c4: 0x3000, 0x27c5: 0x3000, + 0x27c6: 0x3000, 0x27c7: 0x3000, 0x27c8: 0x3000, 0x27c9: 0x3000, 0x27ca: 0x3000, 0x27cb: 0x3000, + 0x27cc: 0x3000, 0x27cd: 0x3000, 0x27ce: 0x3000, 0x27cf: 0x3000, 0x27d0: 0x3000, 0x27d1: 0x3000, + 0x27d2: 0x3000, 0x27d3: 0x3000, 0x27d4: 0x3000, 0x27d5: 0x3000, 0x27d6: 0x3000, 0x27d7: 0x3000, + 0x27d8: 0x3000, 0x27d9: 0x3000, 0x27da: 0x3000, 0x27db: 0x3000, 0x27dc: 0x3000, + 0x27de: 0x3000, 0x27df: 0x3000, 0x27e2: 0x3000, + 0x27e5: 0x3000, 0x27e6: 0x3000, 0x27e9: 0x3000, + 0x27ea: 0x3000, 0x27eb: 0x3000, 0x27ec: 0x3000, 0x27ee: 0x3000, 0x27ef: 0x3000, + 0x27f0: 0x3000, 0x27f1: 0x3000, 0x27f2: 0x3000, 0x27f3: 0x3000, 0x27f4: 0x3000, 0x27f5: 0x3000, + 0x27f6: 0x3000, 0x27f7: 0x3000, 0x27f8: 0x3000, 0x27f9: 0x3000, 0x27fb: 0x3000, + 0x27fd: 0x3000, 0x27fe: 0x3000, 0x27ff: 0x3000, + // Block 0xa0, offset 0x2800 + 0x2800: 0x3000, 0x2801: 0x3000, 0x2802: 0x3000, 0x2803: 0x3000, 0x2805: 0x3000, + 0x2806: 0x3000, 0x2807: 0x3000, 0x2808: 0x3000, 0x2809: 0x3000, 0x280a: 0x3000, 0x280b: 0x3000, + 0x280c: 0x3000, 0x280d: 0x3000, 0x280e: 0x3000, 0x280f: 0x3000, 0x2810: 0x3000, 0x2811: 0x3000, + 0x2812: 0x3000, 0x2813: 0x3000, 0x2814: 0x3000, 0x2815: 0x3000, 0x2816: 0x3000, 0x2817: 0x3000, + 0x2818: 0x3000, 0x2819: 0x3000, 0x281a: 0x3000, 0x281b: 0x3000, 0x281c: 0x3000, 0x281d: 0x3000, + 0x281e: 0x3000, 0x281f: 0x3000, 0x2820: 0x3000, 0x2821: 0x3000, 0x2822: 0x3000, 0x2823: 0x3000, + 0x2824: 0x3000, 0x2825: 0x3000, 0x2826: 0x3000, 0x2827: 0x3000, 0x2828: 0x3000, 0x2829: 0x3000, + 0x282a: 0x3000, 0x282b: 0x3000, 0x282c: 0x3000, 0x282d: 0x3000, 0x282e: 0x3000, 0x282f: 0x3000, + 0x2830: 0x3000, 0x2831: 0x3000, 0x2832: 0x3000, 0x2833: 0x3000, 0x2834: 0x3000, 0x2835: 0x3000, + 0x2836: 0x3000, 0x2837: 0x3000, 0x2838: 0x3000, 0x2839: 0x3000, 0x283a: 0x3000, 0x283b: 0x3000, + 0x283c: 0x3000, 0x283d: 0x3000, 0x283e: 0x3000, 0x283f: 0x3000, + // Block 0xa1, offset 0x2840 + 0x2840: 0x3000, 0x2841: 0x3000, 0x2842: 0x3000, 0x2843: 0x3000, 0x2844: 0x3000, 0x2845: 0x3000, + 0x2847: 0x3000, 0x2848: 0x3000, 0x2849: 0x3000, 0x284a: 0x3000, + 0x284d: 0x3000, 0x284e: 0x3000, 0x284f: 0x3000, 0x2850: 0x3000, 0x2851: 0x3000, + 0x2852: 0x3000, 0x2853: 0x3000, 0x2854: 0x3000, 0x2856: 0x3000, 0x2857: 0x3000, + 0x2858: 0x3000, 0x2859: 0x3000, 0x285a: 0x3000, 0x285b: 0x3000, 0x285c: 0x3000, + 0x285e: 0x3000, 0x285f: 0x3000, 0x2860: 0x3000, 0x2861: 0x3000, 0x2862: 0x3000, 0x2863: 0x3000, + 0x2864: 0x3000, 0x2865: 0x3000, 0x2866: 0x3000, 0x2867: 0x3000, 0x2868: 0x3000, 0x2869: 0x3000, + 0x286a: 0x3000, 0x286b: 0x3000, 0x286c: 0x3000, 0x286d: 0x3000, 0x286e: 0x3000, 0x286f: 0x3000, + 0x2870: 0x3000, 0x2871: 0x3000, 0x2872: 0x3000, 0x2873: 0x3000, 0x2874: 0x3000, 0x2875: 0x3000, + 0x2876: 0x3000, 0x2877: 0x3000, 0x2878: 0x3000, 0x2879: 0x3000, 0x287b: 0x3000, + 0x287c: 0x3000, 0x287d: 0x3000, 0x287e: 0x3000, + // Block 0xa2, offset 0x2880 + 0x2880: 0x3000, 0x2881: 0x3000, 0x2882: 0x3000, 0x2883: 0x3000, 0x2884: 0x3000, + 0x2886: 0x3000, 0x288a: 0x3000, 0x288b: 0x3000, + 0x288c: 0x3000, 0x288d: 0x3000, 0x288e: 0x3000, 0x288f: 0x3000, 0x2890: 0x3000, + 0x2892: 0x3000, 0x2893: 0x3000, 0x2894: 0x3000, 0x2895: 0x3000, 0x2896: 0x3000, 0x2897: 0x3000, + 0x2898: 0x3000, 0x2899: 0x3000, 0x289a: 0x3000, 0x289b: 0x3000, 0x289c: 0x3000, 0x289d: 0x3000, + 0x289e: 0x3000, 0x289f: 0x3000, 0x28a0: 0x3000, 0x28a1: 0x3000, 0x28a2: 0x3000, 0x28a3: 0x3000, + 0x28a4: 0x3000, 0x28a5: 0x3000, 0x28a6: 0x3000, 0x28a7: 0x3000, 0x28a8: 0x3000, 0x28a9: 0x3000, + 0x28aa: 0x3000, 0x28ab: 0x3000, 0x28ac: 0x3000, 0x28ad: 0x3000, 0x28ae: 0x3000, 0x28af: 0x3000, + 0x28b0: 0x3000, 0x28b1: 0x3000, 0x28b2: 0x3000, 0x28b3: 0x3000, 0x28b4: 0x3000, 0x28b5: 0x3000, + 0x28b6: 0x3000, 0x28b7: 0x3000, 0x28b8: 0x3000, 0x28b9: 0x3000, 0x28ba: 0x3000, 0x28bb: 0x3000, + 0x28bc: 0x3000, 0x28bd: 0x3000, 0x28be: 0x3000, 0x28bf: 0x3000, + // Block 0xa3, offset 0x28c0 + 0x28c0: 0x3000, 0x28c1: 0x3000, 0x28c2: 0x3000, 0x28c3: 0x3000, 0x28c4: 0x3000, 0x28c5: 0x3000, + 0x28c6: 0x3000, 0x28c7: 0x3000, 0x28c8: 0x3000, 0x28c9: 0x3000, 0x28ca: 0x3000, 0x28cb: 0x3000, + 0x28cc: 0x3000, 0x28cd: 0x3000, 0x28ce: 0x3000, 0x28cf: 0x3000, 0x28d0: 0x3000, 0x28d1: 0x3000, + 0x28d2: 0x3000, 0x28d3: 0x3000, 0x28d4: 0x3000, 0x28d5: 0x3000, 0x28d6: 0x3000, 0x28d7: 0x3000, + 0x28d8: 0x3000, 0x28d9: 0x3000, 0x28da: 0x3000, 0x28db: 0x3000, 0x28dc: 0x3000, 0x28dd: 0x3000, + 0x28de: 0x3000, 0x28df: 0x3000, 0x28e0: 0x3000, 0x28e1: 0x3000, 0x28e2: 0x3000, 0x28e3: 0x3000, + 0x28e4: 0x3000, 0x28e5: 0x3000, 0x28e8: 0x3000, 0x28e9: 0x3000, + 0x28ea: 0x3000, 0x28eb: 0x3000, 0x28ec: 0x3000, 0x28ed: 0x3000, 0x28ee: 0x3000, 0x28ef: 0x3000, + 0x28f0: 0x3000, 0x28f1: 0x3000, 0x28f2: 0x3000, 0x28f3: 0x3000, 0x28f4: 0x3000, 0x28f5: 0x3000, + 0x28f6: 0x3000, 0x28f7: 0x3000, 0x28f8: 0x3000, 0x28f9: 0x3000, 0x28fa: 0x3000, 0x28fb: 0x3000, + 0x28fc: 0x3000, 0x28fd: 0x3000, 0x28fe: 0x3000, 0x28ff: 0x3000, + // Block 0xa4, offset 0x2900 + 0x2900: 0x3000, 0x2901: 0x3000, 0x2902: 0x3000, 0x2903: 0x3000, 0x2904: 0x3000, 0x2905: 0x3000, + 0x2906: 0x3000, 0x2907: 0x3000, 0x2908: 0x3000, 0x2909: 0x3000, 0x290a: 0x3000, 0x290b: 0x3000, + 0x290e: 0x3000, 0x290f: 0x3000, 0x2910: 0x3000, 0x2911: 0x3000, + 0x2912: 0x3000, 0x2913: 0x3000, 0x2914: 0x3000, 0x2915: 0x3000, 0x2916: 0x3000, 0x2917: 0x3000, + 0x2918: 0x3000, 0x2919: 0x3000, 0x291a: 0x3000, 0x291b: 0x3000, 0x291c: 0x3000, 0x291d: 0x3000, + 0x291e: 0x3000, 0x291f: 0x3000, 0x2920: 0x3000, 0x2921: 0x3000, 0x2922: 0x3000, 0x2923: 0x3000, + 0x2924: 0x3000, 0x2925: 0x3000, 0x2926: 0x3000, 0x2927: 0x3000, 0x2928: 0x3000, 0x2929: 0x3000, + 0x292a: 0x3000, 0x292b: 0x3000, 0x292c: 0x3000, 0x292d: 0x3000, 0x292e: 0x3000, 0x292f: 0x3000, + 0x2930: 0x3000, 0x2931: 0x3000, 0x2932: 0x3000, 0x2933: 0x3000, 0x2934: 0x3000, 0x2935: 0x3000, + 0x2936: 0x3000, 0x2937: 0x3000, 0x2938: 0x3000, 0x2939: 0x3000, 0x293a: 0x3000, 0x293b: 0x3000, + 0x293c: 0x3000, 0x293d: 0x3000, 0x293e: 0x3000, 0x293f: 0x3000, + // Block 0xa5, offset 0x2940 + 0x2940: 0x3000, 0x2941: 0x3000, 0x2942: 0x3000, 0x2943: 0x3000, 0x2944: 0x3000, 0x2945: 0x3000, + 0x2946: 0x3000, 0x2947: 0x3000, 0x2948: 0x3000, 0x2949: 0x3000, 0x294a: 0x3000, + 0x2950: 0x3000, 0x2951: 0x3000, + 0x2952: 0x3000, 0x2953: 0x3000, 0x2954: 0x3000, 0x2955: 0x3000, 0x2956: 0x3000, 0x2957: 0x3000, + 0x2958: 0x3000, 0x2959: 0x3000, 0x295a: 0x3000, 0x295b: 0x3000, 0x295c: 0x3000, 0x295d: 0x3000, + 0x295e: 0x3000, 0x295f: 0x3000, 0x2960: 0x3000, 0x2961: 0x3000, 0x2962: 0x3000, 0x2963: 0x3000, + 0x2964: 0x3000, 0x2965: 0x3000, 0x2966: 0x3000, 0x2967: 0x3000, 0x2968: 0x3000, 0x2969: 0x3000, + 0x296a: 0x3000, 0x296b: 0x3000, 0x296c: 0x3000, 0x296d: 0x3000, 0x296e: 0x3000, + 0x2970: 0x3000, 0x2971: 0x3000, 0x2972: 0x3000, 0x2973: 0x3000, 0x2974: 0x3000, 0x2975: 0x3000, + 0x2976: 0x3000, 0x2977: 0x3000, 0x2978: 0x3000, 0x2979: 0x3000, 0x297a: 0x3000, 0x297b: 0x3000, + 0x297c: 0x3000, 0x297d: 0x3000, 0x297e: 0x3000, 0x297f: 0x3000, + // Block 0xa6, offset 0x2980 + 0x2980: 0x3000, 0x2981: 0x3000, 0x2982: 0x3000, 0x2983: 0x3000, 0x2984: 0x3000, 0x2985: 0x3000, + 0x2986: 0x3000, 0x2987: 0x3000, 0x2988: 0x3000, 0x2989: 0x3000, 0x298a: 0x3000, 0x298b: 0x3000, + 0x298c: 0x3000, 0x298d: 0x3000, 0x298e: 0x3000, 0x298f: 0x3000, + // Block 0xa7, offset 0x29c0 + 0x29d0: 0x3000, + // Block 0xa8, offset 0x2a00 + 0x2a00: 0x3000, 0x2a01: 0x3000, 0x2a02: 0x3000, + 0x2a10: 0x3000, 0x2a11: 0x3000, + 0x2a12: 0x3000, 0x2a13: 0x3000, 0x2a14: 0x3000, 0x2a15: 0x3000, 0x2a16: 0x3000, 0x2a17: 0x3000, + 0x2a18: 0x3000, 0x2a19: 0x3000, 0x2a1a: 0x3000, 0x2a1b: 0x3000, 0x2a1c: 0x3000, 0x2a1d: 0x3000, + 0x2a1e: 0x3000, 0x2a1f: 0x3000, 0x2a20: 0x3000, 0x2a21: 0x3000, 0x2a22: 0x3000, 0x2a23: 0x3000, + 0x2a24: 0x3000, 0x2a25: 0x3000, 0x2a26: 0x3000, 0x2a27: 0x3000, 0x2a28: 0x3000, 0x2a29: 0x3000, + 0x2a2a: 0x3000, 0x2a2b: 0x3000, 0x2a2c: 0x3000, 0x2a2d: 0x3000, 0x2a2e: 0x3000, 0x2a2f: 0x3000, + 0x2a30: 0x3000, 0x2a31: 0x3000, 0x2a32: 0x3000, 0x2a33: 0x3000, 0x2a34: 0x3000, 0x2a35: 0x3000, + 0x2a36: 0x3000, 0x2a37: 0x3000, 0x2a38: 0x3000, 0x2a39: 0x3000, 0x2a3a: 0x3000, + // Block 0xa9, offset 0x2a40 + 0x2a40: 0x3000, 0x2a41: 0x3000, 0x2a42: 0x3000, 0x2a43: 0x3000, 0x2a44: 0x3000, 0x2a45: 0x3000, + 0x2a46: 0x3000, 0x2a47: 0x3000, 0x2a48: 0x3000, + 0x2a50: 0x3000, 0x2a51: 0x3000, + // Block 0xaa, offset 0x2a80 + 0x2a80: 0x3300, 0x2a81: 0x3300, 0x2a82: 0x3300, 0x2a83: 0x3300, 0x2a84: 0x3300, 0x2a85: 0x3300, + 0x2a86: 0x3300, 0x2a87: 0x3300, 0x2a88: 0x3300, 0x2a89: 0x3300, 0x2a8a: 0x3300, 0x2a8b: 0x3300, + 0x2a8c: 0x3300, 0x2a8d: 0x3300, 0x2a8e: 0x3300, 0x2a8f: 0x3300, 0x2a90: 0x3300, 0x2a91: 0x3300, + 0x2a92: 0x3300, 0x2a93: 0x3300, 0x2a94: 0x3300, 0x2a95: 0x3300, 0x2a96: 0x3300, 0x2a97: 0x3300, + 0x2a98: 0x3300, 0x2a99: 0x3300, 0x2a9a: 0x3300, 0x2a9b: 0x3300, 0x2a9c: 0x3300, 0x2a9d: 0x3300, +} + +// charInfoLookup: 1152 bytes +// Block 0 is the null block. +var charInfoLookup = [1152]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c2: 0x03, 0x0c3: 0x04, 0x0c4: 0x05, 0x0c5: 0x06, 0x0c6: 0x07, 0x0c7: 0x08, + 0x0c8: 0x09, 0x0ca: 0x0a, 0x0cb: 0x0b, 0x0cc: 0x0c, 0x0cd: 0x0d, 0x0ce: 0x0e, 0x0cf: 0x0f, + 0x0d0: 0x10, 0x0d1: 0x11, 0x0d2: 0x12, 0x0d3: 0x13, 0x0d6: 0x14, 0x0d7: 0x15, + 0x0d8: 0x16, 0x0d9: 0x17, 0x0db: 0x18, 0x0dc: 0x19, 0x0dd: 0x1a, 0x0df: 0x1b, + 0x0e0: 0x04, 0x0e1: 0x05, 0x0e2: 0x06, 0x0e3: 0x07, + 0x0ea: 0x08, 0x0eb: 0x09, 0x0ec: 0x09, 0x0ed: 0x0a, 0x0ef: 0x0b, + 0x0f0: 0x11, + // Block 0x4, offset 0x100 + 0x120: 0x1c, 0x121: 0x1d, 0x124: 0x1e, 0x125: 0x1f, 0x126: 0x20, 0x127: 0x21, + 0x128: 0x22, 0x129: 0x23, 0x12a: 0x24, 0x12b: 0x25, 0x12c: 0x20, 0x12d: 0x26, 0x12e: 0x27, 0x12f: 0x28, + 0x131: 0x29, 0x132: 0x2a, 0x133: 0x2b, 0x134: 0x2c, 0x135: 0x28, 0x137: 0x2d, + 0x138: 0x2e, 0x139: 0x2f, 0x13a: 0x30, 0x13b: 0x31, 0x13c: 0x32, 0x13d: 0x33, 0x13e: 0x34, 0x13f: 0x35, + // Block 0x5, offset 0x140 + 0x140: 0x36, 0x142: 0x37, 0x143: 0x38, 0x145: 0x39, 0x146: 0x3a, 0x147: 0x3b, + 0x14d: 0x3c, + 0x15c: 0x3d, 0x15f: 0x3e, + 0x162: 0x3f, 0x164: 0x40, + 0x168: 0x41, 0x169: 0x42, 0x16c: 0x43, 0x16d: 0x44, 0x16e: 0x45, 0x16f: 0x46, + 0x170: 0x47, 0x173: 0x48, 0x174: 0x49, 0x175: 0x4a, 0x176: 0x4b, 0x177: 0x4c, + 0x178: 0x4d, 0x179: 0x4e, 0x17a: 0x4f, 0x17b: 0x50, 0x17c: 0x51, 0x17d: 0x52, 0x17e: 0x53, 0x17f: 0x54, + // Block 0x6, offset 0x180 + 0x180: 0x55, 0x181: 0x56, 0x182: 0x57, 0x183: 0x58, 0x184: 0x59, 0x185: 0x5a, 0x186: 0x5b, 0x187: 0x5c, + 0x188: 0x5d, 0x189: 0x5e, 0x18a: 0x5f, 0x18b: 0x60, 0x18c: 0x61, + 0x191: 0x62, 0x192: 0x63, 0x193: 0x64, + 0x1a8: 0x65, 0x1a9: 0x66, 0x1ab: 0x67, + 0x1b1: 0x68, 0x1b3: 0x69, 0x1b5: 0x6a, 0x1b7: 0x6b, + 0x1ba: 0x6c, 0x1bb: 0x6d, 0x1bc: 0x63, 0x1bd: 0x63, 0x1be: 0x63, 0x1bf: 0x6e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x6f, 0x1c1: 0x70, 0x1c2: 0x71, 0x1c3: 0x72, 0x1c4: 0x73, 0x1c5: 0x63, 0x1c6: 0x74, + 0x1c8: 0x75, 0x1c9: 0x76, 0x1ca: 0x63, 0x1cb: 0x77, 0x1cc: 0x63, 0x1cd: 0x63, 0x1ce: 0x63, 0x1cf: 0x63, + // Block 0x8, offset 0x200 + 0x219: 0x78, 0x21b: 0x79, 0x21d: 0x7a, + 0x220: 0x7b, 0x223: 0x7c, 0x224: 0x7d, 0x225: 0x7e, 0x226: 0x7f, 0x227: 0x80, + 0x22a: 0x81, 0x22b: 0x82, 0x22f: 0x83, + 0x230: 0x84, 0x231: 0x84, 0x232: 0x84, 0x233: 0x84, 0x234: 0x84, 0x235: 0x84, 0x236: 0x84, 0x237: 0x84, + 0x238: 0x84, 0x239: 0x84, 0x23a: 0x84, 0x23b: 0x84, 0x23c: 0x84, 0x23d: 0x84, 0x23e: 0x84, 0x23f: 0x84, + // Block 0x9, offset 0x240 + 0x240: 0x84, 0x241: 0x84, 0x242: 0x84, 0x243: 0x84, 0x244: 0x84, 0x245: 0x84, 0x246: 0x84, 0x247: 0x84, + 0x248: 0x84, 0x249: 0x84, 0x24a: 0x84, 0x24b: 0x84, 0x24c: 0x84, 0x24d: 0x84, 0x24e: 0x84, 0x24f: 0x84, + 0x250: 0x84, 0x251: 0x84, 0x252: 0x84, 0x253: 0x84, 0x254: 0x84, 0x255: 0x84, 0x256: 0x84, 0x257: 0x84, + 0x258: 0x84, 0x259: 0x84, 0x25a: 0x84, 0x25b: 0x84, 0x25c: 0x84, 0x25d: 0x84, 0x25e: 0x84, 0x25f: 0x84, + 0x260: 0x84, 0x261: 0x84, 0x262: 0x84, 0x263: 0x84, 0x264: 0x84, 0x265: 0x84, 0x266: 0x84, 0x267: 0x84, + 0x268: 0x84, 0x269: 0x84, 0x26a: 0x84, 0x26b: 0x84, 0x26c: 0x84, 0x26d: 0x84, 0x26e: 0x84, 0x26f: 0x84, + 0x270: 0x84, 0x271: 0x84, 0x272: 0x84, 0x273: 0x84, 0x274: 0x84, 0x275: 0x84, 0x276: 0x84, 0x277: 0x84, + 0x278: 0x84, 0x279: 0x84, 0x27a: 0x84, 0x27b: 0x84, 0x27c: 0x84, 0x27d: 0x84, 0x27e: 0x84, 0x27f: 0x84, + // Block 0xa, offset 0x280 + 0x280: 0x84, 0x281: 0x84, 0x282: 0x84, 0x283: 0x84, 0x284: 0x84, 0x285: 0x84, 0x286: 0x84, 0x287: 0x84, + 0x288: 0x84, 0x289: 0x84, 0x28a: 0x84, 0x28b: 0x84, 0x28c: 0x84, 0x28d: 0x84, 0x28e: 0x84, 0x28f: 0x84, + 0x290: 0x84, 0x291: 0x84, 0x292: 0x84, 0x293: 0x84, 0x294: 0x84, 0x295: 0x84, 0x296: 0x84, 0x297: 0x84, + 0x298: 0x84, 0x299: 0x84, 0x29a: 0x84, 0x29b: 0x84, 0x29c: 0x84, 0x29d: 0x84, 0x29e: 0x85, + // Block 0xb, offset 0x2c0 + 0x2e4: 0x86, 0x2e5: 0x86, 0x2e6: 0x86, 0x2e7: 0x86, + 0x2e8: 0x87, 0x2e9: 0x88, 0x2ea: 0x86, 0x2eb: 0x89, 0x2ec: 0x8a, 0x2ed: 0x8b, 0x2ee: 0x8c, 0x2ef: 0x8d, + 0x2f0: 0x63, 0x2f1: 0x63, 0x2f2: 0x63, 0x2f3: 0x63, 0x2f4: 0x8e, 0x2f5: 0x8f, 0x2f6: 0x90, 0x2f7: 0x91, + 0x2f8: 0x92, 0x2f9: 0x93, 0x2fa: 0x63, 0x2fb: 0x94, 0x2fc: 0x95, 0x2fd: 0x63, 0x2fe: 0x77, 0x2ff: 0x96, + // Block 0xc, offset 0x300 + 0x307: 0x97, + 0x328: 0x98, + // Block 0xd, offset 0x340 + 0x341: 0x7b, 0x342: 0x99, + // Block 0xe, offset 0x380 + 0x385: 0x9a, 0x386: 0x9b, 0x387: 0x9c, + 0x389: 0x9d, + 0x390: 0x63, 0x391: 0x9e, 0x392: 0x9f, 0x393: 0xa0, 0x394: 0xa1, 0x395: 0xa2, 0x396: 0x63, 0x397: 0x63, + 0x398: 0x63, 0x399: 0x63, 0x39a: 0xa3, 0x39b: 0x63, 0x39c: 0x63, 0x39d: 0x63, 0x39e: 0x63, 0x39f: 0xa4, + // Block 0xf, offset 0x3c0 + 0x3c4: 0xa5, 0x3c5: 0xa6, 0x3c6: 0xa7, + 0x3c8: 0xa8, 0x3c9: 0xa9, + // Block 0x10, offset 0x400 + 0x420: 0x86, 0x421: 0x86, 0x422: 0x86, 0x423: 0x86, 0x424: 0x86, 0x425: 0x86, 0x426: 0x86, 0x427: 0x86, + 0x428: 0xaa, + // Block 0x11, offset 0x440 + 0x450: 0x0c, 0x451: 0x0d, + 0x45d: 0x0e, 0x45f: 0x0f, + 0x46f: 0x10, +} + +var charInfoTrie = trie{charInfoLookup[:], charInfoValues[:]} + +// Total size of tables: 78KB (80234 bytes) diff --git a/src/pkg/exp/norm/trie.go b/src/pkg/exp/norm/trie.go new file mode 100644 index 000000000..6b6540187 --- /dev/null +++ b/src/pkg/exp/norm/trie.go @@ -0,0 +1,234 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +type trie struct { + index []uint8 + values []uint16 +} + +const ( + t1 = 0x00 // 0000 0000 + tx = 0x80 // 1000 0000 + t2 = 0xC0 // 1100 0000 + t3 = 0xE0 // 1110 0000 + t4 = 0xF0 // 1111 0000 + t5 = 0xF8 // 1111 1000 + t6 = 0xFC // 1111 1100 + te = 0xFE // 1111 1110 + + maskx = 0x3F // 0011 1111 + mask2 = 0x1F // 0001 1111 + mask3 = 0x0F // 0000 1111 + mask4 = 0x07 // 0000 0111 +) + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *trie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < tx: + return t.values[c0], 1 + case c0 < t2: + return 0, 1 + case c0 < t3: + if len(s) < 2 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + return t.values[o], 2 + case c0 < t4: + if len(s) < 3 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + return t.values[o], 3 + case c0 < t5: + if len(s) < 4 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + i = t.index[o] + c3 := s[3] + if c3 < tx || t2 <= c3 { + return 0, 3 + } + o = uint16(i)<<6 + uint16(c3)&maskx + return t.values[o], 4 + case c0 < t6: + if len(s) < 5 { + return 0, 0 + } + return 0, 5 + case c0 < te: + if len(s) < 6 { + return 0, 0 + } + return 0, 6 + } + // Illegal rune + return 0, 1 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *trie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < tx: + return t.values[c0], 1 + case c0 < t2: + return 0, 1 + case c0 < t3: + if len(s) < 2 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + return t.values[o], 2 + case c0 < t4: + if len(s) < 3 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + return t.values[o], 3 + case c0 < t5: + if len(s) < 4 { + return 0, 0 + } + i := t.index[c0] + c1 := s[1] + if c1 < tx || t2 <= c1 { + return 0, 1 + } + o := uint16(i)<<6 + uint16(c1)&maskx + i = t.index[o] + c2 := s[2] + if c2 < tx || t2 <= c2 { + return 0, 2 + } + o = uint16(i)<<6 + uint16(c2)&maskx + i = t.index[o] + c3 := s[3] + if c3 < tx || t2 <= c3 { + return 0, 3 + } + o = uint16(i)<<6 + uint16(c3)&maskx + return t.values[o], 4 + case c0 < t6: + if len(s) < 5 { + return 0, 0 + } + return 0, 5 + case c0 < te: + if len(s) < 6 { + return 0, 0 + } + return 0, 6 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must hold a full encoding. +func (t *trie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < tx { + return t.values[c0] + } + if c0 < t2 { + return 0 + } + i := t.index[c0] + o := uint16(i)<<6 + uint16(s[1])&maskx + if c0 < t3 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[2])&maskx + if c0 < t4 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[3])&maskx + if c0 < t5 { + return t.values[o] + } + return 0 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must hold a full encoding. +func (t *trie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < tx { + return t.values[c0] + } + if c0 < t2 { + return 0 + } + i := t.index[c0] + o := uint16(i)<<6 + uint16(s[1])&maskx + if c0 < t3 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[2])&maskx + if c0 < t4 { + return t.values[o] + } + i = t.index[o] + o = uint16(i)<<6 + uint16(s[3])&maskx + if c0 < t5 { + return t.values[o] + } + return 0 +} diff --git a/src/pkg/exp/norm/trie_test.go b/src/pkg/exp/norm/trie_test.go new file mode 100644 index 000000000..ad87d972b --- /dev/null +++ b/src/pkg/exp/norm/trie_test.go @@ -0,0 +1,107 @@ +package norm + +import ( + "testing" + "utf8" +) + +// Test data is located in triedata_test.go; generated by maketesttables. +var testdata = testdataTrie + +// Test cases for illegal runes. +type trietest struct { + size int + bytes []byte +} + +var tests = []trietest{ + // illegal runes + {1, []byte{0x80}}, + {1, []byte{0xFF}}, + {1, []byte{t2, tx - 1}}, + {1, []byte{t2, t2}}, + {2, []byte{t3, tx, tx - 1}}, + {2, []byte{t3, tx, t2}}, + {1, []byte{t3, tx - 1, tx}}, + {3, []byte{t4, tx, tx, tx - 1}}, + {3, []byte{t4, tx, tx, t2}}, + {1, []byte{t4, t2, tx, tx - 1}}, + {2, []byte{t4, tx, t2, tx - 1}}, + + // short runes + {0, []byte{t2}}, + {0, []byte{t3, tx}}, + {0, []byte{t4, tx, tx}}, + {0, []byte{t5, tx, tx, tx}}, + {0, []byte{t6, tx, tx, tx, tx}}, +} + +func mkUtf8(rune int) ([]byte, int) { + var b [utf8.UTFMax]byte + sz := utf8.EncodeRune(b[:], rune) + return b[:sz], sz +} + +func TestLookup(t *testing.T) { + for i, tt := range testRunes { + b, szg := mkUtf8(tt) + v, szt := testdata.lookup(b) + if int(v) != i { + t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i) + } + if szt != szg { + t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg) + } + } + for i, tt := range tests { + v, sz := testdata.lookup(tt.bytes) + if int(v) != 0 { + t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v) + } + if sz != tt.size { + t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size) + } + } +} + +func TestLookupUnsafe(t *testing.T) { + for i, tt := range testRunes { + b, _ := mkUtf8(tt) + v := testdata.lookupUnsafe(b) + if int(v) != i { + t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i) + } + } +} + +func TestLookupString(t *testing.T) { + for i, tt := range testRunes { + b, szg := mkUtf8(tt) + v, szt := testdata.lookupString(string(b)) + if int(v) != i { + t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i) + } + if szt != szg { + t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg) + } + } + for i, tt := range tests { + v, sz := testdata.lookupString(string(tt.bytes)) + if int(v) != 0 { + t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v) + } + if sz != tt.size { + t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size) + } + } +} + +func TestLookupStringUnsafe(t *testing.T) { + for i, tt := range testRunes { + b, _ := mkUtf8(tt) + v := testdata.lookupStringUnsafe(string(b)) + if int(v) != i { + t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i) + } + } +} diff --git a/src/pkg/exp/norm/triedata_test.go b/src/pkg/exp/norm/triedata_test.go new file mode 100644 index 000000000..f886e6004 --- /dev/null +++ b/src/pkg/exp/norm/triedata_test.go @@ -0,0 +1,63 @@ +// Generated by running +// maketesttables +// DO NOT EDIT + +package norm + +var testRunes = []int{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111} + +// testdataValues: 768 entries, 1536 bytes +// Block 2 is the null block. +var testdataValues = [768]uint16{ + // Block 0x0, offset 0x0 + 0x000c: 0x0001, + // Block 0x1, offset 0x40 + 0x007f: 0x0002, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x00c0: 0x0003, + // Block 0x4, offset 0x100 + 0x0100: 0x0004, + // Block 0x5, offset 0x140 + 0x017f: 0x0005, + // Block 0x6, offset 0x180 + 0x0180: 0x0006, + // Block 0x7, offset 0x1c0 + 0x01d9: 0x0007, + // Block 0x8, offset 0x200 + 0x023f: 0x0008, + // Block 0x9, offset 0x240 + 0x0240: 0x0009, + // Block 0xa, offset 0x280 + 0x0281: 0x000a, + // Block 0xb, offset 0x2c0 + 0x02ff: 0x000b, +} + +// testdataLookup: 640 bytes +// Block 0 is the null block. +var testdataLookup = [640]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0x0c2: 0x03, 0x0c4: 0x04, + 0x0df: 0x05, + 0x0e0: 0x04, + 0x0ef: 0x05, + 0x0f0: 0x07, 0x0f4: 0x09, + // Block 0x4, offset 0x100 + 0x120: 0x06, 0x126: 0x07, + // Block 0x5, offset 0x140 + 0x17f: 0x08, + // Block 0x6, offset 0x180 + 0x180: 0x09, 0x184: 0x0a, + // Block 0x7, offset 0x1c0 + 0x1d0: 0x06, + // Block 0x8, offset 0x200 + 0x23f: 0x0b, + // Block 0x9, offset 0x240 + 0x24f: 0x08, +} + +var testdataTrie = trie{testdataLookup[:], testdataValues[:]} diff --git a/src/pkg/exp/norm/triegen.go b/src/pkg/exp/norm/triegen.go new file mode 100644 index 000000000..2b7eeee17 --- /dev/null +++ b/src/pkg/exp/norm/triegen.go @@ -0,0 +1,211 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Trie table generator. +// Used by make*tables tools to generate a go file with trie data structures +// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte +// sequence are used to lookup offsets in the index table to be used for the +// next byte. The last byte is used to index into a table with 16-bit values. + +package main + +import ( + "fmt" + "hash/crc32" + "log" + "utf8" +) + +// Intermediate trie structure +type trieNode struct { + table [256]*trieNode + value uint16 + b byte + leaf bool +} + +func newNode() *trieNode { + return new(trieNode) +} + +func (n trieNode) String() string { + s := fmt.Sprint("trieNode{table: { non-nil at index: ") + for i, v := range n.table { + if v != nil { + s += fmt.Sprintf("%d, ", i) + } + } + s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf) + return s +} + +func (n trieNode) isInternal() bool { + internal := true + for i := 0; i < 256; i++ { + if nn := n.table[i]; nn != nil { + if !internal && !nn.leaf { + log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n) + } + internal = internal && !nn.leaf + } + } + return internal +} + +func (n *trieNode) insert(rune int, value uint16) { + var p [utf8.UTFMax]byte + sz := utf8.EncodeRune(p[:], rune) + + for i := 0; i < sz; i++ { + if n.leaf { + log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n) + } + nn := n.table[p[i]] + if nn == nil { + nn = newNode() + nn.b = p[i] + n.table[p[i]] = nn + } + n = nn + } + n.value = value + n.leaf = true +} + +type nodeIndex struct { + lookupBlocks []*trieNode + valueBlocks []*trieNode + + lookupBlockIdx map[uint32]uint16 + valueBlockIdx map[uint32]uint16 +} + +func newIndex() *nodeIndex { + index := &nodeIndex{} + index.lookupBlocks = make([]*trieNode, 0) + index.valueBlocks = make([]*trieNode, 0) + index.lookupBlockIdx = make(map[uint32]uint16) + index.valueBlockIdx = make(map[uint32]uint16) + return index +} + +func computeOffsets(index *nodeIndex, n *trieNode) uint16 { + if n.leaf { + return n.value + } + hasher := crc32.New(crc32.MakeTable(crc32.IEEE)) + // We only index continuation bytes. + for i := 0; i < 64; i++ { + var v uint16 = 0 + if nn := n.table[0x80+i]; nn != nil { + v = computeOffsets(index, nn) + } + hasher.Write([]byte{uint8(v >> 8), uint8(v)}) + } + h := hasher.Sum32() + if n.isInternal() { + v, ok := index.lookupBlockIdx[h] + if !ok { + v = uint16(len(index.lookupBlocks)) + index.lookupBlocks = append(index.lookupBlocks, n) + index.lookupBlockIdx[h] = v + } + n.value = v + } else { + v, ok := index.valueBlockIdx[h] + if !ok { + v = uint16(len(index.valueBlocks)) + index.valueBlocks = append(index.valueBlocks, n) + index.valueBlockIdx[h] = v + } + n.value = v + } + return n.value +} + +func printValueBlock(nr int, n *trieNode, offset int) { + boff := nr * 64 + fmt.Printf("\n// Block %#x, offset %#x", nr, boff) + var printnewline bool + for i := 0; i < 64; i++ { + if i%6 == 0 { + printnewline = true + } + v := uint16(0) + if nn := n.table[i+offset]; nn != nil { + v = nn.value + } + if v != 0 { + if printnewline { + fmt.Printf("\n") + printnewline = false + } + fmt.Printf("%#04x:%#04x, ", nr*64+i, v) + } + } +} + +func printLookupBlock(nr int, n *trieNode, offset int) { + boff := nr * 64 + fmt.Printf("\n// Block %#x, offset %#x", nr, boff) + var printnewline bool + for i := 0; i < 64; i++ { + if i%8 == 0 { + printnewline = true + } + v := uint16(0) + if nn := n.table[i+offset]; nn != nil { + v = nn.value + } + if v != 0 { + if printnewline { + fmt.Printf("\n") + printnewline = false + } + fmt.Printf("%#03x:%#02x, ", boff+i, v) + } + } +} + +// printTables returns the size in bytes of the generated tables. +func (t *trieNode) printTables(name string) int { + index := newIndex() + // Values for 7-bit ASCII are stored in first two block, followed by nil block. + index.valueBlocks = append(index.valueBlocks, nil, nil, nil) + // First byte of multi-byte UTF-8 codepoints are indexed in 4th block. + index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil) + // Index starter bytes of multi-byte UTF-8. + for i := 0xC0; i < 0x100; i++ { + if t.table[i] != nil { + computeOffsets(index, t.table[i]) + } + } + + nv := len(index.valueBlocks) * 64 + fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2) + fmt.Printf("// Block 2 is the null block.\n") + fmt.Printf("var %sValues = [%d]uint16 {", name, nv) + printValueBlock(0, t, 0) + printValueBlock(1, t, 64) + printValueBlock(2, newNode(), 0) + for i := 3; i < len(index.valueBlocks); i++ { + printValueBlock(i, index.valueBlocks[i], 0x80) + } + fmt.Print("\n}\n\n") + + ni := len(index.lookupBlocks) * 64 + fmt.Printf("// %sLookup: %d bytes\n", name, ni) + fmt.Printf("// Block 0 is the null block.\n") + fmt.Printf("var %sLookup = [%d]uint8 {", name, ni) + printLookupBlock(0, newNode(), 0) + printLookupBlock(1, newNode(), 0) + printLookupBlock(2, newNode(), 0) + printLookupBlock(3, t, 0xC0) + for i := 4; i < len(index.lookupBlocks); i++ { + printLookupBlock(i, index.lookupBlocks[i], 0x80) + } + fmt.Print("\n}\n\n") + fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:] }\n\n", name, name, name) + return nv*2 + ni +} diff --git a/src/pkg/exp/regexp/Makefile b/src/pkg/exp/regexp/Makefile new file mode 100644 index 000000000..fc90df320 --- /dev/null +++ b/src/pkg/exp/regexp/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=exp/regexp +GOFILES=\ + exec.go\ + regexp.go\ + +include ../../../Make.pkg diff --git a/src/pkg/exp/regexp/all_test.go b/src/pkg/exp/regexp/all_test.go new file mode 100644 index 000000000..77f32ca1a --- /dev/null +++ b/src/pkg/exp/regexp/all_test.go @@ -0,0 +1,429 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regexp + +import ( + "os" + "strings" + "testing" +) + +var good_re = []string{ + ``, + `.`, + `^.$`, + `a`, + `a*`, + `a+`, + `a?`, + `a|b`, + `a*|b*`, + `(a*|b)(c*|d)`, + `[a-z]`, + `[a-abc-c\-\]\[]`, + `[a-z]+`, + `[abc]`, + `[^1234]`, + `[^\n]`, + `\!\\`, +} + +/* +type stringError struct { + re string + err os.Error +} + +var bad_re = []stringError{ + {`*`, ErrBareClosure}, + {`+`, ErrBareClosure}, + {`?`, ErrBareClosure}, + {`(abc`, ErrUnmatchedLpar}, + {`abc)`, ErrUnmatchedRpar}, + {`x[a-z`, ErrUnmatchedLbkt}, + {`abc]`, ErrUnmatchedRbkt}, + {`[z-a]`, ErrBadRange}, + {`abc\`, ErrExtraneousBackslash}, + {`a**`, ErrBadClosure}, + {`a*+`, ErrBadClosure}, + {`a??`, ErrBadClosure}, + {`\x`, ErrBadBackslash}, +} +*/ + +func compileTest(t *testing.T, expr string, error os.Error) *Regexp { + re, err := Compile(expr) + if err != error { + t.Error("compiling `", expr, "`; unexpected error: ", err.String()) + } + return re +} + +func TestGoodCompile(t *testing.T) { + for i := 0; i < len(good_re); i++ { + compileTest(t, good_re[i], nil) + } +} + +/* +func TestBadCompile(t *testing.T) { + for i := 0; i < len(bad_re); i++ { + compileTest(t, bad_re[i].re, bad_re[i].err) + } +} +*/ + +func matchTest(t *testing.T, test *FindTest) { + re := compileTest(t, test.pat, nil) + if re == nil { + return + } + m := re.MatchString(test.text) + if m != (len(test.matches) > 0) { + t.Errorf("MatchString failure on %s: %t should be %t", test, m, len(test.matches) > 0) + } + // now try bytes + m = re.Match([]byte(test.text)) + if m != (len(test.matches) > 0) { + t.Errorf("Match failure on %s: %t should be %t", test, m, len(test.matches) > 0) + } +} + +func TestMatch(t *testing.T) { + for _, test := range findTests { + matchTest(t, &test) + } +} + +func matchFunctionTest(t *testing.T, test *FindTest) { + m, err := MatchString(test.pat, test.text) + if err == nil { + return + } + if m != (len(test.matches) > 0) { + t.Errorf("Match failure on %s: %t should be %t", test, m, len(test.matches) > 0) + } +} + +func TestMatchFunction(t *testing.T) { + for _, test := range findTests { + matchFunctionTest(t, &test) + } +} + +type ReplaceTest struct { + pattern, replacement, input, output string +} + +var replaceTests = []ReplaceTest{ + // Test empty input and/or replacement, with pattern that matches the empty string. + {"", "", "", ""}, + {"", "x", "", "x"}, + {"", "", "abc", "abc"}, + {"", "x", "abc", "xaxbxcx"}, + + // Test empty input and/or replacement, with pattern that does not match the empty string. + {"b", "", "", ""}, + {"b", "x", "", ""}, + {"b", "", "abc", "ac"}, + {"b", "x", "abc", "axc"}, + {"y", "", "", ""}, + {"y", "x", "", ""}, + {"y", "", "abc", "abc"}, + {"y", "x", "abc", "abc"}, + + // Multibyte characters -- verify that we don't try to match in the middle + // of a character. + {"[a-c]*", "x", "\u65e5", "x\u65e5x"}, + {"[^\u65e5]", "x", "abc\u65e5def", "xxx\u65e5xxx"}, + + // Start and end of a string. + {"^[a-c]*", "x", "abcdabc", "xdabc"}, + {"[a-c]*$", "x", "abcdabc", "abcdx"}, + {"^[a-c]*$", "x", "abcdabc", "abcdabc"}, + {"^[a-c]*", "x", "abc", "x"}, + {"[a-c]*$", "x", "abc", "x"}, + {"^[a-c]*$", "x", "abc", "x"}, + {"^[a-c]*", "x", "dabce", "xdabce"}, + {"[a-c]*$", "x", "dabce", "dabcex"}, + {"^[a-c]*$", "x", "dabce", "dabce"}, + {"^[a-c]*", "x", "", "x"}, + {"[a-c]*$", "x", "", "x"}, + {"^[a-c]*$", "x", "", "x"}, + + {"^[a-c]+", "x", "abcdabc", "xdabc"}, + {"[a-c]+$", "x", "abcdabc", "abcdx"}, + {"^[a-c]+$", "x", "abcdabc", "abcdabc"}, + {"^[a-c]+", "x", "abc", "x"}, + {"[a-c]+$", "x", "abc", "x"}, + {"^[a-c]+$", "x", "abc", "x"}, + {"^[a-c]+", "x", "dabce", "dabce"}, + {"[a-c]+$", "x", "dabce", "dabce"}, + {"^[a-c]+$", "x", "dabce", "dabce"}, + {"^[a-c]+", "x", "", ""}, + {"[a-c]+$", "x", "", ""}, + {"^[a-c]+$", "x", "", ""}, + + // Other cases. + {"abc", "def", "abcdefg", "defdefg"}, + {"bc", "BC", "abcbcdcdedef", "aBCBCdcdedef"}, + {"abc", "", "abcdabc", "d"}, + {"x", "xXx", "xxxXxxx", "xXxxXxxXxXxXxxXxxXx"}, + {"abc", "d", "", ""}, + {"abc", "d", "abc", "d"}, + {".+", "x", "abc", "x"}, + {"[a-c]*", "x", "def", "xdxexfx"}, + {"[a-c]+", "x", "abcbcdcdedef", "xdxdedef"}, + {"[a-c]*", "x", "abcbcdcdedef", "xdxdxexdxexfx"}, +} + +type ReplaceFuncTest struct { + pattern string + replacement func(string) string + input, output string +} + +var replaceFuncTests = []ReplaceFuncTest{ + {"[a-c]", func(s string) string { return "x" + s + "y" }, "defabcdef", "defxayxbyxcydef"}, + {"[a-c]+", func(s string) string { return "x" + s + "y" }, "defabcdef", "defxabcydef"}, + {"[a-c]*", func(s string) string { return "x" + s + "y" }, "defabcdef", "xydxyexyfxabcydxyexyfxy"}, +} + +func TestReplaceAll(t *testing.T) { + for _, tc := range replaceTests { + re, err := Compile(tc.pattern) + if err != nil { + t.Errorf("Unexpected error compiling %q: %v", tc.pattern, err) + continue + } + actual := re.ReplaceAllString(tc.input, tc.replacement) + if actual != tc.output { + t.Errorf("%q.Replace(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + // now try bytes + actual = string(re.ReplaceAll([]byte(tc.input), []byte(tc.replacement))) + if actual != tc.output { + t.Errorf("%q.Replace(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + } +} + +func TestReplaceAllFunc(t *testing.T) { + for _, tc := range replaceFuncTests { + re, err := Compile(tc.pattern) + if err != nil { + t.Errorf("Unexpected error compiling %q: %v", tc.pattern, err) + continue + } + actual := re.ReplaceAllStringFunc(tc.input, tc.replacement) + if actual != tc.output { + t.Errorf("%q.ReplaceFunc(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + // now try bytes + actual = string(re.ReplaceAllFunc([]byte(tc.input), func(s []byte) []byte { return []byte(tc.replacement(string(s))) })) + if actual != tc.output { + t.Errorf("%q.ReplaceFunc(%q,%q) = %q; want %q", + tc.pattern, tc.input, tc.replacement, actual, tc.output) + } + } +} + +type MetaTest struct { + pattern, output, literal string + isLiteral bool +} + +var metaTests = []MetaTest{ + {``, ``, ``, true}, + {`foo`, `foo`, `foo`, true}, + {`foo\.\$`, `foo\\\.\\\$`, `foo.$`, true}, // has meta but no operator + {`foo.\$`, `foo\.\\\$`, `foo`, false}, // has escaped operators and real operators + {`!@#$%^&*()_+-=[{]}\|,<.>/?~`, `!@#\$%\^&\*\(\)_\+-=\[\{\]\}\\\|,<\.>/\?~`, `!@#`, false}, +} + +func TestQuoteMeta(t *testing.T) { + for _, tc := range metaTests { + // Verify that QuoteMeta returns the expected string. + quoted := QuoteMeta(tc.pattern) + if quoted != tc.output { + t.Errorf("QuoteMeta(`%s`) = `%s`; want `%s`", + tc.pattern, quoted, tc.output) + continue + } + + // Verify that the quoted string is in fact treated as expected + // by Compile -- i.e. that it matches the original, unquoted string. + if tc.pattern != "" { + re, err := Compile(quoted) + if err != nil { + t.Errorf("Unexpected error compiling QuoteMeta(`%s`): %v", tc.pattern, err) + continue + } + src := "abc" + tc.pattern + "def" + repl := "xyz" + replaced := re.ReplaceAllString(src, repl) + expected := "abcxyzdef" + if replaced != expected { + t.Errorf("QuoteMeta(`%s`).Replace(`%s`,`%s`) = `%s`; want `%s`", + tc.pattern, src, repl, replaced, expected) + } + } + } +} + +func TestLiteralPrefix(t *testing.T) { + for _, tc := range metaTests { + // Literal method needs to scan the pattern. + re := MustCompile(tc.pattern) + str, complete := re.LiteralPrefix() + if complete != tc.isLiteral { + t.Errorf("LiteralPrefix(`%s`) = %t; want %t", tc.pattern, complete, tc.isLiteral) + } + if str != tc.literal { + t.Errorf("LiteralPrefix(`%s`) = `%s`; want `%s`", tc.pattern, str, tc.literal) + } + } +} + +type numSubexpCase struct { + input string + expected int +} + +var numSubexpCases = []numSubexpCase{ + {``, 0}, + {`.*`, 0}, + {`abba`, 0}, + {`ab(b)a`, 1}, + {`ab(.*)a`, 1}, + {`(.*)ab(.*)a`, 2}, + {`(.*)(ab)(.*)a`, 3}, + {`(.*)((a)b)(.*)a`, 4}, + {`(.*)(\(ab)(.*)a`, 3}, + {`(.*)(\(a\)b)(.*)a`, 3}, +} + +func TestNumSubexp(t *testing.T) { + for _, c := range numSubexpCases { + re := MustCompile(c.input) + n := re.NumSubexp() + if n != c.expected { + t.Errorf("NumSubexp for %q returned %d, expected %d", c.input, n, c.expected) + } + } +} + +func BenchmarkLiteral(b *testing.B) { + x := strings.Repeat("x", 50) + "y" + b.StopTimer() + re := MustCompile("y") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkNotLiteral(b *testing.B) { + x := strings.Repeat("x", 50) + "y" + b.StopTimer() + re := MustCompile(".y") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkMatchClass(b *testing.B) { + b.StopTimer() + x := strings.Repeat("xxxx", 20) + "w" + re := MustCompile("[abcdw]") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkMatchClass_InRange(b *testing.B) { + b.StopTimer() + // 'b' is between 'a' and 'c', so the charclass + // range checking is no help here. + x := strings.Repeat("bbbb", 20) + "c" + re := MustCompile("[ac]") + b.StartTimer() + for i := 0; i < b.N; i++ { + if !re.MatchString(x) { + println("no match!") + break + } + } +} + +func BenchmarkReplaceAll(b *testing.B) { + x := "abcdefghijklmnopqrstuvwxyz" + b.StopTimer() + re := MustCompile("[cjrw]") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.ReplaceAllString(x, "") + } +} + +func BenchmarkAnchoredLiteralShortNonMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + re := MustCompile("^zbc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} + +func BenchmarkAnchoredLiteralLongNonMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + for i := 0; i < 15; i++ { + x = append(x, x...) + } + re := MustCompile("^zbc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} + +func BenchmarkAnchoredShortMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + re := MustCompile("^.bc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} + +func BenchmarkAnchoredLongMatch(b *testing.B) { + b.StopTimer() + x := []byte("abcdefghijklmnopqrstuvwxyz") + for i := 0; i < 15; i++ { + x = append(x, x...) + } + re := MustCompile("^.bc(d|e)") + b.StartTimer() + for i := 0; i < b.N; i++ { + re.Match(x) + } +} diff --git a/src/pkg/exp/regexp/exec.go b/src/pkg/exp/regexp/exec.go new file mode 100644 index 000000000..0670bb9b1 --- /dev/null +++ b/src/pkg/exp/regexp/exec.go @@ -0,0 +1,295 @@ +package regexp + +import "exp/regexp/syntax" + +// A queue is a 'sparse array' holding pending threads of execution. +// See http://research.swtch.com/2008/03/using-uninitialized-memory-for-fun-and.html +type queue struct { + sparse []uint32 + dense []entry +} + +// A entry is an entry on a queue. +// It holds both the instruction pc and the actual thread. +// Some queue entries are just place holders so that the machine +// knows it has considered that pc. Such entries have t == nil. +type entry struct { + pc uint32 + t *thread +} + +// A thread is the state of a single path through the machine: +// an instruction and a corresponding capture array. +// See http://swtch.com/~rsc/regexp/regexp2.html +type thread struct { + inst *syntax.Inst + cap []int +} + +// A machine holds all the state during an NFA simulation for p. +type machine struct { + re *Regexp // corresponding Regexp + p *syntax.Prog // compiled program + q0, q1 queue // two queues for runq, nextq + pool []*thread // pool of available threads + matched bool // whether a match was found + matchcap []int // capture information for the match +} + +// progMachine returns a new machine running the prog p. +func progMachine(p *syntax.Prog) *machine { + m := &machine{p: p} + n := len(m.p.Inst) + m.q0 = queue{make([]uint32, n), make([]entry, 0, n)} + m.q1 = queue{make([]uint32, n), make([]entry, 0, n)} + ncap := p.NumCap + if ncap < 2 { + ncap = 2 + } + m.matchcap = make([]int, ncap) + return m +} + +// alloc allocates a new thread with the given instruction. +// It uses the free pool if possible. +func (m *machine) alloc(i *syntax.Inst) *thread { + var t *thread + if n := len(m.pool); n > 0 { + t = m.pool[n-1] + m.pool = m.pool[:n-1] + } else { + t = new(thread) + t.cap = make([]int, cap(m.matchcap)) + } + t.cap = t.cap[:len(m.matchcap)] + t.inst = i + return t +} + +// free returns t to the free pool. +func (m *machine) free(t *thread) { + m.pool = append(m.pool, t) +} + +// match runs the machine over the input starting at pos. +// It reports whether a match was found. +// If so, m.matchcap holds the submatch information. +func (m *machine) match(i input, pos int) bool { + startCond := m.re.cond + if startCond == ^syntax.EmptyOp(0) { // impossible + return false + } + m.matched = false + for i := range m.matchcap { + m.matchcap[i] = -1 + } + runq, nextq := &m.q0, &m.q1 + rune, rune1 := endOfText, endOfText + width, width1 := 0, 0 + rune, width = i.step(pos) + if rune != endOfText { + rune1, width1 = i.step(pos + width) + } + // TODO: Let caller specify the initial flag setting. + // For now assume pos == 0 is beginning of text and + // pos != 0 is not even beginning of line. + // TODO: Word boundary. + var flag syntax.EmptyOp + if pos == 0 { + flag = syntax.EmptyBeginText | syntax.EmptyBeginLine + } + + // Update flag using lookahead rune. + if rune1 == '\n' { + flag |= syntax.EmptyEndLine + } + if rune1 == endOfText { + flag |= syntax.EmptyEndText + } + + for { + if len(runq.dense) == 0 { + if startCond&syntax.EmptyBeginText != 0 && pos != 0 { + // Anchored match, past beginning of text. + break + } + if m.matched { + // Have match; finished exploring alternatives. + break + } + if len(m.re.prefix) > 0 && rune1 != m.re.prefixRune && i.canCheckPrefix() { + // Match requires literal prefix; fast search for it. + advance := i.index(m.re, pos) + if advance < 0 { + break + } + pos += advance + rune, width = i.step(pos) + rune1, width1 = i.step(pos + width) + } + } + if !m.matched { + if len(m.matchcap) > 0 { + m.matchcap[0] = pos + } + m.add(runq, uint32(m.p.Start), pos, m.matchcap, flag) + } + // TODO: word boundary + flag = 0 + if rune == '\n' { + flag |= syntax.EmptyBeginLine + } + if rune1 == '\n' { + flag |= syntax.EmptyEndLine + } + if rune1 == endOfText { + flag |= syntax.EmptyEndText + } + m.step(runq, nextq, pos, pos+width, rune, flag) + if width == 0 { + break + } + pos += width + rune, width = rune1, width1 + if rune != endOfText { + rune1, width1 = i.step(pos + width) + } + runq, nextq = nextq, runq + } + m.clear(nextq) + return m.matched +} + +// clear frees all threads on the thread queue. +func (m *machine) clear(q *queue) { + for _, d := range q.dense { + if d.t != nil { + m.free(d.t) + } + } + q.dense = q.dense[:0] +} + +// step executes one step of the machine, running each of the threads +// on runq and appending new threads to nextq. +// The step processes the rune c (which may be endOfText), +// which starts at position pos and ends at nextPos. +// nextCond gives the setting for the empty-width flags after c. +func (m *machine) step(runq, nextq *queue, pos, nextPos, c int, nextCond syntax.EmptyOp) { + for j := 0; j < len(runq.dense); j++ { + d := &runq.dense[j] + t := d.t + if t == nil { + continue + } + /* + * If we support leftmost-longest matching: + if longest && matched && match[0] < t.cap[0] { + m.free(t) + continue + } + */ + + i := t.inst + switch i.Op { + default: + panic("bad inst") + + case syntax.InstMatch: + if len(t.cap) > 0 { + t.cap[1] = pos + copy(m.matchcap, t.cap) + } + m.matched = true + for _, d := range runq.dense[j+1:] { + if d.t != nil { + m.free(d.t) + } + } + runq.dense = runq.dense[:0] + + case syntax.InstRune: + if i.MatchRune(c) { + m.add(nextq, i.Out, nextPos, t.cap, nextCond) + } + } + m.free(t) + } + runq.dense = runq.dense[:0] +} + +// add adds an entry to q for pc, unless the q already has such an entry. +// It also recursively adds an entry for all instructions reachable from pc by following +// empty-width conditions satisfied by cond. pos gives the current position +// in the input. +func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.EmptyOp) { + if pc == 0 { + return + } + if j := q.sparse[pc]; j < uint32(len(q.dense)) && q.dense[j].pc == pc { + return + } + + j := len(q.dense) + q.dense = q.dense[:j+1] + d := &q.dense[j] + d.t = nil + d.pc = pc + q.sparse[pc] = uint32(j) + + i := &m.p.Inst[pc] + switch i.Op { + default: + panic("unhandled") + case syntax.InstFail: + // nothing + case syntax.InstAlt, syntax.InstAltMatch: + m.add(q, i.Out, pos, cap, cond) + m.add(q, i.Arg, pos, cap, cond) + case syntax.InstEmptyWidth: + if syntax.EmptyOp(i.Arg)&^cond == 0 { + m.add(q, i.Out, pos, cap, cond) + } + case syntax.InstNop: + m.add(q, i.Out, pos, cap, cond) + case syntax.InstCapture: + if int(i.Arg) < len(cap) { + opos := cap[i.Arg] + cap[i.Arg] = pos + m.add(q, i.Out, pos, cap, cond) + cap[i.Arg] = opos + } else { + m.add(q, i.Out, pos, cap, cond) + } + case syntax.InstMatch, syntax.InstRune: + t := m.alloc(i) + if len(t.cap) > 0 { + copy(t.cap, cap) + } + d.t = t + } +} + +// empty is a non-nil 0-element slice, +// so doExecute can avoid an allocation +// when 0 captures are requested from a successful match. +var empty = make([]int, 0) + +// doExecute finds the leftmost match in the input and returns +// the position of its subexpressions. +func (re *Regexp) doExecute(i input, pos int, ncap int) []int { + m := re.get() + m.matchcap = m.matchcap[:ncap] + if !m.match(i, pos) { + re.put(m) + return nil + } + if ncap == 0 { + re.put(m) + return empty // empty but not nil + } + cap := make([]int, ncap) + copy(cap, m.matchcap) + re.put(m) + return cap +} diff --git a/src/pkg/exp/regexp/find_test.go b/src/pkg/exp/regexp/find_test.go new file mode 100644 index 000000000..dddc3484c --- /dev/null +++ b/src/pkg/exp/regexp/find_test.go @@ -0,0 +1,472 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regexp + +import ( + "fmt" + "strings" + "testing" +) + +// For each pattern/text pair, what is the expected output of each function? +// We can derive the textual results from the indexed results, the non-submatch +// results from the submatched results, the single results from the 'all' results, +// and the byte results from the string results. Therefore the table includes +// only the FindAllStringSubmatchIndex result. +type FindTest struct { + pat string + text string + matches [][]int +} + +func (t FindTest) String() string { + return fmt.Sprintf("pat: %#q text: %#q", t.pat, t.text) +} + +var findTests = []FindTest{ + {``, ``, build(1, 0, 0)}, + {`^abcdefg`, "abcdefg", build(1, 0, 7)}, + {`a+`, "baaab", build(1, 1, 4)}, + {"abcd..", "abcdef", build(1, 0, 6)}, + {`a`, "a", build(1, 0, 1)}, + {`x`, "y", nil}, + {`b`, "abc", build(1, 1, 2)}, + {`.`, "a", build(1, 0, 1)}, + {`.*`, "abcdef", build(1, 0, 6)}, + {`^`, "abcde", build(1, 0, 0)}, + {`$`, "abcde", build(1, 5, 5)}, + {`^abcd$`, "abcd", build(1, 0, 4)}, + {`^bcd'`, "abcdef", nil}, + {`^abcd$`, "abcde", nil}, + {`a+`, "baaab", build(1, 1, 4)}, + {`a*`, "baaab", build(3, 0, 0, 1, 4, 5, 5)}, + {`[a-z]+`, "abcd", build(1, 0, 4)}, + {`[^a-z]+`, "ab1234cd", build(1, 2, 6)}, + {`[a\-\]z]+`, "az]-bcz", build(2, 0, 4, 6, 7)}, + {`[^\n]+`, "abcd\n", build(1, 0, 4)}, + {`[日本語]+`, "日本語日本語", build(1, 0, 18)}, + {`日本語+`, "日本語", build(1, 0, 9)}, + {`日本語+`, "日本語語語語", build(1, 0, 18)}, + {`()`, "", build(1, 0, 0, 0, 0)}, + {`(a)`, "a", build(1, 0, 1, 0, 1)}, + {`(.)(.)`, "日a", build(1, 0, 4, 0, 3, 3, 4)}, + {`(.*)`, "", build(1, 0, 0, 0, 0)}, + {`(.*)`, "abcd", build(1, 0, 4, 0, 4)}, + {`(..)(..)`, "abcd", build(1, 0, 4, 0, 2, 2, 4)}, + {`(([^xyz]*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 3, 4)}, + {`((a|b|c)*(d))`, "abcd", build(1, 0, 4, 0, 4, 2, 3, 3, 4)}, + {`(((a|b|c)*)(d))`, "abcd", build(1, 0, 4, 0, 4, 0, 3, 2, 3, 3, 4)}, + {`\a\f\n\r\t\v`, "\a\f\n\r\t\v", build(1, 0, 6)}, + {`[\a\f\n\r\t\v]+`, "\a\f\n\r\t\v", build(1, 0, 6)}, + + {`a*(|(b))c*`, "aacc", build(1, 0, 4, 2, 2, -1, -1)}, + {`(.*).*`, "ab", build(1, 0, 2, 0, 2)}, + {`[.]`, ".", build(1, 0, 1)}, + {`/$`, "/abc/", build(1, 4, 5)}, + {`/$`, "/abc", nil}, + + // multiple matches + {`.`, "abc", build(3, 0, 1, 1, 2, 2, 3)}, + {`(.)`, "abc", build(3, 0, 1, 0, 1, 1, 2, 1, 2, 2, 3, 2, 3)}, + {`.(.)`, "abcd", build(2, 0, 2, 1, 2, 2, 4, 3, 4)}, + {`ab*`, "abbaab", build(3, 0, 3, 3, 4, 4, 6)}, + {`a(b*)`, "abbaab", build(3, 0, 3, 1, 3, 3, 4, 4, 4, 4, 6, 5, 6)}, + + // fixed bugs + {`ab$`, "cab", build(1, 1, 3)}, + {`axxb$`, "axxcb", nil}, + {`data`, "daXY data", build(1, 5, 9)}, + {`da(.)a$`, "daXY data", build(1, 5, 9, 7, 8)}, + {`zx+`, "zzx", build(1, 1, 3)}, + + // can backslash-escape any punctuation + {`\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\{\|\}\~`, + `!"#$%&'()*+,-./:;<=>?@[\]^_{|}~`, build(1, 0, 31)}, + {`[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\{\|\}\~]+`, + `!"#$%&'()*+,-./:;<=>?@[\]^_{|}~`, build(1, 0, 31)}, + {"\\`", "`", build(1, 0, 1)}, + {"[\\`]+", "`", build(1, 0, 1)}, + + // long set of matches (longer than startSize) + { + ".", + "qwertyuiopasdfghjklzxcvbnm1234567890", + build(36, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, + 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, + 20, 21, 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, + 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36), + }, +} + +// build is a helper to construct a [][]int by extracting n sequences from x. +// This represents n matches with len(x)/n submatches each. +func build(n int, x ...int) [][]int { + ret := make([][]int, n) + runLength := len(x) / n + j := 0 + for i := range ret { + ret[i] = make([]int, runLength) + copy(ret[i], x[j:]) + j += runLength + if j > len(x) { + panic("invalid build entry") + } + } + return ret +} + +// First the simple cases. + +func TestFind(t *testing.T) { + for _, test := range findTests { + re := MustCompile(test.pat) + if re.String() != test.pat { + t.Errorf("String() = `%s`; should be `%s`", re.String(), test.pat) + } + result := re.Find([]byte(test.text)) + switch { + case len(test.matches) == 0 && len(result) == 0: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + expect := test.text[test.matches[0][0]:test.matches[0][1]] + if expect != string(result) { + t.Errorf("expected %q got %q: %s", expect, result, test) + } + } + } +} + +func TestFindString(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindString(test.text) + switch { + case len(test.matches) == 0 && len(result) == 0: + // ok + case test.matches == nil && result != "": + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == "": + // Tricky because an empty result has two meanings: no match or empty match. + if test.matches[0][0] != test.matches[0][1] { + t.Errorf("expected match; got none: %s", test) + } + case test.matches != nil && result != "": + expect := test.text[test.matches[0][0]:test.matches[0][1]] + if expect != result { + t.Errorf("expected %q got %q: %s", expect, result, test) + } + } + } +} + +func testFindIndex(test *FindTest, result []int, t *testing.T) { + switch { + case len(test.matches) == 0 && len(result) == 0: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + expect := test.matches[0] + if expect[0] != result[0] || expect[1] != result[1] { + t.Errorf("expected %v got %v: %s", expect, result, test) + } + } +} + +func TestFindIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindIndex([]byte(test.text)), t) + } +} + +func TestFindStringIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindStringIndex(test.text), t) + } +} + +func TestFindReaderIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindReaderIndex(strings.NewReader(test.text)), t) + } +} + +// Now come the simple All cases. + +func TestFindAll(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAll([]byte(test.text), -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Fatalf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + if len(test.matches) != len(result) { + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + continue + } + for k, e := range test.matches { + expect := test.text[e[0]:e[1]] + if expect != string(result[k]) { + t.Errorf("match %d: expected %q got %q: %s", k, expect, result[k], test) + } + } + } + } +} + +func TestFindAllString(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAllString(test.text, -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + if len(test.matches) != len(result) { + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + continue + } + for k, e := range test.matches { + expect := test.text[e[0]:e[1]] + if expect != result[k] { + t.Errorf("expected %q got %q: %s", expect, result, test) + } + } + } + } +} + +func testFindAllIndex(test *FindTest, result [][]int, t *testing.T) { + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + if len(test.matches) != len(result) { + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + return + } + for k, e := range test.matches { + if e[0] != result[k][0] || e[1] != result[k][1] { + t.Errorf("match %d: expected %v got %v: %s", k, e, result[k], test) + } + } + } +} + +func TestFindAllIndex(t *testing.T) { + for _, test := range findTests { + testFindAllIndex(&test, MustCompile(test.pat).FindAllIndex([]byte(test.text), -1), t) + } +} + +func TestFindAllStringIndex(t *testing.T) { + for _, test := range findTests { + testFindAllIndex(&test, MustCompile(test.pat).FindAllStringIndex(test.text, -1), t) + } +} + +// Now come the Submatch cases. + +func testSubmatchBytes(test *FindTest, n int, submatches []int, result [][]byte, t *testing.T) { + if len(submatches) != len(result)*2 { + t.Errorf("match %d: expected %d submatches; got %d: %s", n, len(submatches)/2, len(result), test) + return + } + for k := 0; k < len(submatches); k += 2 { + if submatches[k] == -1 { + if result[k/2] != nil { + t.Errorf("match %d: expected nil got %q: %s", n, result, test) + } + continue + } + expect := test.text[submatches[k]:submatches[k+1]] + if expect != string(result[k/2]) { + t.Errorf("match %d: expected %q got %q: %s", n, expect, result, test) + return + } + } +} + +func TestFindSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindSubmatch([]byte(test.text)) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + testSubmatchBytes(&test, 0, test.matches[0], result, t) + } + } +} + +func testSubmatchString(test *FindTest, n int, submatches []int, result []string, t *testing.T) { + if len(submatches) != len(result)*2 { + t.Errorf("match %d: expected %d submatches; got %d: %s", n, len(submatches)/2, len(result), test) + return + } + for k := 0; k < len(submatches); k += 2 { + if submatches[k] == -1 { + if result[k/2] != "" { + t.Errorf("match %d: expected nil got %q: %s", n, result, test) + } + continue + } + expect := test.text[submatches[k]:submatches[k+1]] + if expect != result[k/2] { + t.Errorf("match %d: expected %q got %q: %s", n, expect, result, test) + return + } + } +} + +func TestFindStringSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindStringSubmatch(test.text) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + testSubmatchString(&test, 0, test.matches[0], result, t) + } + } +} + +func testSubmatchIndices(test *FindTest, n int, expect, result []int, t *testing.T) { + if len(expect) != len(result) { + t.Errorf("match %d: expected %d matches; got %d: %s", n, len(expect)/2, len(result)/2, test) + return + } + for k, e := range expect { + if e != result[k] { + t.Errorf("match %d: submatch error: expected %v got %v: %s", n, expect, result, test) + } + } +} + +func testFindSubmatchIndex(test *FindTest, result []int, t *testing.T) { + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case test.matches != nil && result != nil: + testSubmatchIndices(test, 0, test.matches[0], result, t) + } +} + +func TestFindSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindSubmatchIndex([]byte(test.text)), t) + } +} + +func TestFindStringSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindStringSubmatchIndex(test.text), t) + } +} + +func TestFindReaderSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindReaderSubmatchIndex(strings.NewReader(test.text)), t) + } +} + +// Now come the monster AllSubmatch cases. + +func TestFindAllSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAllSubmatch([]byte(test.text), -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case len(test.matches) != len(result): + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + case test.matches != nil && result != nil: + for k, match := range test.matches { + testSubmatchBytes(&test, k, match, result[k], t) + } + } + } +} + +func TestFindAllStringSubmatch(t *testing.T) { + for _, test := range findTests { + result := MustCompile(test.pat).FindAllStringSubmatch(test.text, -1) + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case len(test.matches) != len(result): + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + case test.matches != nil && result != nil: + for k, match := range test.matches { + testSubmatchString(&test, k, match, result[k], t) + } + } + } +} + +func testFindAllSubmatchIndex(test *FindTest, result [][]int, t *testing.T) { + switch { + case test.matches == nil && result == nil: + // ok + case test.matches == nil && result != nil: + t.Errorf("expected no match; got one: %s", test) + case test.matches != nil && result == nil: + t.Errorf("expected match; got none: %s", test) + case len(test.matches) != len(result): + t.Errorf("expected %d matches; got %d: %s", len(test.matches), len(result), test) + case test.matches != nil && result != nil: + for k, match := range test.matches { + testSubmatchIndices(test, k, match, result[k], t) + } + } +} + +func TestFindAllSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindAllSubmatchIndex(&test, MustCompile(test.pat).FindAllSubmatchIndex([]byte(test.text), -1), t) + } +} + +func TestFindAllStringSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindAllSubmatchIndex(&test, MustCompile(test.pat).FindAllStringSubmatchIndex(test.text, -1), t) + } +} diff --git a/src/pkg/exp/regexp/regexp.go b/src/pkg/exp/regexp/regexp.go new file mode 100644 index 000000000..1b75900f8 --- /dev/null +++ b/src/pkg/exp/regexp/regexp.go @@ -0,0 +1,795 @@ +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package regexp implements a simple regular expression library. +// +// The syntax of the regular expressions accepted is the same +// general syntax used by Perl, Python, and other languages. +// More precisely, it is the syntax accepted by RE2 and described at +// http://code.google.com/p/re2/wiki/Syntax, except for \C. +// +// All characters are UTF-8-encoded code points. +// +// There are 16 methods of Regexp that match a regular expression and identify +// the matched text. Their names are matched by this regular expression: +// +// Find(All)?(String)?(Submatch)?(Index)? +// +// If 'All' is present, the routine matches successive non-overlapping +// matches of the entire expression. Empty matches abutting a preceding +// match are ignored. The return value is a slice containing the successive +// return values of the corresponding non-'All' routine. These routines take +// an extra integer argument, n; if n >= 0, the function returns at most n +// matches/submatches. +// +// If 'String' is present, the argument is a string; otherwise it is a slice +// of bytes; return values are adjusted as appropriate. +// +// If 'Submatch' is present, the return value is a slice identifying the +// successive submatches of the expression. Submatches are matches of +// parenthesized subexpressions within the regular expression, numbered from +// left to right in order of opening parenthesis. Submatch 0 is the match of +// the entire expression, submatch 1 the match of the first parenthesized +// subexpression, and so on. +// +// If 'Index' is present, matches and submatches are identified by byte index +// pairs within the input string: result[2*n:2*n+1] identifies the indexes of +// the nth submatch. The pair for n==0 identifies the match of the entire +// expression. If 'Index' is not present, the match is identified by the +// text of the match/submatch. If an index is negative, it means that +// subexpression did not match any string in the input. +// +// There is also a subset of the methods that can be applied to text read +// from a RuneReader: +// +// MatchReader, FindReaderIndex, FindReaderSubmatchIndex +// +// This set may grow. Note that regular expression matches may need to +// examine text beyond the text returned by a match, so the methods that +// match text from a RuneReader may read arbitrarily far into the input +// before returning. +// +// (There are a few other methods that do not match this pattern.) +// +package regexp + +import ( + "bytes" + "exp/regexp/syntax" + "io" + "os" + "strings" + "sync" + "utf8" +) + +var debug = false + +// Error is the local type for a parsing error. +type Error string + +func (e Error) String() string { + return string(e) +} + +// Regexp is the representation of a compiled regular expression. +// The public interface is entirely through methods. +// A Regexp is safe for concurrent use by multiple goroutines. +type Regexp struct { + // read-only after Compile + expr string // as passed to Compile + prog *syntax.Prog // compiled program + prefix string // required prefix in unanchored matches + prefixBytes []byte // prefix, as a []byte + prefixComplete bool // prefix is the entire regexp + prefixRune int // first rune in prefix + cond syntax.EmptyOp // empty-width conditions required at start of match + + // cache of machines for running regexp + mu sync.Mutex + machine []*machine +} + +// String returns the source text used to compile the regular expression. +func (re *Regexp) String() string { + return re.expr +} + +// Compile parses a regular expression and returns, if successful, a Regexp +// object that can be used to match against text. +func Compile(expr string) (*Regexp, os.Error) { + re, err := syntax.Parse(expr, syntax.Perl) + if err != nil { + return nil, err + } + prog, err := syntax.Compile(re) + if err != nil { + return nil, err + } + regexp := &Regexp{ + expr: expr, + prog: prog, + } + regexp.prefix, regexp.prefixComplete = prog.Prefix() + if regexp.prefix != "" { + // TODO(rsc): Remove this allocation by adding + // IndexString to package bytes. + regexp.prefixBytes = []byte(regexp.prefix) + regexp.prefixRune, _ = utf8.DecodeRuneInString(regexp.prefix) + } + regexp.cond = prog.StartCond() + return regexp, nil +} + +// get returns a machine to use for matching re. +// It uses the re's machine cache if possible, to avoid +// unnecessary allocation. +func (re *Regexp) get() *machine { + re.mu.Lock() + if n := len(re.machine); n > 0 { + z := re.machine[n-1] + re.machine = re.machine[:n-1] + re.mu.Unlock() + return z + } + re.mu.Unlock() + z := progMachine(re.prog) + z.re = re + return z +} + +// put returns a machine to the re's machine cache. +// There is no attempt to limit the size of the cache, so it will +// grow to the maximum number of simultaneous matches +// run using re. (The cache empties when re gets garbage collected.) +func (re *Regexp) put(z *machine) { + re.mu.Lock() + re.machine = append(re.machine, z) + re.mu.Unlock() +} + +// MustCompile is like Compile but panics if the expression cannot be parsed. +// It simplifies safe initialization of global variables holding compiled regular +// expressions. +func MustCompile(str string) *Regexp { + regexp, error := Compile(str) + if error != nil { + panic(`regexp: compiling "` + str + `": ` + error.String()) + } + return regexp +} + +// NumSubexp returns the number of parenthesized subexpressions in this Regexp. +func (re *Regexp) NumSubexp() int { + // NumCap/2 because captures count ( and ) separately. + // -1 because NumCap counts $0 but NumSubexp does not. + return re.prog.NumCap/2 - 1 +} + +const endOfText = -1 + +// input abstracts different representations of the input text. It provides +// one-character lookahead. +type input interface { + step(pos int) (rune int, width int) // advance one rune + canCheckPrefix() bool // can we look ahead without losing info? + hasPrefix(re *Regexp) bool + index(re *Regexp, pos int) int +} + +// inputString scans a string. +type inputString struct { + str string +} + +func newInputString(str string) *inputString { + return &inputString{str: str} +} + +func (i *inputString) step(pos int) (int, int) { + if pos < len(i.str) { + return utf8.DecodeRuneInString(i.str[pos:len(i.str)]) + } + return endOfText, 0 +} + +func (i *inputString) canCheckPrefix() bool { + return true +} + +func (i *inputString) hasPrefix(re *Regexp) bool { + return strings.HasPrefix(i.str, re.prefix) +} + +func (i *inputString) index(re *Regexp, pos int) int { + return strings.Index(i.str[pos:], re.prefix) +} + +// inputBytes scans a byte slice. +type inputBytes struct { + str []byte +} + +func newInputBytes(str []byte) *inputBytes { + return &inputBytes{str: str} +} + +func (i *inputBytes) step(pos int) (int, int) { + if pos < len(i.str) { + return utf8.DecodeRune(i.str[pos:len(i.str)]) + } + return endOfText, 0 +} + +func (i *inputBytes) canCheckPrefix() bool { + return true +} + +func (i *inputBytes) hasPrefix(re *Regexp) bool { + return bytes.HasPrefix(i.str, re.prefixBytes) +} + +func (i *inputBytes) index(re *Regexp, pos int) int { + return bytes.Index(i.str[pos:], re.prefixBytes) +} + +// inputReader scans a RuneReader. +type inputReader struct { + r io.RuneReader + atEOT bool + pos int +} + +func newInputReader(r io.RuneReader) *inputReader { + return &inputReader{r: r} +} + +func (i *inputReader) step(pos int) (int, int) { + if !i.atEOT && pos != i.pos { + return endOfText, 0 + + } + r, w, err := i.r.ReadRune() + if err != nil { + i.atEOT = true + return endOfText, 0 + } + i.pos += w + return r, w +} + +func (i *inputReader) canCheckPrefix() bool { + return false +} + +func (i *inputReader) hasPrefix(re *Regexp) bool { + return false +} + +func (i *inputReader) index(re *Regexp, pos int) int { + return -1 +} + +// LiteralPrefix returns a literal string that must begin any match +// of the regular expression re. It returns the boolean true if the +// literal string comprises the entire regular expression. +func (re *Regexp) LiteralPrefix() (prefix string, complete bool) { + return re.prefix, re.prefixComplete +} + +// MatchReader returns whether the Regexp matches the text read by the +// RuneReader. The return value is a boolean: true for match, false for no +// match. +func (re *Regexp) MatchReader(r io.RuneReader) bool { + return re.doExecute(newInputReader(r), 0, 0) != nil +} + +// MatchString returns whether the Regexp matches the string s. +// The return value is a boolean: true for match, false for no match. +func (re *Regexp) MatchString(s string) bool { + return re.doExecute(newInputString(s), 0, 0) != nil +} + +// Match returns whether the Regexp matches the byte slice b. +// The return value is a boolean: true for match, false for no match. +func (re *Regexp) Match(b []byte) bool { + return re.doExecute(newInputBytes(b), 0, 0) != nil +} + +// MatchReader checks whether a textual regular expression matches the text +// read by the RuneReader. More complicated queries need to use Compile and +// the full Regexp interface. +func MatchReader(pattern string, r io.RuneReader) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.MatchReader(r), nil +} + +// MatchString checks whether a textual regular expression +// matches a string. More complicated queries need +// to use Compile and the full Regexp interface. +func MatchString(pattern string, s string) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.MatchString(s), nil +} + +// Match checks whether a textual regular expression +// matches a byte slice. More complicated queries need +// to use Compile and the full Regexp interface. +func Match(pattern string, b []byte) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.Match(b), nil +} + +// ReplaceAllString returns a copy of src in which all matches for the Regexp +// have been replaced by repl. No support is provided for expressions +// (e.g. \1 or $1) in the replacement string. +func (re *Regexp) ReplaceAllString(src, repl string) string { + return re.ReplaceAllStringFunc(src, func(string) string { return repl }) +} + +// ReplaceAllStringFunc returns a copy of src in which all matches for the +// Regexp have been replaced by the return value of of function repl (whose +// first argument is the matched string). No support is provided for +// expressions (e.g. \1 or $1) in the replacement string. +func (re *Regexp) ReplaceAllStringFunc(src string, repl func(string) string) string { + lastMatchEnd := 0 // end position of the most recent match + searchPos := 0 // position where we next look for a match + buf := new(bytes.Buffer) + for searchPos <= len(src) { + a := re.doExecute(newInputString(src), searchPos, 2) + if len(a) == 0 { + break // no more matches + } + + // Copy the unmatched characters before this match. + io.WriteString(buf, src[lastMatchEnd:a[0]]) + + // Now insert a copy of the replacement string, but not for a + // match of the empty string immediately after another match. + // (Otherwise, we get double replacement for patterns that + // match both empty and nonempty strings.) + if a[1] > lastMatchEnd || a[0] == 0 { + io.WriteString(buf, repl(src[a[0]:a[1]])) + } + lastMatchEnd = a[1] + + // Advance past this match; always advance at least one character. + _, width := utf8.DecodeRuneInString(src[searchPos:]) + if searchPos+width > a[1] { + searchPos += width + } else if searchPos+1 > a[1] { + // This clause is only needed at the end of the input + // string. In that case, DecodeRuneInString returns width=0. + searchPos++ + } else { + searchPos = a[1] + } + } + + // Copy the unmatched characters after the last match. + io.WriteString(buf, src[lastMatchEnd:]) + + return buf.String() +} + +// ReplaceAll returns a copy of src in which all matches for the Regexp +// have been replaced by repl. No support is provided for expressions +// (e.g. \1 or $1) in the replacement text. +func (re *Regexp) ReplaceAll(src, repl []byte) []byte { + return re.ReplaceAllFunc(src, func([]byte) []byte { return repl }) +} + +// ReplaceAllFunc returns a copy of src in which all matches for the +// Regexp have been replaced by the return value of of function repl (whose +// first argument is the matched []byte). No support is provided for +// expressions (e.g. \1 or $1) in the replacement string. +func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { + lastMatchEnd := 0 // end position of the most recent match + searchPos := 0 // position where we next look for a match + buf := new(bytes.Buffer) + for searchPos <= len(src) { + a := re.doExecute(newInputBytes(src), searchPos, 2) + if len(a) == 0 { + break // no more matches + } + + // Copy the unmatched characters before this match. + buf.Write(src[lastMatchEnd:a[0]]) + + // Now insert a copy of the replacement string, but not for a + // match of the empty string immediately after another match. + // (Otherwise, we get double replacement for patterns that + // match both empty and nonempty strings.) + if a[1] > lastMatchEnd || a[0] == 0 { + buf.Write(repl(src[a[0]:a[1]])) + } + lastMatchEnd = a[1] + + // Advance past this match; always advance at least one character. + _, width := utf8.DecodeRune(src[searchPos:]) + if searchPos+width > a[1] { + searchPos += width + } else if searchPos+1 > a[1] { + // This clause is only needed at the end of the input + // string. In that case, DecodeRuneInString returns width=0. + searchPos++ + } else { + searchPos = a[1] + } + } + + // Copy the unmatched characters after the last match. + buf.Write(src[lastMatchEnd:]) + + return buf.Bytes() +} + +var specialBytes = []byte(`\.+*?()|[]{}^$`) + +func special(b byte) bool { + return bytes.IndexByte(specialBytes, b) >= 0 +} + +// QuoteMeta returns a string that quotes all regular expression metacharacters +// inside the argument text; the returned string is a regular expression matching +// the literal text. For example, QuoteMeta(`[foo]`) returns `\[foo\]`. +func QuoteMeta(s string) string { + b := make([]byte, 2*len(s)) + + // A byte loop is correct because all metacharacters are ASCII. + j := 0 + for i := 0; i < len(s); i++ { + if special(s[i]) { + b[j] = '\\' + j++ + } + b[j] = s[i] + j++ + } + return string(b[0:j]) +} + +// Find matches in slice b if b is non-nil, otherwise find matches in string s. +func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { + var end int + if b == nil { + end = len(s) + } else { + end = len(b) + } + + for pos, i, prevMatchEnd := 0, 0, -1; i < n && pos <= end; { + var in input + if b == nil { + in = newInputString(s) + } else { + in = newInputBytes(b) + } + matches := re.doExecute(in, pos, re.prog.NumCap) + if len(matches) == 0 { + break + } + + accept := true + if matches[1] == pos { + // We've found an empty match. + if matches[0] == prevMatchEnd { + // We don't allow an empty match right + // after a previous match, so ignore it. + accept = false + } + var width int + // TODO: use step() + if b == nil { + _, width = utf8.DecodeRuneInString(s[pos:end]) + } else { + _, width = utf8.DecodeRune(b[pos:end]) + } + if width > 0 { + pos += width + } else { + pos = end + 1 + } + } else { + pos = matches[1] + } + prevMatchEnd = matches[1] + + if accept { + deliver(matches) + i++ + } + } +} + +// Find returns a slice holding the text of the leftmost match in b of the regular expression. +// A return value of nil indicates no match. +func (re *Regexp) Find(b []byte) []byte { + a := re.doExecute(newInputBytes(b), 0, 2) + if a == nil { + return nil + } + return b[a[0]:a[1]] +} + +// FindIndex returns a two-element slice of integers defining the location of +// the leftmost match in b of the regular expression. The match itself is at +// b[loc[0]:loc[1]]. +// A return value of nil indicates no match. +func (re *Regexp) FindIndex(b []byte) (loc []int) { + a := re.doExecute(newInputBytes(b), 0, 2) + if a == nil { + return nil + } + return a[0:2] +} + +// FindString returns a string holding the text of the leftmost match in s of the regular +// expression. If there is no match, the return value is an empty string, +// but it will also be empty if the regular expression successfully matches +// an empty string. Use FindStringIndex or FindStringSubmatch if it is +// necessary to distinguish these cases. +func (re *Regexp) FindString(s string) string { + a := re.doExecute(newInputString(s), 0, 2) + if a == nil { + return "" + } + return s[a[0]:a[1]] +} + +// FindStringIndex returns a two-element slice of integers defining the +// location of the leftmost match in s of the regular expression. The match +// itself is at s[loc[0]:loc[1]]. +// A return value of nil indicates no match. +func (re *Regexp) FindStringIndex(s string) []int { + a := re.doExecute(newInputString(s), 0, 2) + if a == nil { + return nil + } + return a[0:2] +} + +// FindReaderIndex returns a two-element slice of integers defining the +// location of the leftmost match of the regular expression in text read from +// the RuneReader. The match itself is at s[loc[0]:loc[1]]. A return +// value of nil indicates no match. +func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { + a := re.doExecute(newInputReader(r), 0, 2) + if a == nil { + return nil + } + return a[0:2] +} + +// FindSubmatch returns a slice of slices holding the text of the leftmost +// match of the regular expression in b and the matches, if any, of its +// subexpressions, as defined by the 'Submatch' descriptions in the package +// comment. +// A return value of nil indicates no match. +func (re *Regexp) FindSubmatch(b []byte) [][]byte { + a := re.doExecute(newInputBytes(b), 0, re.prog.NumCap) + if a == nil { + return nil + } + ret := make([][]byte, len(a)/2) + for i := range ret { + if a[2*i] >= 0 { + ret[i] = b[a[2*i]:a[2*i+1]] + } + } + return ret +} + +// FindSubmatchIndex returns a slice holding the index pairs identifying the +// leftmost match of the regular expression in b and the matches, if any, of +// its subexpressions, as defined by the 'Submatch' and 'Index' descriptions +// in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindSubmatchIndex(b []byte) []int { + return re.doExecute(newInputBytes(b), 0, re.prog.NumCap) +} + +// FindStringSubmatch returns a slice of strings holding the text of the +// leftmost match of the regular expression in s and the matches, if any, of +// its subexpressions, as defined by the 'Submatch' description in the +// package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindStringSubmatch(s string) []string { + a := re.doExecute(newInputString(s), 0, re.prog.NumCap) + if a == nil { + return nil + } + ret := make([]string, len(a)/2) + for i := range ret { + if a[2*i] >= 0 { + ret[i] = s[a[2*i]:a[2*i+1]] + } + } + return ret +} + +// FindStringSubmatchIndex returns a slice holding the index pairs +// identifying the leftmost match of the regular expression in s and the +// matches, if any, of its subexpressions, as defined by the 'Submatch' and +// 'Index' descriptions in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindStringSubmatchIndex(s string) []int { + return re.doExecute(newInputString(s), 0, re.prog.NumCap) +} + +// FindReaderSubmatchIndex returns a slice holding the index pairs +// identifying the leftmost match of the regular expression of text read by +// the RuneReader, and the matches, if any, of its subexpressions, as defined +// by the 'Submatch' and 'Index' descriptions in the package comment. A +// return value of nil indicates no match. +func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { + return re.doExecute(newInputReader(r), 0, re.prog.NumCap) +} + +const startSize = 10 // The size at which to start a slice in the 'All' routines. + +// FindAll is the 'All' version of Find; it returns a slice of all successive +// matches of the expression, as defined by the 'All' description in the +// package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAll(b []byte, n int) [][]byte { + if n < 0 { + n = len(b) + 1 + } + result := make([][]byte, 0, startSize) + re.allMatches("", b, n, func(match []int) { + result = append(result, b[match[0]:match[1]]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllIndex is the 'All' version of FindIndex; it returns a slice of all +// successive matches of the expression, as defined by the 'All' description +// in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllIndex(b []byte, n int) [][]int { + if n < 0 { + n = len(b) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches("", b, n, func(match []int) { + result = append(result, match[0:2]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllString is the 'All' version of FindString; it returns a slice of all +// successive matches of the expression, as defined by the 'All' description +// in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllString(s string, n int) []string { + if n < 0 { + n = len(s) + 1 + } + result := make([]string, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + result = append(result, s[match[0]:match[1]]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllStringIndex is the 'All' version of FindStringIndex; it returns a +// slice of all successive matches of the expression, as defined by the 'All' +// description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllStringIndex(s string, n int) [][]int { + if n < 0 { + n = len(s) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + result = append(result, match[0:2]) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllSubmatch is the 'All' version of FindSubmatch; it returns a slice +// of all successive matches of the expression, as defined by the 'All' +// description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllSubmatch(b []byte, n int) [][][]byte { + if n < 0 { + n = len(b) + 1 + } + result := make([][][]byte, 0, startSize) + re.allMatches("", b, n, func(match []int) { + slice := make([][]byte, len(match)/2) + for j := range slice { + if match[2*j] >= 0 { + slice[j] = b[match[2*j]:match[2*j+1]] + } + } + result = append(result, slice) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllSubmatchIndex is the 'All' version of FindSubmatchIndex; it returns +// a slice of all successive matches of the expression, as defined by the +// 'All' description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllSubmatchIndex(b []byte, n int) [][]int { + if n < 0 { + n = len(b) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches("", b, n, func(match []int) { + result = append(result, match) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllStringSubmatch is the 'All' version of FindStringSubmatch; it +// returns a slice of all successive matches of the expression, as defined by +// the 'All' description in the package comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllStringSubmatch(s string, n int) [][]string { + if n < 0 { + n = len(s) + 1 + } + result := make([][]string, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + slice := make([]string, len(match)/2) + for j := range slice { + if match[2*j] >= 0 { + slice[j] = s[match[2*j]:match[2*j+1]] + } + } + result = append(result, slice) + }) + if len(result) == 0 { + return nil + } + return result +} + +// FindAllStringSubmatchIndex is the 'All' version of +// FindStringSubmatchIndex; it returns a slice of all successive matches of +// the expression, as defined by the 'All' description in the package +// comment. +// A return value of nil indicates no match. +func (re *Regexp) FindAllStringSubmatchIndex(s string, n int) [][]int { + if n < 0 { + n = len(s) + 1 + } + result := make([][]int, 0, startSize) + re.allMatches(s, nil, n, func(match []int) { + result = append(result, match) + }) + if len(result) == 0 { + return nil + } + return result +} diff --git a/src/pkg/exp/regexp/syntax/Makefile b/src/pkg/exp/regexp/syntax/Makefile new file mode 100644 index 000000000..97d4ad6ca --- /dev/null +++ b/src/pkg/exp/regexp/syntax/Makefile @@ -0,0 +1,16 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=exp/regexp/syntax +GOFILES=\ + compile.go\ + parse.go\ + perl_groups.go\ + prog.go\ + regexp.go\ + simplify.go\ + +include ../../../../Make.pkg diff --git a/src/pkg/exp/regexp/syntax/compile.go b/src/pkg/exp/regexp/syntax/compile.go new file mode 100644 index 000000000..5ea2425c3 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/compile.go @@ -0,0 +1,269 @@ +package syntax + +import ( + "os" + "unicode" +) + +// A patchList is a list of instruction pointers that need to be filled in (patched). +// Because the pointers haven't been filled in yet, we can reuse their storage +// to hold the list. It's kind of sleazy, but works well in practice. +// See http://swtch.com/~rsc/regexp/regexp1.html for inspiration. +// +// These aren't really pointers: they're integers, so we can reinterpret them +// this way without using package unsafe. A value l denotes +// p.inst[l>>1].Out (l&1==0) or .Arg (l&1==1). +// l == 0 denotes the empty list, okay because we start every program +// with a fail instruction, so we'll never want to point at its output link. +type patchList uint32 + +func (l patchList) next(p *Prog) patchList { + i := &p.Inst[l>>1] + if l&1 == 0 { + return patchList(i.Out) + } + return patchList(i.Arg) +} + +func (l patchList) patch(p *Prog, val uint32) { + for l != 0 { + i := &p.Inst[l>>1] + if l&1 == 0 { + l = patchList(i.Out) + i.Out = val + } else { + l = patchList(i.Arg) + i.Arg = val + } + } +} + +func (l1 patchList) append(p *Prog, l2 patchList) patchList { + if l1 == 0 { + return l2 + } + if l2 == 0 { + return l1 + } + + last := l1 + for { + next := last.next(p) + if next == 0 { + break + } + last = next + } + + i := &p.Inst[last>>1] + if last&1 == 0 { + i.Out = uint32(l2) + } else { + i.Arg = uint32(l2) + } + return l1 +} + +// A frag represents a compiled program fragment. +type frag struct { + i uint32 // index of first instruction + out patchList // where to record end instruction +} + +type compiler struct { + p *Prog +} + +// Compile compiles the regexp into a program to be executed. +func Compile(re *Regexp) (*Prog, os.Error) { + var c compiler + c.init() + f := c.compile(re) + f.out.patch(c.p, c.inst(InstMatch).i) + c.p.Start = int(f.i) + return c.p, nil +} + +func (c *compiler) init() { + c.p = new(Prog) + c.p.NumCap = 2 // implicit ( and ) for whole match $0 + c.inst(InstFail) +} + +var anyRuneNotNL = []int{0, '\n' - 1, '\n' - 1, unicode.MaxRune} +var anyRune = []int{0, unicode.MaxRune} + +func (c *compiler) compile(re *Regexp) frag { + switch re.Op { + case OpNoMatch: + return c.fail() + case OpEmptyMatch: + return c.nop() + case OpLiteral: + if len(re.Rune) == 0 { + return c.nop() + } + var f frag + for j := range re.Rune { + f1 := c.rune(re.Rune[j : j+1]) + if j == 0 { + f = f1 + } else { + f = c.cat(f, f1) + } + } + return f + case OpCharClass: + return c.rune(re.Rune) + case OpAnyCharNotNL: + return c.rune(anyRuneNotNL) + case OpAnyChar: + return c.rune(anyRune) + case OpBeginLine: + return c.empty(EmptyBeginLine) + case OpEndLine: + return c.empty(EmptyEndLine) + case OpBeginText: + return c.empty(EmptyBeginText) + case OpEndText: + return c.empty(EmptyEndText) + case OpWordBoundary: + return c.empty(EmptyWordBoundary) + case OpNoWordBoundary: + return c.empty(EmptyNoWordBoundary) + case OpCapture: + bra := c.cap(uint32(re.Cap << 1)) + sub := c.compile(re.Sub[0]) + ket := c.cap(uint32(re.Cap<<1 | 1)) + return c.cat(c.cat(bra, sub), ket) + case OpStar: + return c.star(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpPlus: + return c.plus(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpQuest: + return c.quest(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpConcat: + if len(re.Sub) == 0 { + return c.nop() + } + var f frag + for i, sub := range re.Sub { + if i == 0 { + f = c.compile(sub) + } else { + f = c.cat(f, c.compile(sub)) + } + } + return f + case OpAlternate: + var f frag + for _, sub := range re.Sub { + f = c.alt(f, c.compile(sub)) + } + return f + } + panic("regexp: unhandled case in compile") +} + +func (c *compiler) inst(op InstOp) frag { + // TODO: impose length limit + f := frag{i: uint32(len(c.p.Inst))} + c.p.Inst = append(c.p.Inst, Inst{Op: op}) + return f +} + +func (c *compiler) nop() frag { + f := c.inst(InstNop) + f.out = patchList(f.i << 1) + return f +} + +func (c *compiler) fail() frag { + return frag{} +} + +func (c *compiler) cap(arg uint32) frag { + f := c.inst(InstCapture) + f.out = patchList(f.i << 1) + c.p.Inst[f.i].Arg = arg + + if c.p.NumCap < int(arg)+1 { + c.p.NumCap = int(arg) + 1 + } + return f +} + +func (c *compiler) cat(f1, f2 frag) frag { + // concat of failure is failure + if f1.i == 0 || f2.i == 0 { + return frag{} + } + + // TODO: elide nop + + f1.out.patch(c.p, f2.i) + return frag{f1.i, f2.out} +} + +func (c *compiler) alt(f1, f2 frag) frag { + // alt of failure is other + if f1.i == 0 { + return f2 + } + if f2.i == 0 { + return f1 + } + + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + i.Out = f1.i + i.Arg = f2.i + f.out = f1.out.append(c.p, f2.out) + return f +} + +func (c *compiler) quest(f1 frag, nongreedy bool) frag { + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + if nongreedy { + i.Arg = f1.i + f.out = patchList(f.i << 1) + } else { + i.Out = f1.i + f.out = patchList(f.i<<1 | 1) + } + f.out = f.out.append(c.p, f1.out) + return f +} + +func (c *compiler) star(f1 frag, nongreedy bool) frag { + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + if nongreedy { + i.Arg = f1.i + f.out = patchList(f.i << 1) + } else { + i.Out = f1.i + f.out = patchList(f.i<<1 | 1) + } + f1.out.patch(c.p, f.i) + return f +} + +func (c *compiler) plus(f1 frag, nongreedy bool) frag { + return frag{f1.i, c.star(f1, nongreedy).out} +} + +func (c *compiler) empty(op EmptyOp) frag { + f := c.inst(InstEmptyWidth) + c.p.Inst[f.i].Arg = uint32(op) + f.out = patchList(f.i << 1) + return f +} + +func (c *compiler) rune(rune []int) frag { + f := c.inst(InstRune) + c.p.Inst[f.i].Rune = rune + f.out = patchList(f.i << 1) + return f +} diff --git a/src/pkg/exp/regexp/syntax/make_perl_groups.pl b/src/pkg/exp/regexp/syntax/make_perl_groups.pl new file mode 100755 index 000000000..6d1b84b10 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/make_perl_groups.pl @@ -0,0 +1,103 @@ +#!/usr/bin/perl +# Copyright 2008 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Modified version of RE2's make_perl_groups.pl. + +# Generate table entries giving character ranges +# for POSIX/Perl character classes. Rather than +# figure out what the definition is, it is easier to ask +# Perl about each letter from 0-128 and write down +# its answer. + +@posixclasses = ( + "[:alnum:]", + "[:alpha:]", + "[:ascii:]", + "[:blank:]", + "[:cntrl:]", + "[:digit:]", + "[:graph:]", + "[:lower:]", + "[:print:]", + "[:punct:]", + "[:space:]", + "[:upper:]", + "[:word:]", + "[:xdigit:]", +); + +@perlclasses = ( + "\\d", + "\\s", + "\\w", +); + +sub ComputeClass($) { + my @ranges; + my ($class) = @_; + my $regexp = "[$class]"; + my $start = -1; + for (my $i=0; $i<=129; $i++) { + if ($i == 129) { $i = 256; } + if ($i <= 128 && chr($i) =~ $regexp) { + if ($start < 0) { + $start = $i; + } + } else { + if ($start >= 0) { + push @ranges, [$start, $i-1]; + } + $start = -1; + } + } + return @ranges; +} + +sub PrintClass($$@) { + my ($cname, $name, @ranges) = @_; + print "var code$cname = []int{ /* $name */\n"; + for (my $i=0; $i<@ranges; $i++) { + my @a = @{$ranges[$i]}; + printf "\t0x%x, 0x%x,\n", $a[0], $a[1]; + } + print "}\n\n"; + my $n = @ranges; + $negname = $name; + if ($negname =~ /:/) { + $negname =~ s/:/:^/; + } else { + $negname =~ y/a-z/A-Z/; + } + return "\t`$name`: {+1, code$cname},\n" . + "\t`$negname`: {-1, code$cname},\n"; +} + +my $gen = 0; + +sub PrintClasses($@) { + my ($cname, @classes) = @_; + my @entries; + foreach my $cl (@classes) { + my @ranges = ComputeClass($cl); + push @entries, PrintClass(++$gen, $cl, @ranges); + } + print "var ${cname}Group = map[string]charGroup{\n"; + foreach my $e (@entries) { + print $e; + } + print "}\n"; + my $count = @entries; +} + +print <perl_groups.go + +package syntax + +EOF + +PrintClasses("perl", @perlclasses); +PrintClasses("posix", @posixclasses); diff --git a/src/pkg/exp/regexp/syntax/parse.go b/src/pkg/exp/regexp/syntax/parse.go new file mode 100644 index 000000000..4eed18268 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/parse.go @@ -0,0 +1,1797 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import ( + "os" + "sort" + "strings" + "unicode" + "utf8" +) + +// An Error describes a failure to parse a regular expression +// and gives the offending expression. +type Error struct { + Code ErrorCode + Expr string +} + +func (e *Error) String() string { + return "error parsing regexp: " + e.Code.String() + ": `" + e.Expr + "`" +} + +// An ErrorCode describes a failure to parse a regular expression. +type ErrorCode string + +const ( + // Unexpected error + ErrInternalError ErrorCode = "regexp/syntax: internal error" + + // Parse errors + ErrInvalidCharClass ErrorCode = "invalid character class" + ErrInvalidCharRange ErrorCode = "invalid character class range" + ErrInvalidEscape ErrorCode = "invalid escape sequence" + ErrInvalidNamedCapture ErrorCode = "invalid named capture" + ErrInvalidPerlOp ErrorCode = "invalid or unsupported Perl syntax" + ErrInvalidRepeatOp ErrorCode = "invalid nested repetition operator" + ErrInvalidRepeatSize ErrorCode = "invalid repeat count" + ErrInvalidUTF8 ErrorCode = "invalid UTF-8" + ErrMissingBracket ErrorCode = "missing closing ]" + ErrMissingParen ErrorCode = "missing closing )" + ErrMissingRepeatArgument ErrorCode = "missing argument to repetition operator" + ErrTrailingBackslash ErrorCode = "trailing backslash at end of expression" +) + +func (e ErrorCode) String() string { + return string(e) +} + +// Flags control the behavior of the parser and record information about regexp context. +type Flags uint16 + +const ( + FoldCase Flags = 1 << iota // case-insensitive match + Literal // treat pattern as literal string + ClassNL // allow character classes like [^a-z] and [[:space:]] to match newline + DotNL // allow . to match newline + OneLine // treat ^ and $ as only matching at beginning and end of text + NonGreedy // make repetition operators default to non-greedy + PerlX // allow Perl extensions + UnicodeGroups // allow \p{Han}, \P{Han} for Unicode group and negation + WasDollar // regexp OpEndText was $, not \z + Simple // regexp contains no counted repetition + + MatchNL = ClassNL | DotNL + + Perl = ClassNL | OneLine | PerlX | UnicodeGroups // as close to Perl as possible + POSIX Flags = 0 // POSIX syntax +) + +// Pseudo-ops for parsing stack. +const ( + opLeftParen = opPseudo + iota + opVerticalBar +) + +type parser struct { + flags Flags // parse mode flags + stack []*Regexp // stack of parsed expressions + free *Regexp + numCap int // number of capturing groups seen + wholeRegexp string + tmpClass []int // temporary char class work space +} + +func (p *parser) newRegexp(op Op) *Regexp { + re := p.free + if re != nil { + p.free = re.Sub0[0] + *re = Regexp{} + } else { + re = new(Regexp) + } + re.Op = op + return re +} + +func (p *parser) reuse(re *Regexp) { + re.Sub0[0] = p.free + p.free = re +} + +// Parse stack manipulation. + +// push pushes the regexp re onto the parse stack and returns the regexp. +func (p *parser) push(re *Regexp) *Regexp { + if re.Op == OpCharClass && len(re.Rune) == 2 && re.Rune[0] == re.Rune[1] { + // Single rune. + if p.maybeConcat(re.Rune[0], p.flags&^FoldCase) { + return nil + } + re.Op = OpLiteral + re.Rune = re.Rune[:1] + re.Flags = p.flags &^ FoldCase + } else if re.Op == OpCharClass && len(re.Rune) == 4 && + re.Rune[0] == re.Rune[1] && re.Rune[2] == re.Rune[3] && + unicode.SimpleFold(re.Rune[0]) == re.Rune[2] && + unicode.SimpleFold(re.Rune[2]) == re.Rune[0] || + re.Op == OpCharClass && len(re.Rune) == 2 && + re.Rune[0]+1 == re.Rune[1] && + unicode.SimpleFold(re.Rune[0]) == re.Rune[1] && + unicode.SimpleFold(re.Rune[1]) == re.Rune[0] { + // Case-insensitive rune like [Aa] or [Δδ]. + if p.maybeConcat(re.Rune[0], p.flags|FoldCase) { + return nil + } + + // Rewrite as (case-insensitive) literal. + re.Op = OpLiteral + re.Rune = re.Rune[:1] + re.Flags = p.flags | FoldCase + } else { + // Incremental concatenation. + p.maybeConcat(-1, 0) + } + + p.stack = append(p.stack, re) + return re +} + +// maybeConcat implements incremental concatenation +// of literal runes into string nodes. The parser calls this +// before each push, so only the top fragment of the stack +// might need processing. Since this is called before a push, +// the topmost literal is no longer subject to operators like * +// (Otherwise ab* would turn into (ab)*.) +// If r >= 0 and there's a node left over, maybeConcat uses it +// to push r with the given flags. +// maybeConcat reports whether r was pushed. +func (p *parser) maybeConcat(r int, flags Flags) bool { + n := len(p.stack) + if n < 2 { + return false + } + + re1 := p.stack[n-1] + re2 := p.stack[n-2] + if re1.Op != OpLiteral || re2.Op != OpLiteral || re1.Flags&FoldCase != re2.Flags&FoldCase { + return false + } + + // Push re1 into re2. + re2.Rune = append(re2.Rune, re1.Rune...) + + // Reuse re1 if possible. + if r >= 0 { + re1.Rune = re1.Rune0[:1] + re1.Rune[0] = r + re1.Flags = flags + return true + } + + p.stack = p.stack[:n-1] + p.reuse(re1) + return false // did not push r +} + +// newLiteral returns a new OpLiteral Regexp with the given flags +func (p *parser) newLiteral(r int, flags Flags) *Regexp { + re := p.newRegexp(OpLiteral) + re.Flags = flags + re.Rune0[0] = r + re.Rune = re.Rune0[:1] + return re +} + +// literal pushes a literal regexp for the rune r on the stack +// and returns that regexp. +func (p *parser) literal(r int) { + p.push(p.newLiteral(r, p.flags)) +} + +// op pushes a regexp with the given op onto the stack +// and returns that regexp. +func (p *parser) op(op Op) *Regexp { + re := p.newRegexp(op) + re.Flags = p.flags + return p.push(re) +} + +// repeat replaces the top stack element with itself repeated +// according to op. +func (p *parser) repeat(op Op, min, max int, opstr, t, lastRepeat string) (string, os.Error) { + flags := p.flags + if p.flags&PerlX != 0 { + if len(t) > 0 && t[0] == '?' { + t = t[1:] + flags ^= NonGreedy + } + if lastRepeat != "" { + // In Perl it is not allowed to stack repetition operators: + // a** is a syntax error, not a doubled star, and a++ means + // something else entirely, which we don't support! + return "", &Error{ErrInvalidRepeatOp, lastRepeat[:len(lastRepeat)-len(t)]} + } + } + n := len(p.stack) + if n == 0 { + return "", &Error{ErrMissingRepeatArgument, opstr} + } + sub := p.stack[n-1] + re := p.newRegexp(op) + re.Min = min + re.Max = max + re.Flags = flags + re.Sub = re.Sub0[:1] + re.Sub[0] = sub + p.stack[n-1] = re + return t, nil +} + +// concat replaces the top of the stack (above the topmost '|' or '(') with its concatenation. +func (p *parser) concat() *Regexp { + p.maybeConcat(-1, 0) + + // Scan down to find pseudo-operator | or (. + i := len(p.stack) + for i > 0 && p.stack[i-1].Op < opPseudo { + i-- + } + subs := p.stack[i:] + p.stack = p.stack[:i] + + // Empty concatenation is special case. + if len(subs) == 0 { + return p.push(p.newRegexp(OpEmptyMatch)) + } + + return p.push(p.collapse(subs, OpConcat)) +} + +// alternate replaces the top of the stack (above the topmost '(') with its alternation. +func (p *parser) alternate() *Regexp { + // Scan down to find pseudo-operator (. + // There are no | above (. + i := len(p.stack) + for i > 0 && p.stack[i-1].Op < opPseudo { + i-- + } + subs := p.stack[i:] + p.stack = p.stack[:i] + + // Make sure top class is clean. + // All the others already are (see swapVerticalBar). + if len(subs) > 0 { + cleanAlt(subs[len(subs)-1]) + } + + // Empty alternate is special case + // (shouldn't happen but easy to handle). + if len(subs) == 0 { + return p.push(p.newRegexp(OpNoMatch)) + } + + return p.push(p.collapse(subs, OpAlternate)) +} + +// cleanAlt cleans re for eventual inclusion in an alternation. +func cleanAlt(re *Regexp) { + switch re.Op { + case OpCharClass: + re.Rune = cleanClass(&re.Rune) + if len(re.Rune) == 2 && re.Rune[0] == 0 && re.Rune[1] == unicode.MaxRune { + re.Rune = nil + re.Op = OpAnyChar + return + } + if len(re.Rune) == 4 && re.Rune[0] == 0 && re.Rune[1] == '\n'-1 && re.Rune[2] == '\n'+1 && re.Rune[3] == unicode.MaxRune { + re.Rune = nil + re.Op = OpAnyCharNotNL + return + } + if cap(re.Rune)-len(re.Rune) > 100 { + // re.Rune will not grow any more. + // Make a copy or inline to reclaim storage. + re.Rune = append(re.Rune0[:0], re.Rune...) + } + } +} + +// collapse returns the result of applying op to sub. +// If sub contains op nodes, they all get hoisted up +// so that there is never a concat of a concat or an +// alternate of an alternate. +func (p *parser) collapse(subs []*Regexp, op Op) *Regexp { + if len(subs) == 1 { + return subs[0] + } + re := p.newRegexp(op) + re.Sub = re.Sub0[:0] + for _, sub := range subs { + if sub.Op == op { + re.Sub = append(re.Sub, sub.Sub...) + p.reuse(sub) + } else { + re.Sub = append(re.Sub, sub) + } + } + if op == OpAlternate { + re.Sub = p.factor(re.Sub, re.Flags) + if len(re.Sub) == 1 { + old := re + re = re.Sub[0] + p.reuse(old) + } + } + return re +} + +// factor factors common prefixes from the alternation list sub. +// It returns a replacement list that reuses the same storage and +// frees (passes to p.reuse) any removed *Regexps. +// +// For example, +// ABC|ABD|AEF|BCX|BCY +// simplifies by literal prefix extraction to +// A(B(C|D)|EF)|BC(X|Y) +// which simplifies by character class introduction to +// A(B[CD]|EF)|BC[XY] +// +func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { + if len(sub) < 2 { + return sub + } + + // Round 1: Factor out common literal prefixes. + var str []int + var strflags Flags + start := 0 + out := sub[:0] + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that all begin + // with str as modified by strflags. + var istr []int + var iflags Flags + if i < len(sub) { + istr, iflags = p.leadingString(sub[i]) + if iflags == strflags { + same := 0 + for same < len(str) && same < len(istr) && str[same] == istr[same] { + same++ + } + if same > 0 { + // Matches at least one rune in current range. + // Keep going around. + str = str[:same] + continue + } + } + } + + // Found end of a run with common leading literal string: + // sub[start:i] all begin with str[0:len(str)], but sub[i] + // does not even begin with str[0]. + // + // Factor out common string and append factored expression to out. + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + // Just one: don't bother factoring. + out = append(out, sub[start]) + } else { + // Construct factored form: prefix(suffix1|suffix2|...) + prefix := p.newRegexp(OpLiteral) + prefix.Flags = strflags + prefix.Rune = append(prefix.Rune[:0], str...) + + for j := start; j < i; j++ { + sub[j] = p.removeLeadingString(sub[j], len(str)) + } + suffix := p.collapse(sub[start:i], OpAlternate) // recurse + + re := p.newRegexp(OpConcat) + re.Sub = append(re.Sub[:0], prefix, suffix) + out = append(out, re) + } + + // Prepare for next iteration. + start = i + str = istr + strflags = iflags + } + sub = out + + // Round 2: Factor out common complex prefixes, + // just the first piece of each concatenation, + // whatever it is. This is good enough a lot of the time. + start = 0 + out = sub[:0] + var first *Regexp + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that all begin + // with str as modified by strflags. + var ifirst *Regexp + if i < len(sub) { + ifirst = p.leadingRegexp(sub[i]) + if first != nil && first.Equal(ifirst) { + continue + } + } + + // Found end of a run with common leading regexp: + // sub[start:i] all begin with first but sub[i] does not. + // + // Factor out common regexp and append factored expression to out. + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + // Just one: don't bother factoring. + out = append(out, sub[start]) + } else { + // Construct factored form: prefix(suffix1|suffix2|...) + prefix := first + + for j := start; j < i; j++ { + reuse := j != start // prefix came from sub[start] + sub[j] = p.removeLeadingRegexp(sub[j], reuse) + } + suffix := p.collapse(sub[start:i], OpAlternate) // recurse + + re := p.newRegexp(OpConcat) + re.Sub = append(re.Sub[:0], prefix, suffix) + out = append(out, re) + } + + // Prepare for next iteration. + start = i + first = ifirst + } + sub = out + + // Round 3: Collapse runs of single literals into character classes. + start = 0 + out = sub[:0] + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that are either + // literal runes or character classes. + if i < len(sub) && isCharClass(sub[i]) { + continue + } + + // sub[i] is not a char or char class; + // emit char class for sub[start:i]... + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + out = append(out, sub[start]) + } else { + // Make new char class. + // Start with most complex regexp in sub[start]. + max := start + for j := start + 1; j < i; j++ { + if sub[max].Op < sub[j].Op || sub[max].Op == sub[j].Op && len(sub[max].Rune) < len(sub[j].Rune) { + max = j + } + } + sub[start], sub[max] = sub[max], sub[start] + + for j := start + 1; j < i; j++ { + mergeCharClass(sub[start], sub[j]) + p.reuse(sub[j]) + } + cleanAlt(sub[start]) + out = append(out, sub[start]) + } + + // ... and then emit sub[i]. + if i < len(sub) { + out = append(out, sub[i]) + } + start = i + 1 + } + sub = out + + // Round 4: Collapse runs of empty matches into a single empty match. + start = 0 + out = sub[:0] + for i := range sub { + if i+1 < len(sub) && sub[i].Op == OpEmptyMatch && sub[i+1].Op == OpEmptyMatch { + continue + } + out = append(out, sub[i]) + } + sub = out + + return sub +} + +// leadingString returns the leading literal string that re begins with. +// The string refers to storage in re or its children. +func (p *parser) leadingString(re *Regexp) ([]int, Flags) { + if re.Op == OpConcat && len(re.Sub) > 0 { + re = re.Sub[0] + } + if re.Op != OpLiteral { + return nil, 0 + } + return re.Rune, re.Flags & FoldCase +} + +// removeLeadingString removes the first n leading runes +// from the beginning of re. It returns the replacement for re. +func (p *parser) removeLeadingString(re *Regexp, n int) *Regexp { + if re.Op == OpConcat && len(re.Sub) > 0 { + // Removing a leading string in a concatenation + // might simplify the concatenation. + sub := re.Sub[0] + sub = p.removeLeadingString(sub, n) + re.Sub[0] = sub + if sub.Op == OpEmptyMatch { + p.reuse(sub) + switch len(re.Sub) { + case 0, 1: + // Impossible but handle. + re.Op = OpEmptyMatch + re.Sub = nil + case 2: + old := re + re = re.Sub[1] + p.reuse(old) + default: + copy(re.Sub, re.Sub[1:]) + re.Sub = re.Sub[:len(re.Sub)-1] + } + } + return re + } + + if re.Op == OpLiteral { + re.Rune = re.Rune[:copy(re.Rune, re.Rune[n:])] + if len(re.Rune) == 0 { + re.Op = OpEmptyMatch + } + } + return re +} + +// leadingRegexp returns the leading regexp that re begins with. +// The regexp refers to storage in re or its children. +func (p *parser) leadingRegexp(re *Regexp) *Regexp { + if re.Op == OpEmptyMatch { + return nil + } + if re.Op == OpConcat && len(re.Sub) > 0 { + sub := re.Sub[0] + if sub.Op == OpEmptyMatch { + return nil + } + return sub + } + return re +} + +// removeLeadingRegexp removes the leading regexp in re. +// It returns the replacement for re. +// If reuse is true, it passes the removed regexp (if no longer needed) to p.reuse. +func (p *parser) removeLeadingRegexp(re *Regexp, reuse bool) *Regexp { + if re.Op == OpConcat && len(re.Sub) > 0 { + if reuse { + p.reuse(re.Sub[0]) + } + re.Sub = re.Sub[:copy(re.Sub, re.Sub[1:])] + switch len(re.Sub) { + case 0: + re.Op = OpEmptyMatch + re.Sub = nil + case 1: + old := re + re = re.Sub[0] + p.reuse(old) + } + return re + } + re.Op = OpEmptyMatch + return re +} + +func literalRegexp(s string, flags Flags) *Regexp { + re := &Regexp{Op: OpLiteral} + re.Flags = flags + re.Rune = re.Rune0[:0] // use local storage for small strings + for _, c := range s { + if len(re.Rune) >= cap(re.Rune) { + // string is too long to fit in Rune0. let Go handle it + re.Rune = []int(s) + break + } + re.Rune = append(re.Rune, c) + } + return re +} + +// Parsing. + +func Parse(s string, flags Flags) (*Regexp, os.Error) { + if flags&Literal != 0 { + // Trivial parser for literal string. + if err := checkUTF8(s); err != nil { + return nil, err + } + return literalRegexp(s, flags), nil + } + + // Otherwise, must do real work. + var ( + p parser + err os.Error + c int + op Op + lastRepeat string + min, max int + ) + p.flags = flags + p.wholeRegexp = s + t := s + for t != "" { + repeat := "" + BigSwitch: + switch t[0] { + default: + if c, t, err = nextRune(t); err != nil { + return nil, err + } + p.literal(c) + + case '(': + if p.flags&PerlX != 0 && len(t) >= 2 && t[1] == '?' { + // Flag changes and non-capturing groups. + if t, err = p.parsePerlFlags(t); err != nil { + return nil, err + } + break + } + p.numCap++ + p.op(opLeftParen).Cap = p.numCap + t = t[1:] + case '|': + if err = p.parseVerticalBar(); err != nil { + return nil, err + } + t = t[1:] + case ')': + if err = p.parseRightParen(); err != nil { + return nil, err + } + t = t[1:] + case '^': + if p.flags&OneLine != 0 { + p.op(OpBeginText) + } else { + p.op(OpBeginLine) + } + t = t[1:] + case '$': + if p.flags&OneLine != 0 { + p.op(OpEndText).Flags |= WasDollar + } else { + p.op(OpEndLine) + } + t = t[1:] + case '.': + if p.flags&DotNL != 0 { + p.op(OpAnyChar) + } else { + p.op(OpAnyCharNotNL) + } + t = t[1:] + case '[': + if t, err = p.parseClass(t); err != nil { + return nil, err + } + case '*', '+', '?': + switch t[0] { + case '*': + op = OpStar + case '+': + op = OpPlus + case '?': + op = OpQuest + } + if t, err = p.repeat(op, min, max, t[:1], t[1:], lastRepeat); err != nil { + return nil, err + } + case '{': + op = OpRepeat + min, max, tt, ok := p.parseRepeat(t) + if !ok { + // If the repeat cannot be parsed, { is a literal. + p.literal('{') + t = t[1:] + break + } + if t, err = p.repeat(op, min, max, t[:len(t)-len(tt)], tt, lastRepeat); err != nil { + return nil, err + } + case '\\': + if p.flags&PerlX != 0 && len(t) >= 2 { + switch t[1] { + case 'A': + p.op(OpBeginText) + t = t[2:] + break BigSwitch + case 'b': + p.op(OpWordBoundary) + t = t[2:] + break BigSwitch + case 'B': + p.op(OpNoWordBoundary) + t = t[2:] + break BigSwitch + case 'C': + // any byte; not supported + return nil, &Error{ErrInvalidEscape, t[:2]} + case 'Q': + // \Q ... \E: the ... is always literals + var lit string + if i := strings.Index(t, `\E`); i < 0 { + lit = t[2:] + t = "" + } else { + lit = t[2:i] + t = t[i+2:] + } + p.push(literalRegexp(lit, p.flags)) + break BigSwitch + case 'z': + p.op(OpEndText) + t = t[2:] + break BigSwitch + } + } + + re := p.newRegexp(OpCharClass) + re.Flags = p.flags + + // Look for Unicode character group like \p{Han} + if len(t) >= 2 && (t[1] == 'p' || t[1] == 'P') { + r, rest, err := p.parseUnicodeClass(t, re.Rune0[:0]) + if err != nil { + return nil, err + } + if r != nil { + re.Rune = r + t = rest + p.push(re) + break BigSwitch + } + } + + // Perl character class escape. + if r, rest := p.parsePerlClassEscape(t, re.Rune0[:0]); r != nil { + re.Rune = r + t = rest + p.push(re) + break BigSwitch + } + p.reuse(re) + + // Ordinary single-character escape. + if c, t, err = p.parseEscape(t); err != nil { + return nil, err + } + p.literal(c) + } + lastRepeat = repeat + } + + p.concat() + if p.swapVerticalBar() { + // pop vertical bar + p.stack = p.stack[:len(p.stack)-1] + } + p.alternate() + + n := len(p.stack) + if n != 1 { + return nil, &Error{ErrMissingParen, s} + } + return p.stack[0], nil +} + +// parseRepeat parses {min} (max=min) or {min,} (max=-1) or {min,max}. +// If s is not of that form, it returns ok == false. +func (p *parser) parseRepeat(s string) (min, max int, rest string, ok bool) { + if s == "" || s[0] != '{' { + return + } + s = s[1:] + if min, s, ok = p.parseInt(s); !ok { + return + } + if s == "" { + return + } + if s[0] != ',' { + max = min + } else { + s = s[1:] + if s == "" { + return + } + if s[0] == '}' { + max = -1 + } else if max, s, ok = p.parseInt(s); !ok { + return + } + } + if s == "" || s[0] != '}' { + return + } + rest = s[1:] + ok = true + return +} + +// parsePerlFlags parses a Perl flag setting or non-capturing group or both, +// like (?i) or (?: or (?i:. It removes the prefix from s and updates the parse state. +// The caller must have ensured that s begins with "(?". +func (p *parser) parsePerlFlags(s string) (rest string, err os.Error) { + t := s + + // Check for named captures, first introduced in Python's regexp library. + // As usual, there are three slightly different syntaxes: + // + // (?Pexpr) the original, introduced by Python + // (?expr) the .NET alteration, adopted by Perl 5.10 + // (?'name'expr) another .NET alteration, adopted by Perl 5.10 + // + // Perl 5.10 gave in and implemented the Python version too, + // but they claim that the last two are the preferred forms. + // PCRE and languages based on it (specifically, PHP and Ruby) + // support all three as well. EcmaScript 4 uses only the Python form. + // + // In both the open source world (via Code Search) and the + // Google source tree, (?Pname) is the dominant form, + // so that's the one we implement. One is enough. + if len(t) > 4 && t[2] == 'P' && t[3] == '<' { + // Pull out name. + end := strings.IndexRune(t, '>') + if end < 0 { + if err = checkUTF8(t); err != nil { + return "", err + } + return "", &Error{ErrInvalidNamedCapture, s} + } + + capture := t[:end+1] // "(?P" + name := t[4:end] // "name" + if err = checkUTF8(name); err != nil { + return "", err + } + if !isValidCaptureName(name) { + return "", &Error{ErrInvalidNamedCapture, capture} + } + + // Like ordinary capture, but named. + p.numCap++ + re := p.op(opLeftParen) + re.Cap = p.numCap + re.Name = name + return t[end+1:], nil + } + + // Non-capturing group. Might also twiddle Perl flags. + var c int + t = t[2:] // skip (? + flags := p.flags + sign := +1 + sawFlag := false +Loop: + for t != "" { + if c, t, err = nextRune(t); err != nil { + return "", err + } + switch c { + default: + break Loop + + // Flags. + case 'i': + flags |= FoldCase + sawFlag = true + case 'm': + flags &^= OneLine + sawFlag = true + case 's': + flags |= DotNL + sawFlag = true + case 'U': + flags |= NonGreedy + sawFlag = true + + // Switch to negation. + case '-': + if sign < 0 { + break Loop + } + sign = -1 + // Invert flags so that | above turn into &^ and vice versa. + // We'll invert flags again before using it below. + flags = ^flags + sawFlag = false + + // End of flags, starting group or not. + case ':', ')': + if sign < 0 { + if !sawFlag { + break Loop + } + flags = ^flags + } + if c == ':' { + // Open new group + p.op(opLeftParen) + } + p.flags = flags + return t, nil + } + } + + return "", &Error{ErrInvalidPerlOp, s[:len(s)-len(t)]} +} + +// isValidCaptureName reports whether name +// is a valid capture name: [A-Za-z0-9_]+. +// PCRE limits names to 32 bytes. +// Python rejects names starting with digits. +// We don't enforce either of those. +func isValidCaptureName(name string) bool { + if name == "" { + return false + } + for _, c := range name { + if c != '_' && !isalnum(c) { + return false + } + } + return true +} + +// parseInt parses a decimal integer. +func (p *parser) parseInt(s string) (n int, rest string, ok bool) { + if s == "" || s[0] < '0' || '9' < s[0] { + return + } + // Disallow leading zeros. + if len(s) >= 2 && s[0] == '0' && '0' <= s[1] && s[1] <= '9' { + return + } + for s != "" && '0' <= s[0] && s[0] <= '9' { + // Avoid overflow. + if n >= 1e8 { + return + } + n = n*10 + int(s[0]) - '0' + s = s[1:] + } + rest = s + ok = true + return +} + +// can this be represented as a character class? +// single-rune literal string, char class, ., and .|\n. +func isCharClass(re *Regexp) bool { + return re.Op == OpLiteral && len(re.Rune) == 1 || + re.Op == OpCharClass || + re.Op == OpAnyCharNotNL || + re.Op == OpAnyChar +} + +// does re match r? +func matchRune(re *Regexp, r int) bool { + switch re.Op { + case OpLiteral: + return len(re.Rune) == 1 && re.Rune[0] == r + case OpCharClass: + for i := 0; i < len(re.Rune); i += 2 { + if re.Rune[i] <= r && r <= re.Rune[i+1] { + return true + } + } + return false + case OpAnyCharNotNL: + return r != '\n' + case OpAnyChar: + return true + } + return false +} + +// parseVerticalBar handles a | in the input. +func (p *parser) parseVerticalBar() os.Error { + p.concat() + + // The concatenation we just parsed is on top of the stack. + // If it sits above an opVerticalBar, swap it below + // (things below an opVerticalBar become an alternation). + // Otherwise, push a new vertical bar. + if !p.swapVerticalBar() { + p.op(opVerticalBar) + } + + return nil +} + +// mergeCharClass makes dst = dst|src. +// The caller must ensure that dst.Op >= src.Op, +// to reduce the amount of copying. +func mergeCharClass(dst, src *Regexp) { + switch dst.Op { + case OpAnyChar: + // src doesn't add anything. + case OpAnyCharNotNL: + // src might add \n + if matchRune(src, '\n') { + dst.Op = OpAnyChar + } + case OpCharClass: + // src is simpler, so either literal or char class + if src.Op == OpLiteral { + dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + } else { + dst.Rune = appendClass(dst.Rune, src.Rune) + } + case OpLiteral: + // both literal + if src.Rune[0] == dst.Rune[0] { + break + } + dst.Op = OpCharClass + dst.Rune = append(dst.Rune, dst.Rune[0]) + dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + } +} + +// If the top of the stack is an element followed by an opVerticalBar +// swapVerticalBar swaps the two and returns true. +// Otherwise it returns false. +func (p *parser) swapVerticalBar() bool { + // If above and below vertical bar are literal or char class, + // can merge into a single char class. + n := len(p.stack) + if n >= 3 && p.stack[n-2].Op == opVerticalBar && isCharClass(p.stack[n-1]) && isCharClass(p.stack[n-3]) { + re1 := p.stack[n-1] + re3 := p.stack[n-3] + // Make re3 the more complex of the two. + if re1.Op > re3.Op { + re1, re3 = re3, re1 + p.stack[n-3] = re3 + } + mergeCharClass(re3, re1) + p.reuse(re1) + p.stack = p.stack[:n-1] + return true + } + + if n >= 2 { + re1 := p.stack[n-1] + re2 := p.stack[n-2] + if re2.Op == opVerticalBar { + if n >= 3 { + // Now out of reach. + // Clean opportunistically. + cleanAlt(p.stack[n-3]) + } + p.stack[n-2] = re1 + p.stack[n-1] = re2 + return true + } + } + return false +} + +// parseRightParen handles a ) in the input. +func (p *parser) parseRightParen() os.Error { + p.concat() + if p.swapVerticalBar() { + // pop vertical bar + p.stack = p.stack[:len(p.stack)-1] + } + p.alternate() + + n := len(p.stack) + if n < 2 { + return &Error{ErrInternalError, ""} + } + re1 := p.stack[n-1] + re2 := p.stack[n-2] + p.stack = p.stack[:n-2] + if re2.Op != opLeftParen { + return &Error{ErrMissingParen, p.wholeRegexp} + } + if re2.Cap == 0 { + // Just for grouping. + p.push(re1) + } else { + re2.Op = OpCapture + re2.Sub = re2.Sub0[:1] + re2.Sub[0] = re1 + p.push(re2) + } + return nil +} + +// parseEscape parses an escape sequence at the beginning of s +// and returns the rune. +func (p *parser) parseEscape(s string) (r int, rest string, err os.Error) { + t := s[1:] + if t == "" { + return 0, "", &Error{ErrTrailingBackslash, ""} + } + c, t, err := nextRune(t) + if err != nil { + return 0, "", err + } + +Switch: + switch c { + default: + if c < utf8.RuneSelf && !isalnum(c) { + // Escaped non-word characters are always themselves. + // PCRE is not quite so rigorous: it accepts things like + // \q, but we don't. We once rejected \_, but too many + // programs and people insist on using it, so allow \_. + return c, t, nil + } + + // Octal escapes. + case '1', '2', '3', '4', '5', '6', '7': + // Single non-zero digit is a backreference; not supported + if t == "" || t[0] < '0' || t[0] > '7' { + break + } + fallthrough + case '0': + // Consume up to three octal digits; already have one. + r = c - '0' + for i := 1; i < 3; i++ { + if t == "" || t[0] < '0' || t[0] > '7' { + break + } + r = r*8 + int(t[0]) - '0' + t = t[1:] + } + return r, t, nil + + // Hexadecimal escapes. + case 'x': + if t == "" { + break + } + if c, t, err = nextRune(t); err != nil { + return 0, "", err + } + if c == '{' { + // Any number of digits in braces. + // Perl accepts any text at all; it ignores all text + // after the first non-hex digit. We require only hex digits, + // and at least one. + nhex := 0 + r = 0 + for { + if t == "" { + break Switch + } + if c, t, err = nextRune(t); err != nil { + return 0, "", err + } + if c == '}' { + break + } + v := unhex(c) + if v < 0 { + break Switch + } + r = r*16 + v + if r > unicode.MaxRune { + break Switch + } + nhex++ + } + if nhex == 0 { + break Switch + } + return r, t, nil + } + + // Easy case: two hex digits. + x := unhex(c) + if c, t, err = nextRune(t); err != nil { + return 0, "", err + } + y := unhex(c) + if x < 0 || y < 0 { + break + } + return x*16 + y, t, nil + + // C escapes. There is no case 'b', to avoid misparsing + // the Perl word-boundary \b as the C backspace \b + // when in POSIX mode. In Perl, /\b/ means word-boundary + // but /[\b]/ means backspace. We don't support that. + // If you want a backspace, embed a literal backspace + // character or use \x08. + case 'a': + return '\a', t, err + case 'f': + return '\f', t, err + case 'n': + return '\n', t, err + case 'r': + return '\r', t, err + case 't': + return '\t', t, err + case 'v': + return '\v', t, err + } + return 0, "", &Error{ErrInvalidEscape, s[:len(s)-len(t)]} +} + +// parseClassChar parses a character class character at the beginning of s +// and returns it. +func (p *parser) parseClassChar(s, wholeClass string) (r int, rest string, err os.Error) { + if s == "" { + return 0, "", &Error{Code: ErrMissingBracket, Expr: wholeClass} + } + + // Allow regular escape sequences even though + // many need not be escaped in this context. + if s[0] == '\\' { + return p.parseEscape(s) + } + + return nextRune(s) +} + +type charGroup struct { + sign int + class []int +} + +// parsePerlClassEscape parses a leading Perl character class escape like \d +// from the beginning of s. If one is present, it appends the characters to r +// and returns the new slice r and the remainder of the string. +func (p *parser) parsePerlClassEscape(s string, r []int) (out []int, rest string) { + if p.flags&PerlX == 0 || len(s) < 2 || s[0] != '\\' { + return + } + g := perlGroup[s[0:2]] + if g.sign == 0 { + return + } + return p.appendGroup(r, g), s[2:] +} + +// parseNamedClass parses a leading POSIX named character class like [:alnum:] +// from the beginning of s. If one is present, it appends the characters to r +// and returns the new slice r and the remainder of the string. +func (p *parser) parseNamedClass(s string, r []int) (out []int, rest string, err os.Error) { + if len(s) < 2 || s[0] != '[' || s[1] != ':' { + return + } + + i := strings.Index(s[2:], ":]") + if i < 0 { + return + } + i += 2 + name, s := s[0:i+2], s[i+2:] + g := posixGroup[name] + if g.sign == 0 { + return nil, "", &Error{ErrInvalidCharRange, name} + } + return p.appendGroup(r, g), s, nil +} + +func (p *parser) appendGroup(r []int, g charGroup) []int { + if p.flags&FoldCase == 0 { + if g.sign < 0 { + r = appendNegatedClass(r, g.class) + } else { + r = appendClass(r, g.class) + } + } else { + tmp := p.tmpClass[:0] + tmp = appendFoldedClass(tmp, g.class) + p.tmpClass = tmp + tmp = cleanClass(&p.tmpClass) + if g.sign < 0 { + r = appendNegatedClass(r, tmp) + } else { + r = appendClass(r, tmp) + } + } + return r +} + +// unicodeTable returns the unicode.RangeTable identified by name +// and the table of additional fold-equivalent code points. +func unicodeTable(name string) (*unicode.RangeTable, *unicode.RangeTable) { + if t := unicode.Categories[name]; t != nil { + return t, unicode.FoldCategory[name] + } + if t := unicode.Scripts[name]; t != nil { + return t, unicode.FoldScript[name] + } + return nil, nil +} + +// parseUnicodeClass parses a leading Unicode character class like \p{Han} +// from the beginning of s. If one is present, it appends the characters to r +// and returns the new slice r and the remainder of the string. +func (p *parser) parseUnicodeClass(s string, r []int) (out []int, rest string, err os.Error) { + if p.flags&UnicodeGroups == 0 || len(s) < 2 || s[0] != '\\' || s[1] != 'p' && s[1] != 'P' { + return + } + + // Committed to parse or return error. + sign := +1 + if s[1] == 'P' { + sign = -1 + } + t := s[2:] + c, t, err := nextRune(t) + if err != nil { + return + } + var seq, name string + if c != '{' { + // Single-letter name. + seq = s[:len(s)-len(t)] + name = seq[2:] + } else { + // Name is in braces. + end := strings.IndexRune(s, '}') + if end < 0 { + if err = checkUTF8(s); err != nil { + return + } + return nil, "", &Error{ErrInvalidCharRange, s} + } + seq, t = s[:end+1], s[end+1:] + name = s[3:end] + if err = checkUTF8(name); err != nil { + return + } + } + + // Group can have leading negation too. \p{^Han} == \P{Han}, \P{^Han} == \p{Han}. + if name != "" && name[0] == '^' { + sign = -sign + name = name[1:] + } + + tab, fold := unicodeTable(name) + if tab == nil { + return nil, "", &Error{ErrInvalidCharRange, seq} + } + + if p.flags&FoldCase == 0 || fold == nil { + if sign > 0 { + r = appendTable(r, tab) + } else { + r = appendNegatedTable(r, tab) + } + } else { + // Merge and clean tab and fold in a temporary buffer. + // This is necessary for the negative case and just tidy + // for the positive case. + tmp := p.tmpClass[:0] + tmp = appendTable(tmp, tab) + tmp = appendTable(tmp, fold) + p.tmpClass = tmp + tmp = cleanClass(&p.tmpClass) + if sign > 0 { + r = appendClass(r, tmp) + } else { + r = appendNegatedClass(r, tmp) + } + } + return r, t, nil +} + +// parseClass parses a character class at the beginning of s +// and pushes it onto the parse stack. +func (p *parser) parseClass(s string) (rest string, err os.Error) { + t := s[1:] // chop [ + re := p.newRegexp(OpCharClass) + re.Flags = p.flags + re.Rune = re.Rune0[:0] + + sign := +1 + if t != "" && t[0] == '^' { + sign = -1 + t = t[1:] + + // If character class does not match \n, add it here, + // so that negation later will do the right thing. + if p.flags&ClassNL == 0 { + re.Rune = append(re.Rune, '\n', '\n') + } + } + + class := re.Rune + first := true // ] and - are okay as first char in class + for t == "" || t[0] != ']' || first { + // POSIX: - is only okay unescaped as first or last in class. + // Perl: - is okay anywhere. + if t != "" && t[0] == '-' && p.flags&PerlX == 0 && !first && (len(t) == 1 || t[1] != ']') { + _, size := utf8.DecodeRuneInString(t[1:]) + return "", &Error{Code: ErrInvalidCharRange, Expr: t[:1+size]} + } + first = false + + // Look for POSIX [:alnum:] etc. + if len(t) > 2 && t[0] == '[' && t[1] == ':' { + nclass, nt, err := p.parseNamedClass(t, class) + if err != nil { + return "", err + } + if nclass != nil { + class, t = nclass, nt + continue + } + } + + // Look for Unicode character group like \p{Han}. + nclass, nt, err := p.parseUnicodeClass(t, class) + if err != nil { + return "", err + } + if nclass != nil { + class, t = nclass, nt + continue + } + + // Look for Perl character class symbols (extension). + if nclass, nt := p.parsePerlClassEscape(t, class); nclass != nil { + class, t = nclass, nt + continue + } + + // Single character or simple range. + rng := t + var lo, hi int + if lo, t, err = p.parseClassChar(t, s); err != nil { + return "", err + } + hi = lo + // [a-] means (a|-) so check for final ]. + if len(t) >= 2 && t[0] == '-' && t[1] != ']' { + t = t[1:] + if hi, t, err = p.parseClassChar(t, s); err != nil { + return "", err + } + if hi < lo { + rng = rng[:len(rng)-len(t)] + return "", &Error{Code: ErrInvalidCharRange, Expr: rng} + } + } + if p.flags&FoldCase == 0 { + class = appendRange(class, lo, hi) + } else { + class = appendFoldedRange(class, lo, hi) + } + } + t = t[1:] // chop ] + + // Use &re.Rune instead of &class to avoid allocation. + re.Rune = class + class = cleanClass(&re.Rune) + if sign < 0 { + class = negateClass(class) + } + re.Rune = class + p.push(re) + return t, nil +} + +// cleanClass sorts the ranges (pairs of elements of r), +// merges them, and eliminates duplicates. +func cleanClass(rp *[]int) []int { + + // Sort by lo increasing, hi decreasing to break ties. + sort.Sort(ranges{rp}) + + r := *rp + if len(r) < 2 { + return r + } + + // Merge abutting, overlapping. + w := 2 // write index + for i := 2; i < len(r); i += 2 { + lo, hi := r[i], r[i+1] + if lo <= r[w-1]+1 { + // merge with previous range + if hi > r[w-1] { + r[w-1] = hi + } + continue + } + // new disjoint range + r[w] = lo + r[w+1] = hi + w += 2 + } + + return r[:w] +} + +// appendRange returns the result of appending the range lo-hi to the class r. +func appendRange(r []int, lo, hi int) []int { + // Expand last range or next to last range if it overlaps or abuts. + // Checking two ranges helps when appending case-folded + // alphabets, so that one range can be expanding A-Z and the + // other expanding a-z. + n := len(r) + for i := 2; i <= 4; i += 2 { // twice, using i=2, i=4 + if n >= i { + rlo, rhi := r[n-i], r[n-i+1] + if lo <= rhi+1 && rlo <= hi+1 { + if lo < rlo { + r[n-i] = lo + } + if hi > rhi { + r[n-i+1] = hi + } + return r + } + } + } + + return append(r, lo, hi) +} + +const ( + // minimum and maximum runes involved in folding. + // checked during test. + minFold = 0x0041 + maxFold = 0x1044f +) + +// appendFoldedRange returns the result of appending the range lo-hi +// and its case folding-equivalent runes to the class r. +func appendFoldedRange(r []int, lo, hi int) []int { + // Optimizations. + if lo <= minFold && hi >= maxFold { + // Range is full: folding can't add more. + return appendRange(r, lo, hi) + } + if hi < minFold || lo > maxFold { + // Range is outside folding possibilities. + return appendRange(r, lo, hi) + } + if lo < minFold { + // [lo, minFold-1] needs no folding. + r = appendRange(r, lo, minFold-1) + lo = minFold + } + if hi > maxFold { + // [maxFold+1, hi] needs no folding. + r = appendRange(r, maxFold+1, hi) + hi = maxFold + } + + // Brute force. Depend on appendRange to coalesce ranges on the fly. + for c := lo; c <= hi; c++ { + r = appendRange(r, c, c) + f := unicode.SimpleFold(c) + for f != c { + r = appendRange(r, f, f) + f = unicode.SimpleFold(f) + } + } + return r +} + +// appendClass returns the result of appending the class x to the class r. +// It assume x is clean. +func appendClass(r []int, x []int) []int { + for i := 0; i < len(x); i += 2 { + r = appendRange(r, x[i], x[i+1]) + } + return r +} + +// appendFolded returns the result of appending the case folding of the class x to the class r. +func appendFoldedClass(r []int, x []int) []int { + for i := 0; i < len(x); i += 2 { + r = appendFoldedRange(r, x[i], x[i+1]) + } + return r +} + +// appendNegatedClass returns the result of appending the negation of the class x to the class r. +// It assumes x is clean. +func appendNegatedClass(r []int, x []int) []int { + nextLo := 0 + for i := 0; i < len(x); i += 2 { + lo, hi := x[i], x[i+1] + if nextLo <= lo-1 { + r = appendRange(r, nextLo, lo-1) + } + nextLo = hi + 1 + } + if nextLo <= unicode.MaxRune { + r = appendRange(r, nextLo, unicode.MaxRune) + } + return r +} + +// appendTable returns the result of appending x to the class r. +func appendTable(r []int, x *unicode.RangeTable) []int { + for _, xr := range x.R16 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + r = appendRange(r, lo, hi) + continue + } + for c := lo; c <= hi; c += stride { + r = appendRange(r, c, c) + } + } + for _, xr := range x.R32 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + r = appendRange(r, lo, hi) + continue + } + for c := lo; c <= hi; c += stride { + r = appendRange(r, c, c) + } + } + return r +} + +// appendNegatedTable returns the result of appending the negation of x to the class r. +func appendNegatedTable(r []int, x *unicode.RangeTable) []int { + nextLo := 0 // lo end of next class to add + for _, xr := range x.R16 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + if nextLo <= lo-1 { + r = appendRange(r, nextLo, lo-1) + } + nextLo = hi + 1 + continue + } + for c := lo; c <= hi; c += stride { + if nextLo <= c-1 { + r = appendRange(r, nextLo, c-1) + } + nextLo = c + 1 + } + } + for _, xr := range x.R32 { + lo, hi, stride := int(xr.Lo), int(xr.Hi), int(xr.Stride) + if stride == 1 { + if nextLo <= lo-1 { + r = appendRange(r, nextLo, lo-1) + } + nextLo = hi + 1 + continue + } + for c := lo; c <= hi; c += stride { + if nextLo <= c-1 { + r = appendRange(r, nextLo, c-1) + } + nextLo = c + 1 + } + } + if nextLo <= unicode.MaxRune { + r = appendRange(r, nextLo, unicode.MaxRune) + } + return r +} + +// negateClass overwrites r and returns r's negation. +// It assumes the class r is already clean. +func negateClass(r []int) []int { + nextLo := 0 // lo end of next class to add + w := 0 // write index + for i := 0; i < len(r); i += 2 { + lo, hi := r[i], r[i+1] + if nextLo <= lo-1 { + r[w] = nextLo + r[w+1] = lo - 1 + w += 2 + } + nextLo = hi + 1 + } + r = r[:w] + if nextLo <= unicode.MaxRune { + // It's possible for the negation to have one more + // range - this one - than the original class, so use append. + r = append(r, nextLo, unicode.MaxRune) + } + return r +} + +// ranges implements sort.Interface on a []rune. +// The choice of receiver type definition is strange +// but avoids an allocation since we already have +// a *[]int. +type ranges struct { + p *[]int +} + +func (ra ranges) Less(i, j int) bool { + p := *ra.p + i *= 2 + j *= 2 + return p[i] < p[j] || p[i] == p[j] && p[i+1] > p[j+1] +} + +func (ra ranges) Len() int { + return len(*ra.p) / 2 +} + +func (ra ranges) Swap(i, j int) { + p := *ra.p + i *= 2 + j *= 2 + p[i], p[i+1], p[j], p[j+1] = p[j], p[j+1], p[i], p[i+1] +} + +func checkUTF8(s string) os.Error { + for s != "" { + rune, size := utf8.DecodeRuneInString(s) + if rune == utf8.RuneError && size == 1 { + return &Error{Code: ErrInvalidUTF8, Expr: s} + } + s = s[size:] + } + return nil +} + +func nextRune(s string) (c int, t string, err os.Error) { + c, size := utf8.DecodeRuneInString(s) + if c == utf8.RuneError && size == 1 { + return 0, "", &Error{Code: ErrInvalidUTF8, Expr: s} + } + return c, s[size:], nil +} + +func isalnum(c int) bool { + return '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' +} + +func unhex(c int) int { + if '0' <= c && c <= '9' { + return c - '0' + } + if 'a' <= c && c <= 'f' { + return c - 'a' + 10 + } + if 'A' <= c && c <= 'F' { + return c - 'A' + 10 + } + return -1 +} diff --git a/src/pkg/exp/regexp/syntax/parse_test.go b/src/pkg/exp/regexp/syntax/parse_test.go new file mode 100644 index 000000000..779b9afde --- /dev/null +++ b/src/pkg/exp/regexp/syntax/parse_test.go @@ -0,0 +1,350 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import ( + "bytes" + "fmt" + "testing" + "unicode" +) + +var parseTests = []struct { + Regexp string + Dump string +}{ + // Base cases + {`a`, `lit{a}`}, + {`a.`, `cat{lit{a}dot{}}`}, + {`a.b`, `cat{lit{a}dot{}lit{b}}`}, + {`ab`, `str{ab}`}, + {`a.b.c`, `cat{lit{a}dot{}lit{b}dot{}lit{c}}`}, + {`abc`, `str{abc}`}, + {`a|^`, `alt{lit{a}bol{}}`}, + {`a|b`, `cc{0x61-0x62}`}, + {`(a)`, `cap{lit{a}}`}, + {`(a)|b`, `alt{cap{lit{a}}lit{b}}`}, + {`a*`, `star{lit{a}}`}, + {`a+`, `plus{lit{a}}`}, + {`a?`, `que{lit{a}}`}, + {`a{2}`, `rep{2,2 lit{a}}`}, + {`a{2,3}`, `rep{2,3 lit{a}}`}, + {`a{2,}`, `rep{2,-1 lit{a}}`}, + {`a*?`, `nstar{lit{a}}`}, + {`a+?`, `nplus{lit{a}}`}, + {`a??`, `nque{lit{a}}`}, + {`a{2}?`, `nrep{2,2 lit{a}}`}, + {`a{2,3}?`, `nrep{2,3 lit{a}}`}, + {`a{2,}?`, `nrep{2,-1 lit{a}}`}, + {``, `emp{}`}, + {`|`, `emp{}`}, // alt{emp{}emp{}} but got factored + {`|x|`, `alt{emp{}lit{x}emp{}}`}, + {`.`, `dot{}`}, + {`^`, `bol{}`}, + {`$`, `eol{}`}, + {`\|`, `lit{|}`}, + {`\(`, `lit{(}`}, + {`\)`, `lit{)}`}, + {`\*`, `lit{*}`}, + {`\+`, `lit{+}`}, + {`\?`, `lit{?}`}, + {`{`, `lit{{}`}, + {`}`, `lit{}}`}, + {`\.`, `lit{.}`}, + {`\^`, `lit{^}`}, + {`\$`, `lit{$}`}, + {`\\`, `lit{\}`}, + {`[ace]`, `cc{0x61 0x63 0x65}`}, + {`[abc]`, `cc{0x61-0x63}`}, + {`[a-z]`, `cc{0x61-0x7a}`}, + {`[a]`, `lit{a}`}, + {`\-`, `lit{-}`}, + {`-`, `lit{-}`}, + {`\_`, `lit{_}`}, + {`abc`, `str{abc}`}, + {`abc|def`, `alt{str{abc}str{def}}`}, + {`abc|def|ghi`, `alt{str{abc}str{def}str{ghi}}`}, + + // Posix and Perl extensions + {`[[:lower:]]`, `cc{0x61-0x7a}`}, + {`[a-z]`, `cc{0x61-0x7a}`}, + {`[^[:lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`}, + {`[[:^lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`}, + {`(?i)[[:lower:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)[a-z]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)[^[:lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`(?i)[[:^lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`\d`, `cc{0x30-0x39}`}, + {`\D`, `cc{0x0-0x2f 0x3a-0x10ffff}`}, + {`\s`, `cc{0x9-0xa 0xc-0xd 0x20}`}, + {`\S`, `cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}`}, + {`\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}`}, + {`\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}`}, + {`(?i)\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`[^\\]`, `cc{0x0-0x5b 0x5d-0x10ffff}`}, + // { `\C`, `byte{}` }, // probably never + + // Unicode, negatives, and a double negative. + {`\p{Braille}`, `cc{0x2800-0x28ff}`}, + {`\P{Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`\p{^Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`\P{^Braille}`, `cc{0x2800-0x28ff}`}, + {`\pZ`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, + {`[\p{Braille}]`, `cc{0x2800-0x28ff}`}, + {`[\P{Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`[\p{^Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`[\P{^Braille}]`, `cc{0x2800-0x28ff}`}, + {`[\pZ]`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, + {`\p{Lu}`, mkCharClass(unicode.IsUpper)}, + {`[\p{Lu}]`, mkCharClass(unicode.IsUpper)}, + {`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)}, + + // Hex, octal. + {`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`}, + {`[\x{41}-\x7a]\x61`, `cat{cc{0x41-0x7a}lit{a}}`}, + + // More interesting regular expressions. + {`a{,2}`, `str{a{,2}}`}, + {`\.\^\$\\`, `str{.^$\}`}, + {`[a-zABC]`, `cc{0x41-0x43 0x61-0x7a}`}, + {`[^a]`, `cc{0x0-0x60 0x62-0x10ffff}`}, + {`[α-ε☺]`, `cc{0x3b1-0x3b5 0x263a}`}, // utf-8 + {`a*{`, `cat{star{lit{a}}lit{{}}`}, + + // Test precedences + {`(?:ab)*`, `star{str{ab}}`}, + {`(ab)*`, `star{cap{str{ab}}}`}, + {`ab|cd`, `alt{str{ab}str{cd}}`}, + {`a(b|c)d`, `cat{lit{a}cap{cc{0x62-0x63}}lit{d}}`}, + + // Test flattening. + {`(?:a)`, `lit{a}`}, + {`(?:ab)(?:cd)`, `str{abcd}`}, + {`(?:a+b+)(?:c+d+)`, `cat{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`}, + {`(?:a+|b+)|(?:c+|d+)`, `alt{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`}, + {`(?:a|b)|(?:c|d)`, `cc{0x61-0x64}`}, + {`a|.`, `dot{}`}, + {`.|a`, `dot{}`}, + {`(?:[abc]|A|Z|hello|world)`, `alt{cc{0x41 0x5a 0x61-0x63}str{hello}str{world}}`}, + {`(?:[abc]|A|Z)`, `cc{0x41 0x5a 0x61-0x63}`}, + + // Test Perl quoted literals + {`\Q+|*?{[\E`, `str{+|*?{[}`}, + {`\Q+\E+`, `plus{lit{+}}`}, + {`\Q\\E`, `lit{\}`}, + {`\Q\\\E`, `str{\\}`}, + + // Test Perl \A and \z + {`(?m)^`, `bol{}`}, + {`(?m)$`, `eol{}`}, + {`(?-m)^`, `bot{}`}, + {`(?-m)$`, `eot{}`}, + {`(?m)\A`, `bot{}`}, + {`(?m)\z`, `eot{\z}`}, + {`(?-m)\A`, `bot{}`}, + {`(?-m)\z`, `eot{\z}`}, + + // Test named captures + {`(?Pa)`, `cap{name:lit{a}}`}, + + // Case-folded literals + {`[Aa]`, `litfold{A}`}, + {`[\x{100}\x{101}]`, `litfold{Ā}`}, + {`[Δδ]`, `litfold{Δ}`}, + + // Strings + {`abcde`, `str{abcde}`}, + {`[Aa][Bb]cd`, `cat{strfold{AB}str{cd}}`}, + + // Factoring. + {`abc|abd|aef|bcx|bcy`, `alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}cat{str{bc}cc{0x78-0x79}}}`}, + {`ax+y|ax+z|ay+w`, `cat{lit{a}alt{cat{plus{lit{x}}cc{0x79-0x7a}}cat{plus{lit{y}}lit{w}}}}`}, +} + +const testFlags = MatchNL | PerlX | UnicodeGroups + +// Test Parse -> Dump. +func TestParseDump(t *testing.T) { + for _, tt := range parseTests { + re, err := Parse(tt.Regexp, testFlags) + if err != nil { + t.Errorf("Parse(%#q): %v", tt.Regexp, err) + continue + } + d := dump(re) + if d != tt.Dump { + t.Errorf("Parse(%#q).Dump() = %#q want %#q", tt.Regexp, d, tt.Dump) + } + } +} + +// dump prints a string representation of the regexp showing +// the structure explicitly. +func dump(re *Regexp) string { + var b bytes.Buffer + dumpRegexp(&b, re) + return b.String() +} + +var opNames = []string{ + OpNoMatch: "no", + OpEmptyMatch: "emp", + OpLiteral: "lit", + OpCharClass: "cc", + OpAnyCharNotNL: "dnl", + OpAnyChar: "dot", + OpBeginLine: "bol", + OpEndLine: "eol", + OpBeginText: "bot", + OpEndText: "eot", + OpWordBoundary: "wb", + OpNoWordBoundary: "nwb", + OpCapture: "cap", + OpStar: "star", + OpPlus: "plus", + OpQuest: "que", + OpRepeat: "rep", + OpConcat: "cat", + OpAlternate: "alt", +} + +// dumpRegexp writes an encoding of the syntax tree for the regexp re to b. +// It is used during testing to distinguish between parses that might print +// the same using re's String method. +func dumpRegexp(b *bytes.Buffer, re *Regexp) { + if int(re.Op) >= len(opNames) || opNames[re.Op] == "" { + fmt.Fprintf(b, "op%d", re.Op) + } else { + switch re.Op { + default: + b.WriteString(opNames[re.Op]) + case OpStar, OpPlus, OpQuest, OpRepeat: + if re.Flags&NonGreedy != 0 { + b.WriteByte('n') + } + b.WriteString(opNames[re.Op]) + case OpLiteral: + if len(re.Rune) > 1 { + b.WriteString("str") + } else { + b.WriteString("lit") + } + if re.Flags&FoldCase != 0 { + for _, r := range re.Rune { + if unicode.SimpleFold(r) != r { + b.WriteString("fold") + break + } + } + } + } + } + b.WriteByte('{') + switch re.Op { + case OpEndText: + if re.Flags&WasDollar == 0 { + b.WriteString(`\z`) + } + case OpLiteral: + for _, r := range re.Rune { + b.WriteRune(r) + } + case OpConcat, OpAlternate: + for _, sub := range re.Sub { + dumpRegexp(b, sub) + } + case OpStar, OpPlus, OpQuest: + dumpRegexp(b, re.Sub[0]) + case OpRepeat: + fmt.Fprintf(b, "%d,%d ", re.Min, re.Max) + dumpRegexp(b, re.Sub[0]) + case OpCapture: + if re.Name != "" { + b.WriteString(re.Name) + b.WriteByte(':') + } + dumpRegexp(b, re.Sub[0]) + case OpCharClass: + sep := "" + for i := 0; i < len(re.Rune); i += 2 { + b.WriteString(sep) + sep = " " + lo, hi := re.Rune[i], re.Rune[i+1] + if lo == hi { + fmt.Fprintf(b, "%#x", lo) + } else { + fmt.Fprintf(b, "%#x-%#x", lo, hi) + } + } + } + b.WriteByte('}') +} + +func mkCharClass(f func(int) bool) string { + re := &Regexp{Op: OpCharClass} + lo := -1 + for i := 0; i <= unicode.MaxRune; i++ { + if f(i) { + if lo < 0 { + lo = i + } + } else { + if lo >= 0 { + re.Rune = append(re.Rune, lo, i-1) + lo = -1 + } + } + } + if lo >= 0 { + re.Rune = append(re.Rune, lo, unicode.MaxRune) + } + return dump(re) +} + +func isUpperFold(rune int) bool { + if unicode.IsUpper(rune) { + return true + } + c := unicode.SimpleFold(rune) + for c != rune { + if unicode.IsUpper(c) { + return true + } + c = unicode.SimpleFold(c) + } + return false +} + +func TestFoldConstants(t *testing.T) { + last := -1 + for i := 0; i <= unicode.MaxRune; i++ { + if unicode.SimpleFold(i) == i { + continue + } + if last == -1 && minFold != i { + t.Errorf("minFold=%#U should be %#U", minFold, i) + } + last = i + } + if maxFold != last { + t.Errorf("maxFold=%#U should be %#U", maxFold, last) + } +} + +func TestAppendRangeCollapse(t *testing.T) { + // AppendRange should collapse each of the new ranges + // into the earlier ones (it looks back two ranges), so that + // the slice never grows very large. + // Note that we are not calling cleanClass. + var r []int + for i := 'A'; i <= 'Z'; i++ { + r = appendRange(r, i, i) + r = appendRange(r, i+'a'-'A', i+'a'-'A') + } + if string(r) != "AZaz" { + t.Errorf("appendRange interlaced A-Z a-z = %s, want AZaz", string(r)) + } +} diff --git a/src/pkg/exp/regexp/syntax/perl_groups.go b/src/pkg/exp/regexp/syntax/perl_groups.go new file mode 100644 index 000000000..05b392c40 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/perl_groups.go @@ -0,0 +1,130 @@ +// GENERATED BY make_perl_groups.pl; DO NOT EDIT. +// make_perl_groups.pl >perl_groups.go + +package syntax + +var code1 = []int{ /* \d */ + 0x30, 0x39, +} + +var code2 = []int{ /* \s */ + 0x9, 0xa, + 0xc, 0xd, + 0x20, 0x20, +} + +var code3 = []int{ /* \w */ + 0x30, 0x39, + 0x41, 0x5a, + 0x5f, 0x5f, + 0x61, 0x7a, +} + +var perlGroup = map[string]charGroup{ + `\d`: {+1, code1}, + `\D`: {-1, code1}, + `\s`: {+1, code2}, + `\S`: {-1, code2}, + `\w`: {+1, code3}, + `\W`: {-1, code3}, +} +var code4 = []int{ /* [:alnum:] */ + 0x30, 0x39, + 0x41, 0x5a, + 0x61, 0x7a, +} + +var code5 = []int{ /* [:alpha:] */ + 0x41, 0x5a, + 0x61, 0x7a, +} + +var code6 = []int{ /* [:ascii:] */ + 0x0, 0x7f, +} + +var code7 = []int{ /* [:blank:] */ + 0x9, 0x9, + 0x20, 0x20, +} + +var code8 = []int{ /* [:cntrl:] */ + 0x0, 0x1f, + 0x7f, 0x7f, +} + +var code9 = []int{ /* [:digit:] */ + 0x30, 0x39, +} + +var code10 = []int{ /* [:graph:] */ + 0x21, 0x7e, +} + +var code11 = []int{ /* [:lower:] */ + 0x61, 0x7a, +} + +var code12 = []int{ /* [:print:] */ + 0x20, 0x7e, +} + +var code13 = []int{ /* [:punct:] */ + 0x21, 0x2f, + 0x3a, 0x40, + 0x5b, 0x60, + 0x7b, 0x7e, +} + +var code14 = []int{ /* [:space:] */ + 0x9, 0xd, + 0x20, 0x20, +} + +var code15 = []int{ /* [:upper:] */ + 0x41, 0x5a, +} + +var code16 = []int{ /* [:word:] */ + 0x30, 0x39, + 0x41, 0x5a, + 0x5f, 0x5f, + 0x61, 0x7a, +} + +var code17 = []int{ /* [:xdigit:] */ + 0x30, 0x39, + 0x41, 0x46, + 0x61, 0x66, +} + +var posixGroup = map[string]charGroup{ + `[:alnum:]`: {+1, code4}, + `[:^alnum:]`: {-1, code4}, + `[:alpha:]`: {+1, code5}, + `[:^alpha:]`: {-1, code5}, + `[:ascii:]`: {+1, code6}, + `[:^ascii:]`: {-1, code6}, + `[:blank:]`: {+1, code7}, + `[:^blank:]`: {-1, code7}, + `[:cntrl:]`: {+1, code8}, + `[:^cntrl:]`: {-1, code8}, + `[:digit:]`: {+1, code9}, + `[:^digit:]`: {-1, code9}, + `[:graph:]`: {+1, code10}, + `[:^graph:]`: {-1, code10}, + `[:lower:]`: {+1, code11}, + `[:^lower:]`: {-1, code11}, + `[:print:]`: {+1, code12}, + `[:^print:]`: {-1, code12}, + `[:punct:]`: {+1, code13}, + `[:^punct:]`: {-1, code13}, + `[:space:]`: {+1, code14}, + `[:^space:]`: {-1, code14}, + `[:upper:]`: {+1, code15}, + `[:^upper:]`: {-1, code15}, + `[:word:]`: {+1, code16}, + `[:^word:]`: {-1, code16}, + `[:xdigit:]`: {+1, code17}, + `[:^xdigit:]`: {-1, code17}, +} diff --git a/src/pkg/exp/regexp/syntax/prog.go b/src/pkg/exp/regexp/syntax/prog.go new file mode 100644 index 000000000..bf85b720d --- /dev/null +++ b/src/pkg/exp/regexp/syntax/prog.go @@ -0,0 +1,237 @@ +package syntax + +import ( + "bytes" + "strconv" +) + +// Compiled program. +// May not belong in this package, but convenient for now. + +// A Prog is a compiled regular expression program. +type Prog struct { + Inst []Inst + Start int // index of start instruction + NumCap int // number of InstCapture insts in re +} + +// An InstOp is an instruction opcode. +type InstOp uint8 + +const ( + InstAlt InstOp = iota + InstAltMatch + InstCapture + InstEmptyWidth + InstMatch + InstFail + InstNop + InstRune +) + +// An EmptyOp specifies a kind or mixture of zero-width assertions. +type EmptyOp uint8 + +const ( + EmptyBeginLine EmptyOp = 1 << iota + EmptyEndLine + EmptyBeginText + EmptyEndText + EmptyWordBoundary + EmptyNoWordBoundary +) + +// An Inst is a single instruction in a regular expression program. +type Inst struct { + Op InstOp + Out uint32 // all but InstMatch, InstFail + Arg uint32 // InstAlt, InstAltMatch, InstCapture, InstEmptyWidth + Rune []int +} + +func (p *Prog) String() string { + var b bytes.Buffer + dumpProg(&b, p) + return b.String() +} + +// skipNop follows any no-op or capturing instructions +// and returns the resulting pc. +func (p *Prog) skipNop(pc uint32) *Inst { + i := &p.Inst[pc] + for i.Op == InstNop || i.Op == InstCapture { + pc = i.Out + i = &p.Inst[pc] + } + return i +} + +// Prefix returns a literal string that all matches for the +// regexp must start with. Complete is true if the prefix +// is the entire match. +func (p *Prog) Prefix() (prefix string, complete bool) { + i := p.skipNop(uint32(p.Start)) + + // Avoid allocation of buffer if prefix is empty. + if i.Op != InstRune || len(i.Rune) != 1 { + return "", i.Op == InstMatch + } + + // Have prefix; gather characters. + var buf bytes.Buffer + for i.Op == InstRune && len(i.Rune) == 1 { + buf.WriteRune(i.Rune[0]) + i = p.skipNop(i.Out) + } + return buf.String(), i.Op == InstMatch +} + +// StartCond returns the leading empty-width conditions that must +// be true in any match. It returns ^EmptyOp(0) if no matches are possible. +func (p *Prog) StartCond() EmptyOp { + var flag EmptyOp + pc := uint32(p.Start) + i := &p.Inst[pc] +Loop: + for { + switch i.Op { + case InstEmptyWidth: + flag |= EmptyOp(i.Arg) + case InstFail: + return ^EmptyOp(0) + case InstCapture, InstNop: + // skip + default: + break Loop + } + pc = i.Out + i = &p.Inst[pc] + } + return flag +} + +// MatchRune returns true if the instruction matches (and consumes) r. +// It should only be called when i.Op == InstRune. +func (i *Inst) MatchRune(r int) bool { + rune := i.Rune + + // Special case: single-rune slice is from literal string, not char class. + // TODO: Case folding. + if len(rune) == 1 { + return r == rune[0] + } + + // Peek at the first few pairs. + // Should handle ASCII well. + for j := 0; j < len(rune) && j <= 8; j += 2 { + if r < rune[j] { + return false + } + if r <= rune[j+1] { + return true + } + } + + // Otherwise binary search. + lo := 0 + hi := len(rune) / 2 + for lo < hi { + m := lo + (hi-lo)/2 + if c := rune[2*m]; c <= r { + if r <= rune[2*m+1] { + return true + } + lo = m + 1 + } else { + hi = m + } + } + return false +} + +// As per re2's Prog::IsWordChar. Determines whether rune is an ASCII word char. +// Since we act on runes, it would be easy to support Unicode here. +func wordRune(rune int) bool { + return rune == '_' || + ('A' <= rune && rune <= 'Z') || + ('a' <= rune && rune <= 'z') || + ('0' <= rune && rune <= '9') +} + +// MatchEmptyWidth returns true if the instruction matches +// an empty string between the runes before and after. +// It should only be called when i.Op == InstEmptyWidth. +func (i *Inst) MatchEmptyWidth(before int, after int) bool { + switch EmptyOp(i.Arg) { + case EmptyBeginLine: + return before == '\n' || before == -1 + case EmptyEndLine: + return after == '\n' || after == -1 + case EmptyBeginText: + return before == -1 + case EmptyEndText: + return after == -1 + case EmptyWordBoundary: + return wordRune(before) != wordRune(after) + case EmptyNoWordBoundary: + return wordRune(before) == wordRune(after) + } + panic("unknown empty width arg") +} + +func (i *Inst) String() string { + var b bytes.Buffer + dumpInst(&b, i) + return b.String() +} + +func bw(b *bytes.Buffer, args ...string) { + for _, s := range args { + b.WriteString(s) + } +} + +func dumpProg(b *bytes.Buffer, p *Prog) { + for j := range p.Inst { + i := &p.Inst[j] + pc := strconv.Itoa(j) + if len(pc) < 3 { + b.WriteString(" "[len(pc):]) + } + if j == p.Start { + pc += "*" + } + bw(b, pc, "\t") + dumpInst(b, i) + bw(b, "\n") + } +} + +func u32(i uint32) string { + return strconv.Uitoa64(uint64(i)) +} + +func dumpInst(b *bytes.Buffer, i *Inst) { + switch i.Op { + case InstAlt: + bw(b, "alt -> ", u32(i.Out), ", ", u32(i.Arg)) + case InstAltMatch: + bw(b, "altmatch -> ", u32(i.Out), ", ", u32(i.Arg)) + case InstCapture: + bw(b, "cap ", u32(i.Arg), " -> ", u32(i.Out)) + case InstEmptyWidth: + bw(b, "empty ", u32(i.Arg), " -> ", u32(i.Out)) + case InstMatch: + bw(b, "match") + case InstFail: + bw(b, "fail") + case InstNop: + bw(b, "nop -> ", u32(i.Out)) + case InstRune: + if i.Rune == nil { + // shouldn't happen + bw(b, "rune ") + } + bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune)), " -> ", u32(i.Out)) + } +} diff --git a/src/pkg/exp/regexp/syntax/prog_test.go b/src/pkg/exp/regexp/syntax/prog_test.go new file mode 100644 index 000000000..7be4281c2 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/prog_test.go @@ -0,0 +1,91 @@ +package syntax + +import ( + "testing" +) + +var compileTests = []struct { + Regexp string + Prog string +}{ + {"a", ` 0 fail + 1* rune "a" -> 2 + 2 match +`}, + {"[A-M][n-z]", ` 0 fail + 1* rune "AM" -> 2 + 2 rune "nz" -> 3 + 3 match +`}, + {"", ` 0 fail + 1* nop -> 2 + 2 match +`}, + {"a?", ` 0 fail + 1 rune "a" -> 3 + 2* alt -> 1, 3 + 3 match +`}, + {"a??", ` 0 fail + 1 rune "a" -> 3 + 2* alt -> 3, 1 + 3 match +`}, + {"a+", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 1, 3 + 3 match +`}, + {"a+?", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 3, 1 + 3 match +`}, + {"a*", ` 0 fail + 1 rune "a" -> 2 + 2* alt -> 1, 3 + 3 match +`}, + {"a*?", ` 0 fail + 1 rune "a" -> 2 + 2* alt -> 3, 1 + 3 match +`}, + {"a+b+", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 1, 3 + 3 rune "b" -> 4 + 4 alt -> 3, 5 + 5 match +`}, + {"(a+)(b+)", ` 0 fail + 1* cap 2 -> 2 + 2 rune "a" -> 3 + 3 alt -> 2, 4 + 4 cap 3 -> 5 + 5 cap 4 -> 6 + 6 rune "b" -> 7 + 7 alt -> 6, 8 + 8 cap 5 -> 9 + 9 match +`}, + {"a+|b+", ` 0 fail + 1 rune "a" -> 2 + 2 alt -> 1, 6 + 3 rune "b" -> 4 + 4 alt -> 3, 6 + 5* alt -> 1, 3 + 6 match +`}, +} + +func TestCompile(t *testing.T) { + for _, tt := range compileTests { + re, _ := Parse(tt.Regexp, Perl) + p, _ := Compile(re) + s := p.String() + if s != tt.Prog { + t.Errorf("compiled %#q:\n--- have\n%s---\n--- want\n%s---", tt.Regexp, s, tt.Prog) + } + } +} diff --git a/src/pkg/exp/regexp/syntax/regexp.go b/src/pkg/exp/regexp/syntax/regexp.go new file mode 100644 index 000000000..00a4addef --- /dev/null +++ b/src/pkg/exp/regexp/syntax/regexp.go @@ -0,0 +1,284 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package syntax parses regular expressions into syntax trees. +// WORK IN PROGRESS. +package syntax + +// Note to implementers: +// In this package, re is always a *Regexp and r is always a rune. + +import ( + "bytes" + "strconv" + "strings" + "unicode" +) + +// A Regexp is a node in a regular expression syntax tree. +type Regexp struct { + Op Op // operator + Flags Flags + Sub []*Regexp // subexpressions, if any + Sub0 [1]*Regexp // storage for short Sub + Rune []int // matched runes, for OpLiteral, OpCharClass + Rune0 [2]int // storage for short Rune + Min, Max int // min, max for OpRepeat + Cap int // capturing index, for OpCapture + Name string // capturing name, for OpCapture +} + +// An Op is a single regular expression operator. +type Op uint8 + +// Operators are listed in precedence order, tightest binding to weakest. +// Character class operators are listed simplest to most complex +// (OpLiteral, OpCharClass, OpAnyCharNotNL, OpAnyChar). + +const ( + OpNoMatch Op = 1 + iota // matches no strings + OpEmptyMatch // matches empty string + OpLiteral // matches Runes sequence + OpCharClass // matches Runes interpreted as range pair list + OpAnyCharNotNL // matches any character + OpAnyChar // matches any character + OpBeginLine // matches empty string at beginning of line + OpEndLine // matches empty string at end of line + OpBeginText // matches empty string at beginning of text + OpEndText // matches empty string at end of text + OpWordBoundary // matches word boundary `\b` + OpNoWordBoundary // matches word non-boundary `\B` + OpCapture // capturing subexpression with index Cap, optional name Name + OpStar // matches Sub[0] zero or more times + OpPlus // matches Sub[0] one or more times + OpQuest // matches Sub[0] zero or one times + OpRepeat // matches Sub[0] at least Min times, at most Max (Max == -1 is no limit) + OpConcat // matches concatenation of Subs + OpAlternate // matches alternation of Subs +) + +const opPseudo Op = 128 // where pseudo-ops start + +// Equal returns true if x and y have identical structure. +func (x *Regexp) Equal(y *Regexp) bool { + if x == nil || y == nil { + return x == y + } + if x.Op != y.Op { + return false + } + switch x.Op { + case OpEndText: + // The parse flags remember whether this is \z or \Z. + if x.Flags&WasDollar != y.Flags&WasDollar { + return false + } + + case OpLiteral, OpCharClass: + if len(x.Rune) != len(y.Rune) { + return false + } + for i, r := range x.Rune { + if r != y.Rune[i] { + return false + } + } + + case OpAlternate, OpConcat: + if len(x.Sub) != len(y.Sub) { + return false + } + for i, sub := range x.Sub { + if !sub.Equal(y.Sub[i]) { + return false + } + } + + case OpStar, OpPlus, OpQuest: + if x.Flags&NonGreedy != y.Flags&NonGreedy || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + + case OpRepeat: + if x.Flags&NonGreedy != y.Flags&NonGreedy || x.Min != y.Min || x.Max != y.Max || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + + case OpCapture: + if x.Cap != y.Cap || x.Name != y.Name || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + } + return true +} + +// writeRegexp writes the Perl syntax for the regular expression re to b. +func writeRegexp(b *bytes.Buffer, re *Regexp) { + switch re.Op { + default: + b.WriteString("") + case OpNoMatch: + b.WriteString(`[^\x00-\x{10FFFF}]`) + case OpEmptyMatch: + b.WriteString(`(?:)`) + case OpLiteral: + if re.Flags&FoldCase != 0 { + b.WriteString(`(?i:`) + } + for _, r := range re.Rune { + escape(b, r, false) + } + if re.Flags&FoldCase != 0 { + b.WriteString(`)`) + } + case OpCharClass: + if len(re.Rune)%2 != 0 { + b.WriteString(`[invalid char class]`) + break + } + b.WriteRune('[') + if len(re.Rune) == 0 { + b.WriteString(`^\x00-\x{10FFFF}`) + } else if re.Rune[0] == 0 && re.Rune[len(re.Rune)-1] == unicode.MaxRune { + // Contains 0 and MaxRune. Probably a negated class. + // Print the gaps. + b.WriteRune('^') + for i := 1; i < len(re.Rune)-1; i += 2 { + lo, hi := re.Rune[i]+1, re.Rune[i+1]-1 + escape(b, lo, lo == '-') + if lo != hi { + b.WriteRune('-') + escape(b, hi, hi == '-') + } + } + } else { + for i := 0; i < len(re.Rune); i += 2 { + lo, hi := re.Rune[i], re.Rune[i+1] + escape(b, lo, lo == '-') + if lo != hi { + b.WriteRune('-') + escape(b, hi, hi == '-') + } + } + } + b.WriteRune(']') + case OpAnyCharNotNL: + b.WriteString(`[^\n]`) + case OpAnyChar: + b.WriteRune('.') + case OpBeginLine: + b.WriteRune('^') + case OpEndLine: + b.WriteRune('$') + case OpBeginText: + b.WriteString(`\A`) + case OpEndText: + b.WriteString(`\z`) + case OpWordBoundary: + b.WriteString(`\b`) + case OpNoWordBoundary: + b.WriteString(`\B`) + case OpCapture: + if re.Name != "" { + b.WriteString(`(?P<`) + b.WriteString(re.Name) + b.WriteRune('>') + } else { + b.WriteRune('(') + } + if re.Sub[0].Op != OpEmptyMatch { + writeRegexp(b, re.Sub[0]) + } + b.WriteRune(')') + case OpStar, OpPlus, OpQuest, OpRepeat: + if sub := re.Sub[0]; sub.Op > OpCapture { + b.WriteString(`(?:`) + writeRegexp(b, sub) + b.WriteString(`)`) + } else { + writeRegexp(b, sub) + } + switch re.Op { + case OpStar: + b.WriteRune('*') + case OpPlus: + b.WriteRune('+') + case OpQuest: + b.WriteRune('?') + case OpRepeat: + b.WriteRune('{') + b.WriteString(strconv.Itoa(re.Min)) + if re.Max != re.Min { + b.WriteRune(',') + if re.Max >= 0 { + b.WriteString(strconv.Itoa(re.Max)) + } + } + b.WriteRune('}') + } + case OpConcat: + for _, sub := range re.Sub { + if sub.Op == OpAlternate { + b.WriteString(`(?:`) + writeRegexp(b, sub) + b.WriteString(`)`) + } else { + writeRegexp(b, sub) + } + } + case OpAlternate: + for i, sub := range re.Sub { + if i > 0 { + b.WriteRune('|') + } + writeRegexp(b, sub) + } + } +} + +func (re *Regexp) String() string { + var b bytes.Buffer + writeRegexp(&b, re) + return b.String() +} + +const meta = `\.+*?()|[]{}^$` + +func escape(b *bytes.Buffer, r int, force bool) { + if unicode.IsPrint(r) { + if strings.IndexRune(meta, r) >= 0 || force { + b.WriteRune('\\') + } + b.WriteRune(r) + return + } + + switch r { + case '\a': + b.WriteString(`\a`) + case '\f': + b.WriteString(`\f`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + case '\v': + b.WriteString(`\v`) + default: + if r < 0x100 { + b.WriteString(`\x`) + s := strconv.Itob(r, 16) + if len(s) == 1 { + b.WriteRune('0') + } + b.WriteString(s) + break + } + b.WriteString(`\x{`) + b.WriteString(strconv.Itob(r, 16)) + b.WriteString(`}`) + } +} diff --git a/src/pkg/exp/regexp/syntax/simplify.go b/src/pkg/exp/regexp/syntax/simplify.go new file mode 100644 index 000000000..72390417b --- /dev/null +++ b/src/pkg/exp/regexp/syntax/simplify.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// Simplify returns a regexp equivalent to re but without counted repetitions +// and with various other simplifications, such as rewriting /(?:a+)+/ to /a+/. +// The resulting regexp will execute correctly but its string representation +// will not produce the same parse tree, because capturing parentheses +// may have been duplicated or removed. For example, the simplified form +// for /(x){1,2}/ is /(x)(x)?/ but both parentheses capture as $1. +// The returned regexp may share structure with or be the original. +func (re *Regexp) Simplify() *Regexp { + if re == nil { + return nil + } + switch re.Op { + case OpCapture, OpConcat, OpAlternate: + // Simplify children, building new Regexp if children change. + nre := re + for i, sub := range re.Sub { + nsub := sub.Simplify() + if nre == re && nsub != sub { + // Start a copy. + nre = new(Regexp) + *nre = *re + nre.Rune = nil + nre.Sub = append(nre.Sub0[:0], re.Sub[:i]...) + } + if nre != re { + nre.Sub = append(nre.Sub, nsub) + } + } + return nre + + case OpStar, OpPlus, OpQuest: + sub := re.Sub[0].Simplify() + return simplify1(re.Op, re.Flags, sub, re) + + case OpRepeat: + // Special special case: x{0} matches the empty string + // and doesn't even need to consider x. + if re.Min == 0 && re.Max == 0 { + return &Regexp{Op: OpEmptyMatch} + } + + // The fun begins. + sub := re.Sub[0].Simplify() + + // x{n,} means at least n matches of x. + if re.Max == -1 { + // Special case: x{0,} is x*. + if re.Min == 0 { + return simplify1(OpStar, re.Flags, sub, nil) + } + + // Special case: x{1,} is x+. + if re.Min == 1 { + return simplify1(OpPlus, re.Flags, sub, nil) + } + + // General case: x{4,} is xxxx+. + nre := &Regexp{Op: OpConcat} + nre.Sub = nre.Sub0[:0] + for i := 0; i < re.Min-1; i++ { + nre.Sub = append(nre.Sub, sub) + } + nre.Sub = append(nre.Sub, simplify1(OpPlus, re.Flags, sub, nil)) + return nre + } + + // Special case x{0} handled above. + + // Special case: x{1} is just x. + if re.Min == 1 && re.Max == 1 { + return sub + } + + // General case: x{n,m} means n copies of x and m copies of x? + // The machine will do less work if we nest the final m copies, + // so that x{2,5} = xx(x(x(x)?)?)? + + // Build leading prefix: xx. + var prefix *Regexp + if re.Min > 0 { + prefix = &Regexp{Op: OpConcat} + prefix.Sub = prefix.Sub0[:0] + for i := 0; i < re.Min; i++ { + prefix.Sub = append(prefix.Sub, sub) + } + } + + // Build and attach suffix: (x(x(x)?)?)? + if re.Max > re.Min { + suffix := simplify1(OpQuest, re.Flags, sub, nil) + for i := re.Min + 1; i < re.Max; i++ { + nre2 := &Regexp{Op: OpConcat} + nre2.Sub = append(nre2.Sub0[:0], sub, suffix) + suffix = simplify1(OpQuest, re.Flags, nre2, nil) + } + if prefix == nil { + return suffix + } + prefix.Sub = append(prefix.Sub, suffix) + } + if prefix != nil { + return prefix + } + + // Some degenerate case like min > max or min < max < 0. + // Handle as impossible match. + return &Regexp{Op: OpNoMatch} + } + + return re +} + +// simplify1 implements Simplify for the unary OpStar, +// OpPlus, and OpQuest operators. It returns the simple regexp +// equivalent to +// +// Regexp{Op: op, Flags: flags, Sub: {sub}} +// +// under the assumption that sub is already simple, and +// without first allocating that structure. If the regexp +// to be returned turns out to be equivalent to re, simplify1 +// returns re instead. +// +// simplify1 is factored out of Simplify because the implementation +// for other operators generates these unary expressions. +// Letting them call simplify1 makes sure the expressions they +// generate are simple. +func simplify1(op Op, flags Flags, sub, re *Regexp) *Regexp { + // Special case: repeat the empty string as much as + // you want, but it's still the empty string. + if sub.Op == OpEmptyMatch { + return sub + } + // The operators are idempotent if the flags match. + if op == sub.Op && flags&NonGreedy == sub.Flags&NonGreedy { + return sub + } + if re != nil && re.Op == op && re.Flags&NonGreedy == flags&NonGreedy && sub == re.Sub[0] { + return re + } + + re = &Regexp{Op: op, Flags: flags} + re.Sub = append(re.Sub0[:0], sub) + return re +} diff --git a/src/pkg/exp/regexp/syntax/simplify_test.go b/src/pkg/exp/regexp/syntax/simplify_test.go new file mode 100644 index 000000000..c8cec2183 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/simplify_test.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import "testing" + +var simplifyTests = []struct { + Regexp string + Simple string +}{ + // Already-simple constructs + {`a`, `a`}, + {`ab`, `ab`}, + {`a|b`, `[a-b]`}, + {`ab|cd`, `ab|cd`}, + {`(ab)*`, `(ab)*`}, + {`(ab)+`, `(ab)+`}, + {`(ab)?`, `(ab)?`}, + {`.`, `.`}, + {`^`, `^`}, + {`$`, `$`}, + {`[ac]`, `[ac]`}, + {`[^ac]`, `[^ac]`}, + + // Posix character classes + {`[[:alnum:]]`, `[0-9A-Za-z]`}, + {`[[:alpha:]]`, `[A-Za-z]`}, + {`[[:blank:]]`, `[\t ]`}, + {`[[:cntrl:]]`, `[\x00-\x1f\x7f]`}, + {`[[:digit:]]`, `[0-9]`}, + {`[[:graph:]]`, `[!-~]`}, + {`[[:lower:]]`, `[a-z]`}, + {`[[:print:]]`, `[ -~]`}, + {`[[:punct:]]`, "[!-/:-@\\[-`\\{-~]"}, + {`[[:space:]]`, `[\t-\r ]`}, + {`[[:upper:]]`, `[A-Z]`}, + {`[[:xdigit:]]`, `[0-9A-Fa-f]`}, + + // Perl character classes + {`\d`, `[0-9]`}, + {`\s`, `[\t-\n\f-\r ]`}, + {`\w`, `[0-9A-Z_a-z]`}, + {`\D`, `[^0-9]`}, + {`\S`, `[^\t-\n\f-\r ]`}, + {`\W`, `[^0-9A-Z_a-z]`}, + {`[\d]`, `[0-9]`}, + {`[\s]`, `[\t-\n\f-\r ]`}, + {`[\w]`, `[0-9A-Z_a-z]`}, + {`[\D]`, `[^0-9]`}, + {`[\S]`, `[^\t-\n\f-\r ]`}, + {`[\W]`, `[^0-9A-Z_a-z]`}, + + // Posix repetitions + {`a{1}`, `a`}, + {`a{2}`, `aa`}, + {`a{5}`, `aaaaa`}, + {`a{0,1}`, `a?`}, + // The next three are illegible because Simplify inserts (?:) + // parens instead of () parens to avoid creating extra + // captured subexpressions. The comments show a version with fewer parens. + {`(a){0,2}`, `(?:(a)(a)?)?`}, // (aa?)? + {`(a){0,4}`, `(?:(a)(?:(a)(?:(a)(a)?)?)?)?`}, // (a(a(aa?)?)?)? + {`(a){2,6}`, `(a)(a)(?:(a)(?:(a)(?:(a)(a)?)?)?)?`}, // aa(a(a(aa?)?)?)? + {`a{0,2}`, `(?:aa?)?`}, // (aa?)? + {`a{0,4}`, `(?:a(?:a(?:aa?)?)?)?`}, // (a(a(aa?)?)?)? + {`a{2,6}`, `aa(?:a(?:a(?:aa?)?)?)?`}, // aa(a(a(aa?)?)?)? + {`a{0,}`, `a*`}, + {`a{1,}`, `a+`}, + {`a{2,}`, `aa+`}, + {`a{5,}`, `aaaaa+`}, + + // Test that operators simplify their arguments. + {`(?:a{1,}){1,}`, `a+`}, + {`(a{1,}b{1,})`, `(a+b+)`}, + {`a{1,}|b{1,}`, `a+|b+`}, + {`(?:a{1,})*`, `(?:a+)*`}, + {`(?:a{1,})+`, `a+`}, + {`(?:a{1,})?`, `(?:a+)?`}, + {``, `(?:)`}, + {`a{0}`, `(?:)`}, + + // Character class simplification + {`[ab]`, `[a-b]`}, + {`[a-za-za-z]`, `[a-z]`}, + {`[A-Za-zA-Za-z]`, `[A-Za-z]`}, + {`[ABCDEFGH]`, `[A-H]`}, + {`[AB-CD-EF-GH]`, `[A-H]`}, + {`[W-ZP-XE-R]`, `[E-Z]`}, + {`[a-ee-gg-m]`, `[a-m]`}, + {`[a-ea-ha-m]`, `[a-m]`}, + {`[a-ma-ha-e]`, `[a-m]`}, + {`[a-zA-Z0-9 -~]`, `[ -~]`}, + + // Empty character classes + {`[^[:cntrl:][:^cntrl:]]`, `[^\x00-\x{10FFFF}]`}, + + // Full character classes + {`[[:cntrl:][:^cntrl:]]`, `.`}, + + // Unicode case folding. + {`(?i)A`, `(?i:A)`}, + {`(?i)a`, `(?i:a)`}, + {`(?i)[A]`, `(?i:A)`}, + {`(?i)[a]`, `(?i:A)`}, + {`(?i)K`, `(?i:K)`}, + {`(?i)k`, `(?i:k)`}, + {`(?i)\x{212a}`, "(?i:\u212A)"}, + {`(?i)[K]`, "[Kk\u212A]"}, + {`(?i)[k]`, "[Kk\u212A]"}, + {`(?i)[\x{212a}]`, "[Kk\u212A]"}, + {`(?i)[a-z]`, "[A-Za-z\u017F\u212A]"}, + {`(?i)[\x00-\x{FFFD}]`, "[\\x00-\uFFFD]"}, + {`(?i)[\x00-\x{10FFFF}]`, `.`}, + + // Empty string as a regular expression. + // The empty string must be preserved inside parens in order + // to make submatches work right, so these tests are less + // interesting than they might otherwise be. String inserts + // explicit (?:) in place of non-parenthesized empty strings, + // to make them easier to spot for other parsers. + {`(a|b|)`, `([a-b]|(?:))`}, + {`(|)`, `()`}, + {`a()`, `a()`}, + {`(()|())`, `(()|())`}, + {`(a|)`, `(a|(?:))`}, + {`ab()cd()`, `ab()cd()`}, + {`()`, `()`}, + {`()*`, `()*`}, + {`()+`, `()+`}, + {`()?`, `()?`}, + {`(){0}`, `(?:)`}, + {`(){1}`, `()`}, + {`(){1,}`, `()+`}, + {`(){0,2}`, `(?:()()?)?`}, +} + +func TestSimplify(t *testing.T) { + for _, tt := range simplifyTests { + re, err := Parse(tt.Regexp, MatchNL|Perl&^OneLine) + if err != nil { + t.Errorf("Parse(%#q) = error %v", tt.Regexp, err) + continue + } + s := re.Simplify().String() + if s != tt.Simple { + t.Errorf("Simplify(%#q) = %#q, want %#q", tt.Regexp, s, tt.Simple) + } + } +} diff --git a/src/pkg/exp/template/html/Makefile b/src/pkg/exp/template/html/Makefile new file mode 100644 index 000000000..2f107da11 --- /dev/null +++ b/src/pkg/exp/template/html/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../../Make.inc + +TARG=exp/template/html +GOFILES=\ + escape.go + +include ../../../../Make.pkg diff --git a/src/pkg/exp/template/html/context.go b/src/pkg/exp/template/html/context.go new file mode 100644 index 000000000..411006883 --- /dev/null +++ b/src/pkg/exp/template/html/context.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// context describes the state an HTML parser must be in when it reaches the +// portion of HTML produced by evaluating a particular template node. +// +// The zero value of type context is the start context for a template that +// produces an HTML fragment as defined at +// http://www.w3.org/TR/html5/the-end.html#parsing-html-fragments +// where the context element is null. +type context struct { + state state + delim delim +} + +func (c context) String() string { + return fmt.Sprintf("context{state: %s, delim: %s", c.state, c.delim) +} + +// eq is true if the two contexts are identical field-wise. +func (c context) eq(d context) bool { + return c.state == d.state && c.delim == d.delim +} + +// state describes a high-level HTML parser state. +// +// It bounds the top of the element stack, and by extension the HTML +// insertion mode, but also contains state that does not correspond to +// anything in the HTML5 parsing algorithm because a single token +// production in the HTML grammar may contain embedded actions in a template. +// For instance, the quoted HTML attribute produced by +//
+// is a single token in HTML's grammar but in a template spans several nodes. +type state uint8 + +const ( + // statePCDATA is parsed character data. An HTML parser is in + // this state when its parse position is outside an HTML tag, + // directive, comment, and special element body. + statePCDATA state = iota + // stateTag occurs before an HTML attribute or the end of a tag. + stateTag + // stateURI occurs inside an HTML attribute whose content is a URI. + stateURI + // stateError is an infectious error state outside any valid + // HTML/CSS/JS construct. + stateError +) + +var stateNames = [...]string{ + statePCDATA: "statePCDATA", + stateTag: "stateTag", + stateURI: "stateURI", + stateError: "stateError", +} + +func (s state) String() string { + if uint(s) < uint(len(stateNames)) { + return stateNames[s] + } + return fmt.Sprintf("illegal state %d", uint(s)) +} + +// delim is the delimiter that will end the current HTML attribute. +type delim uint8 + +const ( + // delimNone occurs outside any attribute. + delimNone delim = iota + // delimDoubleQuote occurs when a double quote (") closes the attribute. + delimDoubleQuote + // delimSingleQuote occurs when a single quote (') closes the attribute. + delimSingleQuote + // delimSpaceOrTagEnd occurs when a space or right angle bracket (>) + // closes the attribute. + delimSpaceOrTagEnd +) + +var delimNames = [...]string{ + delimNone: "delimNone", + delimDoubleQuote: "delimDoubleQuote", + delimSingleQuote: "delimSingleQuote", + delimSpaceOrTagEnd: "delimSpaceOrTagEnd", +} + +func (d delim) String() string { + if uint(d) < uint(len(delimNames)) { + return delimNames[d] + } + return fmt.Sprintf("illegal delim %d", uint(d)) +} diff --git a/src/pkg/exp/template/html/escape.go b/src/pkg/exp/template/html/escape.go new file mode 100644 index 000000000..e0e87b98d --- /dev/null +++ b/src/pkg/exp/template/html/escape.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package html is a specialization of exp/template that automates the +// construction of safe HTML output. +// At the moment, the escaping is naive. All dynamic content is assumed to be +// plain text interpolated in an HTML PCDATA context. +package html + +import ( + "template" + "template/parse" +) + +// Escape rewrites each action in the template to guarantee the output is +// HTML-escaped. +func Escape(t *template.Template) { + // If the parser shares trees based on common-subexpression + // joining then we will need to avoid multiply escaping the same action. + escapeListNode(t.Tree.Root) +} + +// escapeNode dispatches to escape helpers by type. +func escapeNode(node parse.Node) { + switch n := node.(type) { + case *parse.ListNode: + escapeListNode(n) + case *parse.TextNode: + // Nothing to do. + case *parse.ActionNode: + escapeActionNode(n) + case *parse.IfNode: + escapeIfNode(n) + case *parse.RangeNode: + escapeRangeNode(n) + case *parse.TemplateNode: + // Nothing to do. + case *parse.WithNode: + escapeWithNode(n) + default: + panic("handling for " + node.String() + " not implemented") + // TODO: Handle other inner node types. + } +} + +// escapeListNode recursively escapes its input's children. +func escapeListNode(node *parse.ListNode) { + if node == nil { + return + } + children := node.Nodes + for _, child := range children { + escapeNode(child) + } +} + +// escapeActionNode adds a pipeline call to the end that escapes the result +// of the expression before it is interpolated into the template output. +func escapeActionNode(node *parse.ActionNode) { + pipe := node.Pipe + + cmds := pipe.Cmds + nCmds := len(cmds) + + // If it already has an escaping command, do not interfere. + if nCmds != 0 { + if lastCmd := cmds[nCmds-1]; len(lastCmd.Args) != 0 { + // TODO: Recognize url and js as escaping functions once + // we have enough context to know whether additional + // escaping is necessary. + if arg, ok := lastCmd.Args[0].(*parse.IdentifierNode); ok && arg.Ident == "html" { + return + } + } + } + + htmlEscapeCommand := parse.CommandNode{ + NodeType: parse.NodeCommand, + Args: []parse.Node{parse.NewIdentifier("html")}, + } + + node.Pipe.Cmds = append(node.Pipe.Cmds, &htmlEscapeCommand) +} + +// escapeIfNode recursively escapes the if and then clauses but leaves the +// condition unchanged. +func escapeIfNode(node *parse.IfNode) { + escapeListNode(node.List) + escapeListNode(node.ElseList) +} + +// escapeRangeNode recursively escapes the loop body and else clause but +// leaves the series unchanged. +func escapeRangeNode(node *parse.RangeNode) { + escapeListNode(node.List) + escapeListNode(node.ElseList) +} + +// escapeWithNode recursively escapes the scope body and else clause but +// leaves the pipeline unchanged. +func escapeWithNode(node *parse.WithNode) { + escapeListNode(node.List) + escapeListNode(node.ElseList) +} diff --git a/src/pkg/exp/template/html/escape_test.go b/src/pkg/exp/template/html/escape_test.go new file mode 100644 index 000000000..345a752a8 --- /dev/null +++ b/src/pkg/exp/template/html/escape_test.go @@ -0,0 +1,75 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "bytes" + "template" + "testing" +) + +type data struct { + F, T bool + C, G, H string + A, E []string +} + +var testData = data{ + F: false, + T: true, + C: "", + G: "", + H: "", + A: []string{"", ""}, + E: []string{}, +} + +type testCase struct { + name string + input string + output string +} + +var testCases = []testCase{ + {"if", "{{if .T}}Hello{{end}}, {{.C}}!", "Hello, <Cincinatti>!"}, + {"else", "{{if .F}}{{.H}}{{else}}{{.G}}{{end}}!", "<Goodbye>!"}, + {"overescaping", "Hello, {{.C | html}}!", "Hello, <Cincinatti>!"}, + {"assignment", "{{if $x := .H}}{{$x}}{{end}}", "<Hello>"}, + {"withBody", "{{with .H}}{{.}}{{end}}", "<Hello>"}, + {"withElse", "{{with .E}}{{.}}{{else}}{{.H}}{{end}}", "<Hello>"}, + {"rangeBody", "{{range .A}}{{.}}{{end}}", "<a><b>"}, + {"rangeElse", "{{range .E}}{{.}}{{else}}{{.H}}{{end}}", "<Hello>"}, + {"nonStringValue", "{{.T}}", "true"}, + {"constant", ``, ``}, +} + +func TestAutoesc(t *testing.T) { + for _, testCase := range testCases { + name := testCase.name + tmpl := template.New(name) + tmpl, err := tmpl.Parse(testCase.input) + if err != nil { + t.Errorf("%s: failed to parse template: %s", name, err) + continue + } + + Escape(tmpl) + + buffer := new(bytes.Buffer) + + err = tmpl.Execute(buffer, testData) + if err != nil { + t.Errorf("%s: template execution failed: %s", name, err) + continue + } + + output := testCase.output + actual := buffer.String() + if output != actual { + t.Errorf("%s: escaped output: %q != %q", + name, output, actual) + } + } +} diff --git a/src/pkg/exp/wingui/Makefile b/src/pkg/exp/wingui/Makefile new file mode 100644 index 000000000..00f7d234d --- /dev/null +++ b/src/pkg/exp/wingui/Makefile @@ -0,0 +1,29 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +GOOS=windows + +include ../../../Make.inc + +LD:=$(LD) -Hwindowsgui + +TARG=wingui + +GOFILES=\ + gui.go\ + winapi.go\ + zwinapi.go\ + +include ../../../Make.cmd + +zwinapi.go: winapi.go + $(GOROOT)/src/pkg/syscall/mksyscall_windows.pl $< \ + | sed 's/^package.*syscall$$/package main/' \ + | sed '/^import/a \ + import "syscall"' \ + | sed 's/Syscall/syscall.Syscall/' \ + | sed 's/NewLazyDLL/syscall.NewLazyDLL/' \ + | sed 's/EINVAL/syscall.EINVAL/' \ + | gofmt \ + > $@ diff --git a/src/pkg/exp/wingui/gui.go b/src/pkg/exp/wingui/gui.go new file mode 100644 index 000000000..cf392934c --- /dev/null +++ b/src/pkg/exp/wingui/gui.go @@ -0,0 +1,153 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "syscall" + "os" + "unsafe" +) + +// some help functions + +func abortf(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, format, a...) + os.Exit(1) +} + +func abortErrNo(funcname string, err int) { + abortf("%s failed: %d %s\n", funcname, err, syscall.Errstr(err)) +} + +// global vars + +var ( + mh uint32 + bh uint32 +) + +// WinProc called by windows to notify us of all windows events we might be interested in. +func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr { + var rc int32 + switch msg { + case WM_CREATE: + var e int + // CreateWindowEx + bh, e = CreateWindowEx( + 0, + syscall.StringToUTF16Ptr("button"), + syscall.StringToUTF16Ptr("Quit"), + WS_CHILD|WS_VISIBLE|BS_DEFPUSHBUTTON, + 75, 70, 140, 25, + hwnd, 1, mh, 0) + if e != 0 { + abortErrNo("CreateWindowEx", e) + } + fmt.Printf("button handle is %x\n", bh) + rc = DefWindowProc(hwnd, msg, wparam, lparam) + case WM_COMMAND: + switch uint32(lparam) { + case bh: + e := PostMessage(hwnd, WM_CLOSE, 0, 0) + if e != 0 { + abortErrNo("PostMessage", e) + } + default: + rc = DefWindowProc(hwnd, msg, wparam, lparam) + } + case WM_CLOSE: + DestroyWindow(hwnd) + case WM_DESTROY: + PostQuitMessage(0) + default: + rc = DefWindowProc(hwnd, msg, wparam, lparam) + } + //fmt.Printf("WndProc(0x%08x, %d, 0x%08x, 0x%08x) (%d)\n", hwnd, msg, wparam, lparam, rc) + return uintptr(rc) +} + +func rungui() int { + var e int + + // GetModuleHandle + mh, e = GetModuleHandle(nil) + if e != 0 { + abortErrNo("GetModuleHandle", e) + } + + // Get icon we're going to use. + myicon, e := LoadIcon(0, IDI_APPLICATION) + if e != 0 { + abortErrNo("LoadIcon", e) + } + + // Get cursor we're going to use. + mycursor, e := LoadCursor(0, IDC_ARROW) + if e != 0 { + abortErrNo("LoadCursor", e) + } + + // Create callback + wproc := syscall.NewCallback(WndProc) + + // RegisterClassEx + wcname := syscall.StringToUTF16Ptr("myWindowClass") + var wc Wndclassex + wc.Size = uint32(unsafe.Sizeof(wc)) + wc.WndProc = wproc + wc.Instance = mh + wc.Icon = myicon + wc.Cursor = mycursor + wc.Background = COLOR_BTNFACE + 1 + wc.MenuName = nil + wc.ClassName = wcname + wc.IconSm = myicon + if _, e := RegisterClassEx(&wc); e != 0 { + abortErrNo("RegisterClassEx", e) + } + + // CreateWindowEx + wh, e := CreateWindowEx( + WS_EX_CLIENTEDGE, + wcname, + syscall.StringToUTF16Ptr("My window"), + WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, 300, 200, + 0, 0, mh, 0) + if e != 0 { + abortErrNo("CreateWindowEx", e) + } + fmt.Printf("main window handle is %x\n", wh) + + // ShowWindow + ShowWindow(wh, SW_SHOWDEFAULT) + + // UpdateWindow + if e := UpdateWindow(wh); e != 0 { + abortErrNo("UpdateWindow", e) + } + + // Process all windows messages until WM_QUIT. + var m Msg + for { + r, e := GetMessage(&m, 0, 0, 0) + if e != 0 { + abortErrNo("GetMessage", e) + } + if r == 0 { + // WM_QUIT received -> get out + break + } + TranslateMessage(&m) + DispatchMessage(&m) + } + return int(m.Wparam) +} + +func main() { + rc := rungui() + os.Exit(rc) +} diff --git a/src/pkg/exp/wingui/winapi.go b/src/pkg/exp/wingui/winapi.go new file mode 100644 index 000000000..31b57a2cc --- /dev/null +++ b/src/pkg/exp/wingui/winapi.go @@ -0,0 +1,131 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "unsafe" +) + +type Wndclassex struct { + Size uint32 + Style uint32 + WndProc uintptr + ClsExtra int32 + WndExtra int32 + Instance uint32 + Icon uint32 + Cursor uint32 + Background uint32 + MenuName *uint16 + ClassName *uint16 + IconSm uint32 +} + +type Point struct { + X int32 + Y int32 +} + +type Msg struct { + Hwnd uint32 + Message uint32 + Wparam int32 + Lparam int32 + Time uint32 + Pt Point +} + +const ( + // Window styles + WS_OVERLAPPED = 0 + WS_POPUP = 0x80000000 + WS_CHILD = 0x40000000 + WS_MINIMIZE = 0x20000000 + WS_VISIBLE = 0x10000000 + WS_DISABLED = 0x8000000 + WS_CLIPSIBLINGS = 0x4000000 + WS_CLIPCHILDREN = 0x2000000 + WS_MAXIMIZE = 0x1000000 + WS_CAPTION = WS_BORDER | WS_DLGFRAME + WS_BORDER = 0x800000 + WS_DLGFRAME = 0x400000 + WS_VSCROLL = 0x200000 + WS_HSCROLL = 0x100000 + WS_SYSMENU = 0x80000 + WS_THICKFRAME = 0x40000 + WS_GROUP = 0x20000 + WS_TABSTOP = 0x10000 + WS_MINIMIZEBOX = 0x20000 + WS_MAXIMIZEBOX = 0x10000 + WS_TILED = WS_OVERLAPPED + WS_ICONIC = WS_MINIMIZE + WS_SIZEBOX = WS_THICKFRAME + // Common Window Styles + WS_OVERLAPPEDWINDOW = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_THICKFRAME | WS_MINIMIZEBOX | WS_MAXIMIZEBOX + WS_TILEDWINDOW = WS_OVERLAPPEDWINDOW + WS_POPUPWINDOW = WS_POPUP | WS_BORDER | WS_SYSMENU + WS_CHILDWINDOW = WS_CHILD + + WS_EX_CLIENTEDGE = 0x200 + + // Some windows messages + WM_CREATE = 1 + WM_DESTROY = 2 + WM_CLOSE = 16 + WM_COMMAND = 273 + + // Some button control styles + BS_DEFPUSHBUTTON = 1 + + // Some color constants + COLOR_WINDOW = 5 + COLOR_BTNFACE = 15 + + // Default window position + CW_USEDEFAULT = 0x80000000 - 0x100000000 + + // Show window default style + SW_SHOWDEFAULT = 10 +) + +var ( + // Some globally known cursors + IDC_ARROW = MakeIntResource(32512) + IDC_IBEAM = MakeIntResource(32513) + IDC_WAIT = MakeIntResource(32514) + IDC_CROSS = MakeIntResource(32515) + + // Some globally known icons + IDI_APPLICATION = MakeIntResource(32512) + IDI_HAND = MakeIntResource(32513) + IDI_QUESTION = MakeIntResource(32514) + IDI_EXCLAMATION = MakeIntResource(32515) + IDI_ASTERISK = MakeIntResource(32516) + IDI_WINLOGO = MakeIntResource(32517) + IDI_WARNING = IDI_EXCLAMATION + IDI_ERROR = IDI_HAND + IDI_INFORMATION = IDI_ASTERISK +) + +//sys GetModuleHandle(modname *uint16) (handle uint32, errno int) = GetModuleHandleW +//sys RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) = user32.RegisterClassExW +//sys CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) = user32.CreateWindowExW +//sys DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.DefWindowProcW +//sys DestroyWindow(hwnd uint32) (errno int) = user32.DestroyWindow +//sys PostQuitMessage(exitcode int32) = user32.PostQuitMessage +//sys ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) = user32.ShowWindow +//sys UpdateWindow(hwnd uint32) (errno int) = user32.UpdateWindow +//sys GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) [failretval==-1] = user32.GetMessageW +//sys TranslateMessage(msg *Msg) (done bool) = user32.TranslateMessage +//sys DispatchMessage(msg *Msg) (ret int32) = user32.DispatchMessageW +//sys LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) = user32.LoadIconW +//sys LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) = user32.LoadCursorW +//sys SetCursor(cursor uint32) (precursor uint32, errno int) = user32.SetCursor +//sys SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.SendMessageW +//sys PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) = user32.PostMessageW + +func MakeIntResource(id uint16) *uint16 { + return (*uint16)(unsafe.Pointer(uintptr(id))) +} diff --git a/src/pkg/exp/wingui/zwinapi.go b/src/pkg/exp/wingui/zwinapi.go new file mode 100644 index 000000000..4c009dd69 --- /dev/null +++ b/src/pkg/exp/wingui/zwinapi.go @@ -0,0 +1,211 @@ +// mksyscall_windows.pl winapi.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package main + +import "unsafe" +import "syscall" + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + moduser32 = syscall.NewLazyDLL("user32.dll") + + procGetModuleHandleW = modkernel32.NewProc("GetModuleHandleW") + procRegisterClassExW = moduser32.NewProc("RegisterClassExW") + procCreateWindowExW = moduser32.NewProc("CreateWindowExW") + procDefWindowProcW = moduser32.NewProc("DefWindowProcW") + procDestroyWindow = moduser32.NewProc("DestroyWindow") + procPostQuitMessage = moduser32.NewProc("PostQuitMessage") + procShowWindow = moduser32.NewProc("ShowWindow") + procUpdateWindow = moduser32.NewProc("UpdateWindow") + procGetMessageW = moduser32.NewProc("GetMessageW") + procTranslateMessage = moduser32.NewProc("TranslateMessage") + procDispatchMessageW = moduser32.NewProc("DispatchMessageW") + procLoadIconW = moduser32.NewProc("LoadIconW") + procLoadCursorW = moduser32.NewProc("LoadCursorW") + procSetCursor = moduser32.NewProc("SetCursor") + procSendMessageW = moduser32.NewProc("SendMessageW") + procPostMessageW = moduser32.NewProc("PostMessageW") +) + +func GetModuleHandle(modname *uint16) (handle uint32, errno int) { + r0, _, e1 := syscall.Syscall(procGetModuleHandleW.Addr(), 1, uintptr(unsafe.Pointer(modname)), 0, 0) + handle = uint32(r0) + if handle == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) { + r0, _, e1 := syscall.Syscall(procRegisterClassExW.Addr(), 1, uintptr(unsafe.Pointer(wndclass)), 0, 0) + atom = uint16(r0) + if atom == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) { + r0, _, e1 := syscall.Syscall12(procCreateWindowExW.Addr(), 12, uintptr(exstyle), uintptr(unsafe.Pointer(classname)), uintptr(unsafe.Pointer(windowname)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height), uintptr(wndparent), uintptr(menu), uintptr(instance), uintptr(param)) + hwnd = uint32(r0) + if hwnd == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { + r0, _, _ := syscall.Syscall6(procDefWindowProcW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + lresult = int32(r0) + return +} + +func DestroyWindow(hwnd uint32) (errno int) { + r1, _, e1 := syscall.Syscall(procDestroyWindow.Addr(), 1, uintptr(hwnd), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func PostQuitMessage(exitcode int32) { + syscall.Syscall(procPostQuitMessage.Addr(), 1, uintptr(exitcode), 0, 0) + return +} + +func ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) { + r0, _, _ := syscall.Syscall(procShowWindow.Addr(), 2, uintptr(hwnd), uintptr(cmdshow), 0) + wasvisible = bool(r0 != 0) + return +} + +func UpdateWindow(hwnd uint32) (errno int) { + r1, _, e1 := syscall.Syscall(procUpdateWindow.Addr(), 1, uintptr(hwnd), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) { + r0, _, e1 := syscall.Syscall6(procGetMessageW.Addr(), 4, uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(MsgFilterMin), uintptr(MsgFilterMax), 0, 0) + ret = int32(r0) + if ret == -1 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func TranslateMessage(msg *Msg) (done bool) { + r0, _, _ := syscall.Syscall(procTranslateMessage.Addr(), 1, uintptr(unsafe.Pointer(msg)), 0, 0) + done = bool(r0 != 0) + return +} + +func DispatchMessage(msg *Msg) (ret int32) { + r0, _, _ := syscall.Syscall(procDispatchMessageW.Addr(), 1, uintptr(unsafe.Pointer(msg)), 0, 0) + ret = int32(r0) + return +} + +func LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) { + r0, _, e1 := syscall.Syscall(procLoadIconW.Addr(), 2, uintptr(instance), uintptr(unsafe.Pointer(iconname)), 0) + icon = uint32(r0) + if icon == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) { + r0, _, e1 := syscall.Syscall(procLoadCursorW.Addr(), 2, uintptr(instance), uintptr(unsafe.Pointer(cursorname)), 0) + cursor = uint32(r0) + if cursor == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func SetCursor(cursor uint32) (precursor uint32, errno int) { + r0, _, e1 := syscall.Syscall(procSetCursor.Addr(), 1, uintptr(cursor), 0, 0) + precursor = uint32(r0) + if precursor == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { + r0, _, _ := syscall.Syscall6(procSendMessageW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + lresult = int32(r0) + return +} + +func PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) { + r1, _, e1 := syscall.Syscall6(procPostMessageW.Addr(), 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} diff --git a/src/pkg/expvar/Makefile b/src/pkg/expvar/Makefile new file mode 100644 index 000000000..5619630d1 --- /dev/null +++ b/src/pkg/expvar/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=expvar +GOFILES=\ + expvar.go\ + +include ../../Make.pkg diff --git a/src/pkg/expvar/expvar.go b/src/pkg/expvar/expvar.go new file mode 100644 index 000000000..7b733faf6 --- /dev/null +++ b/src/pkg/expvar/expvar.go @@ -0,0 +1,287 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package expvar provides a standardized interface to public variables, such +// as operation counters in servers. It exposes these variables via HTTP at +// /debug/vars in JSON format. +// +// Operations to set or modify these public variables are atomic. +// +// In addition to adding the HTTP handler, this package registers the +// following variables: +// +// cmdline os.Args +// memstats runtime.Memstats +// +// The package is sometimes only imported for the side effect of +// registering its HTTP handler and the above variables. To use it +// this way, simply link this package into your program: +// import _ "expvar" +// +package expvar + +import ( + "bytes" + "fmt" + "http" + "json" + "log" + "os" + "runtime" + "strconv" + "sync" +) + +// Var is an abstract type for all exported variables. +type Var interface { + String() string +} + +// Int is a 64-bit integer variable that satisfies the Var interface. +type Int struct { + i int64 + mu sync.Mutex +} + +func (v *Int) String() string { return strconv.Itoa64(v.i) } + +func (v *Int) Add(delta int64) { + v.mu.Lock() + defer v.mu.Unlock() + v.i += delta +} + +func (v *Int) Set(value int64) { + v.mu.Lock() + defer v.mu.Unlock() + v.i = value +} + +// Float is a 64-bit float variable that satisfies the Var interface. +type Float struct { + f float64 + mu sync.Mutex +} + +func (v *Float) String() string { return strconv.Ftoa64(v.f, 'g', -1) } + +// Add adds delta to v. +func (v *Float) Add(delta float64) { + v.mu.Lock() + defer v.mu.Unlock() + v.f += delta +} + +// Set sets v to value. +func (v *Float) Set(value float64) { + v.mu.Lock() + defer v.mu.Unlock() + v.f = value +} + +// Map is a string-to-Var map variable that satisfies the Var interface. +type Map struct { + m map[string]Var + mu sync.Mutex +} + +// KeyValue represents a single entry in a Map. +type KeyValue struct { + Key string + Value Var +} + +func (v *Map) String() string { + v.mu.Lock() + defer v.mu.Unlock() + b := new(bytes.Buffer) + fmt.Fprintf(b, "{") + first := true + for key, val := range v.m { + if !first { + fmt.Fprintf(b, ", ") + } + fmt.Fprintf(b, "\"%s\": %v", key, val.String()) + first = false + } + fmt.Fprintf(b, "}") + return b.String() +} + +func (v *Map) Init() *Map { + v.m = make(map[string]Var) + return v +} + +func (v *Map) Get(key string) Var { + v.mu.Lock() + defer v.mu.Unlock() + return v.m[key] +} + +func (v *Map) Set(key string, av Var) { + v.mu.Lock() + defer v.mu.Unlock() + v.m[key] = av +} + +func (v *Map) Add(key string, delta int64) { + v.mu.Lock() + defer v.mu.Unlock() + av, ok := v.m[key] + if !ok { + av = new(Int) + v.m[key] = av + } + + // Add to Int; ignore otherwise. + if iv, ok := av.(*Int); ok { + iv.Add(delta) + } +} + +// AddFloat adds delta to the *Float value stored under the given map key. +func (v *Map) AddFloat(key string, delta float64) { + v.mu.Lock() + defer v.mu.Unlock() + av, ok := v.m[key] + if !ok { + av = new(Float) + v.m[key] = av + } + + // Add to Float; ignore otherwise. + if iv, ok := av.(*Float); ok { + iv.Add(delta) + } +} + +// TODO(rsc): Make sure map access in separate thread is safe. +func (v *Map) iterate(c chan<- KeyValue) { + for k, v := range v.m { + c <- KeyValue{k, v} + } + close(c) +} + +func (v *Map) Iter() <-chan KeyValue { + c := make(chan KeyValue) + go v.iterate(c) + return c +} + +// String is a string variable, and satisfies the Var interface. +type String struct { + s string +} + +func (v *String) String() string { return strconv.Quote(v.s) } + +func (v *String) Set(value string) { v.s = value } + +// Func implements Var by calling the function +// and formatting the returned value using JSON. +type Func func() interface{} + +func (f Func) String() string { + v, _ := json.Marshal(f()) + return string(v) +} + +// All published variables. +var vars map[string]Var = make(map[string]Var) +var mutex sync.Mutex + +// Publish declares an named exported variable. This should be called from a +// package's init function when it creates its Vars. If the name is already +// registered then this will log.Panic. +func Publish(name string, v Var) { + mutex.Lock() + defer mutex.Unlock() + if _, existing := vars[name]; existing { + log.Panicln("Reuse of exported var name:", name) + } + vars[name] = v +} + +// Get retrieves a named exported variable. +func Get(name string) Var { + return vars[name] +} + +// RemoveAll removes all exported variables. +// This is for tests; don't call this on a real server. +func RemoveAll() { + mutex.Lock() + defer mutex.Unlock() + vars = make(map[string]Var) +} + +// Convenience functions for creating new exported variables. + +func NewInt(name string) *Int { + v := new(Int) + Publish(name, v) + return v +} + +func NewFloat(name string) *Float { + v := new(Float) + Publish(name, v) + return v +} + +func NewMap(name string) *Map { + v := new(Map).Init() + Publish(name, v) + return v +} + +func NewString(name string) *String { + v := new(String) + Publish(name, v) + return v +} + +// TODO(rsc): Make sure map access in separate thread is safe. +func iterate(c chan<- KeyValue) { + for k, v := range vars { + c <- KeyValue{k, v} + } + close(c) +} + +func Iter() <-chan KeyValue { + c := make(chan KeyValue) + go iterate(c) + return c +} + +func expvarHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, "{\n") + first := true + for name, value := range vars { + if !first { + fmt.Fprintf(w, ",\n") + } + first = false + fmt.Fprintf(w, "%q: %s", name, value) + } + fmt.Fprintf(w, "\n}\n") +} + +func cmdline() interface{} { + return os.Args +} + +func memstats() interface{} { + return runtime.MemStats +} + +func init() { + http.Handle("/debug/vars", http.HandlerFunc(expvarHandler)) + Publish("cmdline", Func(cmdline)) + Publish("memstats", Func(memstats)) +} diff --git a/src/pkg/expvar/expvar_test.go b/src/pkg/expvar/expvar_test.go new file mode 100644 index 000000000..8f7a48168 --- /dev/null +++ b/src/pkg/expvar/expvar_test.go @@ -0,0 +1,128 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package expvar + +import ( + "json" + "testing" +) + +func TestInt(t *testing.T) { + reqs := NewInt("requests") + if reqs.i != 0 { + t.Errorf("reqs.i = %v, want 0", reqs.i) + } + if reqs != Get("requests").(*Int) { + t.Errorf("Get() failed.") + } + + reqs.Add(1) + reqs.Add(3) + if reqs.i != 4 { + t.Errorf("reqs.i = %v, want 4", reqs.i) + } + + if s := reqs.String(); s != "4" { + t.Errorf("reqs.String() = %q, want \"4\"", s) + } + + reqs.Set(-2) + if reqs.i != -2 { + t.Errorf("reqs.i = %v, want -2", reqs.i) + } +} + +func TestFloat(t *testing.T) { + reqs := NewFloat("requests-float") + if reqs.f != 0.0 { + t.Errorf("reqs.f = %v, want 0", reqs.f) + } + if reqs != Get("requests-float").(*Float) { + t.Errorf("Get() failed.") + } + + reqs.Add(1.5) + reqs.Add(1.25) + if reqs.f != 2.75 { + t.Errorf("reqs.f = %v, want 2.75", reqs.f) + } + + if s := reqs.String(); s != "2.75" { + t.Errorf("reqs.String() = %q, want \"4.64\"", s) + } + + reqs.Add(-2) + if reqs.f != 0.75 { + t.Errorf("reqs.f = %v, want 0.75", reqs.f) + } +} + +func TestString(t *testing.T) { + name := NewString("my-name") + if name.s != "" { + t.Errorf("name.s = %q, want \"\"", name.s) + } + + name.Set("Mike") + if name.s != "Mike" { + t.Errorf("name.s = %q, want \"Mike\"", name.s) + } + + if s := name.String(); s != "\"Mike\"" { + t.Errorf("reqs.String() = %q, want \"\"Mike\"\"", s) + } +} + +func TestMapCounter(t *testing.T) { + colors := NewMap("bike-shed-colors") + + colors.Add("red", 1) + colors.Add("red", 2) + colors.Add("blue", 4) + colors.AddFloat("green", 4.125) + if x := colors.m["red"].(*Int).i; x != 3 { + t.Errorf("colors.m[\"red\"] = %v, want 3", x) + } + if x := colors.m["blue"].(*Int).i; x != 4 { + t.Errorf("colors.m[\"blue\"] = %v, want 4", x) + } + if x := colors.m["green"].(*Float).f; x != 4.125 { + t.Errorf("colors.m[\"green\"] = %v, want 3.14", x) + } + + // colors.String() should be '{"red":3, "blue":4}', + // though the order of red and blue could vary. + s := colors.String() + var j interface{} + err := json.Unmarshal([]byte(s), &j) + if err != nil { + t.Errorf("colors.String() isn't valid JSON: %v", err) + } + m, ok := j.(map[string]interface{}) + if !ok { + t.Error("colors.String() didn't produce a map.") + } + red := m["red"] + x, ok := red.(float64) + if !ok { + t.Error("red.Kind() is not a number.") + } + if x != 3 { + t.Errorf("red = %v, want 3", x) + } +} + +func TestFunc(t *testing.T) { + var x interface{} = []string{"a", "b"} + f := Func(func() interface{} { return x }) + if s, exp := f.String(), `["a","b"]`; s != exp { + t.Errorf(`f.String() = %q, want %q`, s, exp) + } + + x = 17 + if s, exp := f.String(), `17`; s != exp { + t.Errorf(`f.String() = %q, want %q`, s, exp) + } +} diff --git a/src/pkg/flag/Makefile b/src/pkg/flag/Makefile new file mode 100644 index 000000000..3408ca474 --- /dev/null +++ b/src/pkg/flag/Makefile @@ -0,0 +1,11 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=flag +GOFILES=\ + flag.go\ + +include ../../Make.pkg diff --git a/src/pkg/flag/export_test.go b/src/pkg/flag/export_test.go new file mode 100644 index 000000000..7b190807a --- /dev/null +++ b/src/pkg/flag/export_test.go @@ -0,0 +1,22 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flag + +import "os" + +// Additional routines compiled into the package only during testing. + +// ResetForTesting clears all flag state and sets the usage function as directed. +// After calling ResetForTesting, parse errors in flag handling will not +// exit the program. +func ResetForTesting(usage func()) { + commandLine = NewFlagSet(os.Args[0], ContinueOnError) + Usage = usage +} + +// CommandLine returns the default FlagSet. +func CommandLine() *FlagSet { + return commandLine +} diff --git a/src/pkg/flag/flag.go b/src/pkg/flag/flag.go new file mode 100644 index 000000000..01bbc3770 --- /dev/null +++ b/src/pkg/flag/flag.go @@ -0,0 +1,705 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package flag implements command-line flag parsing. + + Usage: + + Define flags using flag.String(), Bool(), Int(), etc. Example: + import "flag" + var ip *int = flag.Int("flagname", 1234, "help message for flagname") + If you like, you can bind the flag to a variable using the Var() functions. + var flagvar int + func init() { + flag.IntVar(&flagvar, "flagname", 1234, "help message for flagname") + } + Or you can create custom flags that satisfy the Value interface (with + pointer receivers) and couple them to flag parsing by + flag.Var(&flagVal, "name", "help message for flagname") + For such flags, the default value is just the initial value of the variable. + + After all flags are defined, call + flag.Parse() + to parse the command line into the defined flags. + + Flags may then be used directly. If you're using the flags themselves, + they are all pointers; if you bind to variables, they're values. + fmt.Println("ip has value ", *ip); + fmt.Println("flagvar has value ", flagvar); + + After parsing, the arguments after the flag are available as the + slice flag.Args() or individually as flag.Arg(i). + The arguments are indexed from 0 up to flag.NArg(). + + Command line flag syntax: + -flag + -flag=x + -flag x // non-boolean flags only + One or two minus signs may be used; they are equivalent. + The last form is not permitted for boolean flags because the + meaning of the command + cmd -x * + will change if there is a file called 0, false, etc. You must + use the -flag=false form to turn off a boolean flag. + + Flag parsing stops just before the first non-flag argument + ("-" is a non-flag argument) or after the terminator "--". + + Integer flags accept 1234, 0664, 0x1234 and may be negative. + Boolean flags may be 1, 0, t, f, true, false, TRUE, FALSE, True, False. + + The default set of command-line flags is controlled by + top-level functions. The FlagSet type allows one to define + independent sets of flags, such as to implement subcommands + in a command-line interface. The methods of FlagSet are + analogous to the top-level functions for the command-line + flag set. +*/ +package flag + +import ( + "fmt" + "os" + "sort" + "strconv" +) + +// ErrHelp is the error returned if the flag -help is invoked but no such flag is defined. +var ErrHelp = os.NewError("flag: help requested") + +// -- Bool Value +type boolValue bool + +func newBoolValue(val bool, p *bool) *boolValue { + *p = val + return (*boolValue)(p) +} + +func (b *boolValue) Set(s string) bool { + v, err := strconv.Atob(s) + *b = boolValue(v) + return err == nil +} + +func (b *boolValue) String() string { return fmt.Sprintf("%v", *b) } + +// -- Int Value +type intValue int + +func newIntValue(val int, p *int) *intValue { + *p = val + return (*intValue)(p) +} + +func (i *intValue) Set(s string) bool { + v, err := strconv.Btoi64(s, 0) + *i = intValue(v) + return err == nil +} + +func (i *intValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- Int64 Value +type int64Value int64 + +func newInt64Value(val int64, p *int64) *int64Value { + *p = val + return (*int64Value)(p) +} + +func (i *int64Value) Set(s string) bool { + v, err := strconv.Btoi64(s, 0) + *i = int64Value(v) + return err == nil +} + +func (i *int64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- Uint Value +type uintValue uint + +func newUintValue(val uint, p *uint) *uintValue { + *p = val + return (*uintValue)(p) +} + +func (i *uintValue) Set(s string) bool { + v, err := strconv.Btoui64(s, 0) + *i = uintValue(v) + return err == nil +} + +func (i *uintValue) String() string { return fmt.Sprintf("%v", *i) } + +// -- uint64 Value +type uint64Value uint64 + +func newUint64Value(val uint64, p *uint64) *uint64Value { + *p = val + return (*uint64Value)(p) +} + +func (i *uint64Value) Set(s string) bool { + v, err := strconv.Btoui64(s, 0) + *i = uint64Value(v) + return err == nil +} + +func (i *uint64Value) String() string { return fmt.Sprintf("%v", *i) } + +// -- string Value +type stringValue string + +func newStringValue(val string, p *string) *stringValue { + *p = val + return (*stringValue)(p) +} + +func (s *stringValue) Set(val string) bool { + *s = stringValue(val) + return true +} + +func (s *stringValue) String() string { return fmt.Sprintf("%s", *s) } + +// -- Float64 Value +type float64Value float64 + +func newFloat64Value(val float64, p *float64) *float64Value { + *p = val + return (*float64Value)(p) +} + +func (f *float64Value) Set(s string) bool { + v, err := strconv.Atof64(s) + *f = float64Value(v) + return err == nil +} + +func (f *float64Value) String() string { return fmt.Sprintf("%v", *f) } + +// Value is the interface to the dynamic value stored in a flag. +// (The default value is represented as a string.) +type Value interface { + String() string + Set(string) bool +} + +// ErrorHandling defines how to handle flag parsing errors. +type ErrorHandling int + +const ( + ContinueOnError ErrorHandling = iota + ExitOnError + PanicOnError +) + +// A FlagSet represents a set of defined flags. +type FlagSet struct { + // Usage is the function called when an error occurs while parsing flags. + // The field is a function (not a method) that may be changed to point to + // a custom error handler. + Usage func() + + name string + actual map[string]*Flag + formal map[string]*Flag + args []string // arguments after flags + exitOnError bool // does the program exit if there's an error? + errorHandling ErrorHandling +} + +// A Flag represents the state of a flag. +type Flag struct { + Name string // name as it appears on command line + Usage string // help message + Value Value // value as set + DefValue string // default value (as text); for usage message +} + +// sortFlags returns the flags as a slice in lexicographical sorted order. +func sortFlags(flags map[string]*Flag) []*Flag { + list := make(sort.StringSlice, len(flags)) + i := 0 + for _, f := range flags { + list[i] = f.Name + i++ + } + list.Sort() + result := make([]*Flag, len(list)) + for i, name := range list { + result[i] = flags[name] + } + return result +} + +// VisitAll visits the flags in lexicographical order, calling fn for each. +// It visits all flags, even those not set. +func (f *FlagSet) VisitAll(fn func(*Flag)) { + for _, flag := range sortFlags(f.formal) { + fn(flag) + } +} + +// VisitAll visits the command-line flags in lexicographical order, calling +// fn for each. It visits all flags, even those not set. +func VisitAll(fn func(*Flag)) { + commandLine.VisitAll(fn) +} + +// Visit visits the flags in lexicographical order, calling fn for each. +// It visits only those flags that have been set. +func (f *FlagSet) Visit(fn func(*Flag)) { + for _, flag := range sortFlags(f.actual) { + fn(flag) + } +} + +// Visit visits the command-line flags in lexicographical order, calling fn +// for each. It visits only those flags that have been set. +func Visit(fn func(*Flag)) { + commandLine.Visit(fn) +} + +// Lookup returns the Flag structure of the named flag, returning nil if none exists. +func (f *FlagSet) Lookup(name string) *Flag { + return f.formal[name] +} + +// Lookup returns the Flag structure of the named command-line flag, +// returning nil if none exists. +func Lookup(name string) *Flag { + return commandLine.formal[name] +} + +// Set sets the value of the named flag. It returns true if the set succeeded; false if +// there is no such flag defined. +func (f *FlagSet) Set(name, value string) bool { + flag, ok := f.formal[name] + if !ok { + return false + } + ok = flag.Value.Set(value) + if !ok { + return false + } + f.actual[name] = flag + return true +} + +// Set sets the value of the named command-line flag. It returns true if the +// set succeeded; false if there is no such flag defined. +func Set(name, value string) bool { + return commandLine.Set(name, value) +} + +// PrintDefaults prints to standard error the default values of all defined flags in the set. +func (f *FlagSet) PrintDefaults() { + f.VisitAll(func(f *Flag) { + format := " -%s=%s: %s\n" + if _, ok := f.Value.(*stringValue); ok { + // put quotes on the value + format = " -%s=%q: %s\n" + } + fmt.Fprintf(os.Stderr, format, f.Name, f.DefValue, f.Usage) + }) +} + +// PrintDefaults prints to standard error the default values of all defined command-line flags. +func PrintDefaults() { + commandLine.PrintDefaults() +} + +// defaultUsage is the default function to print a usage message. +func defaultUsage(f *FlagSet) { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", f.name) + f.PrintDefaults() +} + +// Usage prints to standard error a usage message documenting all defined command-line flags. +// The function is a variable that may be changed to point to a custom function. +var Usage = func() { + defaultUsage(commandLine) +} + +// NFlag returns the number of flags that have been set. +func (f *FlagSet) NFlag() int { return len(f.actual) } + +// NFlag returns the number of command-line flags that have been set. +func NFlag() int { return len(commandLine.actual) } + +// Arg returns the i'th argument. Arg(0) is the first remaining argument +// after flags have been processed. +func (f *FlagSet) Arg(i int) string { + if i < 0 || i >= len(f.args) { + return "" + } + return f.args[i] +} + +// Arg returns the i'th command-line argument. Arg(0) is the first remaining argument +// after flags have been processed. +func Arg(i int) string { + return commandLine.Arg(i) +} + +// NArg is the number of arguments remaining after flags have been processed. +func (f *FlagSet) NArg() int { return len(f.args) } + +// NArg is the number of arguments remaining after flags have been processed. +func NArg() int { return len(commandLine.args) } + +// Args returns the non-flag arguments. +func (f *FlagSet) Args() []string { return f.args } + +// Args returns the non-flag command-line arguments. +func Args() []string { return commandLine.args } + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func (f *FlagSet) BoolVar(p *bool, name string, value bool, usage string) { + f.Var(newBoolValue(value, p), name, usage) +} + +// BoolVar defines a bool flag with specified name, default value, and usage string. +// The argument p points to a bool variable in which to store the value of the flag. +func BoolVar(p *bool, name string, value bool, usage string) { + commandLine.Var(newBoolValue(value, p), name, usage) +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func (f *FlagSet) Bool(name string, value bool, usage string) *bool { + p := new(bool) + f.BoolVar(p, name, value, usage) + return p +} + +// Bool defines a bool flag with specified name, default value, and usage string. +// The return value is the address of a bool variable that stores the value of the flag. +func Bool(name string, value bool, usage string) *bool { + return commandLine.Bool(name, value, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func (f *FlagSet) IntVar(p *int, name string, value int, usage string) { + f.Var(newIntValue(value, p), name, usage) +} + +// IntVar defines an int flag with specified name, default value, and usage string. +// The argument p points to an int variable in which to store the value of the flag. +func IntVar(p *int, name string, value int, usage string) { + commandLine.Var(newIntValue(value, p), name, usage) +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func (f *FlagSet) Int(name string, value int, usage string) *int { + p := new(int) + f.IntVar(p, name, value, usage) + return p +} + +// Int defines an int flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +func Int(name string, value int, usage string) *int { + return commandLine.Int(name, value, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func (f *FlagSet) Int64Var(p *int64, name string, value int64, usage string) { + f.Var(newInt64Value(value, p), name, usage) +} + +// Int64Var defines an int64 flag with specified name, default value, and usage string. +// The argument p points to an int64 variable in which to store the value of the flag. +func Int64Var(p *int64, name string, value int64, usage string) { + commandLine.Var(newInt64Value(value, p), name, usage) +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func (f *FlagSet) Int64(name string, value int64, usage string) *int64 { + p := new(int64) + f.Int64Var(p, name, value, usage) + return p +} + +// Int64 defines an int64 flag with specified name, default value, and usage string. +// The return value is the address of an int64 variable that stores the value of the flag. +func Int64(name string, value int64, usage string) *int64 { + return commandLine.Int64(name, value, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func (f *FlagSet) UintVar(p *uint, name string, value uint, usage string) { + f.Var(newUintValue(value, p), name, usage) +} + +// UintVar defines a uint flag with specified name, default value, and usage string. +// The argument p points to a uint variable in which to store the value of the flag. +func UintVar(p *uint, name string, value uint, usage string) { + commandLine.Var(newUintValue(value, p), name, usage) +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func (f *FlagSet) Uint(name string, value uint, usage string) *uint { + p := new(uint) + f.UintVar(p, name, value, usage) + return p +} + +// Uint defines a uint flag with specified name, default value, and usage string. +// The return value is the address of a uint variable that stores the value of the flag. +func Uint(name string, value uint, usage string) *uint { + return commandLine.Uint(name, value, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func (f *FlagSet) Uint64Var(p *uint64, name string, value uint64, usage string) { + f.Var(newUint64Value(value, p), name, usage) +} + +// Uint64Var defines a uint64 flag with specified name, default value, and usage string. +// The argument p points to a uint64 variable in which to store the value of the flag. +func Uint64Var(p *uint64, name string, value uint64, usage string) { + commandLine.Var(newUint64Value(value, p), name, usage) +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func (f *FlagSet) Uint64(name string, value uint64, usage string) *uint64 { + p := new(uint64) + f.Uint64Var(p, name, value, usage) + return p +} + +// Uint64 defines a uint64 flag with specified name, default value, and usage string. +// The return value is the address of a uint64 variable that stores the value of the flag. +func Uint64(name string, value uint64, usage string) *uint64 { + return commandLine.Uint64(name, value, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func (f *FlagSet) StringVar(p *string, name string, value string, usage string) { + f.Var(newStringValue(value, p), name, usage) +} + +// StringVar defines a string flag with specified name, default value, and usage string. +// The argument p points to a string variable in which to store the value of the flag. +func StringVar(p *string, name string, value string, usage string) { + commandLine.Var(newStringValue(value, p), name, usage) +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func (f *FlagSet) String(name string, value string, usage string) *string { + p := new(string) + f.StringVar(p, name, value, usage) + return p +} + +// String defines a string flag with specified name, default value, and usage string. +// The return value is the address of a string variable that stores the value of the flag. +func String(name string, value string, usage string) *string { + return commandLine.String(name, value, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func (f *FlagSet) Float64Var(p *float64, name string, value float64, usage string) { + f.Var(newFloat64Value(value, p), name, usage) +} + +// Float64Var defines a float64 flag with specified name, default value, and usage string. +// The argument p points to a float64 variable in which to store the value of the flag. +func Float64Var(p *float64, name string, value float64, usage string) { + commandLine.Var(newFloat64Value(value, p), name, usage) +} + +// Float64 defines a float64 flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func (f *FlagSet) Float64(name string, value float64, usage string) *float64 { + p := new(float64) + f.Float64Var(p, name, value, usage) + return p +} + +// Float64 defines an int flag with specified name, default value, and usage string. +// The return value is the address of a float64 variable that stores the value of the flag. +func Float64(name string, value float64, usage string) *float64 { + return commandLine.Float64(name, value, usage) +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func (f *FlagSet) Var(value Value, name string, usage string) { + // Remember the default value as a string; it won't change. + flag := &Flag{name, usage, value, value.String()} + _, alreadythere := f.formal[name] + if alreadythere { + fmt.Fprintf(os.Stderr, "%s flag redefined: %s\n", f.name, name) + panic("flag redefinition") // Happens only if flags are declared with identical names + } + f.formal[name] = flag +} + +// Var defines a flag with the specified name and usage string. The type and +// value of the flag are represented by the first argument, of type Value, which +// typically holds a user-defined implementation of Value. For instance, the +// caller could create a flag that turns a comma-separated string into a slice +// of strings by giving the slice the methods of Value; in particular, Set would +// decompose the comma-separated string into the slice. +func Var(value Value, name string, usage string) { + commandLine.Var(value, name, usage) +} + +// failf prints to standard error a formatted error and usage message and +// returns the error. +func (f *FlagSet) failf(format string, a ...interface{}) os.Error { + err := fmt.Errorf(format, a...) + fmt.Fprintln(os.Stderr, err) + f.usage() + return err +} + +// usage calls the Usage method for the flag set, or the usage function if +// the flag set is commandLine. +func (f *FlagSet) usage() { + if f == commandLine { + Usage() + } else { + f.Usage() + } +} + +// parseOne parses one flag. It returns whether a flag was seen. +func (f *FlagSet) parseOne() (bool, os.Error) { + if len(f.args) == 0 { + return false, nil + } + s := f.args[0] + if len(s) == 0 || s[0] != '-' || len(s) == 1 { + return false, nil + } + num_minuses := 1 + if s[1] == '-' { + num_minuses++ + if len(s) == 2 { // "--" terminates the flags + f.args = f.args[1:] + return false, nil + } + } + name := s[num_minuses:] + if len(name) == 0 || name[0] == '-' || name[0] == '=' { + return false, f.failf("bad flag syntax: %s", s) + } + + // it's a flag. does it have an argument? + f.args = f.args[1:] + has_value := false + value := "" + for i := 1; i < len(name); i++ { // equals cannot be first + if name[i] == '=' { + value = name[i+1:] + has_value = true + name = name[0:i] + break + } + } + m := f.formal + flag, alreadythere := m[name] // BUG + if !alreadythere { + if name == "help" || name == "h" { // special case for nice help message. + f.usage() + return false, ErrHelp + } + return false, f.failf("flag provided but not defined: -%s", name) + } + if fv, ok := flag.Value.(*boolValue); ok { // special case: doesn't need an arg + if has_value { + if !fv.Set(value) { + f.failf("invalid boolean value %q for flag: -%s", value, name) + } + } else { + fv.Set("true") + } + } else { + // It must have a value, which might be the next argument. + if !has_value && len(f.args) > 0 { + // value is the next arg + has_value = true + value, f.args = f.args[0], f.args[1:] + } + if !has_value { + return false, f.failf("flag needs an argument: -%s", name) + } + ok = flag.Value.Set(value) + if !ok { + return false, f.failf("invalid value %q for flag: -%s", value, name) + } + } + f.actual[name] = flag + return true, nil +} + +// Parse parses flag definitions from the argument list, which should not +// include the command name. Must be called after all flags in the FlagSet +// are defined and before flags are accessed by the program. +// The return value will be ErrHelp if -help was set but not defined. +func (f *FlagSet) Parse(arguments []string) os.Error { + f.args = arguments + for { + seen, err := f.parseOne() + if seen { + continue + } + if err == nil { + break + } + switch f.errorHandling { + case ContinueOnError: + return err + case ExitOnError: + os.Exit(2) + case PanicOnError: + panic(err) + } + } + return nil +} + +// Parse parses the command-line flags from os.Args[1:]. Must be called +// after all flags are defined and before flags are accessed by the program. +func Parse() { + // Ignore errors; commandLine is set for ExitOnError. + commandLine.Parse(os.Args[1:]) +} + +// The default set of command-line flags, parsed from os.Args. +var commandLine = NewFlagSet(os.Args[0], ExitOnError) + +// NewFlagSet returns a new, empty flag set with the specified name and +// error handling property. +func NewFlagSet(name string, errorHandling ErrorHandling) *FlagSet { + f := &FlagSet{ + name: name, + actual: make(map[string]*Flag), + formal: make(map[string]*Flag), + errorHandling: errorHandling, + } + f.Usage = func() { defaultUsage(f) } + return f +} diff --git a/src/pkg/flag/flag_test.go b/src/pkg/flag/flag_test.go new file mode 100644 index 000000000..63d0a9fc8 --- /dev/null +++ b/src/pkg/flag/flag_test.go @@ -0,0 +1,255 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flag_test + +import ( + . "flag" + "fmt" + "os" + "sort" + "testing" +) + +var ( + test_bool = Bool("test_bool", false, "bool value") + test_int = Int("test_int", 0, "int value") + test_int64 = Int64("test_int64", 0, "int64 value") + test_uint = Uint("test_uint", 0, "uint value") + test_uint64 = Uint64("test_uint64", 0, "uint64 value") + test_string = String("test_string", "0", "string value") + test_float64 = Float64("test_float64", 0, "float64 value") +) + +func boolString(s string) string { + if s == "0" { + return "false" + } + return "true" +} + +func TestEverything(t *testing.T) { + m := make(map[string]*Flag) + desired := "0" + visitor := func(f *Flag) { + if len(f.Name) > 5 && f.Name[0:5] == "test_" { + m[f.Name] = f + ok := false + switch { + case f.Value.String() == desired: + ok = true + case f.Name == "test_bool" && f.Value.String() == boolString(desired): + ok = true + } + if !ok { + t.Error("Visit: bad value", f.Value.String(), "for", f.Name) + } + } + } + VisitAll(visitor) + if len(m) != 7 { + t.Error("VisitAll misses some flags") + for k, v := range m { + t.Log(k, *v) + } + } + m = make(map[string]*Flag) + Visit(visitor) + if len(m) != 0 { + t.Errorf("Visit sees unset flags") + for k, v := range m { + t.Log(k, *v) + } + } + // Now set all flags + Set("test_bool", "true") + Set("test_int", "1") + Set("test_int64", "1") + Set("test_uint", "1") + Set("test_uint64", "1") + Set("test_string", "1") + Set("test_float64", "1") + desired = "1" + Visit(visitor) + if len(m) != 7 { + t.Error("Visit fails after set") + for k, v := range m { + t.Log(k, *v) + } + } + // Now test they're visited in sort order. + var flagNames []string + Visit(func(f *Flag) { flagNames = append(flagNames, f.Name) }) + if !sort.StringsAreSorted(flagNames) { + t.Errorf("flag names not sorted: %v", flagNames) + } +} + +func TestUsage(t *testing.T) { + called := false + ResetForTesting(func() { called = true }) + if CommandLine().Parse([]string{"-x"}) == nil { + t.Error("parse did not fail for unknown flag") + } + if !called { + t.Error("did not call Usage for unknown flag") + } +} + +func testParse(f *FlagSet, t *testing.T) { + boolFlag := f.Bool("bool", false, "bool value") + bool2Flag := f.Bool("bool2", false, "bool2 value") + intFlag := f.Int("int", 0, "int value") + int64Flag := f.Int64("int64", 0, "int64 value") + uintFlag := f.Uint("uint", 0, "uint value") + uint64Flag := f.Uint64("uint64", 0, "uint64 value") + stringFlag := f.String("string", "0", "string value") + float64Flag := f.Float64("float64", 0, "float64 value") + extra := "one-extra-argument" + args := []string{ + "-bool", + "-bool2=true", + "--int", "22", + "--int64", "0x23", + "-uint", "24", + "--uint64", "25", + "-string", "hello", + "-float64", "2718e28", + extra, + } + if err := f.Parse(args); err != nil { + t.Fatal(err) + } + if *boolFlag != true { + t.Error("bool flag should be true, is ", *boolFlag) + } + if *bool2Flag != true { + t.Error("bool2 flag should be true, is ", *bool2Flag) + } + if *intFlag != 22 { + t.Error("int flag should be 22, is ", *intFlag) + } + if *int64Flag != 0x23 { + t.Error("int64 flag should be 0x23, is ", *int64Flag) + } + if *uintFlag != 24 { + t.Error("uint flag should be 24, is ", *uintFlag) + } + if *uint64Flag != 25 { + t.Error("uint64 flag should be 25, is ", *uint64Flag) + } + if *stringFlag != "hello" { + t.Error("string flag should be `hello`, is ", *stringFlag) + } + if *float64Flag != 2718e28 { + t.Error("float64 flag should be 2718e28, is ", *float64Flag) + } + if len(f.Args()) != 1 { + t.Error("expected one argument, got", len(f.Args())) + } else if f.Args()[0] != extra { + t.Errorf("expected argument %q got %q", extra, f.Args()[0]) + } +} + +func TestParse(t *testing.T) { + ResetForTesting(func() { t.Error("bad parse") }) + testParse(CommandLine(), t) +} + +func TestFlagSetParse(t *testing.T) { + testParse(NewFlagSet("test", ContinueOnError), t) +} + +// Declare a user-defined flag type. +type flagVar []string + +func (f *flagVar) String() string { + return fmt.Sprint([]string(*f)) +} + +func (f *flagVar) Set(value string) bool { + *f = append(*f, value) + return true +} + +func TestUserDefined(t *testing.T) { + flags := NewFlagSet("test", ContinueOnError) + var v flagVar + flags.Var(&v, "v", "usage") + if err := flags.Parse([]string{"-v", "1", "-v", "2", "-v=3"}); err != nil { + t.Error(err) + } + if len(v) != 3 { + t.Fatal("expected 3 args; got ", len(v)) + } + expect := "[1 2 3]" + if v.String() != expect { + t.Errorf("expected value %q got %q", expect, v.String()) + } +} + +// This tests that one can reset the flags. This still works but not well, and is +// superseded by FlagSet. +func TestChangingArgs(t *testing.T) { + ResetForTesting(func() { t.Fatal("bad parse") }) + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "-before", "subcmd", "-after", "args"} + before := Bool("before", false, "") + if err := CommandLine().Parse(os.Args[1:]); err != nil { + t.Fatal(err) + } + cmd := Arg(0) + os.Args = Args() + after := Bool("after", false, "") + Parse() + args := Args() + + if !*before || cmd != "subcmd" || !*after || len(args) != 1 || args[0] != "args" { + t.Fatalf("expected true subcmd true [args] got %v %v %v %v", *before, cmd, *after, args) + } +} + +// Test that -help invokes the usage message and returns ErrHelp. +func TestHelp(t *testing.T) { + var helpCalled = false + fs := NewFlagSet("help test", ContinueOnError) + fs.Usage = func() { helpCalled = true } + var flag bool + fs.BoolVar(&flag, "flag", false, "regular flag") + // Regular flag invocation should work + err := fs.Parse([]string{"-flag=true"}) + if err != nil { + t.Fatal("expected no error; got ", err) + } + if !flag { + t.Error("flag was not set by -flag") + } + if helpCalled { + t.Error("help called for regular flag") + helpCalled = false // reset for next test + } + // Help flag should work as expected. + err = fs.Parse([]string{"-help"}) + if err == nil { + t.Fatal("error expected") + } + if err != ErrHelp { + t.Fatal("expected ErrHelp; got ", err) + } + if !helpCalled { + t.Fatal("help was not called") + } + // If we define a help flag, that should override. + var help bool + fs.BoolVar(&help, "help", false, "help flag") + helpCalled = false + err = fs.Parse([]string{"-help"}) + if err != nil { + t.Fatal("expected no error for defined -help; got ", err) + } + if helpCalled { + t.Fatal("help was called; should not have been for defined help flag") + } +} diff --git a/src/pkg/fmt/Makefile b/src/pkg/fmt/Makefile new file mode 100644 index 000000000..44b48bc67 --- /dev/null +++ b/src/pkg/fmt/Makefile @@ -0,0 +1,14 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=fmt +GOFILES=\ + doc.go\ + format.go\ + print.go\ + scan.go\ + +include ../../Make.pkg diff --git a/src/pkg/fmt/doc.go b/src/pkg/fmt/doc.go new file mode 100644 index 000000000..35a11e19f --- /dev/null +++ b/src/pkg/fmt/doc.go @@ -0,0 +1,181 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package fmt implements formatted I/O with functions analogous + to C's printf and scanf. The format 'verbs' are derived from C's but + are simpler. + + Printing: + + The verbs: + + General: + %v the value in a default format. + when printing structs, the plus flag (%+v) adds field names + %#v a Go-syntax representation of the value + %T a Go-syntax representation of the type of the value + %% a literal percent sign; consumes no value + + Boolean: + %t the word true or false + Integer: + %b base 2 + %c the character represented by the corresponding Unicode code point + %d base 10 + %o base 8 + %q a single-quoted character literal safely escaped with Go syntax. + %x base 16, with lower-case letters for a-f + %X base 16, with upper-case letters for A-F + %U Unicode format: U+1234; same as "U+%04X" + Floating-point and complex constituents: + %b decimalless scientific notation with exponent a power + of two, in the manner of strconv.Ftoa32, e.g. -123456p-78 + %e scientific notation, e.g. -1234.456e+78 + %E scientific notation, e.g. -1234.456E+78 + %f decimal point but no exponent, e.g. 123.456 + %g whichever of %e or %f produces more compact output + %G whichever of %E or %f produces more compact output + String and slice of bytes: + %s the uninterpreted bytes of the string or slice + %q a double-quoted string safely escaped with Go syntax + %x base 16, lower-case, two characters per byte + %X base 16, upper-case, two characters per byte + Pointer: + %p base 16 notation, with leading 0x + + There is no 'u' flag. Integers are printed unsigned if they have unsigned type. + Similarly, there is no need to specify the size of the operand (int8, int64). + + The width and precision control formatting and are in units of Unicode + code points. (This differs from C's printf where the units are numbers + of bytes.) Either or both of the flags may be replaced with the + character '*', causing their values to be obtained from the next + operand, which must be of type int. + + For numeric values, width sets the width of the field and precision + sets the number of places after the decimal, if appropriate. For + example, the format %6.2f prints 123.45. + + For strings, width is the minimum number of characters to output, + padding with spaces if necessary, and precision is the maximum + number of characters to output, truncating if necessary. + + Other flags: + + always print a sign for numeric values; + guarantee ASCII-only output for %q (%+q) + - pad with spaces on the right rather than the left (left-justify the field) + # alternate format: add leading 0 for octal (%#o), 0x for hex (%#x); + 0X for hex (%#X); suppress 0x for %p (%#p); + print a raw (backquoted) string if possible for %q (%#q); + write e.g. U+0078 'x' if the character is printable for %U (%#U). + ' ' (space) leave a space for elided sign in numbers (% d); + put spaces between bytes printing strings or slices in hex (% x, % X) + 0 pad with leading zeros rather than spaces + + For each Printf-like function, there is also a Print function + that takes no format and is equivalent to saying %v for every + operand. Another variant Println inserts blanks between + operands and appends a newline. + + Regardless of the verb, if an operand is an interface value, + the internal concrete value is used, not the interface itself. + Thus: + var i interface{} = 23 + fmt.Printf("%v\n", i) + will print 23. + + If an operand implements interface Formatter, that interface + can be used for fine control of formatting. + + If an operand implements method String() string that method + will be used to convert the object to a string, which will then + be formatted as required by the verb (if any). To avoid + recursion in cases such as + type X int + func (x X) String() string { return Sprintf("%d", x) } + cast the value before recurring: + func (x X) String() string { return Sprintf("%d", int(x)) } + + Format errors: + + If an invalid argument is given for a verb, such as providing + a string to %d, the generated string will contain a + description of the problem, as in these examples: + + Wrong type or unknown verb: %!verb(type=value) + Printf("%d", hi): %!d(string=hi) + Too many arguments: %!(EXTRA type=value) + Printf("hi", "guys"): hi%!(EXTRA string=guys) + Too few arguments: %!verb(MISSING) + Printf("hi%d"): hi %!d(MISSING) + Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC) + Printf("%*s", 4.5, "hi"): %!(BADWIDTH)hi + Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi + + All errors begin with the string "%!" followed sometimes + by a single character (the verb) and end with a parenthesized + description. + + Scanning: + + An analogous set of functions scans formatted text to yield + values. Scan, Scanf and Scanln read from os.Stdin; Fscan, + Fscanf and Fscanln read from a specified os.Reader; Sscan, + Sscanf and Sscanln read from an argument string. Scanln, + Fscanln and Sscanln stop scanning at a newline and require that + the items be followed by one; Sscanf, Fscanf and Sscanf require + newlines in the input to match newlines in the format; the other + routines treat newlines as spaces. + + Scanf, Fscanf, and Sscanf parse the arguments according to a + format string, analogous to that of Printf. For example, %x + will scan an integer as a hexadecimal number, and %v will scan + the default representation format for the value. + + The formats behave analogously to those of Printf with the + following exceptions: + + %p is not implemented + %T is not implemented + %e %E %f %F %g %G are all equivalent and scan any floating point or complex value + %s and %v on strings scan a space-delimited token + + The familiar base-setting prefixes 0 (octal) and 0x + (hexadecimal) are accepted when scanning integers without a + format or with the %v verb. + + Width is interpreted in the input text (%5s means at most + five runes of input will be read to scan a string) but there + is no syntax for scanning with a precision (no %5.2f, just + %5f). + + When scanning with a format, all non-empty runs of space + characters (except newline) are equivalent to a single + space in both the format and the input. With that proviso, + text in the format string must match the input text; scanning + stops if it does not, with the return value of the function + indicating the number of arguments scanned. + + In all the scanning functions, if an operand implements method + Scan (that is, it implements the Scanner interface) that + method will be used to scan the text for that operand. Also, + if the number of arguments scanned is less than the number of + arguments provided, an error is returned. + + All arguments to be scanned must be either pointers to basic + types or implementations of the Scanner interface. + + Note: Fscan etc. can read one character (rune) past the input + they return, which means that a loop calling a scan routine + may skip some of the input. This is usually a problem only + when there is no space between input values. If the reader + provided to Fscan implements ReadRune, that method will be used + to read characters. If the reader also implements UnreadRune, + that method will be used to save the character and successive + calls will not lose data. To attach ReadRune and UnreadRune + methods to a reader without that capability, use + bufio.NewReader. +*/ +package fmt diff --git a/src/pkg/fmt/fmt_test.go b/src/pkg/fmt/fmt_test.go new file mode 100644 index 000000000..1142c9f8a --- /dev/null +++ b/src/pkg/fmt/fmt_test.go @@ -0,0 +1,746 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + . "fmt" + "io" + "math" + "runtime" // for the malloc count test only + "strings" + "testing" +) + +type ( + renamedBool bool + renamedInt int + renamedInt8 int8 + renamedInt16 int16 + renamedInt32 int32 + renamedInt64 int64 + renamedUint uint + renamedUint8 uint8 + renamedUint16 uint16 + renamedUint32 uint32 + renamedUint64 uint64 + renamedUintptr uintptr + renamedString string + renamedBytes []byte + renamedFloat32 float32 + renamedFloat64 float64 + renamedComplex64 complex64 + renamedComplex128 complex128 +) + +func TestFmtInterface(t *testing.T) { + var i1 interface{} + i1 = "abc" + s := Sprintf("%s", i1) + if s != "abc" { + t.Errorf(`Sprintf("%%s", empty("abc")) = %q want %q`, s, "abc") + } +} + +const b32 uint32 = 1<<32 - 1 +const b64 uint64 = 1<<64 - 1 + +var array = []int{1, 2, 3, 4, 5} +var iarray = []interface{}{1, "hello", 2.5, nil} + +type A struct { + i int + j uint + s string + x []int +} + +type I int + +func (i I) String() string { return Sprintf("<%d>", int(i)) } + +type B struct { + i I + j int +} + +type C struct { + i int + B +} + +type F int + +func (f F) Format(s State, c int) { + Fprintf(s, "<%c=F(%d)>", c, int(f)) +} + +type G int + +func (g G) GoString() string { + return Sprintf("GoString(%d)", int(g)) +} + +type S struct { + f F // a struct field that Formats + g G // a struct field that GoStrings +} + +// A type with a String method with pointer receiver for testing %p +type P int + +var pValue P + +func (p *P) String() string { + return "String(p)" +} + +var b byte + +var fmttests = []struct { + fmt string + val interface{} + out string +}{ + {"%d", 12345, "12345"}, + {"%v", 12345, "12345"}, + {"%t", true, "true"}, + + // basic string + {"%s", "abc", "abc"}, + {"%x", "abc", "616263"}, + {"%x", "xyz", "78797a"}, + {"%X", "xyz", "78797A"}, + {"%q", "abc", `"abc"`}, + + // basic bytes + {"%s", []byte("abc"), "abc"}, + {"%x", []byte("abc"), "616263"}, + {"% x", []byte("abc\xff"), "61 62 63 ff"}, + {"% X", []byte("abc\xff"), "61 62 63 FF"}, + {"%x", []byte("xyz"), "78797a"}, + {"%X", []byte("xyz"), "78797A"}, + {"%q", []byte("abc"), `"abc"`}, + + // escaped strings + {"%#q", `abc`, "`abc`"}, + {"%#q", `"`, "`\"`"}, + {"1 %#q", `\n`, "1 `\\n`"}, + {"2 %#q", "\n", `2 "\n"`}, + {"%q", `"`, `"\""`}, + {"%q", "\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`}, + {"%q", "abc\xffdef", `"abc\xffdef"`}, + {"%q", "\u263a", `"☺"`}, + {"%+q", "\u263a", `"\u263a"`}, + {"%q", "\U0010ffff", `"\U0010ffff"`}, + + // escaped characters + {"%q", 'x', `'x'`}, + {"%q", 0, `'\x00'`}, + {"%q", '\n', `'\n'`}, + {"%q", '\u0e00', `'\u0e00'`}, // not a printable rune. + {"%q", '\U000c2345', `'\U000c2345'`}, // not a printable rune. + {"%q", int64(0x7FFFFFFF), `%!q(int64=2147483647)`}, + {"%q", uint64(0xFFFFFFFF), `%!q(uint64=4294967295)`}, + {"%q", '"', `'"'`}, + {"%q", '\'', `'\''`}, + {"%q", "\u263a", `"☺"`}, + {"%+q", "\u263a", `"\u263a"`}, + + // width + {"%5s", "abc", " abc"}, + {"%2s", "\u263a", " ☺"}, + {"%-5s", "abc", "abc "}, + {"%-8q", "abc", `"abc" `}, + {"%05s", "abc", "00abc"}, + {"%08q", "abc", `000"abc"`}, + {"%5s", "abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"}, + {"%.5s", "abcdefghijklmnopqrstuvwxyz", "abcde"}, + {"%.5s", "日本語日本語", "日本語日本"}, + {"%.5s", []byte("日本語日本語"), "日本語日本"}, + {"%.5q", "abcdefghijklmnopqrstuvwxyz", `"abcde"`}, + {"%.3q", "日本語日本語", `"日本語"`}, + {"%.3q", []byte("日本語日本語"), `"日本語"`}, + {"%10.1q", "日本語日本語", ` "日"`}, + + // integers + {"%d", 12345, "12345"}, + {"%d", -12345, "-12345"}, + {"%10d", 12345, " 12345"}, + {"%10d", -12345, " -12345"}, + {"%+10d", 12345, " +12345"}, + {"%010d", 12345, "0000012345"}, + {"%010d", -12345, "-000012345"}, + {"%-10d", 12345, "12345 "}, + {"%010.3d", 1, " 001"}, + {"%010.3d", -1, " -001"}, + {"%+d", 12345, "+12345"}, + {"%+d", -12345, "-12345"}, + {"%+d", 0, "+0"}, + {"% d", 0, " 0"}, + {"% d", 12345, " 12345"}, + {"%.0d", 0, ""}, + {"%.d", 0, ""}, + + // unicode format + {"%U", 0x1, "U+0001"}, + {"%U", uint(0x1), "U+0001"}, + {"%.8U", 0x2, "U+00000002"}, + {"%U", 0x1234, "U+1234"}, + {"%U", 0x12345, "U+12345"}, + {"%10.6U", 0xABC, " U+000ABC"}, + {"%-10.6U", 0xABC, "U+000ABC "}, + {"%U", '\n', `U+000A`}, + {"%#U", '\n', `U+000A`}, + {"%U", 'x', `U+0078`}, + {"%#U", 'x', `U+0078 'x'`}, + {"%U", '\u263a', `U+263A`}, + {"%#U", '\u263a', `U+263A '☺'`}, + + // floats + {"%+.3e", 0.0, "+0.000e+00"}, + {"%+.3e", 1.0, "+1.000e+00"}, + {"%+.3f", -1.0, "-1.000"}, + {"% .3E", -1.0, "-1.000E+00"}, + {"% .3e", 1.0, " 1.000e+00"}, + {"%+.3g", 0.0, "+0"}, + {"%+.3g", 1.0, "+1"}, + {"%+.3g", -1.0, "-1"}, + {"% .3g", -1.0, "-1"}, + {"% .3g", 1.0, " 1"}, + + // complex values + {"%+.3e", 0i, "(+0.000e+00+0.000e+00i)"}, + {"%+.3f", 0i, "(+0.000+0.000i)"}, + {"%+.3g", 0i, "(+0+0i)"}, + {"%+.3e", 1 + 2i, "(+1.000e+00+2.000e+00i)"}, + {"%+.3f", 1 + 2i, "(+1.000+2.000i)"}, + {"%+.3g", 1 + 2i, "(+1+2i)"}, + {"%.3e", 0i, "(0.000e+00+0.000e+00i)"}, + {"%.3f", 0i, "(0.000+0.000i)"}, + {"%.3g", 0i, "(0+0i)"}, + {"%.3e", 1 + 2i, "(1.000e+00+2.000e+00i)"}, + {"%.3f", 1 + 2i, "(1.000+2.000i)"}, + {"%.3g", 1 + 2i, "(1+2i)"}, + {"%.3e", -1 - 2i, "(-1.000e+00-2.000e+00i)"}, + {"%.3f", -1 - 2i, "(-1.000-2.000i)"}, + {"%.3g", -1 - 2i, "(-1-2i)"}, + {"% .3E", -1 - 2i, "(-1.000E+00-2.000E+00i)"}, + {"%+.3g", complex64(1 + 2i), "(+1+2i)"}, + {"%+.3g", complex128(1 + 2i), "(+1+2i)"}, + + // erroneous formats + {"", 2, "%!(EXTRA int=2)"}, + {"%d", "hello", "%!d(string=hello)"}, + + // old test/fmt_test.go + {"%d", 1234, "1234"}, + {"%d", -1234, "-1234"}, + {"%d", uint(1234), "1234"}, + {"%d", uint32(b32), "4294967295"}, + {"%d", uint64(b64), "18446744073709551615"}, + {"%o", 01234, "1234"}, + {"%#o", 01234, "01234"}, + {"%o", uint32(b32), "37777777777"}, + {"%o", uint64(b64), "1777777777777777777777"}, + {"%x", 0x1234abcd, "1234abcd"}, + {"%#x", 0x1234abcd, "0x1234abcd"}, + {"%x", b32 - 0x1234567, "fedcba98"}, + {"%X", 0x1234abcd, "1234ABCD"}, + {"%X", b32 - 0x1234567, "FEDCBA98"}, + {"%#X", 0, "0X0"}, + {"%x", b64, "ffffffffffffffff"}, + {"%b", 7, "111"}, + {"%b", b64, "1111111111111111111111111111111111111111111111111111111111111111"}, + {"%b", -6, "-110"}, + {"%e", 1.0, "1.000000e+00"}, + {"%e", 1234.5678e3, "1.234568e+06"}, + {"%e", 1234.5678e-8, "1.234568e-05"}, + {"%e", -7.0, "-7.000000e+00"}, + {"%e", -1e-9, "-1.000000e-09"}, + {"%f", 1234.5678e3, "1234567.800000"}, + {"%f", 1234.5678e-8, "0.000012"}, + {"%f", -7.0, "-7.000000"}, + {"%f", -1e-9, "-0.000000"}, + {"%g", 1234.5678e3, "1.2345678e+06"}, + {"%g", float32(1234.5678e3), "1.2345678e+06"}, + {"%g", 1234.5678e-8, "1.2345678e-05"}, + {"%g", -7.0, "-7"}, + {"%g", -1e-9, "-1e-09"}, + {"%g", float32(-1e-9), "-1e-09"}, + {"%E", 1.0, "1.000000E+00"}, + {"%E", 1234.5678e3, "1.234568E+06"}, + {"%E", 1234.5678e-8, "1.234568E-05"}, + {"%E", -7.0, "-7.000000E+00"}, + {"%E", -1e-9, "-1.000000E-09"}, + {"%G", 1234.5678e3, "1.2345678E+06"}, + {"%G", float32(1234.5678e3), "1.2345678E+06"}, + {"%G", 1234.5678e-8, "1.2345678E-05"}, + {"%G", -7.0, "-7"}, + {"%G", -1e-9, "-1E-09"}, + {"%G", float32(-1e-9), "-1E-09"}, + {"%c", 'x', "x"}, + {"%c", 0xe4, "ä"}, + {"%c", 0x672c, "本"}, + {"%c", '日', "日"}, + {"%20.8d", 1234, " 00001234"}, + {"%20.8d", -1234, " -00001234"}, + {"%20d", 1234, " 1234"}, + {"%-20.8d", 1234, "00001234 "}, + {"%-20.8d", -1234, "-00001234 "}, + {"%-#20.8x", 0x1234abc, "0x01234abc "}, + {"%-#20.8X", 0x1234abc, "0X01234ABC "}, + {"%-#20.8o", 01234, "00001234 "}, + {"%.20b", 7, "00000000000000000111"}, + {"%20.5s", "qwertyuiop", " qwert"}, + {"%.5s", "qwertyuiop", "qwert"}, + {"%-20.5s", "qwertyuiop", "qwert "}, + {"%20c", 'x', " x"}, + {"%-20c", 'x', "x "}, + {"%20.6e", 1.2345e3, " 1.234500e+03"}, + {"%20.6e", 1.2345e-3, " 1.234500e-03"}, + {"%20e", 1.2345e3, " 1.234500e+03"}, + {"%20e", 1.2345e-3, " 1.234500e-03"}, + {"%20.8e", 1.2345e3, " 1.23450000e+03"}, + {"%20f", 1.23456789e3, " 1234.567890"}, + {"%20f", 1.23456789e-3, " 0.001235"}, + {"%20f", 12345678901.23456789, " 12345678901.234568"}, + {"%-20f", 1.23456789e3, "1234.567890 "}, + {"%20.8f", 1.23456789e3, " 1234.56789000"}, + {"%20.8f", 1.23456789e-3, " 0.00123457"}, + {"%g", 1.23456789e3, "1234.56789"}, + {"%g", 1.23456789e-3, "0.00123456789"}, + {"%g", 1.23456789e20, "1.23456789e+20"}, + {"%20e", math.Inf(1), " +Inf"}, + {"%-20f", math.Inf(-1), "-Inf "}, + {"%20g", math.NaN(), " NaN"}, + + // arrays + {"%v", array, "[1 2 3 4 5]"}, + {"%v", iarray, "[1 hello 2.5 ]"}, + {"%v", &array, "&[1 2 3 4 5]"}, + {"%v", &iarray, "&[1 hello 2.5 ]"}, + + // complexes with %v + {"%v", 1 + 2i, "(1+2i)"}, + {"%v", complex64(1 + 2i), "(1+2i)"}, + {"%v", complex128(1 + 2i), "(1+2i)"}, + + // structs + {"%v", A{1, 2, "a", []int{1, 2}}, `{1 2 a [1 2]}`}, + {"%+v", A{1, 2, "a", []int{1, 2}}, `{i:1 j:2 s:a x:[1 2]}`}, + + // +v on structs with Stringable items + {"%+v", B{1, 2}, `{i:<1> j:2}`}, + {"%+v", C{1, B{2, 3}}, `{i:1 B:{i:<2> j:3}}`}, + + // q on Stringable items + {"%s", I(23), `<23>`}, + {"%q", I(23), `"<23>"`}, + {"%x", I(23), `3c32333e`}, + {"%d", I(23), `%!d(string=<23>)`}, + + // go syntax + {"%#v", A{1, 2, "a", []int{1, 2}}, `fmt_test.A{i:1, j:0x2, s:"a", x:[]int{1, 2}}`}, + {"%#v", &b, "(*uint8)(0xPTR)"}, + {"%#v", TestFmtInterface, "(func(*testing.T))(0xPTR)"}, + {"%#v", make(chan int), "(chan int)(0xPTR)"}, + {"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"}, + {"%#v", 1000000000, "1000000000"}, + {"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`}, + {"%#v", map[string]B{"a": {1, 2}, "b": {3, 4}}, `map[string] fmt_test.B{"a":fmt_test.B{i:1, j:2}, "b":fmt_test.B{i:3, j:4}}`}, + {"%#v", []string{"a", "b"}, `[]string{"a", "b"}`}, + + // slices with other formats + {"%#x", []int{1, 2, 15}, `[0x1 0x2 0xf]`}, + {"%x", []int{1, 2, 15}, `[1 2 f]`}, + {"%d", []int{1, 2, 15}, `[1 2 15]`}, + {"%d", []byte{1, 2, 15}, `[1 2 15]`}, + {"%q", []string{"a", "b"}, `["a" "b"]`}, + + // renamings + {"%v", renamedBool(true), "true"}, + {"%d", renamedBool(true), "%!d(fmt_test.renamedBool=true)"}, + {"%o", renamedInt(8), "10"}, + {"%d", renamedInt8(-9), "-9"}, + {"%v", renamedInt16(10), "10"}, + {"%v", renamedInt32(-11), "-11"}, + {"%X", renamedInt64(255), "FF"}, + {"%v", renamedUint(13), "13"}, + {"%o", renamedUint8(14), "16"}, + {"%X", renamedUint16(15), "F"}, + {"%d", renamedUint32(16), "16"}, + {"%X", renamedUint64(17), "11"}, + {"%o", renamedUintptr(18), "22"}, + {"%x", renamedString("thing"), "7468696e67"}, + {"%d", renamedBytes([]byte{1, 2, 15}), `[1 2 15]`}, + {"%q", renamedBytes([]byte("hello")), `"hello"`}, + {"%v", renamedFloat32(22), "22"}, + {"%v", renamedFloat64(33), "33"}, + {"%v", renamedComplex64(3 + 4i), "(3+4i)"}, + {"%v", renamedComplex128(4 - 3i), "(4-3i)"}, + + // Formatter + {"%x", F(1), ""}, + {"%x", G(2), "2"}, + {"%+v", S{F(4), G(5)}, "{f: g:5}"}, + + // GoStringer + {"%#v", G(6), "GoString(6)"}, + {"%#v", S{F(7), G(8)}, "fmt_test.S{f:, g:GoString(8)}"}, + + // %T + {"%T", (4 - 3i), "complex128"}, + {"%T", renamedComplex128(4 - 3i), "fmt_test.renamedComplex128"}, + {"%T", intVal, "int"}, + {"%6T", &intVal, " *int"}, + + // %p + {"p0=%p", new(int), "p0=0xPTR"}, + {"p1=%s", &pValue, "p1=String(p)"}, // String method... + {"p2=%p", &pValue, "p2=0xPTR"}, // ... not called with %p + {"p4=%#p", new(int), "p4=PTR"}, + + // %p on non-pointers + {"%p", make(chan int), "0xPTR"}, + {"%p", make(map[int]int), "0xPTR"}, + {"%p", make([]int, 1), "0xPTR"}, + {"%p", 27, "%!p(int=27)"}, // not a pointer at all + + // erroneous things + {"%s %", "hello", "hello %!(NOVERB)"}, + {"%s %.2", "hello", "hello %!(NOVERB)"}, + {"%d", "hello", "%!d(string=hello)"}, + {"no args", "hello", "no args%!(EXTRA string=hello)"}, + {"%s", nil, "%!s()"}, + {"%T", nil, ""}, + {"%-1", 100, "%!(NOVERB)%!(EXTRA int=100)"}, +} + +func TestSprintf(t *testing.T) { + for _, tt := range fmttests { + s := Sprintf(tt.fmt, tt.val) + if i := strings.Index(tt.out, "PTR"); i >= 0 { + j := i + for ; j < len(s); j++ { + c := s[j] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + break + } + } + s = s[0:i] + "PTR" + s[j:] + } + if s != tt.out { + if _, ok := tt.val.(string); ok { + // Don't requote the already-quoted strings. + // It's too confusing to read the errors. + t.Errorf("Sprintf(%q, %q) = <%s> want <%s>", tt.fmt, tt.val, s, tt.out) + } else { + t.Errorf("Sprintf(%q, %v) = %q want %q", tt.fmt, tt.val, s, tt.out) + } + } + } +} + +func BenchmarkSprintfEmpty(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("") + } +} + +func BenchmarkSprintfString(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%s", "hello") + } +} + +func BenchmarkSprintfInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%d", 5) + } +} + +func BenchmarkSprintfIntInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("%d %d", 5, 6) + } +} + +func BenchmarkSprintfPrefixedInt(b *testing.B) { + for i := 0; i < b.N; i++ { + Sprintf("This is some meaningless prefix text that needs to be scanned %d", 6) + } +} + +func TestCountMallocs(t *testing.T) { + if testing.Short() { + return + } + runtime.UpdateMemStats() + mallocs := 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("") + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("xxx") + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"xxx\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("%x", i) + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%x\"): %d\n", mallocs/100) + runtime.UpdateMemStats() + mallocs = 0 - runtime.MemStats.Mallocs + for i := 0; i < 100; i++ { + Sprintf("%x %x", i, i) + } + runtime.UpdateMemStats() + mallocs += runtime.MemStats.Mallocs + Printf("mallocs per Sprintf(\"%%x %%x\"): %d\n", mallocs/100) +} + +type flagPrinter struct{} + +func (*flagPrinter) Format(f State, c int) { + s := "%" + for i := 0; i < 128; i++ { + if f.Flag(i) { + s += string(i) + } + } + if w, ok := f.Width(); ok { + s += Sprintf("%d", w) + } + if p, ok := f.Precision(); ok { + s += Sprintf(".%d", p) + } + s += string(c) + io.WriteString(f, "["+s+"]") +} + +var flagtests = []struct { + in string + out string +}{ + {"%a", "[%a]"}, + {"%-a", "[%-a]"}, + {"%+a", "[%+a]"}, + {"%#a", "[%#a]"}, + {"% a", "[% a]"}, + {"%0a", "[%0a]"}, + {"%1.2a", "[%1.2a]"}, + {"%-1.2a", "[%-1.2a]"}, + {"%+1.2a", "[%+1.2a]"}, + {"%-+1.2a", "[%+-1.2a]"}, + {"%-+1.2abc", "[%+-1.2a]bc"}, + {"%-1.2abc", "[%-1.2a]bc"}, +} + +func TestFlagParser(t *testing.T) { + var flagprinter flagPrinter + for _, tt := range flagtests { + s := Sprintf(tt.in, &flagprinter) + if s != tt.out { + t.Errorf("Sprintf(%q, &flagprinter) => %q, want %q", tt.in, s, tt.out) + } + } +} + +func TestStructPrinter(t *testing.T) { + var s struct { + a string + b string + c int + } + s.a = "abc" + s.b = "def" + s.c = 123 + var tests = []struct { + fmt string + out string + }{ + {"%v", "{abc def 123}"}, + {"%+v", "{a:abc b:def c:123}"}, + } + for _, tt := range tests { + out := Sprintf(tt.fmt, s) + if out != tt.out { + t.Errorf("Sprintf(%q, &s) = %q, want %q", tt.fmt, out, tt.out) + } + } +} + +// Check map printing using substrings so we don't depend on the print order. +func presentInMap(s string, a []string, t *testing.T) { + for i := 0; i < len(a); i++ { + loc := strings.Index(s, a[i]) + if loc < 0 { + t.Errorf("map print: expected to find %q in %q", a[i], s) + } + // make sure the match ends here + loc += len(a[i]) + if loc >= len(s) || (s[loc] != ' ' && s[loc] != ']') { + t.Errorf("map print: %q not properly terminated in %q", a[i], s) + } + } +} + +func TestMapPrinter(t *testing.T) { + m0 := make(map[int]string) + s := Sprint(m0) + if s != "map[]" { + t.Errorf("empty map printed as %q not %q", s, "map[]") + } + m1 := map[int]string{1: "one", 2: "two", 3: "three"} + a := []string{"1:one", "2:two", "3:three"} + presentInMap(Sprintf("%v", m1), a, t) + presentInMap(Sprint(m1), a, t) +} + +func TestEmptyMap(t *testing.T) { + const emptyMapStr = "map[]" + var m map[string]int + s := Sprint(m) + if s != emptyMapStr { + t.Errorf("nil map printed as %q not %q", s, emptyMapStr) + } + m = make(map[string]int) + s = Sprint(m) + if s != emptyMapStr { + t.Errorf("empty map printed as %q not %q", s, emptyMapStr) + } +} + +// Check that Sprint (and hence Print, Fprint) puts spaces in the right places, +// that is, between arg pairs in which neither is a string. +func TestBlank(t *testing.T) { + got := Sprint("<", 1, ">:", 1, 2, 3, "!") + expect := "<1>:1 2 3!" + if got != expect { + t.Errorf("got %q expected %q", got, expect) + } +} + +// Check that Sprintln (and hence Println, Fprintln) puts spaces in the right places, +// that is, between all arg pairs. +func TestBlankln(t *testing.T) { + got := Sprintln("<", 1, ">:", 1, 2, 3, "!") + expect := "< 1 >: 1 2 3 !\n" + if got != expect { + t.Errorf("got %q expected %q", got, expect) + } +} + +// Check Formatter with Sprint, Sprintln, Sprintf +func TestFormatterPrintln(t *testing.T) { + f := F(1) + expect := "\n" + s := Sprint(f, "\n") + if s != expect { + t.Errorf("Sprint wrong with Formatter: expected %q got %q", expect, s) + } + s = Sprintln(f) + if s != expect { + t.Errorf("Sprintln wrong with Formatter: expected %q got %q", expect, s) + } + s = Sprintf("%v\n", f) + if s != expect { + t.Errorf("Sprintf wrong with Formatter: expected %q got %q", expect, s) + } +} + +func args(a ...interface{}) []interface{} { return a } + +var startests = []struct { + fmt string + in []interface{} + out string +}{ + {"%*d", args(4, 42), " 42"}, + {"%.*d", args(4, 42), "0042"}, + {"%*.*d", args(8, 4, 42), " 0042"}, + {"%0*d", args(4, 42), "0042"}, + {"%-*d", args(4, 42), "42 "}, + + // erroneous + {"%*d", args(nil, 42), "%!(BADWIDTH)42"}, + {"%.*d", args(nil, 42), "%!(BADPREC)42"}, + {"%*d", args(5, "foo"), "%!d(string= foo)"}, + {"%*% %d", args(20, 5), "% 5"}, + {"%*", args(4), "%!(NOVERB)"}, + {"%*d", args(int32(4), 42), "%!(BADWIDTH)42"}, +} + +func TestWidthAndPrecision(t *testing.T) { + for _, tt := range startests { + s := Sprintf(tt.fmt, tt.in...) + if s != tt.out { + t.Errorf("%q: got %q expected %q", tt.fmt, s, tt.out) + } + } +} + +// A type that panics in String. +type Panic struct { + message interface{} +} + +// Value receiver. +func (p Panic) GoString() string { + panic(p.message) +} + +// Value receiver. +func (p Panic) String() string { + panic(p.message) +} + +// A type that panics in Format. +type PanicF struct { + message interface{} +} + +// Value receiver. +func (p PanicF) Format(f State, c int) { + panic(p.message) +} + +var panictests = []struct { + fmt string + in interface{} + out string +}{ + // String + {"%d", (*Panic)(nil), ""}, // nil pointer special case + {"%d", Panic{io.ErrUnexpectedEOF}, "%d(PANIC=unexpected EOF)"}, + {"%d", Panic{3}, "%d(PANIC=3)"}, + // GoString + {"%#v", (*Panic)(nil), ""}, // nil pointer special case + {"%#v", Panic{io.ErrUnexpectedEOF}, "%v(PANIC=unexpected EOF)"}, + {"%#v", Panic{3}, "%v(PANIC=3)"}, + // Format + {"%s", (*PanicF)(nil), ""}, // nil pointer special case + {"%s", PanicF{io.ErrUnexpectedEOF}, "%s(PANIC=unexpected EOF)"}, + {"%s", PanicF{3}, "%s(PANIC=3)"}, +} + +func TestPanics(t *testing.T) { + for _, tt := range panictests { + s := Sprintf(tt.fmt, tt.in) + if s != tt.out { + t.Errorf("%q: got %q expected %q", tt.fmt, s, tt.out) + } + } +} diff --git a/src/pkg/fmt/format.go b/src/pkg/fmt/format.go new file mode 100644 index 000000000..24b15a286 --- /dev/null +++ b/src/pkg/fmt/format.go @@ -0,0 +1,452 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "strconv" + "unicode" + "utf8" +) + +const ( + nByte = 64 + + ldigits = "0123456789abcdef" + udigits = "0123456789ABCDEF" +) + +const ( + signed = true + unsigned = false +) + +var padZeroBytes = make([]byte, nByte) +var padSpaceBytes = make([]byte, nByte) + +var newline = []byte{'\n'} + +func init() { + for i := 0; i < nByte; i++ { + padZeroBytes[i] = '0' + padSpaceBytes[i] = ' ' + } +} + +// A fmt is the raw formatter used by Printf etc. +// It prints into a bytes.Buffer that must be set up externally. +type fmt struct { + intbuf [nByte]byte + buf *bytes.Buffer + // width, precision + wid int + prec int + // flags + widPresent bool + precPresent bool + minus bool + plus bool + sharp bool + space bool + unicode bool + uniQuote bool // Use 'x'= prefix for %U if printable. + zero bool +} + +func (f *fmt) clearflags() { + f.wid = 0 + f.widPresent = false + f.prec = 0 + f.precPresent = false + f.minus = false + f.plus = false + f.sharp = false + f.space = false + f.unicode = false + f.uniQuote = false + f.zero = false +} + +func (f *fmt) init(buf *bytes.Buffer) { + f.buf = buf + f.clearflags() +} + +// Compute left and right padding widths (only one will be non-zero). +func (f *fmt) computePadding(width int) (padding []byte, leftWidth, rightWidth int) { + left := !f.minus + w := f.wid + if w < 0 { + left = false + w = -w + } + w -= width + if w > 0 { + if left && f.zero { + return padZeroBytes, w, 0 + } + if left { + return padSpaceBytes, w, 0 + } else { + // can't be zero padding on the right + return padSpaceBytes, 0, w + } + } + return +} + +// Generate n bytes of padding. +func (f *fmt) writePadding(n int, padding []byte) { + for n > 0 { + m := n + if m > nByte { + m = nByte + } + f.buf.Write(padding[0:m]) + n -= m + } +} + +// Append b to f.buf, padded on left (w > 0) or right (w < 0 or f.minus) +// clear flags afterwards. +func (f *fmt) pad(b []byte) { + var padding []byte + var left, right int + if f.widPresent && f.wid != 0 { + padding, left, right = f.computePadding(len(b)) + } + if left > 0 { + f.writePadding(left, padding) + } + f.buf.Write(b) + if right > 0 { + f.writePadding(right, padding) + } +} + +// append s to buf, padded on left (w > 0) or right (w < 0 or f.minus). +// clear flags afterwards. +func (f *fmt) padString(s string) { + var padding []byte + var left, right int + if f.widPresent && f.wid != 0 { + padding, left, right = f.computePadding(utf8.RuneCountInString(s)) + } + if left > 0 { + f.writePadding(left, padding) + } + f.buf.WriteString(s) + if right > 0 { + f.writePadding(right, padding) + } +} + +func putint(buf []byte, base, val uint64, digits string) int { + i := len(buf) - 1 + for val >= base { + buf[i] = digits[val%base] + i-- + val /= base + } + buf[i] = digits[val] + return i - 1 +} + +// fmt_boolean formats a boolean. +func (f *fmt) fmt_boolean(v bool) { + if v { + f.padString("true") + } else { + f.padString("false") + } +} + +// integer; interprets prec but not wid. Once formatted, result is sent to pad() +// and then flags are cleared. +func (f *fmt) integer(a int64, base uint64, signedness bool, digits string) { + // precision of 0 and value of 0 means "print nothing" + if f.precPresent && f.prec == 0 && a == 0 { + return + } + + var buf []byte = f.intbuf[0:] + negative := signedness == signed && a < 0 + if negative { + a = -a + } + + // two ways to ask for extra leading zero digits: %.3d or %03d. + // apparently the first cancels the second. + prec := 0 + if f.precPresent { + prec = f.prec + f.zero = false + } else if f.zero && f.widPresent && !f.minus && f.wid > 0 { + prec = f.wid + if negative || f.plus || f.space { + prec-- // leave room for sign + } + } + + // format a into buf, ending at buf[i]. (printing is easier right-to-left.) + // a is made into unsigned ua. we could make things + // marginally faster by splitting the 32-bit case out into a separate + // block but it's not worth the duplication, so ua has 64 bits. + i := len(f.intbuf) + ua := uint64(a) + for ua >= base { + i-- + buf[i] = digits[ua%base] + ua /= base + } + i-- + buf[i] = digits[ua] + for i > 0 && prec > nByte-i { + i-- + buf[i] = '0' + } + + // Various prefixes: 0x, -, etc. + if f.sharp { + switch base { + case 8: + if buf[i] != '0' { + i-- + buf[i] = '0' + } + case 16: + i-- + buf[i] = 'x' + digits[10] - 'a' + i-- + buf[i] = '0' + } + } + if f.unicode { + i-- + buf[i] = '+' + i-- + buf[i] = 'U' + } + + if negative { + i-- + buf[i] = '-' + } else if f.plus { + i-- + buf[i] = '+' + } else if f.space { + i-- + buf[i] = ' ' + } + + // If we want a quoted char for %#U, move the data up to make room. + if f.unicode && f.uniQuote && a >= 0 && a <= unicode.MaxRune && unicode.IsPrint(int(a)) { + runeWidth := utf8.RuneLen(int(a)) + width := 1 + 1 + runeWidth + 1 // space, quote, rune, quote + copy(buf[i-width:], buf[i:]) // guaranteed to have enough room. + i -= width + // Now put " 'x'" at the end. + j := len(buf) - width + buf[j] = ' ' + j++ + buf[j] = '\'' + j++ + utf8.EncodeRune(buf[j:], int(a)) + j += runeWidth + buf[j] = '\'' + } + + f.pad(buf[i:]) +} + +// truncate truncates the string to the specified precision, if present. +func (f *fmt) truncate(s string) string { + if f.precPresent && f.prec < utf8.RuneCountInString(s) { + n := f.prec + for i := range s { + if n == 0 { + s = s[:i] + break + } + n-- + } + } + return s +} + +// fmt_s formats a string. +func (f *fmt) fmt_s(s string) { + s = f.truncate(s) + f.padString(s) +} + +// fmt_sx formats a string as a hexadecimal encoding of its bytes. +func (f *fmt) fmt_sx(s string) { + t := "" + for i := 0; i < len(s); i++ { + if i > 0 && f.space { + t += " " + } + v := s[i] + t += string(ldigits[v>>4]) + t += string(ldigits[v&0xF]) + } + f.padString(t) +} + +// fmt_sX formats a string as an uppercase hexadecimal encoding of its bytes. +func (f *fmt) fmt_sX(s string) { + t := "" + for i := 0; i < len(s); i++ { + if i > 0 && f.space { + t += " " + } + v := s[i] + t += string(udigits[v>>4]) + t += string(udigits[v&0xF]) + } + f.padString(t) +} + +// fmt_q formats a string as a double-quoted, escaped Go string constant. +func (f *fmt) fmt_q(s string) { + s = f.truncate(s) + var quoted string + if f.sharp && strconv.CanBackquote(s) { + quoted = "`" + s + "`" + } else { + if f.plus { + quoted = strconv.QuoteToASCII(s) + } else { + quoted = strconv.Quote(s) + } + } + f.padString(quoted) +} + +// fmt_qc formats the integer as a single-quoted, escaped Go character constant. +// If the character is not valid Unicode, it will print '\ufffd'. +func (f *fmt) fmt_qc(c int64) { + var quoted string + if f.plus { + quoted = strconv.QuoteRuneToASCII(int(c)) + } else { + quoted = strconv.QuoteRune(int(c)) + } + f.padString(quoted) +} + +// floating-point + +func doPrec(f *fmt, def int) int { + if f.precPresent { + return f.prec + } + return def +} + +// Add a plus sign or space to the floating-point string representation if missing and required. +func (f *fmt) plusSpace(s string) { + if s[0] != '-' { + if f.plus { + s = "+" + s + } else if f.space { + s = " " + s + } + } + f.padString(s) +} + +// fmt_e64 formats a float64 in the form -1.23e+12. +func (f *fmt) fmt_e64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'e', doPrec(f, 6))) } + +// fmt_E64 formats a float64 in the form -1.23E+12. +func (f *fmt) fmt_E64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'E', doPrec(f, 6))) } + +// fmt_f64 formats a float64 in the form -1.23. +func (f *fmt) fmt_f64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'f', doPrec(f, 6))) } + +// fmt_g64 formats a float64 in the 'f' or 'e' form according to size. +func (f *fmt) fmt_g64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'g', doPrec(f, -1))) } + +// fmt_g64 formats a float64 in the 'f' or 'E' form according to size. +func (f *fmt) fmt_G64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'G', doPrec(f, -1))) } + +// fmt_fb64 formats a float64 in the form -123p3 (exponent is power of 2). +func (f *fmt) fmt_fb64(v float64) { f.plusSpace(strconv.Ftoa64(v, 'b', 0)) } + +// float32 +// cannot defer to float64 versions +// because it will get rounding wrong in corner cases. + +// fmt_e32 formats a float32 in the form -1.23e+12. +func (f *fmt) fmt_e32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'e', doPrec(f, 6))) } + +// fmt_E32 formats a float32 in the form -1.23E+12. +func (f *fmt) fmt_E32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'E', doPrec(f, 6))) } + +// fmt_f32 formats a float32 in the form -1.23. +func (f *fmt) fmt_f32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'f', doPrec(f, 6))) } + +// fmt_g32 formats a float32 in the 'f' or 'e' form according to size. +func (f *fmt) fmt_g32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'g', doPrec(f, -1))) } + +// fmt_G32 formats a float32 in the 'f' or 'E' form according to size. +func (f *fmt) fmt_G32(v float32) { f.plusSpace(strconv.Ftoa32(v, 'G', doPrec(f, -1))) } + +// fmt_fb32 formats a float32 in the form -123p3 (exponent is power of 2). +func (f *fmt) fmt_fb32(v float32) { f.padString(strconv.Ftoa32(v, 'b', 0)) } + +// fmt_c64 formats a complex64 according to the verb. +func (f *fmt) fmt_c64(v complex64, verb int) { + f.buf.WriteByte('(') + r := real(v) + for i := 0; ; i++ { + switch verb { + case 'e': + f.fmt_e32(r) + case 'E': + f.fmt_E32(r) + case 'f': + f.fmt_f32(r) + case 'g': + f.fmt_g32(r) + case 'G': + f.fmt_G32(r) + } + if i != 0 { + break + } + f.plus = true + r = imag(v) + } + f.buf.Write(irparenBytes) +} + +// fmt_c128 formats a complex128 according to the verb. +func (f *fmt) fmt_c128(v complex128, verb int) { + f.buf.WriteByte('(') + r := real(v) + for i := 0; ; i++ { + switch verb { + case 'e': + f.fmt_e64(r) + case 'E': + f.fmt_E64(r) + case 'f': + f.fmt_f64(r) + case 'g': + f.fmt_g64(r) + case 'G': + f.fmt_G64(r) + } + if i != 0 { + break + } + f.plus = true + r = imag(v) + } + f.buf.Write(irparenBytes) +} diff --git a/src/pkg/fmt/print.go b/src/pkg/fmt/print.go new file mode 100644 index 000000000..738734908 --- /dev/null +++ b/src/pkg/fmt/print.go @@ -0,0 +1,996 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "io" + "os" + "reflect" + "unicode" + "utf8" +) + +// Some constants in the form of bytes, to avoid string overhead. +// Needlessly fastidious, I suppose. +var ( + commaSpaceBytes = []byte(", ") + nilAngleBytes = []byte("") + nilParenBytes = []byte("(nil)") + nilBytes = []byte("nil") + mapBytes = []byte("map[") + missingBytes = []byte("(MISSING)") + panicBytes = []byte("(PANIC=") + extraBytes = []byte("%!(EXTRA ") + irparenBytes = []byte("i)") + bytesBytes = []byte("[]byte{") + widthBytes = []byte("%!(BADWIDTH)") + precBytes = []byte("%!(BADPREC)") + noVerbBytes = []byte("%!(NOVERB)") +) + +// State represents the printer state passed to custom formatters. +// It provides access to the io.Writer interface plus information about +// the flags and options for the operand's format specifier. +type State interface { + // Write is the function to call to emit formatted output to be printed. + Write(b []byte) (ret int, err os.Error) + // Width returns the value of the width option and whether it has been set. + Width() (wid int, ok bool) + // Precision returns the value of the precision option and whether it has been set. + Precision() (prec int, ok bool) + + // Flag returns whether the flag c, a character, has been set. + Flag(c int) bool +} + +// Formatter is the interface implemented by values with a custom formatter. +// The implementation of Format may call Sprintf or Fprintf(f) etc. +// to generate its output. +type Formatter interface { + Format(f State, c int) +} + +// Stringer is implemented by any value that has a String method, +// which defines the ``native'' format for that value. +// The String method is used to print values passed as an operand +// to a %s or %v format or to an unformatted printer such as Print. +type Stringer interface { + String() string +} + +// GoStringer is implemented by any value that has a GoString method, +// which defines the Go syntax for that value. +// The GoString method is used to print values passed as an operand +// to a %#v format. +type GoStringer interface { + GoString() string +} + +type pp struct { + n int + panicking bool + buf bytes.Buffer + runeBuf [utf8.UTFMax]byte + fmt fmt +} + +// A cache holds a set of reusable objects. +// The buffered channel holds the currently available objects. +// If more are needed, the cache creates them by calling new. +type cache struct { + saved chan interface{} + new func() interface{} +} + +func (c *cache) put(x interface{}) { + select { + case c.saved <- x: + // saved in cache + default: + // discard + } +} + +func (c *cache) get() interface{} { + select { + case x := <-c.saved: + return x // reused from cache + default: + return c.new() + } + panic("not reached") +} + +func newCache(f func() interface{}) *cache { + return &cache{make(chan interface{}, 100), f} +} + +var ppFree = newCache(func() interface{} { return new(pp) }) + +// Allocate a new pp struct or grab a cached one. +func newPrinter() *pp { + p := ppFree.get().(*pp) + p.panicking = false + p.fmt.init(&p.buf) + return p +} + +// Save used pp structs in ppFree; avoids an allocation per invocation. +func (p *pp) free() { + // Don't hold on to pp structs with large buffers. + if cap(p.buf.Bytes()) > 1024 { + return + } + p.buf.Reset() + ppFree.put(p) +} + +func (p *pp) Width() (wid int, ok bool) { return p.fmt.wid, p.fmt.widPresent } + +func (p *pp) Precision() (prec int, ok bool) { return p.fmt.prec, p.fmt.precPresent } + +func (p *pp) Flag(b int) bool { + switch b { + case '-': + return p.fmt.minus + case '+': + return p.fmt.plus + case '#': + return p.fmt.sharp + case ' ': + return p.fmt.space + case '0': + return p.fmt.zero + } + return false +} + +func (p *pp) add(c int) { + p.buf.WriteRune(c) +} + +// Implement Write so we can call Fprintf on a pp (through State), for +// recursive use in custom verbs. +func (p *pp) Write(b []byte) (ret int, err os.Error) { + return p.buf.Write(b) +} + +// These routines end in 'f' and take a format string. + +// Fprintf formats according to a format specifier and writes to w. +// It returns the number of bytes written and any write error encountered. +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err os.Error) { + p := newPrinter() + p.doPrintf(format, a) + n64, err := p.buf.WriteTo(w) + p.free() + return int(n64), err +} + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err os.Error) { + return Fprintf(os.Stdout, format, a...) +} + +// Sprintf formats according to a format specifier and returns the resulting string. +func Sprintf(format string, a ...interface{}) string { + p := newPrinter() + p.doPrintf(format, a) + s := p.buf.String() + p.free() + return s +} + +// Errorf formats according to a format specifier and returns the string +// converted to an os.ErrorString, which satisfies the os.Error interface. +func Errorf(format string, a ...interface{}) os.Error { + return os.NewError(Sprintf(format, a...)) +} + +// These routines do not take a format string + +// Fprint formats using the default formats for its operands and writes to w. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Fprint(w io.Writer, a ...interface{}) (n int, err os.Error) { + p := newPrinter() + p.doPrint(a, false, false) + n64, err := p.buf.WriteTo(w) + p.free() + return int(n64), err +} + +// Print formats using the default formats for its operands and writes to standard output. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Print(a ...interface{}) (n int, err os.Error) { + return Fprint(os.Stdout, a...) +} + +// Sprint formats using the default formats for its operands and returns the resulting string. +// Spaces are added between operands when neither is a string. +func Sprint(a ...interface{}) string { + p := newPrinter() + p.doPrint(a, false, false) + s := p.buf.String() + p.free() + return s +} + +// These routines end in 'ln', do not take a format string, +// always add spaces between operands, and add a newline +// after the last operand. + +// Fprintln formats using the default formats for its operands and writes to w. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Fprintln(w io.Writer, a ...interface{}) (n int, err os.Error) { + p := newPrinter() + p.doPrint(a, true, true) + n64, err := p.buf.WriteTo(w) + p.free() + return int(n64), err +} + +// Println formats using the default formats for its operands and writes to standard output. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Println(a ...interface{}) (n int, err os.Error) { + return Fprintln(os.Stdout, a...) +} + +// Sprintln formats using the default formats for its operands and returns the resulting string. +// Spaces are always added between operands and a newline is appended. +func Sprintln(a ...interface{}) string { + p := newPrinter() + p.doPrint(a, true, true) + s := p.buf.String() + p.free() + return s +} + +// Get the i'th arg of the struct value. +// If the arg itself is an interface, return a value for +// the thing inside the interface, not the interface itself. +func getField(v reflect.Value, i int) reflect.Value { + val := v.Field(i) + if i := val; i.Kind() == reflect.Interface { + if inter := i.Interface(); inter != nil { + return reflect.ValueOf(inter) + } + } + return val +} + +// Convert ASCII to integer. n is 0 (and got is false) if no number present. +func parsenum(s string, start, end int) (num int, isnum bool, newi int) { + if start >= end { + return 0, false, end + } + for newi = start; newi < end && '0' <= s[newi] && s[newi] <= '9'; newi++ { + num = num*10 + int(s[newi]-'0') + isnum = true + } + return +} + +func (p *pp) unknownType(v interface{}) { + if v == nil { + p.buf.Write(nilAngleBytes) + return + } + p.buf.WriteByte('?') + p.buf.WriteString(reflect.TypeOf(v).String()) + p.buf.WriteByte('?') +} + +func (p *pp) badVerb(verb int, val interface{}) { + p.add('%') + p.add('!') + p.add(verb) + p.add('(') + if val == nil { + p.buf.Write(nilAngleBytes) + } else { + p.buf.WriteString(reflect.TypeOf(val).String()) + p.add('=') + p.printField(val, 'v', false, false, 0) + } + p.add(')') +} + +func (p *pp) fmtBool(v bool, verb int, value interface{}) { + switch verb { + case 't', 'v': + p.fmt.fmt_boolean(v) + default: + p.badVerb(verb, value) + } +} + +// fmtC formats a rune for the 'c' format. +func (p *pp) fmtC(c int64) { + rune := int(c) // Check for overflow. + if int64(rune) != c { + rune = utf8.RuneError + } + w := utf8.EncodeRune(p.runeBuf[0:utf8.UTFMax], rune) + p.fmt.pad(p.runeBuf[0:w]) +} + +func (p *pp) fmtInt64(v int64, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.integer(v, 2, signed, ldigits) + case 'c': + p.fmtC(v) + case 'd', 'v': + p.fmt.integer(v, 10, signed, ldigits) + case 'o': + p.fmt.integer(v, 8, signed, ldigits) + case 'q': + if 0 <= v && v <= unicode.MaxRune { + p.fmt.fmt_qc(v) + } else { + p.badVerb(verb, value) + } + case 'x': + p.fmt.integer(v, 16, signed, ldigits) + case 'U': + p.fmtUnicode(v) + case 'X': + p.fmt.integer(v, 16, signed, udigits) + default: + p.badVerb(verb, value) + } +} + +// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x or +// not, as requested, by temporarily setting the sharp flag. +func (p *pp) fmt0x64(v uint64, leading0x bool) { + sharp := p.fmt.sharp + p.fmt.sharp = leading0x + p.fmt.integer(int64(v), 16, unsigned, ldigits) + p.fmt.sharp = sharp +} + +// fmtUnicode formats a uint64 in U+1234 form by +// temporarily turning on the unicode flag and tweaking the precision. +func (p *pp) fmtUnicode(v int64) { + precPresent := p.fmt.precPresent + sharp := p.fmt.sharp + p.fmt.sharp = false + prec := p.fmt.prec + if !precPresent { + // If prec is already set, leave it alone; otherwise 4 is minimum. + p.fmt.prec = 4 + p.fmt.precPresent = true + } + p.fmt.unicode = true // turn on U+ + p.fmt.uniQuote = sharp + p.fmt.integer(int64(v), 16, unsigned, udigits) + p.fmt.unicode = false + p.fmt.uniQuote = false + p.fmt.prec = prec + p.fmt.precPresent = precPresent + p.fmt.sharp = sharp +} + +func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { + switch verb { + case 'b': + p.fmt.integer(int64(v), 2, unsigned, ldigits) + case 'c': + p.fmtC(int64(v)) + case 'd': + p.fmt.integer(int64(v), 10, unsigned, ldigits) + case 'v': + if goSyntax { + p.fmt0x64(v, true) + } else { + p.fmt.integer(int64(v), 10, unsigned, ldigits) + } + case 'o': + p.fmt.integer(int64(v), 8, unsigned, ldigits) + case 'q': + if 0 <= v && v <= unicode.MaxRune { + p.fmt.fmt_qc(int64(v)) + } else { + p.badVerb(verb, value) + } + case 'x': + p.fmt.integer(int64(v), 16, unsigned, ldigits) + case 'X': + p.fmt.integer(int64(v), 16, unsigned, udigits) + case 'U': + p.fmtUnicode(int64(v)) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtFloat32(v float32, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.fmt_fb32(v) + case 'e': + p.fmt.fmt_e32(v) + case 'E': + p.fmt.fmt_E32(v) + case 'f': + p.fmt.fmt_f32(v) + case 'g', 'v': + p.fmt.fmt_g32(v) + case 'G': + p.fmt.fmt_G32(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtFloat64(v float64, verb int, value interface{}) { + switch verb { + case 'b': + p.fmt.fmt_fb64(v) + case 'e': + p.fmt.fmt_e64(v) + case 'E': + p.fmt.fmt_E64(v) + case 'f': + p.fmt.fmt_f64(v) + case 'g', 'v': + p.fmt.fmt_g64(v) + case 'G': + p.fmt.fmt_G64(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtComplex64(v complex64, verb int, value interface{}) { + switch verb { + case 'e', 'E', 'f', 'F', 'g', 'G': + p.fmt.fmt_c64(v, verb) + case 'v': + p.fmt.fmt_c64(v, 'g') + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtComplex128(v complex128, verb int, value interface{}) { + switch verb { + case 'e', 'E', 'f', 'F', 'g', 'G': + p.fmt.fmt_c128(v, verb) + case 'v': + p.fmt.fmt_c128(v, 'g') + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}) { + switch verb { + case 'v': + if goSyntax { + p.fmt.fmt_q(v) + } else { + p.fmt.fmt_s(v) + } + case 's': + p.fmt.fmt_s(v) + case 'x': + p.fmt.fmt_sx(v) + case 'X': + p.fmt.fmt_sX(v) + case 'q': + p.fmt.fmt_q(v) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interface{}) { + if verb == 'v' || verb == 'd' { + if goSyntax { + p.buf.Write(bytesBytes) + } else { + p.buf.WriteByte('[') + } + for i, c := range v { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(c, 'v', p.fmt.plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + return + } + s := string(v) + switch verb { + case 's': + p.fmt.fmt_s(s) + case 'x': + p.fmt.fmt_sx(s) + case 'X': + p.fmt.fmt_sX(s) + case 'q': + p.fmt.fmt_q(s) + default: + p.badVerb(verb, value) + } +} + +func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSyntax bool) { + var u uintptr + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: + u = value.Pointer() + default: + p.badVerb(verb, field) + return + } + if goSyntax { + p.add('(') + p.buf.WriteString(reflect.TypeOf(field).String()) + p.add(')') + p.add('(') + if u == 0 { + p.buf.Write(nilBytes) + } else { + p.fmt0x64(uint64(u), true) + } + p.add(')') + } else { + p.fmt0x64(uint64(u), !p.fmt.sharp) + } +} + +var ( + intBits = reflect.TypeOf(0).Bits() + floatBits = reflect.TypeOf(0.0).Bits() + complexBits = reflect.TypeOf(1i).Bits() + uintptrBits = reflect.TypeOf(uintptr(0)).Bits() +) + +func (p *pp) catchPanic(val interface{}, verb int) { + if err := recover(); err != nil { + // If it's a nil pointer, just say "". The likeliest causes are a + // Stringer that fails to guard against nil or a nil pointer for a + // value receiver, and in either case, "" is a nice result. + if v := reflect.ValueOf(val); v.Kind() == reflect.Ptr && v.IsNil() { + p.buf.Write(nilAngleBytes) + return + } + // Otherwise print a concise panic message. Most of the time the panic + // value will print itself nicely. + if p.panicking { + // Nested panics; the recursion in printField cannot succeed. + panic(err) + } + p.buf.WriteByte('%') + p.add(verb) + p.buf.Write(panicBytes) + p.panicking = true + p.printField(err, 'v', false, false, 0) + p.panicking = false + p.buf.WriteByte(')') + } +} + +func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth int) (wasString bool) { + if field == nil { + if verb == 'T' || verb == 'v' { + p.buf.Write(nilAngleBytes) + } else { + p.badVerb(verb, field) + } + return false + } + + // Special processing considerations. + // %T (the value's type) and %p (its address) are special; we always do them first. + switch verb { + case 'T': + p.printField(reflect.TypeOf(field).String(), 's', false, false, 0) + return false + case 'p': + p.fmtPointer(field, reflect.ValueOf(field), verb, goSyntax) + return false + } + // Is it a Formatter? + if formatter, ok := field.(Formatter); ok { + defer p.catchPanic(field, verb) + formatter.Format(p, verb) + return false // this value is not a string + + } + // Must not touch flags before Formatter looks at them. + if plus { + p.fmt.plus = false + } + // If we're doing Go syntax and the field knows how to supply it, take care of it now. + if goSyntax { + p.fmt.sharp = false + if stringer, ok := field.(GoStringer); ok { + defer p.catchPanic(field, verb) + // Print the result of GoString unadorned. + p.fmtString(stringer.GoString(), 's', false, field) + return false // this value is not a string + } + } else { + // Is it a Stringer? + if stringer, ok := field.(Stringer); ok { + defer p.catchPanic(field, verb) + p.printField(stringer.String(), verb, plus, false, depth) + return false // this value is not a string + } + } + + // Some types can be done without reflection. + switch f := field.(type) { + case bool: + p.fmtBool(f, verb, field) + return false + case float32: + p.fmtFloat32(f, verb, field) + return false + case float64: + p.fmtFloat64(f, verb, field) + return false + case complex64: + p.fmtComplex64(complex64(f), verb, field) + return false + case complex128: + p.fmtComplex128(f, verb, field) + return false + case int: + p.fmtInt64(int64(f), verb, field) + return false + case int8: + p.fmtInt64(int64(f), verb, field) + return false + case int16: + p.fmtInt64(int64(f), verb, field) + return false + case int32: + p.fmtInt64(int64(f), verb, field) + return false + case int64: + p.fmtInt64(f, verb, field) + return false + case uint: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint8: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint16: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint32: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case uint64: + p.fmtUint64(f, verb, goSyntax, field) + return false + case uintptr: + p.fmtUint64(uint64(f), verb, goSyntax, field) + return false + case string: + p.fmtString(f, verb, goSyntax, field) + return verb == 's' || verb == 'v' + case []byte: + p.fmtBytes(f, verb, goSyntax, depth, field) + return verb == 's' + } + + // Need to use reflection + value := reflect.ValueOf(field) + +BigSwitch: + switch f := value; f.Kind() { + case reflect.Bool: + p.fmtBool(f.Bool(), verb, field) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + p.fmtInt64(f.Int(), verb, field) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + p.fmtUint64(uint64(f.Uint()), verb, goSyntax, field) + case reflect.Float32, reflect.Float64: + if f.Type().Size() == 4 { + p.fmtFloat32(float32(f.Float()), verb, field) + } else { + p.fmtFloat64(float64(f.Float()), verb, field) + } + case reflect.Complex64, reflect.Complex128: + if f.Type().Size() == 8 { + p.fmtComplex64(complex64(f.Complex()), verb, field) + } else { + p.fmtComplex128(complex128(f.Complex()), verb, field) + } + case reflect.String: + p.fmtString(f.String(), verb, goSyntax, field) + case reflect.Map: + if goSyntax { + p.buf.WriteString(f.Type().String()) + p.buf.WriteByte('{') + } else { + p.buf.Write(mapBytes) + } + keys := f.MapKeys() + for i, key := range keys { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(key.Interface(), verb, plus, goSyntax, depth+1) + p.buf.WriteByte(':') + p.printField(f.MapIndex(key).Interface(), verb, plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + case reflect.Struct: + if goSyntax { + p.buf.WriteString(reflect.TypeOf(field).String()) + } + p.add('{') + v := f + t := v.Type() + for i := 0; i < v.NumField(); i++ { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + if plus || goSyntax { + if f := t.Field(i); f.Name != "" { + p.buf.WriteString(f.Name) + p.buf.WriteByte(':') + } + } + p.printField(getField(v, i).Interface(), verb, plus, goSyntax, depth+1) + } + p.buf.WriteByte('}') + case reflect.Interface: + value := f.Elem() + if !value.IsValid() { + if goSyntax { + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.Write(nilParenBytes) + } else { + p.buf.Write(nilAngleBytes) + } + } else { + return p.printField(value.Interface(), verb, plus, goSyntax, depth+1) + } + case reflect.Array, reflect.Slice: + // Byte slices are special. + if f.Type().Elem().Kind() == reflect.Uint8 { + // We know it's a slice of bytes, but we also know it does not have static type + // []byte, or it would have been caught above. Therefore we cannot convert + // it directly in the (slightly) obvious way: f.Interface().([]byte); it doesn't have + // that type, and we can't write an expression of the right type and do a + // conversion because we don't have a static way to write the right type. + // So we build a slice by hand. This is a rare case but it would be nice + // if reflection could help a little more. + bytes := make([]byte, f.Len()) + for i := range bytes { + bytes[i] = byte(f.Index(i).Uint()) + } + p.fmtBytes(bytes, verb, goSyntax, depth, field) + return verb == 's' + } + if goSyntax { + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.WriteByte('{') + } else { + p.buf.WriteByte('[') + } + for i := 0; i < f.Len(); i++ { + if i > 0 { + if goSyntax { + p.buf.Write(commaSpaceBytes) + } else { + p.buf.WriteByte(' ') + } + } + p.printField(f.Index(i).Interface(), verb, plus, goSyntax, depth+1) + } + if goSyntax { + p.buf.WriteByte('}') + } else { + p.buf.WriteByte(']') + } + case reflect.Ptr: + v := f.Pointer() + // pointer to array or slice or struct? ok at top level + // but not embedded (avoid loops) + if v != 0 && depth == 0 { + switch a := f.Elem(); a.Kind() { + case reflect.Array, reflect.Slice: + p.buf.WriteByte('&') + p.printField(a.Interface(), verb, plus, goSyntax, depth+1) + break BigSwitch + case reflect.Struct: + p.buf.WriteByte('&') + p.printField(a.Interface(), verb, plus, goSyntax, depth+1) + break BigSwitch + } + } + if goSyntax { + p.buf.WriteByte('(') + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.WriteByte(')') + p.buf.WriteByte('(') + if v == 0 { + p.buf.Write(nilBytes) + } else { + p.fmt0x64(uint64(v), true) + } + p.buf.WriteByte(')') + break + } + if v == 0 { + p.buf.Write(nilAngleBytes) + break + } + p.fmt0x64(uint64(v), true) + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + p.fmtPointer(field, value, verb, goSyntax) + default: + p.unknownType(f) + } + return false +} + +// intFromArg gets the fieldnumth element of a. On return, isInt reports whether the argument has type int. +func intFromArg(a []interface{}, end, i, fieldnum int) (num int, isInt bool, newi, newfieldnum int) { + newi, newfieldnum = end, fieldnum + if i < end && fieldnum < len(a) { + num, isInt = a[fieldnum].(int) + newi, newfieldnum = i+1, fieldnum+1 + } + return +} + +func (p *pp) doPrintf(format string, a []interface{}) { + end := len(format) + fieldnum := 0 // we process one field per non-trivial format + for i := 0; i < end; { + lasti := i + for i < end && format[i] != '%' { + i++ + } + if i > lasti { + p.buf.WriteString(format[lasti:i]) + } + if i >= end { + // done processing format string + break + } + + // Process one verb + i++ + // flags and widths + p.fmt.clearflags() + F: + for ; i < end; i++ { + switch format[i] { + case '#': + p.fmt.sharp = true + case '0': + p.fmt.zero = true + case '+': + p.fmt.plus = true + case '-': + p.fmt.minus = true + case ' ': + p.fmt.space = true + default: + break F + } + } + // do we have width? + if i < end && format[i] == '*' { + p.fmt.wid, p.fmt.widPresent, i, fieldnum = intFromArg(a, end, i, fieldnum) + if !p.fmt.widPresent { + p.buf.Write(widthBytes) + } + } else { + p.fmt.wid, p.fmt.widPresent, i = parsenum(format, i, end) + } + // do we have precision? + if i < end && format[i] == '.' { + if format[i+1] == '*' { + p.fmt.prec, p.fmt.precPresent, i, fieldnum = intFromArg(a, end, i+1, fieldnum) + if !p.fmt.precPresent { + p.buf.Write(precBytes) + } + } else { + p.fmt.prec, p.fmt.precPresent, i = parsenum(format, i+1, end) + if !p.fmt.precPresent { + p.fmt.prec = 0 + p.fmt.precPresent = true + } + } + } + if i >= end { + p.buf.Write(noVerbBytes) + continue + } + c, w := utf8.DecodeRuneInString(format[i:]) + i += w + // percent is special - absorbs no operand + if c == '%' { + p.buf.WriteByte('%') // We ignore width and prec. + continue + } + if fieldnum >= len(a) { // out of operands + p.buf.WriteByte('%') + p.add(c) + p.buf.Write(missingBytes) + continue + } + field := a[fieldnum] + fieldnum++ + + goSyntax := c == 'v' && p.fmt.sharp + plus := c == 'v' && p.fmt.plus + p.printField(field, c, plus, goSyntax, 0) + } + + if fieldnum < len(a) { + p.buf.Write(extraBytes) + for ; fieldnum < len(a); fieldnum++ { + field := a[fieldnum] + if field != nil { + p.buf.WriteString(reflect.TypeOf(field).String()) + p.buf.WriteByte('=') + } + p.printField(field, 'v', false, false, 0) + if fieldnum+1 < len(a) { + p.buf.Write(commaSpaceBytes) + } + } + p.buf.WriteByte(')') + } +} + +func (p *pp) doPrint(a []interface{}, addspace, addnewline bool) { + prevString := false + for fieldnum := 0; fieldnum < len(a); fieldnum++ { + p.fmt.clearflags() + // always add spaces if we're doing println + field := a[fieldnum] + if fieldnum > 0 { + isString := field != nil && reflect.TypeOf(field).Kind() == reflect.String + if addspace || !isString && !prevString { + p.buf.WriteByte(' ') + } + } + prevString = p.printField(field, 'v', false, false, 0) + } + if addnewline { + p.buf.WriteByte('\n') + } +} diff --git a/src/pkg/fmt/scan.go b/src/pkg/fmt/scan.go new file mode 100644 index 000000000..259451d02 --- /dev/null +++ b/src/pkg/fmt/scan.go @@ -0,0 +1,1112 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "bytes" + "io" + "math" + "os" + "reflect" + "strconv" + "strings" + "unicode" + "utf8" +) + +// runeUnreader is the interface to something that can unread runes. +// If the object provided to Scan does not satisfy this interface, +// a local buffer will be used to back up the input, but its contents +// will be lost when Scan returns. +type runeUnreader interface { + UnreadRune() os.Error +} + +// ScanState represents the scanner state passed to custom scanners. +// Scanners may do rune-at-a-time scanning or ask the ScanState +// to discover the next space-delimited token. +type ScanState interface { + // ReadRune reads the next rune (Unicode code point) from the input. + // If invoked during Scanln, Fscanln, or Sscanln, ReadRune() will + // return EOF after returning the first '\n' or when reading beyond + // the specified width. + ReadRune() (rune int, size int, err os.Error) + // UnreadRune causes the next call to ReadRune to return the same rune. + UnreadRune() os.Error + // SkipSpace skips space in the input. Newlines are treated as space + // unless the scan operation is Scanln, Fscanln or Sscanln, in which case + // a newline is treated as EOF. + SkipSpace() + // Token skips space in the input if skipSpace is true, then returns the + // run of Unicode code points c satisfying f(c). If f is nil, + // !unicode.IsSpace(c) is used; that is, the token will hold non-space + // characters. Newlines are treated as space unless the scan operation + // is Scanln, Fscanln or Sscanln, in which case a newline is treated as + // EOF. The returned slice points to shared data that may be overwritten + // by the next call to Token, a call to a Scan function using the ScanState + // as input, or when the calling Scan method returns. + Token(skipSpace bool, f func(int) bool) (token []byte, err os.Error) + // Width returns the value of the width option and whether it has been set. + // The unit is Unicode code points. + Width() (wid int, ok bool) + // Because ReadRune is implemented by the interface, Read should never be + // called by the scanning routines and a valid implementation of + // ScanState may choose always to return an error from Read. + Read(buf []byte) (n int, err os.Error) +} + +// Scanner is implemented by any value that has a Scan method, which scans +// the input for the representation of a value and stores the result in the +// receiver, which must be a pointer to be useful. The Scan method is called +// for any argument to Scan, Scanf, or Scanln that implements it. +type Scanner interface { + Scan(state ScanState, verb int) os.Error +} + +// Scan scans text read from standard input, storing successive +// space-separated values into successive arguments. Newlines count +// as space. It returns the number of items successfully scanned. +// If that is less than the number of arguments, err will report why. +func Scan(a ...interface{}) (n int, err os.Error) { + return Fscan(os.Stdin, a...) +} + +// Scanln is similar to Scan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Scanln(a ...interface{}) (n int, err os.Error) { + return Fscanln(os.Stdin, a...) +} + +// Scanf scans text read from standard input, storing successive +// space-separated values into successive arguments as determined by +// the format. It returns the number of items successfully scanned. +func Scanf(format string, a ...interface{}) (n int, err os.Error) { + return Fscanf(os.Stdin, format, a...) +} + +// Sscan scans the argument string, storing successive space-separated +// values into successive arguments. Newlines count as space. It +// returns the number of items successfully scanned. If that is less +// than the number of arguments, err will report why. +func Sscan(str string, a ...interface{}) (n int, err os.Error) { + return Fscan(strings.NewReader(str), a...) +} + +// Sscanln is similar to Sscan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Sscanln(str string, a ...interface{}) (n int, err os.Error) { + return Fscanln(strings.NewReader(str), a...) +} + +// Sscanf scans the argument string, storing successive space-separated +// values into successive arguments as determined by the format. It +// returns the number of items successfully parsed. +func Sscanf(str string, format string, a ...interface{}) (n int, err os.Error) { + return Fscanf(strings.NewReader(str), format, a...) +} + +// Fscan scans text read from r, storing successive space-separated +// values into successive arguments. Newlines count as space. It +// returns the number of items successfully scanned. If that is less +// than the number of arguments, err will report why. +func Fscan(r io.Reader, a ...interface{}) (n int, err os.Error) { + s, old := newScanState(r, true, false) + n, err = s.doScan(a) + s.free(old) + return +} + +// Fscanln is similar to Fscan, but stops scanning at a newline and +// after the final item there must be a newline or EOF. +func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { + s, old := newScanState(r, false, true) + n, err = s.doScan(a) + s.free(old) + return +} + +// Fscanf scans text read from r, storing successive space-separated +// values into successive arguments as determined by the format. It +// returns the number of items successfully parsed. +func Fscanf(r io.Reader, format string, a ...interface{}) (n int, err os.Error) { + s, old := newScanState(r, false, false) + n, err = s.doScanf(format, a) + s.free(old) + return +} + +// scanError represents an error generated by the scanning software. +// It's used as a unique signature to identify such errors when recovering. +type scanError struct { + err os.Error +} + +const eof = -1 + +// ss is the internal implementation of ScanState. +type ss struct { + rr io.RuneReader // where to read input + buf bytes.Buffer // token accumulator + peekRune int // one-rune lookahead + prevRune int // last rune returned by ReadRune + count int // runes consumed so far. + atEOF bool // already read EOF + ssave +} + +// ssave holds the parts of ss that need to be +// saved and restored on recursive scans. +type ssave struct { + validSave bool // is or was a part of an actual ss. + nlIsEnd bool // whether newline terminates scan + nlIsSpace bool // whether newline counts as white space + fieldLimit int // max value of ss.count for this field; fieldLimit <= limit + limit int // max value of ss.count. + maxWid int // width of this field. +} + +// The Read method is only in ScanState so that ScanState +// satisfies io.Reader. It will never be called when used as +// intended, so there is no need to make it actually work. +func (s *ss) Read(buf []byte) (n int, err os.Error) { + return 0, os.NewError("ScanState's Read should not be called. Use ReadRune") +} + +func (s *ss) ReadRune() (rune int, size int, err os.Error) { + if s.peekRune >= 0 { + s.count++ + rune = s.peekRune + size = utf8.RuneLen(rune) + s.prevRune = rune + s.peekRune = -1 + return + } + if s.atEOF || s.nlIsEnd && s.prevRune == '\n' || s.count >= s.fieldLimit { + err = os.EOF + return + } + + rune, size, err = s.rr.ReadRune() + if err == nil { + s.count++ + s.prevRune = rune + } else if err == os.EOF { + s.atEOF = true + } + return +} + +func (s *ss) Width() (wid int, ok bool) { + if s.maxWid == hugeWid { + return 0, false + } + return s.maxWid, true +} + +// The public method returns an error; this private one panics. +// If getRune reaches EOF, the return value is EOF (-1). +func (s *ss) getRune() (rune int) { + rune, _, err := s.ReadRune() + if err != nil { + if err == os.EOF { + return eof + } + s.error(err) + } + return +} + +// mustReadRune turns os.EOF into a panic(io.ErrUnexpectedEOF). +// It is called in cases such as string scanning where an EOF is a +// syntax error. +func (s *ss) mustReadRune() (rune int) { + rune = s.getRune() + if rune == eof { + s.error(io.ErrUnexpectedEOF) + } + return +} + +func (s *ss) UnreadRune() os.Error { + if u, ok := s.rr.(runeUnreader); ok { + u.UnreadRune() + } else { + s.peekRune = s.prevRune + } + s.prevRune = -1 + s.count-- + return nil +} + +func (s *ss) error(err os.Error) { + panic(scanError{err}) +} + +func (s *ss) errorString(err string) { + panic(scanError{os.NewError(err)}) +} + +func (s *ss) Token(skipSpace bool, f func(int) bool) (tok []byte, err os.Error) { + defer func() { + if e := recover(); e != nil { + if se, ok := e.(scanError); ok { + err = se.err + } else { + panic(e) + } + } + }() + if f == nil { + f = notSpace + } + s.buf.Reset() + tok = s.token(skipSpace, f) + return +} + +// notSpace is the default scanning function used in Token. +func notSpace(r int) bool { + return !unicode.IsSpace(r) +} + +// skipSpace provides Scan() methods the ability to skip space and newline characters +// in keeping with the current scanning mode set by format strings and Scan()/Scanln(). +func (s *ss) SkipSpace() { + s.skipSpace(false) +} + +// readRune is a structure to enable reading UTF-8 encoded code points +// from an io.Reader. It is used if the Reader given to the scanner does +// not already implement io.RuneReader. +type readRune struct { + reader io.Reader + buf [utf8.UTFMax]byte // used only inside ReadRune + pending int // number of bytes in pendBuf; only >0 for bad UTF-8 + pendBuf [utf8.UTFMax]byte // bytes left over +} + +// readByte returns the next byte from the input, which may be +// left over from a previous read if the UTF-8 was ill-formed. +func (r *readRune) readByte() (b byte, err os.Error) { + if r.pending > 0 { + b = r.pendBuf[0] + copy(r.pendBuf[0:], r.pendBuf[1:]) + r.pending-- + return + } + _, err = r.reader.Read(r.pendBuf[0:1]) + return r.pendBuf[0], err +} + +// unread saves the bytes for the next read. +func (r *readRune) unread(buf []byte) { + copy(r.pendBuf[r.pending:], buf) + r.pending += len(buf) +} + +// ReadRune returns the next UTF-8 encoded code point from the +// io.Reader inside r. +func (r *readRune) ReadRune() (rune int, size int, err os.Error) { + r.buf[0], err = r.readByte() + if err != nil { + return 0, 0, err + } + if r.buf[0] < utf8.RuneSelf { // fast check for common ASCII case + rune = int(r.buf[0]) + return + } + var n int + for n = 1; !utf8.FullRune(r.buf[0:n]); n++ { + r.buf[n], err = r.readByte() + if err != nil { + if err == os.EOF { + err = nil + break + } + return + } + } + rune, size = utf8.DecodeRune(r.buf[0:n]) + if size < n { // an error + r.unread(r.buf[size:n]) + } + return +} + +var ssFree = newCache(func() interface{} { return new(ss) }) + +// Allocate a new ss struct or grab a cached one. +func newScanState(r io.Reader, nlIsSpace, nlIsEnd bool) (s *ss, old ssave) { + // If the reader is a *ss, then we've got a recursive + // call to Scan, so re-use the scan state. + s, ok := r.(*ss) + if ok { + old = s.ssave + s.limit = s.fieldLimit + s.nlIsEnd = nlIsEnd || s.nlIsEnd + s.nlIsSpace = nlIsSpace + return + } + + s = ssFree.get().(*ss) + if rr, ok := r.(io.RuneReader); ok { + s.rr = rr + } else { + s.rr = &readRune{reader: r} + } + s.nlIsSpace = nlIsSpace + s.nlIsEnd = nlIsEnd + s.prevRune = -1 + s.peekRune = -1 + s.atEOF = false + s.limit = hugeWid + s.fieldLimit = hugeWid + s.maxWid = hugeWid + s.validSave = true + return +} + +// Save used ss structs in ssFree; avoid an allocation per invocation. +func (s *ss) free(old ssave) { + // If it was used recursively, just restore the old state. + if old.validSave { + s.ssave = old + return + } + // Don't hold on to ss structs with large buffers. + if cap(s.buf.Bytes()) > 1024 { + return + } + s.buf.Reset() + s.rr = nil + ssFree.put(s) +} + +// skipSpace skips spaces and maybe newlines. +func (s *ss) skipSpace(stopAtNewline bool) { + for { + rune := s.getRune() + if rune == eof { + return + } + if rune == '\n' { + if stopAtNewline { + break + } + if s.nlIsSpace { + continue + } + s.errorString("unexpected newline") + return + } + if !unicode.IsSpace(rune) { + s.UnreadRune() + break + } + } +} + +// token returns the next space-delimited string from the input. It +// skips white space. For Scanln, it stops at newlines. For Scan, +// newlines are treated as spaces. +func (s *ss) token(skipSpace bool, f func(int) bool) []byte { + if skipSpace { + s.skipSpace(false) + } + // read until white space or newline + for { + rune := s.getRune() + if rune == eof { + break + } + if !f(rune) { + s.UnreadRune() + break + } + s.buf.WriteRune(rune) + } + return s.buf.Bytes() +} + +// typeError indicates that the type of the operand did not match the format +func (s *ss) typeError(field interface{}, expected string) { + s.errorString("expected field of type pointer to " + expected + "; found " + reflect.TypeOf(field).String()) +} + +var complexError = os.NewError("syntax error scanning complex number") +var boolError = os.NewError("syntax error scanning boolean") + +// consume reads the next rune in the input and reports whether it is in the ok string. +// If accept is true, it puts the character into the input token. +func (s *ss) consume(ok string, accept bool) bool { + rune := s.getRune() + if rune == eof { + return false + } + if strings.IndexRune(ok, rune) >= 0 { + if accept { + s.buf.WriteRune(rune) + } + return true + } + if rune != eof && accept { + s.UnreadRune() + } + return false +} + +// peek reports whether the next character is in the ok string, without consuming it. +func (s *ss) peek(ok string) bool { + rune := s.getRune() + if rune != eof { + s.UnreadRune() + } + return strings.IndexRune(ok, rune) >= 0 +} + +func (s *ss) notEOF() { + // Guarantee there is data to be read. + if rune := s.getRune(); rune == eof { + panic(os.EOF) + } + s.UnreadRune() +} + +// accept checks the next rune in the input. If it's a byte (sic) in the string, it puts it in the +// buffer and returns true. Otherwise it return false. +func (s *ss) accept(ok string) bool { + return s.consume(ok, true) +} + +// okVerb verifies that the verb is present in the list, setting s.err appropriately if not. +func (s *ss) okVerb(verb int, okVerbs, typ string) bool { + for _, v := range okVerbs { + if v == verb { + return true + } + } + s.errorString("bad verb %" + string(verb) + " for " + typ) + return false +} + +// scanBool returns the value of the boolean represented by the next token. +func (s *ss) scanBool(verb int) bool { + s.skipSpace(false) + s.notEOF() + if !s.okVerb(verb, "tv", "boolean") { + return false + } + // Syntax-checking a boolean is annoying. We're not fastidious about case. + switch s.getRune() { + case '0': + return false + case '1': + return true + case 't', 'T': + if s.accept("rR") && (!s.accept("uU") || !s.accept("eE")) { + s.error(boolError) + } + return true + case 'f', 'F': + if s.accept("aL") && (!s.accept("lL") || !s.accept("sS") || !s.accept("eE")) { + s.error(boolError) + } + return false + } + return false +} + +// Numerical elements +const ( + binaryDigits = "01" + octalDigits = "01234567" + decimalDigits = "0123456789" + hexadecimalDigits = "0123456789aAbBcCdDeEfF" + sign = "+-" + period = "." + exponent = "eEp" +) + +// getBase returns the numeric base represented by the verb and its digit string. +func (s *ss) getBase(verb int) (base int, digits string) { + s.okVerb(verb, "bdoUxXv", "integer") // sets s.err + base = 10 + digits = decimalDigits + switch verb { + case 'b': + base = 2 + digits = binaryDigits + case 'o': + base = 8 + digits = octalDigits + case 'x', 'X', 'U': + base = 16 + digits = hexadecimalDigits + } + return +} + +// scanNumber returns the numerical string with specified digits starting here. +func (s *ss) scanNumber(digits string, haveDigits bool) string { + if !haveDigits { + s.notEOF() + if !s.accept(digits) { + s.errorString("expected integer") + } + } + for s.accept(digits) { + } + return s.buf.String() +} + +// scanRune returns the next rune value in the input. +func (s *ss) scanRune(bitSize int) int64 { + s.notEOF() + rune := int64(s.getRune()) + n := uint(bitSize) + x := (rune << (64 - n)) >> (64 - n) + if x != rune { + s.errorString("overflow on character value " + string(rune)) + } + return rune +} + +// scanBasePrefix reports whether the integer begins with a 0 or 0x, +// and returns the base, digit string, and whether a zero was found. +// It is called only if the verb is %v. +func (s *ss) scanBasePrefix() (base int, digits string, found bool) { + if !s.peek("0") { + return 10, decimalDigits, false + } + s.accept("0") + found = true // We've put a digit into the token buffer. + // Special cases for '0' && '0x' + base, digits = 8, octalDigits + if s.peek("xX") { + s.consume("xX", false) + base, digits = 16, hexadecimalDigits + } + return +} + +// scanInt returns the value of the integer represented by the next +// token, checking for overflow. Any error is stored in s.err. +func (s *ss) scanInt(verb int, bitSize int) int64 { + if verb == 'c' { + return s.scanRune(bitSize) + } + s.skipSpace(false) + s.notEOF() + base, digits := s.getBase(verb) + haveDigits := false + if verb == 'U' { + if !s.consume("U", false) || !s.consume("+", false) { + s.errorString("bad unicode format ") + } + } else { + s.accept(sign) // If there's a sign, it will be left in the token buffer. + if verb == 'v' { + base, digits, haveDigits = s.scanBasePrefix() + } + } + tok := s.scanNumber(digits, haveDigits) + i, err := strconv.Btoi64(tok, base) + if err != nil { + s.error(err) + } + n := uint(bitSize) + x := (i << (64 - n)) >> (64 - n) + if x != i { + s.errorString("integer overflow on token " + tok) + } + return i +} + +// scanUint returns the value of the unsigned integer represented +// by the next token, checking for overflow. Any error is stored in s.err. +func (s *ss) scanUint(verb int, bitSize int) uint64 { + if verb == 'c' { + return uint64(s.scanRune(bitSize)) + } + s.skipSpace(false) + s.notEOF() + base, digits := s.getBase(verb) + haveDigits := false + if verb == 'U' { + if !s.consume("U", false) || !s.consume("+", false) { + s.errorString("bad unicode format ") + } + } else if verb == 'v' { + base, digits, haveDigits = s.scanBasePrefix() + } + tok := s.scanNumber(digits, haveDigits) + i, err := strconv.Btoui64(tok, base) + if err != nil { + s.error(err) + } + n := uint(bitSize) + x := (i << (64 - n)) >> (64 - n) + if x != i { + s.errorString("unsigned integer overflow on token " + tok) + } + return i +} + +// floatToken returns the floating-point number starting here, no longer than swid +// if the width is specified. It's not rigorous about syntax because it doesn't check that +// we have at least some digits, but Atof will do that. +func (s *ss) floatToken() string { + s.buf.Reset() + // NaN? + if s.accept("nN") && s.accept("aA") && s.accept("nN") { + return s.buf.String() + } + // leading sign? + s.accept(sign) + // Inf? + if s.accept("iI") && s.accept("nN") && s.accept("fF") { + return s.buf.String() + } + // digits? + for s.accept(decimalDigits) { + } + // decimal point? + if s.accept(period) { + // fraction? + for s.accept(decimalDigits) { + } + } + // exponent? + if s.accept(exponent) { + // leading sign? + s.accept(sign) + // digits? + for s.accept(decimalDigits) { + } + } + return s.buf.String() +} + +// complexTokens returns the real and imaginary parts of the complex number starting here. +// The number might be parenthesized and has the format (N+Ni) where N is a floating-point +// number and there are no spaces within. +func (s *ss) complexTokens() (real, imag string) { + // TODO: accept N and Ni independently? + parens := s.accept("(") + real = s.floatToken() + s.buf.Reset() + // Must now have a sign. + if !s.accept("+-") { + s.error(complexError) + } + // Sign is now in buffer + imagSign := s.buf.String() + imag = s.floatToken() + if !s.accept("i") { + s.error(complexError) + } + if parens && !s.accept(")") { + s.error(complexError) + } + return real, imagSign + imag +} + +// convertFloat converts the string to a float64value. +func (s *ss) convertFloat(str string, n int) float64 { + if p := strings.Index(str, "p"); p >= 0 { + // Atof doesn't handle power-of-2 exponents, + // but they're easy to evaluate. + f, err := strconv.AtofN(str[:p], n) + if err != nil { + // Put full string into error. + if e, ok := err.(*strconv.NumError); ok { + e.Num = str + } + s.error(err) + } + n, err := strconv.Atoi(str[p+1:]) + if err != nil { + // Put full string into error. + if e, ok := err.(*strconv.NumError); ok { + e.Num = str + } + s.error(err) + } + return math.Ldexp(f, n) + } + f, err := strconv.AtofN(str, n) + if err != nil { + s.error(err) + } + return f +} + +// convertComplex converts the next token to a complex128 value. +// The atof argument is a type-specific reader for the underlying type. +// If we're reading complex64, atof will parse float32s and convert them +// to float64's to avoid reproducing this code for each complex type. +func (s *ss) scanComplex(verb int, n int) complex128 { + if !s.okVerb(verb, floatVerbs, "complex") { + return 0 + } + s.skipSpace(false) + s.notEOF() + sreal, simag := s.complexTokens() + real := s.convertFloat(sreal, n/2) + imag := s.convertFloat(simag, n/2) + return complex(real, imag) +} + +// convertString returns the string represented by the next input characters. +// The format of the input is determined by the verb. +func (s *ss) convertString(verb int) (str string) { + if !s.okVerb(verb, "svqx", "string") { + return "" + } + s.skipSpace(false) + s.notEOF() + switch verb { + case 'q': + str = s.quotedString() + case 'x': + str = s.hexString() + default: + str = string(s.token(true, notSpace)) // %s and %v just return the next word + } + return +} + +// quotedString returns the double- or back-quoted string represented by the next input characters. +func (s *ss) quotedString() string { + s.notEOF() + quote := s.getRune() + switch quote { + case '`': + // Back-quoted: Anything goes until EOF or back quote. + for { + rune := s.mustReadRune() + if rune == quote { + break + } + s.buf.WriteRune(rune) + } + return s.buf.String() + case '"': + // Double-quoted: Include the quotes and let strconv.Unquote do the backslash escapes. + s.buf.WriteRune(quote) + for { + rune := s.mustReadRune() + s.buf.WriteRune(rune) + if rune == '\\' { + // In a legal backslash escape, no matter how long, only the character + // immediately after the escape can itself be a backslash or quote. + // Thus we only need to protect the first character after the backslash. + rune := s.mustReadRune() + s.buf.WriteRune(rune) + } else if rune == '"' { + break + } + } + result, err := strconv.Unquote(s.buf.String()) + if err != nil { + s.error(err) + } + return result + default: + s.errorString("expected quoted string") + } + return "" +} + +// hexDigit returns the value of the hexadecimal digit +func (s *ss) hexDigit(digit int) int { + switch digit { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return digit - '0' + case 'a', 'b', 'c', 'd', 'e', 'f': + return 10 + digit - 'a' + case 'A', 'B', 'C', 'D', 'E', 'F': + return 10 + digit - 'A' + } + s.errorString("Scan: illegal hex digit") + return 0 +} + +// hexByte returns the next hex-encoded (two-character) byte from the input. +// There must be either two hexadecimal digits or a space character in the input. +func (s *ss) hexByte() (b byte, ok bool) { + rune1 := s.getRune() + if rune1 == eof { + return + } + if unicode.IsSpace(rune1) { + s.UnreadRune() + return + } + rune2 := s.mustReadRune() + return byte(s.hexDigit(rune1)<<4 | s.hexDigit(rune2)), true +} + +// hexString returns the space-delimited hexpair-encoded string. +func (s *ss) hexString() string { + s.notEOF() + for { + b, ok := s.hexByte() + if !ok { + break + } + s.buf.WriteByte(b) + } + if s.buf.Len() == 0 { + s.errorString("Scan: no hex data for %x string") + return "" + } + return s.buf.String() +} + +const floatVerbs = "beEfFgGv" + +const hugeWid = 1 << 30 + +// scanOne scans a single value, deriving the scanner from the type of the argument. +func (s *ss) scanOne(verb int, field interface{}) { + s.buf.Reset() + var err os.Error + // If the parameter has its own Scan method, use that. + if v, ok := field.(Scanner); ok { + err = v.Scan(s, verb) + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + s.error(err) + } + return + } + + switch v := field.(type) { + case *bool: + *v = s.scanBool(verb) + case *complex64: + *v = complex64(s.scanComplex(verb, 64)) + case *complex128: + *v = s.scanComplex(verb, 128) + case *int: + *v = int(s.scanInt(verb, intBits)) + case *int8: + *v = int8(s.scanInt(verb, 8)) + case *int16: + *v = int16(s.scanInt(verb, 16)) + case *int32: + *v = int32(s.scanInt(verb, 32)) + case *int64: + *v = s.scanInt(verb, 64) + case *uint: + *v = uint(s.scanUint(verb, intBits)) + case *uint8: + *v = uint8(s.scanUint(verb, 8)) + case *uint16: + *v = uint16(s.scanUint(verb, 16)) + case *uint32: + *v = uint32(s.scanUint(verb, 32)) + case *uint64: + *v = s.scanUint(verb, 64) + case *uintptr: + *v = uintptr(s.scanUint(verb, uintptrBits)) + // Floats are tricky because you want to scan in the precision of the result, not + // scan in high precision and convert, in order to preserve the correct error condition. + case *float32: + if s.okVerb(verb, floatVerbs, "float32") { + s.skipSpace(false) + s.notEOF() + *v = float32(s.convertFloat(s.floatToken(), 32)) + } + case *float64: + if s.okVerb(verb, floatVerbs, "float64") { + s.skipSpace(false) + s.notEOF() + *v = s.convertFloat(s.floatToken(), 64) + } + case *string: + *v = s.convertString(verb) + case *[]byte: + // We scan to string and convert so we get a copy of the data. + // If we scanned to bytes, the slice would point at the buffer. + *v = []byte(s.convertString(verb)) + default: + val := reflect.ValueOf(v) + ptr := val + if ptr.Kind() != reflect.Ptr { + s.errorString("Scan: type not a pointer: " + val.Type().String()) + return + } + switch v := ptr.Elem(); v.Kind() { + case reflect.Bool: + v.SetBool(s.scanBool(verb)) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v.SetInt(s.scanInt(verb, v.Type().Bits())) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + v.SetUint(s.scanUint(verb, v.Type().Bits())) + case reflect.String: + v.SetString(s.convertString(verb)) + case reflect.Slice: + // For now, can only handle (renamed) []byte. + typ := v.Type() + if typ.Elem().Kind() != reflect.Uint8 { + s.errorString("Scan: can't handle type: " + val.Type().String()) + } + str := s.convertString(verb) + v.Set(reflect.MakeSlice(typ, len(str), len(str))) + for i := 0; i < len(str); i++ { + v.Index(i).SetUint(uint64(str[i])) + } + case reflect.Float32, reflect.Float64: + s.skipSpace(false) + s.notEOF() + v.SetFloat(s.convertFloat(s.floatToken(), v.Type().Bits())) + case reflect.Complex64, reflect.Complex128: + v.SetComplex(s.scanComplex(verb, v.Type().Bits())) + default: + s.errorString("Scan: can't handle type: " + val.Type().String()) + } + } +} + +// errorHandler turns local panics into error returns. +func errorHandler(errp *os.Error) { + if e := recover(); e != nil { + if se, ok := e.(scanError); ok { // catch local error + *errp = se.err + } else if eof, ok := e.(os.Error); ok && eof == os.EOF { // out of input + *errp = eof + } else { + panic(e) + } + } +} + +// doScan does the real work for scanning without a format string. +func (s *ss) doScan(a []interface{}) (numProcessed int, err os.Error) { + defer errorHandler(&err) + for _, field := range a { + s.scanOne('v', field) + numProcessed++ + } + // Check for newline if required. + if !s.nlIsSpace { + for { + rune := s.getRune() + if rune == '\n' || rune == eof { + break + } + if !unicode.IsSpace(rune) { + s.errorString("Scan: expected newline") + break + } + } + } + return +} + +// advance determines whether the next characters in the input match +// those of the format. It returns the number of bytes (sic) consumed +// in the format. Newlines included, all runs of space characters in +// either input or format behave as a single space. This routine also +// handles the %% case. If the return value is zero, either format +// starts with a % (with no following %) or the input is empty. +// If it is negative, the input did not match the string. +func (s *ss) advance(format string) (i int) { + for i < len(format) { + fmtc, w := utf8.DecodeRuneInString(format[i:]) + if fmtc == '%' { + // %% acts like a real percent + nextc, _ := utf8.DecodeRuneInString(format[i+w:]) // will not match % if string is empty + if nextc != '%' { + return + } + i += w // skip the first % + } + sawSpace := false + for unicode.IsSpace(fmtc) && i < len(format) { + sawSpace = true + i += w + fmtc, w = utf8.DecodeRuneInString(format[i:]) + } + if sawSpace { + // There was space in the format, so there should be space (EOF) + // in the input. + inputc := s.getRune() + if inputc == eof { + return + } + if !unicode.IsSpace(inputc) { + // Space in format but not in input: error + s.errorString("expected space in input to match format") + } + s.skipSpace(true) + continue + } + inputc := s.mustReadRune() + if fmtc != inputc { + s.UnreadRune() + return -1 + } + i += w + } + return +} + +// doScanf does the real work when scanning with a format string. +// At the moment, it handles only pointers to basic types. +func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.Error) { + defer errorHandler(&err) + end := len(format) - 1 + // We process one item per non-trivial format + for i := 0; i <= end; { + w := s.advance(format[i:]) + if w > 0 { + i += w + continue + } + // Either we failed to advance, we have a percent character, or we ran out of input. + if format[i] != '%' { + // Can't advance format. Why not? + if w < 0 { + s.errorString("input does not match format") + } + // Otherwise at EOF; "too many operands" error handled below + break + } + i++ // % is one byte + + // do we have 20 (width)? + var widPresent bool + s.maxWid, widPresent, i = parsenum(format, i, end) + if !widPresent { + s.maxWid = hugeWid + } + s.fieldLimit = s.limit + if f := s.count + s.maxWid; f < s.fieldLimit { + s.fieldLimit = f + } + + c, w := utf8.DecodeRuneInString(format[i:]) + i += w + + if numProcessed >= len(a) { // out of operands + s.errorString("too few operands for format %" + format[i-w:]) + break + } + field := a[numProcessed] + + s.scanOne(c, field) + numProcessed++ + s.fieldLimit = s.limit + } + if numProcessed < len(a) { + s.errorString("too many operands") + } + return +} diff --git a/src/pkg/fmt/scan_test.go b/src/pkg/fmt/scan_test.go new file mode 100644 index 000000000..3f06e5725 --- /dev/null +++ b/src/pkg/fmt/scan_test.go @@ -0,0 +1,923 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + "bufio" + "bytes" + . "fmt" + "io" + "math" + "os" + "reflect" + "regexp" + "strings" + "testing" + "utf8" +) + +type ScanTest struct { + text string + in interface{} + out interface{} +} + +type ScanfTest struct { + format string + text string + in interface{} + out interface{} +} + +type ScanfMultiTest struct { + format string + text string + in []interface{} + out []interface{} + err string +} + +var ( + boolVal bool + intVal int + int8Val int8 + int16Val int16 + int32Val int32 + int64Val int64 + uintVal uint + uint8Val uint8 + uint16Val uint16 + uint32Val uint32 + uint64Val uint64 + float32Val float32 + float64Val float64 + stringVal string + stringVal1 string + bytesVal []byte + complex64Val complex64 + complex128Val complex128 + renamedBoolVal renamedBool + renamedIntVal renamedInt + renamedInt8Val renamedInt8 + renamedInt16Val renamedInt16 + renamedInt32Val renamedInt32 + renamedInt64Val renamedInt64 + renamedUintVal renamedUint + renamedUint8Val renamedUint8 + renamedUint16Val renamedUint16 + renamedUint32Val renamedUint32 + renamedUint64Val renamedUint64 + renamedUintptrVal renamedUintptr + renamedStringVal renamedString + renamedBytesVal renamedBytes + renamedFloat32Val renamedFloat32 + renamedFloat64Val renamedFloat64 + renamedComplex64Val renamedComplex64 + renamedComplex128Val renamedComplex128 +) + +type FloatTest struct { + text string + in float64 + out float64 +} + +// Xs accepts any non-empty run of the verb character +type Xs string + +func (x *Xs) Scan(state ScanState, verb int) os.Error { + tok, err := state.Token(true, func(r int) bool { return r == verb }) + if err != nil { + return err + } + s := string(tok) + if !regexp.MustCompile("^" + string(verb) + "+$").MatchString(s) { + return os.NewError("syntax error for xs") + } + *x = Xs(s) + return nil +} + +var xVal Xs + +// IntString accepts an integer followed immediately by a string. +// It tests the embedding of a scan within a scan. +type IntString struct { + i int + s string +} + +func (s *IntString) Scan(state ScanState, verb int) os.Error { + if _, err := Fscan(state, &s.i); err != nil { + return err + } + + tok, err := state.Token(true, nil) + if err != nil { + return err + } + s.s = string(tok) + return nil +} + +var intStringVal IntString + +// myStringReader implements Read but not ReadRune, allowing us to test our readRune wrapper +// type that creates something that can read runes given only Read(). +type myStringReader struct { + r *strings.Reader +} + +func (s *myStringReader) Read(p []byte) (n int, err os.Error) { + return s.r.Read(p) +} + +func newReader(s string) *myStringReader { + return &myStringReader{strings.NewReader(s)} +} + +var scanTests = []ScanTest{ + // Basic types + {"T\n", &boolVal, true}, // boolean test vals toggle to be sure they are written + {"F\n", &boolVal, false}, // restored to zero value + {"21\n", &intVal, 21}, + {"0\n", &intVal, 0}, + {"000\n", &intVal, 0}, + {"0x10\n", &intVal, 0x10}, + {"-0x10\n", &intVal, -0x10}, + {"0377\n", &intVal, 0377}, + {"-0377\n", &intVal, -0377}, + {"0\n", &uintVal, uint(0)}, + {"000\n", &uintVal, uint(0)}, + {"0x10\n", &uintVal, uint(0x10)}, + {"0377\n", &uintVal, uint(0377)}, + {"22\n", &int8Val, int8(22)}, + {"23\n", &int16Val, int16(23)}, + {"24\n", &int32Val, int32(24)}, + {"25\n", &int64Val, int64(25)}, + {"127\n", &int8Val, int8(127)}, + {"-21\n", &intVal, -21}, + {"-22\n", &int8Val, int8(-22)}, + {"-23\n", &int16Val, int16(-23)}, + {"-24\n", &int32Val, int32(-24)}, + {"-25\n", &int64Val, int64(-25)}, + {"-128\n", &int8Val, int8(-128)}, + {"+21\n", &intVal, +21}, + {"+22\n", &int8Val, int8(+22)}, + {"+23\n", &int16Val, int16(+23)}, + {"+24\n", &int32Val, int32(+24)}, + {"+25\n", &int64Val, int64(+25)}, + {"+127\n", &int8Val, int8(+127)}, + {"26\n", &uintVal, uint(26)}, + {"27\n", &uint8Val, uint8(27)}, + {"28\n", &uint16Val, uint16(28)}, + {"29\n", &uint32Val, uint32(29)}, + {"30\n", &uint64Val, uint64(30)}, + {"255\n", &uint8Val, uint8(255)}, + {"32767\n", &int16Val, int16(32767)}, + {"2.3\n", &float64Val, 2.3}, + {"2.3e1\n", &float32Val, float32(2.3e1)}, + {"2.3e2\n", &float64Val, 2.3e2}, + {"2.3p2\n", &float64Val, 2.3 * 4}, + {"2.3p+2\n", &float64Val, 2.3 * 4}, + {"2.3p+66\n", &float64Val, 2.3 * (1 << 32) * (1 << 32) * 4}, + {"2.3p-66\n", &float64Val, 2.3 / ((1 << 32) * (1 << 32) * 4)}, + {"2.35\n", &stringVal, "2.35"}, + {"2345678\n", &bytesVal, []byte("2345678")}, + {"(3.4e1-2i)\n", &complex128Val, 3.4e1 - 2i}, + {"-3.45e1-3i\n", &complex64Val, complex64(-3.45e1 - 3i)}, + {"-.45e1-1e2i\n", &complex128Val, complex128(-.45e1 - 100i)}, + {"hello\n", &stringVal, "hello"}, + + // Renamed types + {"true\n", &renamedBoolVal, renamedBool(true)}, + {"F\n", &renamedBoolVal, renamedBool(false)}, + {"101\n", &renamedIntVal, renamedInt(101)}, + {"102\n", &renamedIntVal, renamedInt(102)}, + {"103\n", &renamedUintVal, renamedUint(103)}, + {"104\n", &renamedUintVal, renamedUint(104)}, + {"105\n", &renamedInt8Val, renamedInt8(105)}, + {"106\n", &renamedInt16Val, renamedInt16(106)}, + {"107\n", &renamedInt32Val, renamedInt32(107)}, + {"108\n", &renamedInt64Val, renamedInt64(108)}, + {"109\n", &renamedUint8Val, renamedUint8(109)}, + {"110\n", &renamedUint16Val, renamedUint16(110)}, + {"111\n", &renamedUint32Val, renamedUint32(111)}, + {"112\n", &renamedUint64Val, renamedUint64(112)}, + {"113\n", &renamedUintptrVal, renamedUintptr(113)}, + {"114\n", &renamedStringVal, renamedString("114")}, + {"115\n", &renamedBytesVal, renamedBytes([]byte("115"))}, + + // Custom scanners. + {" vvv ", &xVal, Xs("vvv")}, + {" 1234hello", &intStringVal, IntString{1234, "hello"}}, + + // Fixed bugs + {"2147483648\n", &int64Val, int64(2147483648)}, // was: integer overflow +} + +var scanfTests = []ScanfTest{ + {"%v", "TRUE\n", &boolVal, true}, + {"%t", "false\n", &boolVal, false}, + {"%v", "-71\n", &intVal, -71}, + {"%v", "0377\n", &intVal, 0377}, + {"%v", "0x44\n", &intVal, 0x44}, + {"%d", "72\n", &intVal, 72}, + {"%c", "a\n", &intVal, 'a'}, + {"%c", "\u5072\n", &intVal, 0x5072}, + {"%c", "\u1234\n", &intVal, '\u1234'}, + {"%d", "73\n", &int8Val, int8(73)}, + {"%d", "+74\n", &int16Val, int16(74)}, + {"%d", "75\n", &int32Val, int32(75)}, + {"%d", "76\n", &int64Val, int64(76)}, + {"%b", "1001001\n", &intVal, 73}, + {"%o", "075\n", &intVal, 075}, + {"%x", "a75\n", &intVal, 0xa75}, + {"%v", "71\n", &uintVal, uint(71)}, + {"%d", "72\n", &uintVal, uint(72)}, + {"%d", "73\n", &uint8Val, uint8(73)}, + {"%d", "74\n", &uint16Val, uint16(74)}, + {"%d", "75\n", &uint32Val, uint32(75)}, + {"%d", "76\n", &uint64Val, uint64(76)}, + {"%b", "1001001\n", &uintVal, uint(73)}, + {"%o", "075\n", &uintVal, uint(075)}, + {"%x", "a75\n", &uintVal, uint(0xa75)}, + {"%x", "A75\n", &uintVal, uint(0xa75)}, + {"%U", "U+1234\n", &intVal, int(0x1234)}, + {"%U", "U+4567\n", &uintVal, uint(0x4567)}, + + // Strings + {"%s", "using-%s\n", &stringVal, "using-%s"}, + {"%x", "7573696e672d2578\n", &stringVal, "using-%x"}, + {"%q", `"quoted\twith\\do\u0075bl\x65s"` + "\n", &stringVal, "quoted\twith\\doubles"}, + {"%q", "`quoted with backs`\n", &stringVal, "quoted with backs"}, + + // Byte slices + {"%s", "bytes-%s\n", &bytesVal, []byte("bytes-%s")}, + {"%x", "62797465732d2578\n", &bytesVal, []byte("bytes-%x")}, + {"%q", `"bytes\rwith\vdo\u0075bl\x65s"` + "\n", &bytesVal, []byte("bytes\rwith\vdoubles")}, + {"%q", "`bytes with backs`\n", &bytesVal, []byte("bytes with backs")}, + + // Renamed types + {"%v\n", "true\n", &renamedBoolVal, renamedBool(true)}, + {"%t\n", "F\n", &renamedBoolVal, renamedBool(false)}, + {"%v", "101\n", &renamedIntVal, renamedInt(101)}, + {"%c", "\u0101\n", &renamedIntVal, renamedInt('\u0101')}, + {"%o", "0146\n", &renamedIntVal, renamedInt(102)}, + {"%v", "103\n", &renamedUintVal, renamedUint(103)}, + {"%d", "104\n", &renamedUintVal, renamedUint(104)}, + {"%d", "105\n", &renamedInt8Val, renamedInt8(105)}, + {"%d", "106\n", &renamedInt16Val, renamedInt16(106)}, + {"%d", "107\n", &renamedInt32Val, renamedInt32(107)}, + {"%d", "108\n", &renamedInt64Val, renamedInt64(108)}, + {"%x", "6D\n", &renamedUint8Val, renamedUint8(109)}, + {"%o", "0156\n", &renamedUint16Val, renamedUint16(110)}, + {"%d", "111\n", &renamedUint32Val, renamedUint32(111)}, + {"%d", "112\n", &renamedUint64Val, renamedUint64(112)}, + {"%d", "113\n", &renamedUintptrVal, renamedUintptr(113)}, + {"%s", "114\n", &renamedStringVal, renamedString("114")}, + {"%q", "\"1155\"\n", &renamedBytesVal, renamedBytes([]byte("1155"))}, + {"%g", "116e1\n", &renamedFloat32Val, renamedFloat32(116e1)}, + {"%g", "-11.7e+1", &renamedFloat64Val, renamedFloat64(-11.7e+1)}, + {"%g", "11+6e1i\n", &renamedComplex64Val, renamedComplex64(11 + 6e1i)}, + {"%g", "-11.+7e+1i", &renamedComplex128Val, renamedComplex128(-11. + 7e+1i)}, + + // Interesting formats + {"here is\tthe value:%d", "here is the\tvalue:118\n", &intVal, 118}, + {"%% %%:%d", "% %:119\n", &intVal, 119}, + + // Corner cases + {"%x", "FFFFFFFF\n", &uint32Val, uint32(0xFFFFFFFF)}, + + // Custom scanner. + {"%s", " sss ", &xVal, Xs("sss")}, + {"%2s", "sssss", &xVal, Xs("ss")}, + + // Fixed bugs + {"%d\n", "27\n", &intVal, 27}, // ok + {"%d\n", "28 \n", &intVal, 28}, // was: "unexpected newline" + {"%v", "0", &intVal, 0}, // was: "EOF"; 0 was taken as base prefix and not counted. + {"%v", "0", &uintVal, uint(0)}, // was: "EOF"; 0 was taken as base prefix and not counted. +} + +var overflowTests = []ScanTest{ + {"128", &int8Val, 0}, + {"32768", &int16Val, 0}, + {"-129", &int8Val, 0}, + {"-32769", &int16Val, 0}, + {"256", &uint8Val, 0}, + {"65536", &uint16Val, 0}, + {"1e100", &float32Val, 0}, + {"1e500", &float64Val, 0}, + {"(1e100+0i)", &complex64Val, 0}, + {"(1+1e100i)", &complex64Val, 0}, + {"(1-1e500i)", &complex128Val, 0}, +} + +var i, j, k int +var f float64 +var s, t string +var c complex128 +var x, y Xs +var z IntString + +var multiTests = []ScanfMultiTest{ + {"", "", nil, nil, ""}, + {"%d", "23", args(&i), args(23), ""}, + {"%2s%3s", "22333", args(&s, &t), args("22", "333"), ""}, + {"%2d%3d", "44555", args(&i, &j), args(44, 555), ""}, + {"%2d.%3d", "66.777", args(&i, &j), args(66, 777), ""}, + {"%d, %d", "23, 18", args(&i, &j), args(23, 18), ""}, + {"%3d22%3d", "33322333", args(&i, &j), args(333, 333), ""}, + {"%6vX=%3fY", "3+2iX=2.5Y", args(&c, &f), args((3 + 2i), 2.5), ""}, + {"%d%s", "123abc", args(&i, &s), args(123, "abc"), ""}, + {"%c%c%c", "2\u50c2X", args(&i, &j, &k), args('2', '\u50c2', 'X'), ""}, + + // Custom scanners. + {"%e%f", "eefffff", args(&x, &y), args(Xs("ee"), Xs("fffff")), ""}, + {"%4v%s", "12abcd", args(&z, &s), args(IntString{12, "ab"}, "cd"), ""}, + + // Errors + {"%t", "23 18", args(&i), nil, "bad verb"}, + {"%d %d %d", "23 18", args(&i, &j), args(23, 18), "too few operands"}, + {"%d %d", "23 18 27", args(&i, &j, &k), args(23, 18), "too many operands"}, + {"%c", "\u0100", args(&int8Val), nil, "overflow"}, + {"X%d", "10X", args(&intVal), nil, "input does not match format"}, + + // Bad UTF-8: should see every byte. + {"%c%c%c", "\xc2X\xc2", args(&i, &j, &k), args(utf8.RuneError, 'X', utf8.RuneError), ""}, +} + +func testScan(name string, t *testing.T, scan func(r io.Reader, a ...interface{}) (int, os.Error)) { + for _, test := range scanTests { + var r io.Reader + if name == "StringReader" { + r = strings.NewReader(test.text) + } else { + r = newReader(test.text) + } + n, err := scan(r, test.in) + if err != nil { + m := "" + if n > 0 { + m = Sprintf(" (%d fields ok)", n) + } + t.Errorf("%s got error scanning %q: %s%s", name, test.text, err, m) + continue + } + if n != 1 { + t.Errorf("%s count error on entry %q: got %d", name, test.text, n) + continue + } + // The incoming value may be a pointer + v := reflect.ValueOf(test.in) + if p := v; p.Kind() == reflect.Ptr { + v = p.Elem() + } + val := v.Interface() + if !reflect.DeepEqual(val, test.out) { + t.Errorf("%s scanning %q: expected %v got %v, type %T", name, test.text, test.out, val, val) + } + } +} + +func TestScan(t *testing.T) { + testScan("StringReader", t, Fscan) +} + +func TestMyReaderScan(t *testing.T) { + testScan("myStringReader", t, Fscan) +} + +func TestScanln(t *testing.T) { + testScan("StringReader", t, Fscanln) +} + +func TestMyReaderScanln(t *testing.T) { + testScan("myStringReader", t, Fscanln) +} + +func TestScanf(t *testing.T) { + for _, test := range scanfTests { + n, err := Sscanf(test.text, test.format, test.in) + if err != nil { + t.Errorf("got error scanning (%q, %q): %s", test.format, test.text, err) + continue + } + if n != 1 { + t.Errorf("count error on entry (%q, %q): got %d", test.format, test.text, n) + continue + } + // The incoming value may be a pointer + v := reflect.ValueOf(test.in) + if p := v; p.Kind() == reflect.Ptr { + v = p.Elem() + } + val := v.Interface() + if !reflect.DeepEqual(val, test.out) { + t.Errorf("scanning (%q, %q): expected %v got %v, type %T", test.format, test.text, test.out, val, val) + } + } +} + +func TestScanOverflow(t *testing.T) { + // different machines and different types report errors with different strings. + re := regexp.MustCompile("overflow|too large|out of range|not representable") + for _, test := range overflowTests { + _, err := Sscan(test.text, test.in) + if err == nil { + t.Errorf("expected overflow scanning %q", test.text) + continue + } + if !re.MatchString(err.String()) { + t.Errorf("expected overflow error scanning %q: %s", test.text, err) + } + } +} + +func verifyNaN(str string, t *testing.T) { + var f float64 + var f32 float32 + var f64 float64 + text := str + " " + str + " " + str + n, err := Fscan(strings.NewReader(text), &f, &f32, &f64) + if err != nil { + t.Errorf("got error scanning %q: %s", text, err) + } + if n != 3 { + t.Errorf("count error scanning %q: got %d", text, n) + } + if !math.IsNaN(float64(f)) || !math.IsNaN(float64(f32)) || !math.IsNaN(f64) { + t.Errorf("didn't get NaNs scanning %q: got %g %g %g", text, f, f32, f64) + } +} + +func TestNaN(t *testing.T) { + for _, s := range []string{"nan", "NAN", "NaN"} { + verifyNaN(s, t) + } +} + +func verifyInf(str string, t *testing.T) { + var f float64 + var f32 float32 + var f64 float64 + text := str + " " + str + " " + str + n, err := Fscan(strings.NewReader(text), &f, &f32, &f64) + if err != nil { + t.Errorf("got error scanning %q: %s", text, err) + } + if n != 3 { + t.Errorf("count error scanning %q: got %d", text, n) + } + sign := 1 + if str[0] == '-' { + sign = -1 + } + if !math.IsInf(float64(f), sign) || !math.IsInf(float64(f32), sign) || !math.IsInf(f64, sign) { + t.Errorf("didn't get right Infs scanning %q: got %g %g %g", text, f, f32, f64) + } +} + +func TestInf(t *testing.T) { + for _, s := range []string{"inf", "+inf", "-inf", "INF", "-INF", "+INF", "Inf", "-Inf", "+Inf"} { + verifyInf(s, t) + } +} + +func testScanfMulti(name string, t *testing.T) { + sliceType := reflect.TypeOf(make([]interface{}, 1)) + for _, test := range multiTests { + var r io.Reader + if name == "StringReader" { + r = strings.NewReader(test.text) + } else { + r = newReader(test.text) + } + n, err := Fscanf(r, test.format, test.in...) + if err != nil { + if test.err == "" { + t.Errorf("got error scanning (%q, %q): %q", test.format, test.text, err) + } else if strings.Index(err.String(), test.err) < 0 { + t.Errorf("got wrong error scanning (%q, %q): %q; expected %q", test.format, test.text, err, test.err) + } + continue + } + if test.err != "" { + t.Errorf("expected error %q error scanning (%q, %q)", test.err, test.format, test.text) + } + if n != len(test.out) { + t.Errorf("count error on entry (%q, %q): expected %d got %d", test.format, test.text, len(test.out), n) + continue + } + // Convert the slice of pointers into a slice of values + resultVal := reflect.MakeSlice(sliceType, n, n) + for i := 0; i < n; i++ { + v := reflect.ValueOf(test.in[i]).Elem() + resultVal.Index(i).Set(v) + } + result := resultVal.Interface() + if !reflect.DeepEqual(result, test.out) { + t.Errorf("scanning (%q, %q): expected %v got %v", test.format, test.text, test.out, result) + } + } +} + +func TestScanfMulti(t *testing.T) { + testScanfMulti("StringReader", t) +} + +func TestMyReaderScanfMulti(t *testing.T) { + testScanfMulti("myStringReader", t) +} + +func TestScanMultiple(t *testing.T) { + var a int + var s string + n, err := Sscan("123abc", &a, &s) + if n != 2 { + t.Errorf("Sscan count error: expected 2: got %d", n) + } + if err != nil { + t.Errorf("Sscan expected no error; got %s", err) + } + if a != 123 || s != "abc" { + t.Errorf("Sscan wrong values: got (%d %q) expected (123 \"abc\")", a, s) + } + n, err = Sscan("asdf", &s, &a) + if n != 1 { + t.Errorf("Sscan count error: expected 1: got %d", n) + } + if err == nil { + t.Errorf("Sscan expected error; got none: %s", err) + } + if s != "asdf" { + t.Errorf("Sscan wrong values: got %q expected \"asdf\"", s) + } +} + +// Empty strings are not valid input when scanning a string. +func TestScanEmpty(t *testing.T) { + var s1, s2 string + n, err := Sscan("abc", &s1, &s2) + if n != 1 { + t.Errorf("Sscan count error: expected 1: got %d", n) + } + if err == nil { + t.Error("Sscan expected error; got none") + } + if s1 != "abc" { + t.Errorf("Sscan wrong values: got %q expected \"abc\"", s1) + } + n, err = Sscan("", &s1, &s2) + if n != 0 { + t.Errorf("Sscan count error: expected 0: got %d", n) + } + if err == nil { + t.Error("Sscan expected error; got none") + } + // Quoted empty string is OK. + n, err = Sscanf(`""`, "%q", &s1) + if n != 1 { + t.Errorf("Sscanf count error: expected 1: got %d", n) + } + if err != nil { + t.Errorf("Sscanf expected no error with quoted string; got %s", err) + } +} + +func TestScanNotPointer(t *testing.T) { + r := strings.NewReader("1") + var a int + _, err := Fscan(r, a) + if err == nil { + t.Error("expected error scanning non-pointer") + } else if strings.Index(err.String(), "pointer") < 0 { + t.Errorf("expected pointer error scanning non-pointer, got: %s", err) + } +} + +func TestScanlnNoNewline(t *testing.T) { + var a int + _, err := Sscanln("1 x\n", &a) + if err == nil { + t.Error("expected error scanning string missing newline") + } else if strings.Index(err.String(), "newline") < 0 { + t.Errorf("expected newline error scanning string missing newline, got: %s", err) + } +} + +func TestScanlnWithMiddleNewline(t *testing.T) { + r := strings.NewReader("123\n456\n") + var a, b int + _, err := Fscanln(r, &a, &b) + if err == nil { + t.Error("expected error scanning string with extra newline") + } else if strings.Index(err.String(), "newline") < 0 { + t.Errorf("expected newline error scanning string with extra newline, got: %s", err) + } +} + +// Special Reader that counts reads at end of file. +type eofCounter struct { + reader *strings.Reader + eofCount int +} + +func (ec *eofCounter) Read(b []byte) (n int, err os.Error) { + n, err = ec.reader.Read(b) + if n == 0 { + ec.eofCount++ + } + return +} + +// Verify that when we scan, we see at most EOF once per call to a Scan function, +// and then only when it's really an EOF +func TestEOF(t *testing.T) { + ec := &eofCounter{strings.NewReader("123\n"), 0} + var a int + n, err := Fscanln(ec, &a) + if err != nil { + t.Error("unexpected error", err) + } + if n != 1 { + t.Error("expected to scan one item, got", n) + } + if ec.eofCount != 0 { + t.Error("expected zero EOFs", ec.eofCount) + ec.eofCount = 0 // reset for next test + } + n, err = Fscanln(ec, &a) + if err == nil { + t.Error("expected error scanning empty string") + } + if n != 0 { + t.Error("expected to scan zero items, got", n) + } + if ec.eofCount != 1 { + t.Error("expected one EOF, got", ec.eofCount) + } +} + +// Verify that we see an EOF error if we run out of input. +// This was a buglet: we used to get "expected integer". +func TestEOFAtEndOfInput(t *testing.T) { + var i, j int + n, err := Sscanf("23", "%d %d", &i, &j) + if n != 1 || i != 23 { + t.Errorf("Sscanf expected one value of 23; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscanf expected EOF; got %q", err) + } + n, err = Sscan("234", &i, &j) + if n != 1 || i != 234 { + t.Errorf("Sscan expected one value of 234; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscan expected EOF; got %q", err) + } + // Trailing space is tougher. + n, err = Sscan("234 ", &i, &j) + if n != 1 || i != 234 { + t.Errorf("Sscan expected one value of 234; got %d %d", n, i) + } + if err != os.EOF { + t.Errorf("Sscan expected EOF; got %q", err) + } +} + +var eofTests = []struct { + format string + v interface{} +}{ + {"%s", &stringVal}, + {"%q", &stringVal}, + {"%x", &stringVal}, + {"%v", &stringVal}, + {"%v", &bytesVal}, + {"%v", &intVal}, + {"%v", &uintVal}, + {"%v", &boolVal}, + {"%v", &float32Val}, + {"%v", &complex64Val}, + {"%v", &renamedStringVal}, + {"%v", &renamedBytesVal}, + {"%v", &renamedIntVal}, + {"%v", &renamedUintVal}, + {"%v", &renamedBoolVal}, + {"%v", &renamedFloat32Val}, + {"%v", &renamedComplex64Val}, +} + +func TestEOFAllTypes(t *testing.T) { + for i, test := range eofTests { + if _, err := Sscanf("", test.format, test.v); err != os.EOF { + t.Errorf("#%d: %s %T not eof on empty string: %s", i, test.format, test.v, err) + } + if _, err := Sscanf(" ", test.format, test.v); err != os.EOF { + t.Errorf("#%d: %s %T not eof on trailing blanks: %s", i, test.format, test.v, err) + } + } +} + +// Verify that, at least when using bufio, successive calls to Fscan do not lose runes. +func TestUnreadRuneWithBufio(t *testing.T) { + r := bufio.NewReader(strings.NewReader("123αb")) + var i int + var a string + n, err := Fscanf(r, "%d", &i) + if n != 1 || err != nil { + t.Errorf("reading int expected one item, no errors; got %d %q", n, err) + } + if i != 123 { + t.Errorf("expected 123; got %d", i) + } + n, err = Fscanf(r, "%s", &a) + if n != 1 || err != nil { + t.Errorf("reading string expected one item, no errors; got %d %q", n, err) + } + if a != "αb" { + t.Errorf("expected αb; got %q", a) + } +} + +type TwoLines string + +// Attempt to read two lines into the object. Scanln should prevent this +// because it stops at newline; Scan and Scanf should be fine. +func (t *TwoLines) Scan(state ScanState, verb int) os.Error { + chars := make([]int, 0, 100) + for nlCount := 0; nlCount < 2; { + c, _, err := state.ReadRune() + if err != nil { + return err + } + chars = append(chars, c) + if c == '\n' { + nlCount++ + } + } + *t = TwoLines(string(chars)) + return nil +} + +func TestMultiLine(t *testing.T) { + input := "abc\ndef\n" + // Sscan should work + var tscan TwoLines + n, err := Sscan(input, &tscan) + if n != 1 { + t.Errorf("Sscan: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscan: expected no error; got %s", err) + } + if string(tscan) != input { + t.Errorf("Sscan: expected %q; got %q", input, tscan) + } + // Sscanf should work + var tscanf TwoLines + n, err = Sscanf(input, "%s", &tscanf) + if n != 1 { + t.Errorf("Sscanf: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscanf: expected no error; got %s", err) + } + if string(tscanf) != input { + t.Errorf("Sscanf: expected %q; got %q", input, tscanf) + } + // Sscanln should not work + var tscanln TwoLines + n, err = Sscanln(input, &tscanln) + if n != 0 { + t.Errorf("Sscanln: expected 0 items; got %d: %q", n, tscanln) + } + if err == nil { + t.Error("Sscanln: expected error; got none") + } else if err != io.ErrUnexpectedEOF { + t.Errorf("Sscanln: expected io.ErrUnexpectedEOF (ha!); got %s", err) + } +} + +// RecursiveInt accepts an string matching %d.%d.%d.... +// and parses it into a linked list. +// It allows us to benchmark recursive descent style scanners. +type RecursiveInt struct { + i int + next *RecursiveInt +} + +func (r *RecursiveInt) Scan(state ScanState, verb int) (err os.Error) { + _, err = Fscan(state, &r.i) + if err != nil { + return + } + next := new(RecursiveInt) + _, err = Fscanf(state, ".%v", next) + if err != nil { + if err == os.NewError("input does not match format") || err == io.ErrUnexpectedEOF { + err = nil + } + return + } + r.next = next + return +} + +// Perform the same scanning task as RecursiveInt.Scan +// but without recurring through scanner, so we can compare +// performance more directly. +func scanInts(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + r.next = nil + _, err = Fscan(b, &r.i) + if err != nil { + return + } + var c int + c, _, err = b.ReadRune() + if err != nil { + if err == os.EOF { + err = nil + } + return + } + if c != '.' { + return + } + next := new(RecursiveInt) + err = scanInts(next, b) + if err == nil { + r.next = next + } + return +} + +func makeInts(n int) []byte { + var buf bytes.Buffer + Fprintf(&buf, "1") + for i := 1; i < n; i++ { + Fprintf(&buf, ".%d", i+1) + } + return buf.Bytes() +} + +func TestScanInts(t *testing.T) { + testScanInts(t, scanInts) + testScanInts(t, func(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + _, err = Fscan(b, r) + return + }) +} + +// 800 is small enough to not overflow the stack when using gccgo on a +// platform that does not support split stack. +const intCount = 800 + +func testScanInts(t *testing.T, scan func(*RecursiveInt, *bytes.Buffer) os.Error) { + r := new(RecursiveInt) + ints := makeInts(intCount) + buf := bytes.NewBuffer(ints) + err := scan(r, buf) + if err != nil { + t.Error("unexpected error", err) + } + i := 1 + for ; r != nil; r = r.next { + if r.i != i { + t.Fatalf("bad scan: expected %d got %d", i, r.i) + } + i++ + } + if i-1 != intCount { + t.Fatalf("bad scan count: expected %d got %d", intCount, i-1) + } +} + +func BenchmarkScanInts(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + scanInts(&r, buf) + b.StopTimer() + } +} + +func BenchmarkScanRecursiveInt(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + Fscan(buf, &r) + b.StopTimer() + } +} diff --git a/src/pkg/fmt/stringer_test.go b/src/pkg/fmt/stringer_test.go new file mode 100644 index 000000000..0ca3f522d --- /dev/null +++ b/src/pkg/fmt/stringer_test.go @@ -0,0 +1,61 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt_test + +import ( + . "fmt" + "testing" +) + +type TI int +type TI8 int8 +type TI16 int16 +type TI32 int32 +type TI64 int64 +type TU uint +type TU8 uint8 +type TU16 uint16 +type TU32 uint32 +type TU64 uint64 +type TUI uintptr +type TF float64 +type TF32 float32 +type TF64 float64 +type TB bool +type TS string + +func (v TI) String() string { return Sprintf("I: %d", int(v)) } +func (v TI8) String() string { return Sprintf("I8: %d", int8(v)) } +func (v TI16) String() string { return Sprintf("I16: %d", int16(v)) } +func (v TI32) String() string { return Sprintf("I32: %d", int32(v)) } +func (v TI64) String() string { return Sprintf("I64: %d", int64(v)) } +func (v TU) String() string { return Sprintf("U: %d", uint(v)) } +func (v TU8) String() string { return Sprintf("U8: %d", uint8(v)) } +func (v TU16) String() string { return Sprintf("U16: %d", uint16(v)) } +func (v TU32) String() string { return Sprintf("U32: %d", uint32(v)) } +func (v TU64) String() string { return Sprintf("U64: %d", uint64(v)) } +func (v TUI) String() string { return Sprintf("UI: %d", uintptr(v)) } +func (v TF) String() string { return Sprintf("F: %f", float64(v)) } +func (v TF32) String() string { return Sprintf("F32: %f", float32(v)) } +func (v TF64) String() string { return Sprintf("F64: %f", float64(v)) } +func (v TB) String() string { return Sprintf("B: %t", bool(v)) } +func (v TS) String() string { return Sprintf("S: %q", string(v)) } + +func check(t *testing.T, got, want string) { + if got != want { + t.Error(got, "!=", want) + } +} + +func TestStringer(t *testing.T) { + s := Sprintf("%v %v %v %v %v", TI(0), TI8(1), TI16(2), TI32(3), TI64(4)) + check(t, s, "I: 0 I8: 1 I16: 2 I32: 3 I64: 4") + s = Sprintf("%v %v %v %v %v %v", TU(5), TU8(6), TU16(7), TU32(8), TU64(9), TUI(10)) + check(t, s, "U: 5 U8: 6 U16: 7 U32: 8 U64: 9 UI: 10") + s = Sprintf("%v %v %v", TF(1.0), TF32(2.0), TF64(3.0)) + check(t, s, "F: 1.000000 F32: 2.000000 F64: 3.000000") + s = Sprintf("%v %v", TB(true), TS("x")) + check(t, s, "B: true S: \"x\"") +} diff --git a/src/pkg/go/ast/Makefile b/src/pkg/go/ast/Makefile new file mode 100644 index 000000000..40be10208 --- /dev/null +++ b/src/pkg/go/ast/Makefile @@ -0,0 +1,16 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/ast +GOFILES=\ + ast.go\ + filter.go\ + print.go\ + resolve.go\ + scope.go\ + walk.go\ + +include ../../../Make.pkg diff --git a/src/pkg/go/ast/ast.go b/src/pkg/go/ast/ast.go new file mode 100644 index 000000000..22bd5ee22 --- /dev/null +++ b/src/pkg/go/ast/ast.go @@ -0,0 +1,914 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ast declares the types used to represent syntax trees for Go +// packages. +// +package ast + +import ( + "go/token" + "unicode" + "utf8" +) + +// ---------------------------------------------------------------------------- +// Interfaces +// +// There are 3 main classes of nodes: Expressions and type nodes, +// statement nodes, and declaration nodes. The node names usually +// match the corresponding Go spec production names to which they +// correspond. The node fields correspond to the individual parts +// of the respective productions. +// +// All nodes contain position information marking the beginning of +// the corresponding source text segment; it is accessible via the +// Pos accessor method. Nodes may contain additional position info +// for language constructs where comments may be found between parts +// of the construct (typically any larger, parenthesized subpart). +// That position information is needed to properly position comments +// when printing the construct. + +// All node types implement the Node interface. +type Node interface { + Pos() token.Pos // position of first character belonging to the node + End() token.Pos // position of first character immediately after the node +} + +// All expression nodes implement the Expr interface. +type Expr interface { + Node + exprNode() +} + +// All statement nodes implement the Stmt interface. +type Stmt interface { + Node + stmtNode() +} + +// All declaration nodes implement the Decl interface. +type Decl interface { + Node + declNode() +} + +// ---------------------------------------------------------------------------- +// Comments + +// A Comment node represents a single //-style or /*-style comment. +type Comment struct { + Slash token.Pos // position of "/" starting the comment + Text string // comment text (excluding '\n' for //-style comments) +} + +func (c *Comment) Pos() token.Pos { return c.Slash } +func (c *Comment) End() token.Pos { return token.Pos(int(c.Slash) + len(c.Text)) } + +// A CommentGroup represents a sequence of comments +// with no other tokens and no empty lines between. +// +type CommentGroup struct { + List []*Comment // len(List) > 0 +} + +func (g *CommentGroup) Pos() token.Pos { return g.List[0].Pos() } +func (g *CommentGroup) End() token.Pos { return g.List[len(g.List)-1].End() } + +// ---------------------------------------------------------------------------- +// Expressions and types + +// A Field represents a Field declaration list in a struct type, +// a method list in an interface type, or a parameter/result declaration +// in a signature. +// +type Field struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // field/method/parameter names; or nil if anonymous field + Type Expr // field/method/parameter type + Tag *BasicLit // field tag; or nil + Comment *CommentGroup // line comments; or nil +} + +func (f *Field) Pos() token.Pos { + if len(f.Names) > 0 { + return f.Names[0].Pos() + } + return f.Type.Pos() +} + +func (f *Field) End() token.Pos { + if f.Tag != nil { + return f.Tag.End() + } + return f.Type.End() +} + +// A FieldList represents a list of Fields, enclosed by parentheses or braces. +type FieldList struct { + Opening token.Pos // position of opening parenthesis/brace, if any + List []*Field // field list; or nil + Closing token.Pos // position of closing parenthesis/brace, if any +} + +func (f *FieldList) Pos() token.Pos { + if f.Opening.IsValid() { + return f.Opening + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if len(f.List) > 0 { + return f.List[0].Pos() + } + return token.NoPos +} + +func (f *FieldList) End() token.Pos { + if f.Closing.IsValid() { + return f.Closing + 1 + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if n := len(f.List); n > 0 { + return f.List[n-1].End() + } + return token.NoPos +} + +// NumFields returns the number of (named and anonymous fields) in a FieldList. +func (f *FieldList) NumFields() int { + n := 0 + if f != nil { + for _, g := range f.List { + m := len(g.Names) + if m == 0 { + m = 1 // anonymous field + } + n += m + } + } + return n +} + +// An expression is represented by a tree consisting of one +// or more of the following concrete expression nodes. +// +type ( + // A BadExpr node is a placeholder for expressions containing + // syntax errors for which no correct expression nodes can be + // created. + // + BadExpr struct { + From, To token.Pos // position range of bad expression + } + + // An Ident node represents an identifier. + Ident struct { + NamePos token.Pos // identifier position + Name string // identifier name + Obj *Object // denoted object; or nil + } + + // An Ellipsis node stands for the "..." type in a + // parameter list or the "..." length in an array type. + // + Ellipsis struct { + Ellipsis token.Pos // position of "..." + Elt Expr // ellipsis element type (parameter lists only); or nil + } + + // A BasicLit node represents a literal of basic type. + BasicLit struct { + ValuePos token.Pos // literal position + Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING + Value string // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` + } + + // A FuncLit node represents a function literal. + FuncLit struct { + Type *FuncType // function type + Body *BlockStmt // function body + } + + // A CompositeLit node represents a composite literal. + CompositeLit struct { + Type Expr // literal type; or nil + Lbrace token.Pos // position of "{" + Elts []Expr // list of composite elements; or nil + Rbrace token.Pos // position of "}" + } + + // A ParenExpr node represents a parenthesized expression. + ParenExpr struct { + Lparen token.Pos // position of "(" + X Expr // parenthesized expression + Rparen token.Pos // position of ")" + } + + // A SelectorExpr node represents an expression followed by a selector. + SelectorExpr struct { + X Expr // expression + Sel *Ident // field selector + } + + // An IndexExpr node represents an expression followed by an index. + IndexExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Index Expr // index expression + Rbrack token.Pos // position of "]" + } + + // An SliceExpr node represents an expression followed by slice indices. + SliceExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Low Expr // begin of slice range; or nil + High Expr // end of slice range; or nil + Rbrack token.Pos // position of "]" + } + + // A TypeAssertExpr node represents an expression followed by a + // type assertion. + // + TypeAssertExpr struct { + X Expr // expression + Type Expr // asserted type; nil means type switch X.(type) + } + + // A CallExpr node represents an expression followed by an argument list. + CallExpr struct { + Fun Expr // function expression + Lparen token.Pos // position of "(" + Args []Expr // function arguments; or nil + Ellipsis token.Pos // position of "...", if any + Rparen token.Pos // position of ")" + } + + // A StarExpr node represents an expression of the form "*" Expression. + // Semantically it could be a unary "*" expression, or a pointer type. + // + StarExpr struct { + Star token.Pos // position of "*" + X Expr // operand + } + + // A UnaryExpr node represents a unary expression. + // Unary "*" expressions are represented via StarExpr nodes. + // + UnaryExpr struct { + OpPos token.Pos // position of Op + Op token.Token // operator + X Expr // operand + } + + // A BinaryExpr node represents a binary expression. + BinaryExpr struct { + X Expr // left operand + OpPos token.Pos // position of Op + Op token.Token // operator + Y Expr // right operand + } + + // A KeyValueExpr node represents (key : value) pairs + // in composite literals. + // + KeyValueExpr struct { + Key Expr + Colon token.Pos // position of ":" + Value Expr + } +) + +// The direction of a channel type is indicated by one +// of the following constants. +// +type ChanDir int + +const ( + SEND ChanDir = 1 << iota + RECV +) + +// A type is represented by a tree consisting of one +// or more of the following type-specific expression +// nodes. +// +type ( + // An ArrayType node represents an array or slice type. + ArrayType struct { + Lbrack token.Pos // position of "[" + Len Expr // Ellipsis node for [...]T array types, nil for slice types + Elt Expr // element type + } + + // A StructType node represents a struct type. + StructType struct { + Struct token.Pos // position of "struct" keyword + Fields *FieldList // list of field declarations + Incomplete bool // true if (source) fields are missing in the Fields list + } + + // Pointer types are represented via StarExpr nodes. + + // A FuncType node represents a function type. + FuncType struct { + Func token.Pos // position of "func" keyword + Params *FieldList // (incoming) parameters; or nil + Results *FieldList // (outgoing) results; or nil + } + + // An InterfaceType node represents an interface type. + InterfaceType struct { + Interface token.Pos // position of "interface" keyword + Methods *FieldList // list of methods + Incomplete bool // true if (source) methods are missing in the Methods list + } + + // A MapType node represents a map type. + MapType struct { + Map token.Pos // position of "map" keyword + Key Expr + Value Expr + } + + // A ChanType node represents a channel type. + ChanType struct { + Begin token.Pos // position of "chan" keyword or "<-" (whichever comes first) + Dir ChanDir // channel direction + Value Expr // value type + } +) + +// Pos and End implementations for expression/type nodes. +// +func (x *BadExpr) Pos() token.Pos { return x.From } +func (x *Ident) Pos() token.Pos { return x.NamePos } +func (x *Ellipsis) Pos() token.Pos { return x.Ellipsis } +func (x *BasicLit) Pos() token.Pos { return x.ValuePos } +func (x *FuncLit) Pos() token.Pos { return x.Type.Pos() } +func (x *CompositeLit) Pos() token.Pos { + if x.Type != nil { + return x.Type.Pos() + } + return x.Lbrace +} +func (x *ParenExpr) Pos() token.Pos { return x.Lparen } +func (x *SelectorExpr) Pos() token.Pos { return x.X.Pos() } +func (x *IndexExpr) Pos() token.Pos { return x.X.Pos() } +func (x *SliceExpr) Pos() token.Pos { return x.X.Pos() } +func (x *TypeAssertExpr) Pos() token.Pos { return x.X.Pos() } +func (x *CallExpr) Pos() token.Pos { return x.Fun.Pos() } +func (x *StarExpr) Pos() token.Pos { return x.Star } +func (x *UnaryExpr) Pos() token.Pos { return x.OpPos } +func (x *BinaryExpr) Pos() token.Pos { return x.X.Pos() } +func (x *KeyValueExpr) Pos() token.Pos { return x.Key.Pos() } +func (x *ArrayType) Pos() token.Pos { return x.Lbrack } +func (x *StructType) Pos() token.Pos { return x.Struct } +func (x *FuncType) Pos() token.Pos { return x.Func } +func (x *InterfaceType) Pos() token.Pos { return x.Interface } +func (x *MapType) Pos() token.Pos { return x.Map } +func (x *ChanType) Pos() token.Pos { return x.Begin } + +func (x *BadExpr) End() token.Pos { return x.To } +func (x *Ident) End() token.Pos { return token.Pos(int(x.NamePos) + len(x.Name)) } +func (x *Ellipsis) End() token.Pos { + if x.Elt != nil { + return x.Elt.End() + } + return x.Ellipsis + 3 // len("...") +} +func (x *BasicLit) End() token.Pos { return token.Pos(int(x.ValuePos) + len(x.Value)) } +func (x *FuncLit) End() token.Pos { return x.Body.End() } +func (x *CompositeLit) End() token.Pos { return x.Rbrace + 1 } +func (x *ParenExpr) End() token.Pos { return x.Rparen + 1 } +func (x *SelectorExpr) End() token.Pos { return x.Sel.End() } +func (x *IndexExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *SliceExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *TypeAssertExpr) End() token.Pos { + if x.Type != nil { + return x.Type.End() + } + return x.X.End() +} +func (x *CallExpr) End() token.Pos { return x.Rparen + 1 } +func (x *StarExpr) End() token.Pos { return x.X.End() } +func (x *UnaryExpr) End() token.Pos { return x.X.End() } +func (x *BinaryExpr) End() token.Pos { return x.Y.End() } +func (x *KeyValueExpr) End() token.Pos { return x.Value.End() } +func (x *ArrayType) End() token.Pos { return x.Elt.End() } +func (x *StructType) End() token.Pos { return x.Fields.End() } +func (x *FuncType) End() token.Pos { + if x.Results != nil { + return x.Results.End() + } + return x.Params.End() +} +func (x *InterfaceType) End() token.Pos { return x.Methods.End() } +func (x *MapType) End() token.Pos { return x.Value.End() } +func (x *ChanType) End() token.Pos { return x.Value.End() } + +// exprNode() ensures that only expression/type nodes can be +// assigned to an ExprNode. +// +func (x *BadExpr) exprNode() {} +func (x *Ident) exprNode() {} +func (x *Ellipsis) exprNode() {} +func (x *BasicLit) exprNode() {} +func (x *FuncLit) exprNode() {} +func (x *CompositeLit) exprNode() {} +func (x *ParenExpr) exprNode() {} +func (x *SelectorExpr) exprNode() {} +func (x *IndexExpr) exprNode() {} +func (x *SliceExpr) exprNode() {} +func (x *TypeAssertExpr) exprNode() {} +func (x *CallExpr) exprNode() {} +func (x *StarExpr) exprNode() {} +func (x *UnaryExpr) exprNode() {} +func (x *BinaryExpr) exprNode() {} +func (x *KeyValueExpr) exprNode() {} + +func (x *ArrayType) exprNode() {} +func (x *StructType) exprNode() {} +func (x *FuncType) exprNode() {} +func (x *InterfaceType) exprNode() {} +func (x *MapType) exprNode() {} +func (x *ChanType) exprNode() {} + +// ---------------------------------------------------------------------------- +// Convenience functions for Idents + +var noPos token.Pos + +// NewIdent creates a new Ident without position. +// Useful for ASTs generated by code other than the Go parser. +// +func NewIdent(name string) *Ident { return &Ident{noPos, name, nil} } + +// IsExported returns whether name is an exported Go symbol +// (i.e., whether it begins with an uppercase letter). +// +func IsExported(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(ch) +} + +// IsExported returns whether id is an exported Go symbol +// (i.e., whether it begins with an uppercase letter). +// +func (id *Ident) IsExported() bool { return IsExported(id.Name) } + +func (id *Ident) String() string { + if id != nil { + return id.Name + } + return "" +} + +// ---------------------------------------------------------------------------- +// Statements + +// A statement is represented by a tree consisting of one +// or more of the following concrete statement nodes. +// +type ( + // A BadStmt node is a placeholder for statements containing + // syntax errors for which no correct statement nodes can be + // created. + // + BadStmt struct { + From, To token.Pos // position range of bad statement + } + + // A DeclStmt node represents a declaration in a statement list. + DeclStmt struct { + Decl Decl + } + + // An EmptyStmt node represents an empty statement. + // The "position" of the empty statement is the position + // of the immediately preceding semicolon. + // + EmptyStmt struct { + Semicolon token.Pos // position of preceding ";" + } + + // A LabeledStmt node represents a labeled statement. + LabeledStmt struct { + Label *Ident + Colon token.Pos // position of ":" + Stmt Stmt + } + + // An ExprStmt node represents a (stand-alone) expression + // in a statement list. + // + ExprStmt struct { + X Expr // expression + } + + // A SendStmt node represents a send statement. + SendStmt struct { + Chan Expr + Arrow token.Pos // position of "<-" + Value Expr + } + + // An IncDecStmt node represents an increment or decrement statement. + IncDecStmt struct { + X Expr + TokPos token.Pos // position of Tok + Tok token.Token // INC or DEC + } + + // An AssignStmt node represents an assignment or + // a short variable declaration. + // + AssignStmt struct { + Lhs []Expr + TokPos token.Pos // position of Tok + Tok token.Token // assignment token, DEFINE + Rhs []Expr + } + + // A GoStmt node represents a go statement. + GoStmt struct { + Go token.Pos // position of "go" keyword + Call *CallExpr + } + + // A DeferStmt node represents a defer statement. + DeferStmt struct { + Defer token.Pos // position of "defer" keyword + Call *CallExpr + } + + // A ReturnStmt node represents a return statement. + ReturnStmt struct { + Return token.Pos // position of "return" keyword + Results []Expr // result expressions; or nil + } + + // A BranchStmt node represents a break, continue, goto, + // or fallthrough statement. + // + BranchStmt struct { + TokPos token.Pos // position of Tok + Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH) + Label *Ident // label name; or nil + } + + // A BlockStmt node represents a braced statement list. + BlockStmt struct { + Lbrace token.Pos // position of "{" + List []Stmt + Rbrace token.Pos // position of "}" + } + + // An IfStmt node represents an if statement. + IfStmt struct { + If token.Pos // position of "if" keyword + Init Stmt // initialization statement; or nil + Cond Expr // condition + Body *BlockStmt + Else Stmt // else branch; or nil + } + + // A CaseClause represents a case of an expression or type switch statement. + CaseClause struct { + Case token.Pos // position of "case" or "default" keyword + List []Expr // list of expressions or types; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // A SwitchStmt node represents an expression switch statement. + SwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initialization statement; or nil + Tag Expr // tag expression; or nil + Body *BlockStmt // CaseClauses only + } + + // An TypeSwitchStmt node represents a type switch statement. + TypeSwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initialization statement; or nil + Assign Stmt // x := y.(type) or y.(type) + Body *BlockStmt // CaseClauses only + } + + // A CommClause node represents a case of a select statement. + CommClause struct { + Case token.Pos // position of "case" or "default" keyword + Comm Stmt // send or receive statement; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // An SelectStmt node represents a select statement. + SelectStmt struct { + Select token.Pos // position of "select" keyword + Body *BlockStmt // CommClauses only + } + + // A ForStmt represents a for statement. + ForStmt struct { + For token.Pos // position of "for" keyword + Init Stmt // initialization statement; or nil + Cond Expr // condition; or nil + Post Stmt // post iteration statement; or nil + Body *BlockStmt + } + + // A RangeStmt represents a for statement with a range clause. + RangeStmt struct { + For token.Pos // position of "for" keyword + Key, Value Expr // Value may be nil + TokPos token.Pos // position of Tok + Tok token.Token // ASSIGN, DEFINE + X Expr // value to range over + Body *BlockStmt + } +) + +// Pos and End implementations for statement nodes. +// +func (s *BadStmt) Pos() token.Pos { return s.From } +func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() } +func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon } +func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() } +func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() } +func (s *SendStmt) Pos() token.Pos { return s.Chan.Pos() } +func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() } +func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() } +func (s *GoStmt) Pos() token.Pos { return s.Go } +func (s *DeferStmt) Pos() token.Pos { return s.Defer } +func (s *ReturnStmt) Pos() token.Pos { return s.Return } +func (s *BranchStmt) Pos() token.Pos { return s.TokPos } +func (s *BlockStmt) Pos() token.Pos { return s.Lbrace } +func (s *IfStmt) Pos() token.Pos { return s.If } +func (s *CaseClause) Pos() token.Pos { return s.Case } +func (s *SwitchStmt) Pos() token.Pos { return s.Switch } +func (s *TypeSwitchStmt) Pos() token.Pos { return s.Switch } +func (s *CommClause) Pos() token.Pos { return s.Case } +func (s *SelectStmt) Pos() token.Pos { return s.Select } +func (s *ForStmt) Pos() token.Pos { return s.For } +func (s *RangeStmt) Pos() token.Pos { return s.For } + +func (s *BadStmt) End() token.Pos { return s.To } +func (s *DeclStmt) End() token.Pos { return s.Decl.End() } +func (s *EmptyStmt) End() token.Pos { + return s.Semicolon + 1 /* len(";") */ +} +func (s *LabeledStmt) End() token.Pos { return s.Stmt.End() } +func (s *ExprStmt) End() token.Pos { return s.X.End() } +func (s *SendStmt) End() token.Pos { return s.Value.End() } +func (s *IncDecStmt) End() token.Pos { + return s.TokPos + 2 /* len("++") */ +} +func (s *AssignStmt) End() token.Pos { return s.Rhs[len(s.Rhs)-1].End() } +func (s *GoStmt) End() token.Pos { return s.Call.End() } +func (s *DeferStmt) End() token.Pos { return s.Call.End() } +func (s *ReturnStmt) End() token.Pos { + if n := len(s.Results); n > 0 { + return s.Results[n-1].End() + } + return s.Return + 6 // len("return") +} +func (s *BranchStmt) End() token.Pos { + if s.Label != nil { + return s.Label.End() + } + return token.Pos(int(s.TokPos) + len(s.Tok.String())) +} +func (s *BlockStmt) End() token.Pos { return s.Rbrace + 1 } +func (s *IfStmt) End() token.Pos { + if s.Else != nil { + return s.Else.End() + } + return s.Body.End() +} +func (s *CaseClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SwitchStmt) End() token.Pos { return s.Body.End() } +func (s *TypeSwitchStmt) End() token.Pos { return s.Body.End() } +func (s *CommClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SelectStmt) End() token.Pos { return s.Body.End() } +func (s *ForStmt) End() token.Pos { return s.Body.End() } +func (s *RangeStmt) End() token.Pos { return s.Body.End() } + +// stmtNode() ensures that only statement nodes can be +// assigned to a StmtNode. +// +func (s *BadStmt) stmtNode() {} +func (s *DeclStmt) stmtNode() {} +func (s *EmptyStmt) stmtNode() {} +func (s *LabeledStmt) stmtNode() {} +func (s *ExprStmt) stmtNode() {} +func (s *SendStmt) stmtNode() {} +func (s *IncDecStmt) stmtNode() {} +func (s *AssignStmt) stmtNode() {} +func (s *GoStmt) stmtNode() {} +func (s *DeferStmt) stmtNode() {} +func (s *ReturnStmt) stmtNode() {} +func (s *BranchStmt) stmtNode() {} +func (s *BlockStmt) stmtNode() {} +func (s *IfStmt) stmtNode() {} +func (s *CaseClause) stmtNode() {} +func (s *SwitchStmt) stmtNode() {} +func (s *TypeSwitchStmt) stmtNode() {} +func (s *CommClause) stmtNode() {} +func (s *SelectStmt) stmtNode() {} +func (s *ForStmt) stmtNode() {} +func (s *RangeStmt) stmtNode() {} + +// ---------------------------------------------------------------------------- +// Declarations + +// A Spec node represents a single (non-parenthesized) import, +// constant, type, or variable declaration. +// +type ( + // The Spec type stands for any of *ImportSpec, *ValueSpec, and *TypeSpec. + Spec interface { + Node + specNode() + } + + // An ImportSpec node represents a single package import. + ImportSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // local package name (including "."); or nil + Path *BasicLit // import path + Comment *CommentGroup // line comments; or nil + } + + // A ValueSpec node represents a constant or variable declaration + // (ConstSpec or VarSpec production). + // + ValueSpec struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // value names (len(Names) > 0) + Type Expr // value type; or nil + Values []Expr // initial values; or nil + Comment *CommentGroup // line comments; or nil + } + + // A TypeSpec node represents a type declaration (TypeSpec production). + TypeSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // type name + Type Expr // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes + Comment *CommentGroup // line comments; or nil + } +) + +// Pos and End implementations for spec nodes. +// +func (s *ImportSpec) Pos() token.Pos { + if s.Name != nil { + return s.Name.Pos() + } + return s.Path.Pos() +} +func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() } +func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() } + +func (s *ImportSpec) End() token.Pos { return s.Path.End() } +func (s *ValueSpec) End() token.Pos { + if n := len(s.Values); n > 0 { + return s.Values[n-1].End() + } + if s.Type != nil { + return s.Type.End() + } + return s.Names[len(s.Names)-1].End() +} +func (s *TypeSpec) End() token.Pos { return s.Type.End() } + +// specNode() ensures that only spec nodes can be +// assigned to a Spec. +// +func (s *ImportSpec) specNode() {} +func (s *ValueSpec) specNode() {} +func (s *TypeSpec) specNode() {} + +// A declaration is represented by one of the following declaration nodes. +// +type ( + // A BadDecl node is a placeholder for declarations containing + // syntax errors for which no correct declaration nodes can be + // created. + // + BadDecl struct { + From, To token.Pos // position range of bad declaration + } + + // A GenDecl node (generic declaration node) represents an import, + // constant, type or variable declaration. A valid Lparen position + // (Lparen.Line > 0) indicates a parenthesized declaration. + // + // Relationship between Tok value and Specs element type: + // + // token.IMPORT *ImportSpec + // token.CONST *ValueSpec + // token.TYPE *TypeSpec + // token.VAR *ValueSpec + // + GenDecl struct { + Doc *CommentGroup // associated documentation; or nil + TokPos token.Pos // position of Tok + Tok token.Token // IMPORT, CONST, TYPE, VAR + Lparen token.Pos // position of '(', if any + Specs []Spec + Rparen token.Pos // position of ')', if any + } + + // A FuncDecl node represents a function declaration. + FuncDecl struct { + Doc *CommentGroup // associated documentation; or nil + Recv *FieldList // receiver (methods); or nil (functions) + Name *Ident // function/method name + Type *FuncType // position of Func keyword, parameters and results + Body *BlockStmt // function body; or nil (forward declaration) + } +) + +// Pos and End implementations for declaration nodes. +// +func (d *BadDecl) Pos() token.Pos { return d.From } +func (d *GenDecl) Pos() token.Pos { return d.TokPos } +func (d *FuncDecl) Pos() token.Pos { return d.Type.Pos() } + +func (d *BadDecl) End() token.Pos { return d.To } +func (d *GenDecl) End() token.Pos { + if d.Rparen.IsValid() { + return d.Rparen + 1 + } + return d.Specs[0].End() +} +func (d *FuncDecl) End() token.Pos { + if d.Body != nil { + return d.Body.End() + } + return d.Type.End() +} + +// declNode() ensures that only declaration nodes can be +// assigned to a DeclNode. +// +func (d *BadDecl) declNode() {} +func (d *GenDecl) declNode() {} +func (d *FuncDecl) declNode() {} + +// ---------------------------------------------------------------------------- +// Files and packages + +// A File node represents a Go source file. +// +// The Comments list contains all comments in the source file in order of +// appearance, including the comments that are pointed to from other nodes +// via Doc and Comment fields. +// +type File struct { + Doc *CommentGroup // associated documentation; or nil + Package token.Pos // position of "package" keyword + Name *Ident // package name + Decls []Decl // top-level declarations; or nil + Scope *Scope // package scope (this file only) + Imports []*ImportSpec // imports in this file + Unresolved []*Ident // unresolved identifiers in this file + Comments []*CommentGroup // list of all comments in the source file +} + +func (f *File) Pos() token.Pos { return f.Package } +func (f *File) End() token.Pos { + if n := len(f.Decls); n > 0 { + return f.Decls[n-1].End() + } + return f.Name.End() +} + +// A Package node represents a set of source files +// collectively building a Go package. +// +type Package struct { + Name string // package name + Scope *Scope // package scope across all files + Imports map[string]*Object // map of package id -> package object + Files map[string]*File // Go source files by filename +} + +func (p *Package) Pos() token.Pos { return token.NoPos } +func (p *Package) End() token.Pos { return token.NoPos } diff --git a/src/pkg/go/ast/filter.go b/src/pkg/go/ast/filter.go new file mode 100644 index 000000000..26733430d --- /dev/null +++ b/src/pkg/go/ast/filter.go @@ -0,0 +1,475 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "go/token" + +// ---------------------------------------------------------------------------- +// Export filtering + +func identListExports(list []*Ident) []*Ident { + j := 0 + for _, x := range list { + if x.IsExported() { + list[j] = x + j++ + } + } + return list[0:j] +} + +// fieldName assumes that x is the type of an anonymous field and +// returns the corresponding field name. If x is not an acceptable +// anonymous field, the result is nil. +// +func fieldName(x Expr) *Ident { + switch t := x.(type) { + case *Ident: + return t + case *SelectorExpr: + if _, ok := t.X.(*Ident); ok { + return t.Sel + } + case *StarExpr: + return fieldName(t.X) + } + return nil +} + +func fieldListExports(fields *FieldList) (removedFields bool) { + if fields == nil { + return + } + list := fields.List + j := 0 + for _, f := range list { + exported := false + if len(f.Names) == 0 { + // anonymous field + // (Note that a non-exported anonymous field + // may still refer to a type with exported + // fields, so this is not absolutely correct. + // However, this cannot be done w/o complete + // type information.) + name := fieldName(f.Type) + exported = name != nil && name.IsExported() + } else { + n := len(f.Names) + f.Names = identListExports(f.Names) + if len(f.Names) < n { + removedFields = true + } + exported = len(f.Names) > 0 + } + if exported { + typeExports(f.Type) + list[j] = f + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +func paramListExports(fields *FieldList) { + if fields == nil { + return + } + for _, f := range fields.List { + typeExports(f.Type) + } +} + +func typeExports(typ Expr) { + switch t := typ.(type) { + case *ArrayType: + typeExports(t.Elt) + case *StructType: + if fieldListExports(t.Fields) { + t.Incomplete = true + } + case *FuncType: + paramListExports(t.Params) + paramListExports(t.Results) + case *InterfaceType: + if fieldListExports(t.Methods) { + t.Incomplete = true + } + case *MapType: + typeExports(t.Key) + typeExports(t.Value) + case *ChanType: + typeExports(t.Value) + } +} + +func specExports(spec Spec) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = identListExports(s.Names) + if len(s.Names) > 0 { + typeExports(s.Type) + return true + } + case *TypeSpec: + if s.Name.IsExported() { + typeExports(s.Type) + return true + } + } + return false +} + +func specListExports(list []Spec) []Spec { + j := 0 + for _, s := range list { + if specExports(s) { + list[j] = s + j++ + } + } + return list[0:j] +} + +func declExports(decl Decl) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = specListExports(d.Specs) + return len(d.Specs) > 0 + case *FuncDecl: + d.Body = nil // strip body + return d.Name.IsExported() + } + return false +} + +// FileExports trims the AST for a Go source file in place such that only +// exported nodes remain: all top-level identifiers which are not exported +// and their associated information (such as type, initial value, or function +// body) are removed. Non-exported fields and methods of exported types are +// stripped, and the function bodies of exported functions are set to nil. +// The File.comments list is not changed. +// +// FileExports returns true if there is an exported declaration; it returns +// false otherwise. +// +func FileExports(src *File) bool { + j := 0 + for _, d := range src.Decls { + if declExports(d) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + +// PackageExports trims the AST for a Go package in place such that only +// exported nodes remain. The pkg.Files list is not changed, so that file +// names and top-level package comments don't get lost. +// +// PackageExports returns true if there is an exported declaration; it +// returns false otherwise. +// +func PackageExports(pkg *Package) bool { + hasExports := false + for _, f := range pkg.Files { + if FileExports(f) { + hasExports = true + } + } + return hasExports +} + +// ---------------------------------------------------------------------------- +// General filtering + +type Filter func(string) bool + +func filterIdentList(list []*Ident, f Filter) []*Ident { + j := 0 + for _, x := range list { + if f(x.Name) { + list[j] = x + j++ + } + } + return list[0:j] +} + +func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { + if fields == nil { + return false + } + list := fields.List + j := 0 + for _, f := range list { + keepField := false + if len(f.Names) == 0 { + // anonymous field + name := fieldName(f.Type) + keepField = name != nil && filter(name.Name) + } else { + n := len(f.Names) + f.Names = filterIdentList(f.Names, filter) + if len(f.Names) < n { + removedFields = true + } + keepField = len(f.Names) > 0 + } + if keepField { + list[j] = f + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +func filterSpec(spec Spec, f Filter) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = filterIdentList(s.Names, f) + return len(s.Names) > 0 + case *TypeSpec: + if f(s.Name.Name) { + return true + } + switch t := s.Type.(type) { + case *StructType: + if filterFieldList(t.Fields, f) { + t.Incomplete = true + } + return len(t.Fields.List) > 0 + case *InterfaceType: + if filterFieldList(t.Methods, f) { + t.Incomplete = true + } + return len(t.Methods.List) > 0 + } + } + return false +} + +func filterSpecList(list []Spec, f Filter) []Spec { + j := 0 + for _, s := range list { + if filterSpec(s, f) { + list[j] = s + j++ + } + } + return list[0:j] +} + +// FilterDecl trims the AST for a Go declaration in place by removing +// all names (including struct field and interface method names, but +// not from parameter lists) that don't pass through the filter f. +// +// FilterDecl returns true if there are any declared names left after +// filtering; it returns false otherwise. +// +func FilterDecl(decl Decl, f Filter) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = filterSpecList(d.Specs, f) + return len(d.Specs) > 0 + case *FuncDecl: + return f(d.Name.Name) + } + return false +} + +// FilterFile trims the AST for a Go file in place by removing all +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. +// The File.comments list is not changed. +// +// FilterFile returns true if there are any top-level declarations +// left after filtering; it returns false otherwise. +// +func FilterFile(src *File, f Filter) bool { + j := 0 + for _, d := range src.Decls { + if FilterDecl(d, f) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + +// FilterPackage trims the AST for a Go package in place by removing all +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. +// The pkg.Files list is not changed, so that file names and top-level +// package comments don't get lost. +// +// FilterPackage returns true if there are any top-level declarations +// left after filtering; it returns false otherwise. +// +func FilterPackage(pkg *Package, f Filter) bool { + hasDecls := false + for _, src := range pkg.Files { + if FilterFile(src, f) { + hasDecls = true + } + } + return hasDecls +} + +// ---------------------------------------------------------------------------- +// Merging of package files + +// The MergeMode flags control the behavior of MergePackageFiles. +type MergeMode uint + +const ( + // If set, duplicate function declarations are excluded. + FilterFuncDuplicates MergeMode = 1 << iota + // If set, comments that are not associated with a specific + // AST node (as Doc or Comment) are excluded. + FilterUnassociatedComments +) + +// separator is an empty //-style comment that is interspersed between +// different comment groups when they are concatenated into a single group +// +var separator = &Comment{noPos, "//"} + +// MergePackageFiles creates a file AST by merging the ASTs of the +// files belonging to a package. The mode flags control merging behavior. +// +func MergePackageFiles(pkg *Package, mode MergeMode) *File { + // Count the number of package docs, comments and declarations across + // all package files. + ndocs := 0 + ncomments := 0 + ndecls := 0 + for _, f := range pkg.Files { + if f.Doc != nil { + ndocs += len(f.Doc.List) + 1 // +1 for separator + } + ncomments += len(f.Comments) + ndecls += len(f.Decls) + } + + // Collect package comments from all package files into a single + // CommentGroup - the collected package documentation. The order + // is unspecified. In general there should be only one file with + // a package comment; but it's better to collect extra comments + // than drop them on the floor. + var doc *CommentGroup + var pos token.Pos + if ndocs > 0 { + list := make([]*Comment, ndocs-1) // -1: no separator before first group + i := 0 + for _, f := range pkg.Files { + if f.Doc != nil { + if i > 0 { + // not the first group - add separator + list[i] = separator + i++ + } + for _, c := range f.Doc.List { + list[i] = c + i++ + } + if f.Package > pos { + // Keep the maximum package clause position as + // position for the package clause of the merged + // files. + pos = f.Package + } + } + } + doc = &CommentGroup{list} + } + + // Collect declarations from all package files. + var decls []Decl + if ndecls > 0 { + decls = make([]Decl, ndecls) + funcs := make(map[string]int) // map of global function name -> decls index + i := 0 // current index + n := 0 // number of filtered entries + for _, f := range pkg.Files { + for _, d := range f.Decls { + if mode&FilterFuncDuplicates != 0 { + // A language entity may be declared multiple + // times in different package files; only at + // build time declarations must be unique. + // For now, exclude multiple declarations of + // functions - keep the one with documentation. + // + // TODO(gri): Expand this filtering to other + // entities (const, type, vars) if + // multiple declarations are common. + if f, isFun := d.(*FuncDecl); isFun { + name := f.Name.Name + if j, exists := funcs[name]; exists { + // function declared already + if decls[j] != nil && decls[j].(*FuncDecl).Doc == nil { + // existing declaration has no documentation; + // ignore the existing declaration + decls[j] = nil + } else { + // ignore the new declaration + d = nil + } + n++ // filtered an entry + } else { + funcs[name] = i + } + } + } + decls[i] = d + i++ + } + } + + // Eliminate nil entries from the decls list if entries were + // filtered. We do this using a 2nd pass in order to not disturb + // the original declaration order in the source (otherwise, this + // would also invalidate the monotonically increasing position + // info within a single file). + if n > 0 { + i = 0 + for _, d := range decls { + if d != nil { + decls[i] = d + i++ + } + } + decls = decls[0:i] + } + } + + // Collect comments from all package files. + var comments []*CommentGroup + if mode&FilterUnassociatedComments == 0 { + comments = make([]*CommentGroup, ncomments) + i := 0 + for _, f := range pkg.Files { + i += copy(comments[i:], f.Comments) + } + } + + // TODO(gri) need to compute pkgScope and unresolved identifiers! + // TODO(gri) need to compute imports! + return &File{doc, pos, NewIdent(pkg.Name), decls, nil, nil, nil, comments} +} diff --git a/src/pkg/go/ast/print.go b/src/pkg/go/ast/print.go new file mode 100644 index 000000000..62a30481d --- /dev/null +++ b/src/pkg/go/ast/print.go @@ -0,0 +1,224 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains printing suppport for ASTs. + +package ast + +import ( + "fmt" + "go/token" + "io" + "os" + "reflect" +) + +// A FieldFilter may be provided to Fprint to control the output. +type FieldFilter func(name string, value reflect.Value) bool + +// NotNilFilter returns true for field values that are not nil; +// it returns false otherwise. +func NotNilFilter(_ string, v reflect.Value) bool { + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return !v.IsNil() + } + return true +} + +// Fprint prints the (sub-)tree starting at AST node x to w. +// If fset != nil, position information is interpreted relative +// to that file set. Otherwise positions are printed as integer +// values (file set specific offsets). +// +// A non-nil FieldFilter f may be provided to control the output: +// struct fields for which f(fieldname, fieldvalue) is true are +// are printed; all others are filtered from the output. +// +func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (n int, err os.Error) { + // setup printer + p := printer{ + output: w, + fset: fset, + filter: f, + ptrmap: make(map[interface{}]int), + last: '\n', // force printing of line number on first line + } + + // install error handler + defer func() { + n = p.written + if e := recover(); e != nil { + err = e.(localError).err // re-panics if it's not a localError + } + }() + + // print x + if x == nil { + p.printf("nil\n") + return + } + p.print(reflect.ValueOf(x)) + p.printf("\n") + + return +} + +// Print prints x to standard output, skipping nil fields. +// Print(fset, x) is the same as Fprint(os.Stdout, fset, x, NotNilFilter). +func Print(fset *token.FileSet, x interface{}) (int, os.Error) { + return Fprint(os.Stdout, fset, x, NotNilFilter) +} + +type printer struct { + output io.Writer + fset *token.FileSet + filter FieldFilter + ptrmap map[interface{}]int // *T -> line number + written int // number of bytes written to output + indent int // current indentation level + last byte // the last byte processed by Write + line int // current line number +} + +var indent = []byte(". ") + +func (p *printer) Write(data []byte) (n int, err os.Error) { + var m int + for i, b := range data { + // invariant: data[0:n] has been written + if b == '\n' { + m, err = p.output.Write(data[n : i+1]) + n += m + if err != nil { + return + } + p.line++ + } else if p.last == '\n' { + _, err = fmt.Fprintf(p.output, "%6d ", p.line) + if err != nil { + return + } + for j := p.indent; j > 0; j-- { + _, err = p.output.Write(indent) + if err != nil { + return + } + } + } + p.last = b + } + m, err = p.output.Write(data[n:]) + n += m + return +} + +// localError wraps locally caught os.Errors so we can distinguish +// them from genuine panics which we don't want to return as errors. +type localError struct { + err os.Error +} + +// printf is a convenience wrapper that takes care of print errors. +func (p *printer) printf(format string, args ...interface{}) { + n, err := fmt.Fprintf(p, format, args...) + p.written += n + if err != nil { + panic(localError{err}) + } +} + +// Implementation note: Print is written for AST nodes but could be +// used to print arbitrary data structures; such a version should +// probably be in a different package. +// +// Note: This code detects (some) cycles created via pointers but +// not cycles that are created via slices or maps containing the +// same slice or map. Code for general data structures probably +// should catch those as well. + +func (p *printer) print(x reflect.Value) { + if !NotNilFilter("", x) { + p.printf("nil") + return + } + + switch x.Kind() { + case reflect.Interface: + p.print(x.Elem()) + + case reflect.Map: + p.printf("%s (len = %d) {\n", x.Type().String(), x.Len()) + p.indent++ + for _, key := range x.MapKeys() { + p.print(key) + p.printf(": ") + p.print(x.MapIndex(key)) + p.printf("\n") + } + p.indent-- + p.printf("}") + + case reflect.Ptr: + p.printf("*") + // type-checked ASTs may contain cycles - use ptrmap + // to keep track of objects that have been printed + // already and print the respective line number instead + ptr := x.Interface() + if line, exists := p.ptrmap[ptr]; exists { + p.printf("(obj @ %d)", line) + } else { + p.ptrmap[ptr] = p.line + p.print(x.Elem()) + } + + case reflect.Slice: + if s, ok := x.Interface().([]byte); ok { + p.printf("%#q", s) + return + } + p.printf("%s (len = %d) {\n", x.Type().String(), x.Len()) + p.indent++ + for i, n := 0, x.Len(); i < n; i++ { + p.printf("%d: ", i) + p.print(x.Index(i)) + p.printf("\n") + } + p.indent-- + p.printf("}") + + case reflect.Struct: + p.printf("%s {\n", x.Type().String()) + p.indent++ + t := x.Type() + for i, n := 0, t.NumField(); i < n; i++ { + name := t.Field(i).Name + value := x.Field(i) + if p.filter == nil || p.filter(name, value) { + p.printf("%s: ", name) + p.print(value) + p.printf("\n") + } + } + p.indent-- + p.printf("}") + + default: + v := x.Interface() + switch v := v.(type) { + case string: + // print strings in quotes + p.printf("%q", v) + return + case token.Pos: + // position values can be printed nicely if we have a file set + if p.fset != nil { + p.printf("%s", p.fset.Position(v)) + return + } + } + // default + p.printf("%v", v) + } +} diff --git a/src/pkg/go/ast/print_test.go b/src/pkg/go/ast/print_test.go new file mode 100644 index 000000000..f4e8f7a78 --- /dev/null +++ b/src/pkg/go/ast/print_test.go @@ -0,0 +1,77 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "bytes" + "strings" + "testing" +) + +var tests = []struct { + x interface{} // x is printed as s + s string +}{ + // basic types + {nil, "0 nil"}, + {true, "0 true"}, + {42, "0 42"}, + {3.14, "0 3.14"}, + {1 + 2.718i, "0 (1+2.718i)"}, + {"foobar", "0 \"foobar\""}, + + // maps + {map[string]int{"a": 1, "b": 2}, + `0 map[string] int (len = 2) { + 1 . "a": 1 + 2 . "b": 2 + 3 }`}, + + // pointers + {new(int), "0 *0"}, + + // slices + {[]int{1, 2, 3}, + `0 []int (len = 3) { + 1 . 0: 1 + 2 . 1: 2 + 3 . 2: 3 + 4 }`}, + + // structs + {struct{ x, y int }{42, 991}, + `0 struct { x int; y int } { + 1 . x: 42 + 2 . y: 991 + 3 }`}, +} + +// Split s into lines, trim whitespace from all lines, and return +// the concatenated non-empty lines. +func trim(s string) string { + lines := strings.Split(s, "\n") + i := 0 + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + lines[i] = line + i++ + } + } + return strings.Join(lines[0:i], "\n") +} + +func TestPrint(t *testing.T) { + var buf bytes.Buffer + for _, test := range tests { + buf.Reset() + if _, err := Fprint(&buf, nil, test.x, nil); err != nil { + t.Errorf("Fprint failed: %s", err) + } + if s, ts := trim(buf.String()), trim(test.s); s != ts { + t.Errorf("got:\n%s\nexpected:\n%s\n", s, ts) + } + } +} diff --git a/src/pkg/go/ast/resolve.go b/src/pkg/go/ast/resolve.go new file mode 100644 index 000000000..3927a799e --- /dev/null +++ b/src/pkg/go/ast/resolve.go @@ -0,0 +1,174 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements NewPackage. + +package ast + +import ( + "fmt" + "go/scanner" + "go/token" + "os" + "strconv" +) + +type pkgBuilder struct { + scanner.ErrorVector + fset *token.FileSet +} + +func (p *pkgBuilder) error(pos token.Pos, msg string) { + p.Error(p.fset.Position(pos), msg) +} + +func (p *pkgBuilder) errorf(pos token.Pos, format string, args ...interface{}) { + p.error(pos, fmt.Sprintf(format, args...)) +} + +func (p *pkgBuilder) declare(scope, altScope *Scope, obj *Object) { + alt := scope.Insert(obj) + if alt == nil && altScope != nil { + // see if there is a conflicting declaration in altScope + alt = altScope.Lookup(obj.Name) + } + if alt != nil { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.fset.Position(pos)) + } + p.error(obj.Pos(), fmt.Sprintf("%s redeclared in this block%s", obj.Name, prevDecl)) + } +} + +func resolve(scope *Scope, ident *Ident) bool { + for ; scope != nil; scope = scope.Outer { + if obj := scope.Lookup(ident.Name); obj != nil { + ident.Obj = obj + return true + } + } + return false +} + +// An Importer resolves import paths to package Objects. +// The imports map records the packages already imported, +// indexed by package id (canonical import path). +// An Importer must determine the canonical import path and +// check the map to see if it is already present in the imports map. +// If so, the Importer can return the map entry. Otherwise, the +// Importer should load the package data for the given path into +// a new *Object (pkg), record pkg in the imports map, and then +// return pkg. +type Importer func(imports map[string]*Object, path string) (pkg *Object, err os.Error) + +// NewPackage creates a new Package node from a set of File nodes. It resolves +// unresolved identifiers across files and updates each file's Unresolved list +// accordingly. If a non-nil importer and universe scope are provided, they are +// used to resolve identifiers not declared in any of the package files. Any +// remaining unresolved identifiers are reported as undeclared. If the files +// belong to different packages, one package name is selected and files with +// different package names are reported and then ignored. +// The result is a package node and a scanner.ErrorList if there were errors. +// +func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, universe *Scope) (*Package, os.Error) { + var p pkgBuilder + p.fset = fset + + // complete package scope + pkgName := "" + pkgScope := NewScope(universe) + for _, file := range files { + // package names must match + switch name := file.Name.Name; { + case pkgName == "": + pkgName = name + case name != pkgName: + p.errorf(file.Package, "package %s; expected %s", name, pkgName) + continue // ignore this file + } + + // collect top-level file objects in package scope + for _, obj := range file.Scope.Objects { + p.declare(pkgScope, nil, obj) + } + } + + // package global mapping of imported package ids to package objects + imports := make(map[string]*Object) + + // complete file scopes with imports and resolve identifiers + for _, file := range files { + // ignore file if it belongs to a different package + // (error has already been reported) + if file.Name.Name != pkgName { + continue + } + + // build file scope by processing all imports + importErrors := false + fileScope := NewScope(pkgScope) + for _, spec := range file.Imports { + if importer == nil { + importErrors = true + continue + } + path, _ := strconv.Unquote(string(spec.Path.Value)) + pkg, err := importer(imports, path) + if err != nil { + p.errorf(spec.Path.Pos(), "could not import %s (%s)", path, err) + importErrors = true + continue + } + // TODO(gri) If a local package name != "." is provided, + // global identifier resolution could proceed even if the + // import failed. Consider adjusting the logic here a bit. + + // local name overrides imported package name + name := pkg.Name + if spec.Name != nil { + name = spec.Name.Name + } + + // add import to file scope + if name == "." { + // merge imported scope with file scope + for _, obj := range pkg.Data.(*Scope).Objects { + p.declare(fileScope, pkgScope, obj) + } + } else { + // declare imported package object in file scope + // (do not re-use pkg in the file scope but create + // a new object instead; the Decl field is different + // for different files) + obj := NewObj(Pkg, name) + obj.Decl = spec + obj.Data = pkg.Data + p.declare(fileScope, pkgScope, obj) + } + } + + // resolve identifiers + if importErrors { + // don't use the universe scope without correct imports + // (objects in the universe may be shadowed by imports; + // with missing imports, identifiers might get resolved + // incorrectly to universe objects) + pkgScope.Outer = nil + } + i := 0 + for _, ident := range file.Unresolved { + if !resolve(fileScope, ident) { + p.errorf(ident.Pos(), "undeclared name: %s", ident.Name) + file.Unresolved[i] = ident + i++ + } + + } + file.Unresolved = file.Unresolved[0:i] + pkgScope.Outer = universe // reset universe scope + } + + return &Package{pkgName, pkgScope, imports, files}, p.GetError(scanner.Sorted) +} diff --git a/src/pkg/go/ast/scope.go b/src/pkg/go/ast/scope.go new file mode 100644 index 000000000..92e366980 --- /dev/null +++ b/src/pkg/go/ast/scope.go @@ -0,0 +1,156 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements scopes and the objects they contain. + +package ast + +import ( + "bytes" + "fmt" + "go/token" +) + +// A Scope maintains the set of named language entities declared +// in the scope and a link to the immediately surrounding (outer) +// scope. +// +type Scope struct { + Outer *Scope + Objects map[string]*Object +} + +// NewScope creates a new scope nested in the outer scope. +func NewScope(outer *Scope) *Scope { + const n = 4 // initial scope capacity + return &Scope{outer, make(map[string]*Object, n)} +} + +// Lookup returns the object with the given name if it is +// found in scope s, otherwise it returns nil. Outer scopes +// are ignored. +// +func (s *Scope) Lookup(name string) *Object { + return s.Objects[name] +} + +// Insert attempts to insert a named object obj into the scope s. +// If the scope already contains an object alt with the same name, +// Insert leaves the scope unchanged and returns alt. Otherwise +// it inserts obj and returns nil." +// +func (s *Scope) Insert(obj *Object) (alt *Object) { + if alt = s.Objects[obj.Name]; alt == nil { + s.Objects[obj.Name] = obj + } + return +} + +// Debugging support +func (s *Scope) String() string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "scope %p {", s) + if s != nil && len(s.Objects) > 0 { + fmt.Fprintln(&buf) + for _, obj := range s.Objects { + fmt.Fprintf(&buf, "\t%s %s\n", obj.Kind, obj.Name) + } + } + fmt.Fprintf(&buf, "}\n") + return buf.String() +} + +// ---------------------------------------------------------------------------- +// Objects + +// TODO(gri) Consider replacing the Object struct with an interface +// and a corresponding set of object implementations. + +// An Object describes a named language entity such as a package, +// constant, type, variable, function (incl. methods), or label. +// +// The Data fields contains object-specific data: +// +// Kind Data type Data value +// Pkg *Scope package scope +// Con int iota for the respective declaration +// Con != nil constant value +// +type Object struct { + Kind ObjKind + Name string // declared name + Decl interface{} // corresponding Field, XxxSpec, FuncDecl, or LabeledStmt; or nil + Data interface{} // object-specific data; or nil + Type interface{} // place holder for type information; may be nil +} + +// NewObj creates a new object of a given kind and name. +func NewObj(kind ObjKind, name string) *Object { + return &Object{Kind: kind, Name: name} +} + +// Pos computes the source position of the declaration of an object name. +// The result may be an invalid position if it cannot be computed +// (obj.Decl may be nil or not correct). +func (obj *Object) Pos() token.Pos { + name := obj.Name + switch d := obj.Decl.(type) { + case *Field: + for _, n := range d.Names { + if n.Name == name { + return n.Pos() + } + } + case *ImportSpec: + if d.Name != nil && d.Name.Name == name { + return d.Name.Pos() + } + return d.Path.Pos() + case *ValueSpec: + for _, n := range d.Names { + if n.Name == name { + return n.Pos() + } + } + case *TypeSpec: + if d.Name.Name == name { + return d.Name.Pos() + } + case *FuncDecl: + if d.Name.Name == name { + return d.Name.Pos() + } + case *LabeledStmt: + if d.Label.Name == name { + return d.Label.Pos() + } + } + return token.NoPos +} + +// ObKind describes what an object represents. +type ObjKind int + +// The list of possible Object kinds. +const ( + Bad ObjKind = iota // for error handling + Pkg // package + Con // constant + Typ // type + Var // variable + Fun // function or method + Lbl // label +) + +var objKindStrings = [...]string{ + Bad: "bad", + Pkg: "package", + Con: "const", + Typ: "type", + Var: "var", + Fun: "func", + Lbl: "label", +} + +func (kind ObjKind) String() string { return objKindStrings[kind] } diff --git a/src/pkg/go/ast/walk.go b/src/pkg/go/ast/walk.go new file mode 100644 index 000000000..181cfd149 --- /dev/null +++ b/src/pkg/go/ast/walk.go @@ -0,0 +1,382 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "fmt" + +// A Visitor's Visit method is invoked for each node encountered by Walk. +// If the result visitor w is not nil, Walk visits each of the children +// of node with the visitor w, followed by a call of w.Visit(nil). +type Visitor interface { + Visit(node Node) (w Visitor) +} + +// Helper functions for common node lists. They may be empty. + +func walkIdentList(v Visitor, list []*Ident) { + for _, x := range list { + Walk(v, x) + } +} + +func walkExprList(v Visitor, list []Expr) { + for _, x := range list { + Walk(v, x) + } +} + +func walkStmtList(v Visitor, list []Stmt) { + for _, x := range list { + Walk(v, x) + } +} + +func walkDeclList(v Visitor, list []Decl) { + for _, x := range list { + Walk(v, x) + } +} + +// TODO(gri): Investigate if providing a closure to Walk leads to +// simpler use (and may help eliminate Inspect in turn). + +// Walk traverses an AST in depth-first order: It starts by calling +// v.Visit(node); node must not be nil. If the visitor w returned by +// v.Visit(node) is not nil, Walk is invoked recursively with visitor +// w for each of the non-nil children of node, followed by a call of +// w.Visit(nil). +// +func Walk(v Visitor, node Node) { + if v = v.Visit(node); v == nil { + return + } + + // walk children + // (the order of the cases matches the order + // of the corresponding node types in ast.go) + switch n := node.(type) { + // Comments and fields + case *Comment: + // nothing to do + + case *CommentGroup: + for _, c := range n.List { + Walk(v, c) + } + + case *Field: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + Walk(v, n.Type) + if n.Tag != nil { + Walk(v, n.Tag) + } + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *FieldList: + for _, f := range n.List { + Walk(v, f) + } + + // Expressions + case *BadExpr, *Ident, *BasicLit: + // nothing to do + + case *Ellipsis: + if n.Elt != nil { + Walk(v, n.Elt) + } + + case *FuncLit: + Walk(v, n.Type) + Walk(v, n.Body) + + case *CompositeLit: + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Elts) + + case *ParenExpr: + Walk(v, n.X) + + case *SelectorExpr: + Walk(v, n.X) + Walk(v, n.Sel) + + case *IndexExpr: + Walk(v, n.X) + Walk(v, n.Index) + + case *SliceExpr: + Walk(v, n.X) + if n.Low != nil { + Walk(v, n.Low) + } + if n.High != nil { + Walk(v, n.High) + } + + case *TypeAssertExpr: + Walk(v, n.X) + if n.Type != nil { + Walk(v, n.Type) + } + + case *CallExpr: + Walk(v, n.Fun) + walkExprList(v, n.Args) + + case *StarExpr: + Walk(v, n.X) + + case *UnaryExpr: + Walk(v, n.X) + + case *BinaryExpr: + Walk(v, n.X) + Walk(v, n.Y) + + case *KeyValueExpr: + Walk(v, n.Key) + Walk(v, n.Value) + + // Types + case *ArrayType: + if n.Len != nil { + Walk(v, n.Len) + } + Walk(v, n.Elt) + + case *StructType: + Walk(v, n.Fields) + + case *FuncType: + Walk(v, n.Params) + if n.Results != nil { + Walk(v, n.Results) + } + + case *InterfaceType: + Walk(v, n.Methods) + + case *MapType: + Walk(v, n.Key) + Walk(v, n.Value) + + case *ChanType: + Walk(v, n.Value) + + // Statements + case *BadStmt: + // nothing to do + + case *DeclStmt: + Walk(v, n.Decl) + + case *EmptyStmt: + // nothing to do + + case *LabeledStmt: + Walk(v, n.Label) + Walk(v, n.Stmt) + + case *ExprStmt: + Walk(v, n.X) + + case *SendStmt: + Walk(v, n.Chan) + Walk(v, n.Value) + + case *IncDecStmt: + Walk(v, n.X) + + case *AssignStmt: + walkExprList(v, n.Lhs) + walkExprList(v, n.Rhs) + + case *GoStmt: + Walk(v, n.Call) + + case *DeferStmt: + Walk(v, n.Call) + + case *ReturnStmt: + walkExprList(v, n.Results) + + case *BranchStmt: + if n.Label != nil { + Walk(v, n.Label) + } + + case *BlockStmt: + walkStmtList(v, n.List) + + case *IfStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Cond) + Walk(v, n.Body) + if n.Else != nil { + Walk(v, n.Else) + } + + case *CaseClause: + walkExprList(v, n.List) + walkStmtList(v, n.Body) + + case *SwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Tag != nil { + Walk(v, n.Tag) + } + Walk(v, n.Body) + + case *TypeSwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Assign) + Walk(v, n.Body) + + case *CommClause: + if n.Comm != nil { + Walk(v, n.Comm) + } + walkStmtList(v, n.Body) + + case *SelectStmt: + Walk(v, n.Body) + + case *ForStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Cond != nil { + Walk(v, n.Cond) + } + if n.Post != nil { + Walk(v, n.Post) + } + Walk(v, n.Body) + + case *RangeStmt: + Walk(v, n.Key) + if n.Value != nil { + Walk(v, n.Value) + } + Walk(v, n.X) + Walk(v, n.Body) + + // Declarations + case *ImportSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Name != nil { + Walk(v, n.Name) + } + Walk(v, n.Path) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *ValueSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Values) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *TypeSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *BadDecl: + // nothing to do + + case *GenDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + for _, s := range n.Specs { + Walk(v, s) + } + + case *FuncDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Recv != nil { + Walk(v, n.Recv) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Body != nil { + Walk(v, n.Body) + } + + // Files and packages + case *File: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + walkDeclList(v, n.Decls) + for _, g := range n.Comments { + Walk(v, g) + } + // don't walk n.Comments - they have been + // visited already through the individual + // nodes + + case *Package: + for _, f := range n.Files { + Walk(v, f) + } + + default: + fmt.Printf("ast.Walk: unexpected node type %T", n) + panic("ast.Walk") + } + + v.Visit(nil) +} + +type inspector func(Node) bool + +func (f inspector) Visit(node Node) Visitor { + if f(node) { + return f + } + return nil +} + +// Inspect traverses an AST in depth-first order: It starts by calling +// f(node); node must not be nil. If f returns true, Inspect invokes f +// for all the non-nil children of node, recursively. +// +func Inspect(node Node, f func(Node) bool) { + Walk(inspector(f), node) +} diff --git a/src/pkg/go/build/Makefile b/src/pkg/go/build/Makefile new file mode 100644 index 000000000..349e00e80 --- /dev/null +++ b/src/pkg/go/build/Makefile @@ -0,0 +1,22 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/build +GOFILES=\ + build.go\ + dir.go\ + path.go\ + syslist.go\ + +CLEANFILES+=syslist.go pkgtest/_obj cmdtest/_obj cgotest/_obj + +include ../../../Make.pkg + +syslist.go: ../../../Make.inc Makefile + echo '// Generated automatically by make.' >$@ + echo 'package build' >>$@ + echo 'const goosList = "$(GOOS_LIST)"' >>$@ + echo 'const goarchList = "$(GOARCH_LIST)"' >>$@ diff --git a/src/pkg/go/build/build.go b/src/pkg/go/build/build.go new file mode 100644 index 000000000..97f92bfb6 --- /dev/null +++ b/src/pkg/go/build/build.go @@ -0,0 +1,444 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package build provides tools for building Go packages. +package build + +import ( + "bytes" + "exec" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// Build produces a build Script for the given package. +func Build(tree *Tree, pkg string, info *DirInfo) (*Script, os.Error) { + s := &Script{} + b := &build{ + script: s, + path: filepath.Join(tree.SrcDir(), pkg), + } + b.obj = b.abs("_obj") + string(filepath.Separator) + + b.goarch = runtime.GOARCH + if g := os.Getenv("GOARCH"); g != "" { + b.goarch = g + } + var err os.Error + b.arch, err = ArchChar(b.goarch) + if err != nil { + return nil, err + } + + // add import object files to list of Inputs + for _, pkg := range info.Imports { + t, p, err := FindTree(pkg) + if err != nil && err != ErrNotFound { + // FindTree should always be able to suggest an import + // path and tree. The path must be malformed + // (for example, an absolute or relative path). + return nil, os.NewError("build: invalid import: " + pkg) + } + s.addInput(filepath.Join(t.PkgDir(), p+".a")) + } + + // .go files to be built with gc + gofiles := b.abss(info.GoFiles...) + s.addInput(gofiles...) + + var ofiles []string // object files to be linked or packed + + // make build directory + b.mkdir(b.obj) + s.addIntermediate(b.obj) + + // cgo + if len(info.CgoFiles) > 0 { + cgoFiles := b.abss(info.CgoFiles...) + s.addInput(cgoFiles...) + cgoCFiles := b.abss(info.CFiles...) + s.addInput(cgoCFiles...) + outGo, outObj := b.cgo(cgoFiles, cgoCFiles) + gofiles = append(gofiles, outGo...) + ofiles = append(ofiles, outObj...) + s.addIntermediate(outGo...) + s.addIntermediate(outObj...) + } + + // compile + if len(gofiles) > 0 { + ofile := b.obj + "_go_." + b.arch + b.gc(ofile, gofiles...) + ofiles = append(ofiles, ofile) + s.addIntermediate(ofile) + } + + // assemble + for _, sfile := range info.SFiles { + ofile := b.obj + sfile[:len(sfile)-1] + b.arch + sfile = b.abs(sfile) + s.addInput(sfile) + b.asm(ofile, sfile) + ofiles = append(ofiles, ofile) + s.addIntermediate(ofile) + } + + if len(ofiles) == 0 { + return nil, os.NewError("make: no object files to build") + } + + // choose target file + var targ string + if info.IsCommand() { + // use the last part of the import path as binary name + _, bin := filepath.Split(pkg) + if runtime.GOOS == "windows" { + bin += ".exe" + } + targ = filepath.Join(tree.BinDir(), bin) + } else { + targ = filepath.Join(tree.PkgDir(), pkg+".a") + } + + // make target directory + targDir, _ := filepath.Split(targ) + b.mkdir(targDir) + + // link binary or pack object + if info.IsCommand() { + b.ld(targ, ofiles...) + } else { + b.gopack(targ, ofiles...) + } + s.Output = append(s.Output, targ) + + return b.script, nil +} + +// A Script describes the build process for a Go package. +// The Input, Intermediate, and Output fields are lists of absolute paths. +type Script struct { + Cmd []*Cmd + Input []string + Intermediate []string + Output []string +} + +func (s *Script) addInput(file ...string) { + s.Input = append(s.Input, file...) +} + +func (s *Script) addIntermediate(file ...string) { + s.Intermediate = append(s.Intermediate, file...) +} + +// Run runs the Script's Cmds in order. +func (s *Script) Run() os.Error { + for _, c := range s.Cmd { + if err := c.Run(); err != nil { + return err + } + } + return nil +} + +// Stale returns true if the build's inputs are newer than its outputs. +func (s *Script) Stale() bool { + var latest int64 + // get latest mtime of outputs + for _, file := range s.Output { + fi, err := os.Stat(file) + if err != nil { + // any error reading output files means stale + return true + } + if m := fi.Mtime_ns; m > latest { + latest = m + } + } + for _, file := range s.Input { + fi, err := os.Stat(file) + if err != nil || fi.Mtime_ns > latest { + // any error reading input files means stale + // (attempt to rebuild to figure out why) + return true + } + } + return false +} + +// Clean removes the Script's Intermediate files. +// It tries to remove every file and returns the first error it encounters. +func (s *Script) Clean() (err os.Error) { + // Reverse order so that directories get removed after the files they contain. + for i := len(s.Intermediate) - 1; i >= 0; i-- { + if e := os.Remove(s.Intermediate[i]); err == nil { + err = e + } + } + return +} + +// Nuke removes the Script's Intermediate and Output files. +// It tries to remove every file and returns the first error it encounters. +func (s *Script) Nuke() (err os.Error) { + // Reverse order so that directories get removed after the files they contain. + for i := len(s.Output) - 1; i >= 0; i-- { + if e := os.Remove(s.Output[i]); err == nil { + err = e + } + } + if e := s.Clean(); err == nil { + err = e + } + return +} + +// A Cmd describes an individual build command. +type Cmd struct { + Args []string // command-line + Stdout string // write standard output to this file, "" is passthrough + Dir string // working directory + Env []string // environment + Input []string // file paths (dependencies) + Output []string // file paths +} + +func (c *Cmd) String() string { + return strings.Join(c.Args, " ") +} + +// Run executes the Cmd. +func (c *Cmd) Run() os.Error { + if c.Args[0] == "mkdir" { + for _, p := range c.Output { + if err := os.MkdirAll(p, 0777); err != nil { + return fmt.Errorf("command %q: %v", c, err) + } + } + return nil + } + out := new(bytes.Buffer) + cmd := exec.Command(c.Args[0], c.Args[1:]...) + cmd.Dir = c.Dir + cmd.Env = c.Env + cmd.Stdout = out + cmd.Stderr = out + if c.Stdout != "" { + f, err := os.Create(c.Stdout) + if err != nil { + return err + } + defer f.Close() + cmd.Stdout = f + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("command %q: %v\n%v", c, err, out) + } + return nil +} + +// ArchChar returns the architecture character for the given goarch. +// For example, ArchChar("amd64") returns "6". +func ArchChar(goarch string) (string, os.Error) { + switch goarch { + case "386": + return "8", nil + case "amd64": + return "6", nil + case "arm": + return "5", nil + } + return "", os.NewError("unsupported GOARCH " + goarch) +} + +type build struct { + script *Script + path string + obj string + goarch string + arch string +} + +func (b *build) abs(file string) string { + if filepath.IsAbs(file) { + return file + } + return filepath.Join(b.path, file) +} + +func (b *build) abss(file ...string) []string { + s := make([]string, len(file)) + for i, f := range file { + s[i] = b.abs(f) + } + return s +} + +func (b *build) add(c Cmd) { + b.script.Cmd = append(b.script.Cmd, &c) +} + +func (b *build) mkdir(name string) { + b.add(Cmd{ + Args: []string{"mkdir", "-p", name}, + Output: []string{name}, + }) +} + +func (b *build) gc(ofile string, gofiles ...string) { + gc := b.arch + "g" + args := append([]string{gc, "-o", ofile}, gcImportArgs...) + args = append(args, gofiles...) + b.add(Cmd{ + Args: args, + Input: gofiles, + Output: []string{ofile}, + }) +} + +func (b *build) asm(ofile string, sfile string) { + asm := b.arch + "a" + b.add(Cmd{ + Args: []string{asm, "-o", ofile, sfile}, + Input: []string{sfile}, + Output: []string{ofile}, + }) +} + +func (b *build) ld(targ string, ofiles ...string) { + ld := b.arch + "l" + args := append([]string{ld, "-o", targ}, ldImportArgs...) + args = append(args, ofiles...) + b.add(Cmd{ + Args: args, + Input: ofiles, + Output: []string{targ}, + }) +} + +func (b *build) gopack(targ string, ofiles ...string) { + b.add(Cmd{ + Args: append([]string{"gopack", "grc", targ}, ofiles...), + Input: ofiles, + Output: []string{targ}, + }) +} + +func (b *build) cc(ofile string, cfiles ...string) { + cc := b.arch + "c" + dir := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) + inc := filepath.Join(runtime.GOROOT(), "pkg", dir) + args := []string{cc, "-FVw", "-I", inc, "-o", ofile} + b.add(Cmd{ + Args: append(args, cfiles...), + Input: cfiles, + Output: []string{ofile}, + }) +} + +func (b *build) gccCompile(ofile, cfile string) { + b.add(Cmd{ + Args: b.gccArgs("-o", ofile, "-c", cfile), + Input: []string{cfile}, + Output: []string{ofile}, + }) +} + +func (b *build) gccLink(ofile string, ofiles ...string) { + b.add(Cmd{ + Args: append(b.gccArgs("-o", ofile), ofiles...), + Input: ofiles, + Output: []string{ofile}, + }) +} + +func (b *build) gccArgs(args ...string) []string { + // TODO(adg): HOST_CC + a := []string{"gcc", "-I", b.path, "-g", "-fPIC", "-O2"} + switch b.arch { + case "8": + a = append(a, "-m32") + case "6": + a = append(a, "-m64") + } + return append(a, args...) +} + +var cgoRe = regexp.MustCompile(`[/\\:]`) + +func (b *build) cgo(cgofiles, cgocfiles []string) (outGo, outObj []string) { + // cgo + // TODO(adg): CGOPKGPATH + // TODO(adg): CGO_FLAGS + gofiles := []string{b.obj + "_cgo_gotypes.go"} + cfiles := []string{b.obj + "_cgo_main.c", b.obj + "_cgo_export.c"} + for _, fn := range cgofiles { + f := b.obj + cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") + gofiles = append(gofiles, f+"cgo1.go") + cfiles = append(cfiles, f+"cgo2.c") + } + defunC := b.obj + "_cgo_defun.c" + output := append([]string{defunC}, cfiles...) + output = append(output, gofiles...) + b.add(Cmd{ + Args: append([]string{"cgo", "--"}, cgofiles...), + Dir: b.path, + Env: append(os.Environ(), "GOARCH="+b.goarch), + Input: cgofiles, + Output: output, + }) + outGo = append(outGo, gofiles...) + exportH := filepath.Join(b.path, "_cgo_export.h") + b.script.addIntermediate(defunC, exportH, b.obj+"_cgo_flags") + b.script.addIntermediate(cfiles...) + + // cc _cgo_defun.c + defunObj := b.obj + "_cgo_defun." + b.arch + b.cc(defunObj, defunC) + outObj = append(outObj, defunObj) + + // gcc + linkobj := make([]string, 0, len(cfiles)) + for _, cfile := range cfiles { + ofile := cfile[:len(cfile)-1] + "o" + b.gccCompile(ofile, cfile) + linkobj = append(linkobj, ofile) + if !strings.HasSuffix(ofile, "_cgo_main.o") { + outObj = append(outObj, ofile) + } else { + b.script.addIntermediate(ofile) + } + } + for _, cfile := range cgocfiles { + ofile := b.obj + cgoRe.ReplaceAllString(cfile[:len(cfile)-1], "_") + "o" + b.gccCompile(ofile, cfile) + linkobj = append(linkobj, ofile) + outObj = append(outObj, ofile) + } + dynObj := b.obj + "_cgo_.o" + b.gccLink(dynObj, linkobj...) + b.script.addIntermediate(dynObj) + + // cgo -dynimport + importC := b.obj + "_cgo_import.c" + b.add(Cmd{ + Args: []string{"cgo", "-dynimport", dynObj}, + Stdout: importC, + Input: []string{dynObj}, + Output: []string{importC}, + }) + b.script.addIntermediate(importC) + + // cc _cgo_import.ARCH + importObj := b.obj + "_cgo_import." + b.arch + b.cc(importObj, importC) + outObj = append(outObj, importObj) + + return +} diff --git a/src/pkg/go/build/build_test.go b/src/pkg/go/build/build_test.go new file mode 100644 index 000000000..e59d87672 --- /dev/null +++ b/src/pkg/go/build/build_test.go @@ -0,0 +1,61 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "exec" + "path/filepath" + "testing" +) + +var buildPkgs = []string{ + "go/build/pkgtest", + "go/build/cmdtest", + "go/build/cgotest", +} + +const cmdtestOutput = "3" + +func TestBuild(t *testing.T) { + for _, pkg := range buildPkgs { + tree := Path[0] // Goroot + dir := filepath.Join(tree.SrcDir(), pkg) + + info, err := ScanDir(dir, true) + if err != nil { + t.Error("ScanDir:", err) + continue + } + + s, err := Build(tree, pkg, info) + if err != nil { + t.Error("Build:", err) + continue + } + + if err := s.Run(); err != nil { + t.Error("Run:", err) + continue + } + + if pkg == "go/build/cmdtest" { + bin := s.Output[0] + b, err := exec.Command(bin).CombinedOutput() + if err != nil { + t.Errorf("exec: %s: %v", bin, err) + continue + } + if string(b) != cmdtestOutput { + t.Errorf("cmdtest output: %s want: %s", b, cmdtestOutput) + } + } + + defer func(s *Script) { + if err := s.Nuke(); err != nil { + t.Errorf("nuking: %v", err) + } + }(s) + } +} diff --git a/src/pkg/go/build/cgotest/cgotest.c b/src/pkg/go/build/cgotest/cgotest.c new file mode 100644 index 000000000..b13acb227 --- /dev/null +++ b/src/pkg/go/build/cgotest/cgotest.c @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +int +Add(int x, int y, int *sum) +{ + sum = x+y; +} diff --git a/src/pkg/go/build/cgotest/cgotest.go b/src/pkg/go/build/cgotest/cgotest.go new file mode 100644 index 000000000..93bbf0688 --- /dev/null +++ b/src/pkg/go/build/cgotest/cgotest.go @@ -0,0 +1,19 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +char* greeting = "hello, world"; +*/ +// #include "cgotest.h" +import "C" +import "unsafe" + +var Greeting = C.GoString(C.greeting) + +func DoAdd(x, y int) (sum int) { + C.Add(C.int(x), C.int(y), (*C.int)(unsafe.Pointer(&sum))) + return +} diff --git a/src/pkg/go/build/cgotest/cgotest.h b/src/pkg/go/build/cgotest/cgotest.h new file mode 100644 index 000000000..9c73643b6 --- /dev/null +++ b/src/pkg/go/build/cgotest/cgotest.h @@ -0,0 +1,5 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +extern int Add(int, int, int *); diff --git a/src/pkg/go/build/cmdtest/main.go b/src/pkg/go/build/cmdtest/main.go new file mode 100644 index 000000000..bed4f485a --- /dev/null +++ b/src/pkg/go/build/cmdtest/main.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "go/build/pkgtest" + +func main() { + pkgtest.Foo() + print(int(pkgtest.Sqrt(9))) +} diff --git a/src/pkg/go/build/dir.go b/src/pkg/go/build/dir.go new file mode 100644 index 000000000..e0000b534 --- /dev/null +++ b/src/pkg/go/build/dir.go @@ -0,0 +1,172 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "runtime" +) + +type DirInfo struct { + GoFiles []string // .go files in dir (excluding CgoFiles) + CgoFiles []string // .go files that import "C" + CFiles []string // .c files in dir + SFiles []string // .s files in dir + Imports []string // All packages imported by goFiles + PkgName string // Name of package in dir +} + +func (d *DirInfo) IsCommand() bool { + return d.PkgName == "main" +} + +// ScanDir returns a structure with details about the Go content found +// in the given directory. The file lists exclude: +// +// - files in package main (unless allowMain is true) +// - files in package documentation +// - files ending in _test.go +// - files starting with _ or . +// +// Only files that satisfy the goodOSArch function are included. +func ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) { + f, err := os.Open(dir) + if err != nil { + return nil, err + } + dirs, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + + var di DirInfo + imported := make(map[string]bool) + fset := token.NewFileSet() + for i := range dirs { + d := &dirs[i] + if strings.HasPrefix(d.Name, "_") || + strings.HasPrefix(d.Name, ".") { + continue + } + if !goodOSArch(d.Name) { + continue + } + + switch filepath.Ext(d.Name) { + case ".go": + if strings.HasSuffix(d.Name, "_test.go") { + continue + } + case ".c": + di.CFiles = append(di.CFiles, d.Name) + continue + case ".s": + di.SFiles = append(di.SFiles, d.Name) + continue + default: + continue + } + + filename := filepath.Join(dir, d.Name) + pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly) + if err != nil { + return nil, err + } + s := string(pf.Name.Name) + if s == "main" && !allowMain { + continue + } + if s == "documentation" { + continue + } + if di.PkgName == "" { + di.PkgName = s + } else if di.PkgName != s { + // Only if all files in the directory are in package main + // do we return PkgName=="main". + // A mix of main and another package reverts + // to the original (allowMain=false) behaviour. + if s == "main" || di.PkgName == "main" { + return ScanDir(dir, false) + } + return nil, os.NewError("multiple package names in " + dir) + } + isCgo := false + for _, spec := range pf.Imports { + quoted := string(spec.Path.Value) + path, err := strconv.Unquote(quoted) + if err != nil { + log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) + } + imported[path] = true + if path == "C" { + isCgo = true + } + } + if isCgo { + di.CgoFiles = append(di.CgoFiles, d.Name) + } else { + di.GoFiles = append(di.GoFiles, d.Name) + } + } + di.Imports = make([]string, len(imported)) + i := 0 + for p := range imported { + di.Imports[i] = p + i++ + } + return &di, nil +} + +// goodOSArch returns false if the filename contains a $GOOS or $GOARCH +// suffix which does not match the current system. +// The recognized filename formats are: +// +// name_$(GOOS).* +// name_$(GOARCH).* +// name_$(GOOS)_$(GOARCH).* +// +func goodOSArch(filename string) bool { + if dot := strings.Index(filename, "."); dot != -1 { + filename = filename[:dot] + } + l := strings.Split(filename, "_") + n := len(l) + if n == 0 { + return true + } + if good, known := goodOS[l[n-1]]; known { + return good + } + if good, known := goodArch[l[n-1]]; known { + if !good || n < 2 { + return false + } + good, known = goodOS[l[n-2]] + return good || !known + } + return true +} + +var goodOS = make(map[string]bool) +var goodArch = make(map[string]bool) + +func init() { + goodOS = make(map[string]bool) + goodArch = make(map[string]bool) + for _, v := range strings.Fields(goosList) { + goodOS[v] = v == runtime.GOOS + } + for _, v := range strings.Fields(goarchList) { + goodArch[v] = v == runtime.GOARCH + } +} diff --git a/src/pkg/go/build/path.go b/src/pkg/go/build/path.go new file mode 100644 index 000000000..e39b5f8fa --- /dev/null +++ b/src/pkg/go/build/path.go @@ -0,0 +1,182 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" +) + +// Path is a validated list of Trees derived from $GOROOT and $GOPATH at init. +var Path []*Tree + +// Tree describes a Go source tree, either $GOROOT or one from $GOPATH. +type Tree struct { + Path string + Goroot bool +} + +func newTree(p string) (*Tree, os.Error) { + if !filepath.IsAbs(p) { + return nil, os.NewError("must be absolute") + } + ep, err := filepath.EvalSymlinks(p) + if err != nil { + return nil, err + } + return &Tree{Path: ep}, nil +} + +// SrcDir returns the tree's package source directory. +func (t *Tree) SrcDir() string { + if t.Goroot { + return filepath.Join(t.Path, "src", "pkg") + } + return filepath.Join(t.Path, "src") +} + +// PkgDir returns the tree's package object directory. +func (t *Tree) PkgDir() string { + goos, goarch := runtime.GOOS, runtime.GOARCH + if e := os.Getenv("GOOS"); e != "" { + goos = e + } + if e := os.Getenv("GOARCH"); e != "" { + goarch = e + } + return filepath.Join(t.Path, "pkg", goos+"_"+goarch) +} + +// BinDir returns the tree's binary executable directory. +func (t *Tree) BinDir() string { + if t.Goroot { + if gobin := os.Getenv("GOBIN"); gobin != "" { + return gobin + } + } + return filepath.Join(t.Path, "bin") +} + +// HasSrc returns whether the given package's +// source can be found inside this Tree. +func (t *Tree) HasSrc(pkg string) bool { + fi, err := os.Stat(filepath.Join(t.SrcDir(), pkg)) + if err != nil { + return false + } + return fi.IsDirectory() +} + +// HasPkg returns whether the given package's +// object file can be found inside this Tree. +func (t *Tree) HasPkg(pkg string) bool { + fi, err := os.Stat(filepath.Join(t.PkgDir(), pkg+".a")) + if err != nil { + return false + } + return fi.IsRegular() + // TODO(adg): check object version is consistent +} + +var ( + ErrNotFound = os.NewError("go/build: package could not be found locally") + ErrTreeNotFound = os.NewError("go/build: no valid GOROOT or GOPATH could be found") +) + +// FindTree takes an import or filesystem path and returns the +// tree where the package source should be and the package import path. +func FindTree(path string) (tree *Tree, pkg string, err os.Error) { + if isLocalPath(path) { + if path, err = filepath.Abs(path); err != nil { + return + } + if path, err = filepath.EvalSymlinks(path); err != nil { + return + } + for _, t := range Path { + tpath := t.SrcDir() + string(filepath.Separator) + if !filepath.HasPrefix(path, tpath) { + continue + } + tree = t + pkg = path[len(tpath):] + return + } + err = fmt.Errorf("path %q not inside a GOPATH", path) + return + } + tree = defaultTree + pkg = path + for _, t := range Path { + if t.HasSrc(pkg) { + tree = t + return + } + } + if tree == nil { + err = ErrTreeNotFound + } else { + err = ErrNotFound + } + return +} + +// isLocalPath returns whether the given path is local (/foo ./foo ../foo . ..) +// Windows paths that starts with drive letter (c:\foo c:foo) are considered local. +func isLocalPath(s string) bool { + const sep = string(filepath.Separator) + return s == "." || s == ".." || + filepath.HasPrefix(s, sep) || + filepath.HasPrefix(s, "."+sep) || filepath.HasPrefix(s, ".."+sep) || + filepath.VolumeName(s) != "" +} + +var ( + // argument lists used by the build's gc and ld methods + gcImportArgs []string + ldImportArgs []string + + // default tree for remote packages + defaultTree *Tree +) + +// set up Path: parse and validate GOROOT and GOPATH variables +func init() { + root := runtime.GOROOT() + t, err := newTree(root) + if err != nil { + log.Printf("go/build: invalid GOROOT %q: %v", root, err) + } else { + t.Goroot = true + Path = []*Tree{t} + } + + for _, p := range filepath.SplitList(os.Getenv("GOPATH")) { + if p == "" { + continue + } + t, err := newTree(p) + if err != nil { + log.Printf("go/build: invalid GOPATH %q: %v", p, err) + continue + } + Path = append(Path, t) + gcImportArgs = append(gcImportArgs, "-I", t.PkgDir()) + ldImportArgs = append(ldImportArgs, "-L", t.PkgDir()) + + // select first GOPATH entry as default + if defaultTree == nil { + defaultTree = t + } + } + + // use GOROOT if no valid GOPATH specified + if defaultTree == nil && len(Path) > 0 { + defaultTree = Path[0] + } +} diff --git a/src/pkg/go/build/pkgtest/pkgtest.go b/src/pkg/go/build/pkgtest/pkgtest.go new file mode 100644 index 000000000..9322f5ebd --- /dev/null +++ b/src/pkg/go/build/pkgtest/pkgtest.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgtest + +func Foo() {} + +func Sqrt(x float64) float64 diff --git a/src/pkg/go/build/pkgtest/sqrt_386.s b/src/pkg/go/build/pkgtest/sqrt_386.s new file mode 100644 index 000000000..d0a428d52 --- /dev/null +++ b/src/pkg/go/build/pkgtest/sqrt_386.s @@ -0,0 +1,10 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + FMOVD x+0(FP),F0 + FSQRT + FMOVDP F0,r+8(FP) + RET diff --git a/src/pkg/go/build/pkgtest/sqrt_amd64.s b/src/pkg/go/build/pkgtest/sqrt_amd64.s new file mode 100644 index 000000000..f5b329e70 --- /dev/null +++ b/src/pkg/go/build/pkgtest/sqrt_amd64.s @@ -0,0 +1,9 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + SQRTSD x+0(FP), X0 + MOVSD X0, r+8(FP) + RET diff --git a/src/pkg/go/build/pkgtest/sqrt_arm.s b/src/pkg/go/build/pkgtest/sqrt_arm.s new file mode 100644 index 000000000..befbb8a89 --- /dev/null +++ b/src/pkg/go/build/pkgtest/sqrt_arm.s @@ -0,0 +1,10 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// func Sqrt(x float64) float64 +TEXT ·Sqrt(SB),7,$0 + MOVD x+0(FP),F0 + SQRTD F0,F0 + MOVD F0,r+8(FP) + RET diff --git a/src/pkg/go/build/syslist_test.go b/src/pkg/go/build/syslist_test.go new file mode 100644 index 000000000..eb0e5dcb6 --- /dev/null +++ b/src/pkg/go/build/syslist_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "runtime" + "testing" +) + +var ( + thisOS = runtime.GOOS + thisArch = runtime.GOARCH + otherOS = anotherOS() + otherArch = anotherArch() +) + +func anotherOS() string { + if thisOS != "darwin" { + return "darwin" + } + return "linux" +} + +func anotherArch() string { + if thisArch != "amd64" { + return "amd64" + } + return "386" +} + +type GoodFileTest struct { + name string + result bool +} + +var tests = []GoodFileTest{ + {"file.go", true}, + {"file.c", true}, + {"file_foo.go", true}, + {"file_" + thisArch + ".go", true}, + {"file_" + otherArch + ".go", false}, + {"file_" + thisOS + ".go", true}, + {"file_" + otherOS + ".go", false}, + {"file_" + thisOS + "_" + thisArch + ".go", true}, + {"file_" + otherOS + "_" + thisArch + ".go", false}, + {"file_" + thisOS + "_" + otherArch + ".go", false}, + {"file_" + otherOS + "_" + otherArch + ".go", false}, + {"file_foo_" + thisArch + ".go", true}, + {"file_foo_" + otherArch + ".go", false}, + {"file_" + thisOS + ".c", true}, + {"file_" + otherOS + ".c", false}, +} + +func TestGoodOSArch(t *testing.T) { + for _, test := range tests { + if goodOSArch(test.name) != test.result { + t.Fatalf("goodOSArch(%q) != %v", test.name, test.result) + } + } +} diff --git a/src/pkg/go/doc/Makefile b/src/pkg/go/doc/Makefile new file mode 100644 index 000000000..a5152c793 --- /dev/null +++ b/src/pkg/go/doc/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=go/doc +GOFILES=\ + comment.go\ + doc.go\ + +include ../../../Make.pkg diff --git a/src/pkg/go/doc/comment.go b/src/pkg/go/doc/comment.go new file mode 100644 index 000000000..e1989226b --- /dev/null +++ b/src/pkg/go/doc/comment.go @@ -0,0 +1,345 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Godoc comment extraction and comment -> HTML formatting. + +package doc + +import ( + "go/ast" + "io" + "regexp" + "strings" + "template" // for HTMLEscape +) + +func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } + +func stripTrailingWhitespace(s string) string { + i := len(s) + for i > 0 && isWhitespace(s[i-1]) { + i-- + } + return s[0:i] +} + +// CommentText returns the text of comment, +// with the comment markers - //, /*, and */ - removed. +func CommentText(comment *ast.CommentGroup) string { + if comment == nil { + return "" + } + comments := make([]string, len(comment.List)) + for i, c := range comment.List { + comments[i] = string(c.Text) + } + + lines := make([]string, 0, 10) // most comments are less than 10 lines + for _, c := range comments { + // Remove comment markers. + // The parser has given us exactly the comment text. + switch c[1] { + case '/': + //-style comment + c = c[2:] + // Remove leading space after //, if there is one. + // TODO(gri) This appears to be necessary in isolated + // cases (bignum.RatFromString) - why? + if len(c) > 0 && c[0] == ' ' { + c = c[1:] + } + case '*': + /*-style comment */ + c = c[2 : len(c)-2] + } + + // Split on newlines. + cl := strings.Split(c, "\n") + + // Walk lines, stripping trailing white space and adding to list. + for _, l := range cl { + lines = append(lines, stripTrailingWhitespace(l)) + } + } + + // Remove leading blank lines; convert runs of + // interior blank lines to a single blank line. + n := 0 + for _, line := range lines { + if line != "" || n > 0 && lines[n-1] != "" { + lines[n] = line + n++ + } + } + lines = lines[0:n] + + // Add final "" entry to get trailing newline from Join. + if n > 0 && lines[n-1] != "" { + lines = append(lines, "") + } + + return strings.Join(lines, "\n") +} + +// Split bytes into lines. +func split(text []byte) [][]byte { + // count lines + n := 0 + last := 0 + for i, c := range text { + if c == '\n' { + last = i + 1 + n++ + } + } + if last < len(text) { + n++ + } + + // split + out := make([][]byte, n) + last = 0 + n = 0 + for i, c := range text { + if c == '\n' { + out[n] = text[last : i+1] + last = i + 1 + n++ + } + } + if last < len(text) { + out[n] = text[last:] + } + + return out +} + +var ( + ldquo = []byte("“") + rdquo = []byte("”") +) + +// Escape comment text for HTML. If nice is set, +// also turn `` into “ and '' into ”. +func commentEscape(w io.Writer, s []byte, nice bool) { + last := 0 + if nice { + for i := 0; i < len(s)-1; i++ { + ch := s[i] + if ch == s[i+1] && (ch == '`' || ch == '\'') { + template.HTMLEscape(w, s[last:i]) + last = i + 2 + switch ch { + case '`': + w.Write(ldquo) + case '\'': + w.Write(rdquo) + } + i++ // loop will add one more + } + } + } + template.HTMLEscape(w, s[last:]) +} + +const ( + // Regexp for Go identifiers + identRx = `[a-zA-Z_][a-zA-Z_0-9]*` // TODO(gri) ASCII only for now - fix this + + // Regexp for URLs + protocol = `(https?|ftp|file|gopher|mailto|news|nntp|telnet|wais|prospero):` + hostPart = `[a-zA-Z0-9_@\-]+` + filePart = `[a-zA-Z0-9_?%#~&/\-+=]+` + urlRx = protocol + `//` + // http:// + hostPart + `([.:]` + hostPart + `)*/?` + // //www.google.com:8080/ + filePart + `([:.,]` + filePart + `)*` +) + +var matchRx = regexp.MustCompile(`(` + identRx + `)|(` + urlRx + `)`) + +var ( + html_a = []byte(``) + html_enda = []byte("") + html_i = []byte("") + html_endi = []byte("") + html_p = []byte("

\n") + html_endp = []byte("

\n") + html_pre = []byte("
")
+	html_endpre = []byte("
\n") +) + +// Emphasize and escape a line of text for HTML. URLs are converted into links; +// if the URL also appears in the words map, the link is taken from the map (if +// the corresponding map value is the empty string, the URL is not converted +// into a link). Go identifiers that appear in the words map are italicized; if +// the corresponding map value is not the empty string, it is considered a URL +// and the word is converted into a link. If nice is set, the remaining text's +// appearance is improved where it makes sense (e.g., `` is turned into “ +// and '' into ”). +func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) { + for { + m := matchRx.FindSubmatchIndex(line) + if m == nil { + break + } + // m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is identRx) + + // write text before match + commentEscape(w, line[0:m[0]], nice) + + // analyze match + match := line[m[0]:m[1]] + url := "" + italics := false + if words != nil { + url, italics = words[string(match)] + } + if m[2] < 0 { + // didn't match against first parenthesized sub-regexp; must be match against urlRx + if !italics { + // no alternative URL in words list, use match instead + url = string(match) + } + italics = false // don't italicize URLs + } + + // write match + if len(url) > 0 { + w.Write(html_a) + template.HTMLEscape(w, []byte(url)) + w.Write(html_aq) + } + if italics { + w.Write(html_i) + } + commentEscape(w, match, nice) + if italics { + w.Write(html_endi) + } + if len(url) > 0 { + w.Write(html_enda) + } + + // advance + line = line[m[1]:] + } + commentEscape(w, line, nice) +} + +func indentLen(s []byte) int { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + return i +} + +func isBlank(s []byte) bool { return len(s) == 0 || (len(s) == 1 && s[0] == '\n') } + +func commonPrefix(a, b []byte) []byte { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return a[0:i] +} + +func unindent(block [][]byte) { + if len(block) == 0 { + return + } + + // compute maximum common white prefix + prefix := block[0][0:indentLen(block[0])] + for _, line := range block { + if !isBlank(line) { + prefix = commonPrefix(prefix, line[0:indentLen(line)]) + } + } + n := len(prefix) + + // remove + for i, line := range block { + if !isBlank(line) { + block[i] = line[n:] + } + } +} + +// Convert comment text to formatted HTML. +// The comment was prepared by DocReader, +// so it is known not to have leading, trailing blank lines +// nor to have trailing spaces at the end of lines. +// The comment markers have already been removed. +// +// Turn each run of multiple \n into

. +// Turn each run of indented lines into a 

 block without indent.
+//
+// URLs in the comment text are converted into links; if the URL also appears
+// in the words map, the link is taken from the map (if the corresponding map
+// value is the empty string, the URL is not converted into a link).
+//
+// Go identifiers that appear in the words map are italicized; if the corresponding
+// map value is not the empty string, it is considered a URL and the word is converted
+// into a link.
+func ToHTML(w io.Writer, s []byte, words map[string]string) {
+	inpara := false
+
+	close := func() {
+		if inpara {
+			w.Write(html_endp)
+			inpara = false
+		}
+	}
+	open := func() {
+		if !inpara {
+			w.Write(html_p)
+			inpara = true
+		}
+	}
+
+	lines := split(s)
+	unindent(lines)
+	for i := 0; i < len(lines); {
+		line := lines[i]
+		if isBlank(line) {
+			// close paragraph
+			close()
+			i++
+			continue
+		}
+		if indentLen(line) > 0 {
+			// close paragraph
+			close()
+
+			// count indented or blank lines
+			j := i + 1
+			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
+				j++
+			}
+			// but not trailing blank lines
+			for j > i && isBlank(lines[j-1]) {
+				j--
+			}
+			block := lines[i:j]
+			i = j
+
+			unindent(block)
+
+			// put those lines in a pre block
+			w.Write(html_pre)
+			for _, line := range block {
+				emphasize(w, line, nil, false) // no nice text formatting
+			}
+			w.Write(html_endpre)
+			continue
+		}
+		// open paragraph
+		open()
+		emphasize(w, lines[i], words, true) // nice text formatting
+		i++
+	}
+	close()
+}
diff --git a/src/pkg/go/doc/doc.go b/src/pkg/go/doc/doc.go
new file mode 100644
index 000000000..c7fed9784
--- /dev/null
+++ b/src/pkg/go/doc/doc.go
@@ -0,0 +1,641 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package doc extracts source code documentation from a Go AST.
+package doc
+
+import (
+	"go/ast"
+	"go/token"
+	"regexp"
+	"sort"
+)
+
+// ----------------------------------------------------------------------------
+
+type typeDoc struct {
+	// len(decl.Specs) == 1, and the element type is *ast.TypeSpec
+	// if the type declaration hasn't been seen yet, decl is nil
+	decl *ast.GenDecl
+	// values, factory functions, and methods associated with the type
+	values    []*ast.GenDecl // consts and vars
+	factories map[string]*ast.FuncDecl
+	methods   map[string]*ast.FuncDecl
+}
+
+// docReader accumulates documentation for a single package.
+// It modifies the AST: Comments (declaration documentation)
+// that have been collected by the DocReader are set to nil
+// in the respective AST nodes so that they are not printed
+// twice (once when printing the documentation and once when
+// printing the corresponding AST node).
+//
+type docReader struct {
+	doc     *ast.CommentGroup // package documentation, if any
+	pkgName string
+	values  []*ast.GenDecl // consts and vars
+	types   map[string]*typeDoc
+	funcs   map[string]*ast.FuncDecl
+	bugs    []*ast.CommentGroup
+}
+
+func (doc *docReader) init(pkgName string) {
+	doc.pkgName = pkgName
+	doc.types = make(map[string]*typeDoc)
+	doc.funcs = make(map[string]*ast.FuncDecl)
+}
+
+func (doc *docReader) addDoc(comments *ast.CommentGroup) {
+	if doc.doc == nil {
+		// common case: just one package comment
+		doc.doc = comments
+		return
+	}
+
+	// More than one package comment: Usually there will be only
+	// one file with a package comment, but it's better to collect
+	// all comments than drop them on the floor.
+	// (This code isn't particularly clever - no amortized doubling is
+	// used - but this situation occurs rarely and is not time-critical.)
+	n1 := len(doc.doc.List)
+	n2 := len(comments.List)
+	list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
+	copy(list, doc.doc.List)
+	list[n1] = &ast.Comment{token.NoPos, "//"} // separator line
+	copy(list[n1+1:], comments.List)
+	doc.doc = &ast.CommentGroup{list}
+}
+
+func (doc *docReader) addType(decl *ast.GenDecl) {
+	spec := decl.Specs[0].(*ast.TypeSpec)
+	typ := doc.lookupTypeDoc(spec.Name.Name)
+	// typ should always be != nil since declared types
+	// are always named - be conservative and check
+	if typ != nil {
+		// a type should be added at most once, so typ.decl
+		// should be nil - if it isn't, simply overwrite it
+		typ.decl = decl
+	}
+}
+
+func (doc *docReader) lookupTypeDoc(name string) *typeDoc {
+	if name == "" {
+		return nil // no type docs for anonymous types
+	}
+	if tdoc, found := doc.types[name]; found {
+		return tdoc
+	}
+	// type wasn't found - add one without declaration
+	tdoc := &typeDoc{nil, nil, make(map[string]*ast.FuncDecl), make(map[string]*ast.FuncDecl)}
+	doc.types[name] = tdoc
+	return tdoc
+}
+
+func baseTypeName(typ ast.Expr) string {
+	switch t := typ.(type) {
+	case *ast.Ident:
+		// if the type is not exported, the effect to
+		// a client is as if there were no type name
+		if t.IsExported() {
+			return t.Name
+		}
+	case *ast.StarExpr:
+		return baseTypeName(t.X)
+	}
+	return ""
+}
+
+func (doc *docReader) addValue(decl *ast.GenDecl) {
+	// determine if decl should be associated with a type
+	// Heuristic: For each typed entry, determine the type name, if any.
+	//            If there is exactly one type name that is sufficiently
+	//            frequent, associate the decl with the respective type.
+	domName := ""
+	domFreq := 0
+	prev := ""
+	for _, s := range decl.Specs {
+		if v, ok := s.(*ast.ValueSpec); ok {
+			name := ""
+			switch {
+			case v.Type != nil:
+				// a type is present; determine its name
+				name = baseTypeName(v.Type)
+			case decl.Tok == token.CONST:
+				// no type is present but we have a constant declaration;
+				// use the previous type name (w/o more type information
+				// we cannot handle the case of unnamed variables with
+				// initializer expressions except for some trivial cases)
+				name = prev
+			}
+			if name != "" {
+				// entry has a named type
+				if domName != "" && domName != name {
+					// more than one type name - do not associate
+					// with any type
+					domName = ""
+					break
+				}
+				domName = name
+				domFreq++
+			}
+			prev = name
+		}
+	}
+
+	// determine values list
+	const threshold = 0.75
+	values := &doc.values
+	if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) {
+		// typed entries are sufficiently frequent
+		typ := doc.lookupTypeDoc(domName)
+		if typ != nil {
+			values = &typ.values // associate with that type
+		}
+	}
+
+	*values = append(*values, decl)
+}
+
+// Helper function to set the table entry for function f. Makes sure that
+// at least one f with associated documentation is stored in table, if there
+// are multiple f's with the same name.
+func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
+	name := f.Name.Name
+	if g, exists := table[name]; exists && g.Doc != nil {
+		// a function with the same name has already been registered;
+		// since it has documentation, assume f is simply another
+		// implementation and ignore it
+		// TODO(gri) consider collecting all functions, or at least
+		//           all comments
+		return
+	}
+	// function doesn't exist or has no documentation; use f
+	table[name] = f
+}
+
+func (doc *docReader) addFunc(fun *ast.FuncDecl) {
+	name := fun.Name.Name
+
+	// determine if it should be associated with a type
+	if fun.Recv != nil {
+		// method
+		typ := doc.lookupTypeDoc(baseTypeName(fun.Recv.List[0].Type))
+		if typ != nil {
+			// exported receiver type
+			setFunc(typ.methods, fun)
+		}
+		// otherwise don't show the method
+		// TODO(gri): There may be exported methods of non-exported types
+		// that can be called because of exported values (consts, vars, or
+		// function results) of that type. Could determine if that is the
+		// case and then show those methods in an appropriate section.
+		return
+	}
+
+	// perhaps a factory function
+	// determine result type, if any
+	if fun.Type.Results.NumFields() >= 1 {
+		res := fun.Type.Results.List[0]
+		if len(res.Names) <= 1 {
+			// exactly one (named or anonymous) result associated
+			// with the first type in result signature (there may
+			// be more than one result)
+			tname := baseTypeName(res.Type)
+			typ := doc.lookupTypeDoc(tname)
+			if typ != nil {
+				// named and exported result type
+
+				// Work-around for failure of heuristic: In package os
+				// too many functions are considered factory functions
+				// for the Error type. Eliminate manually for now as
+				// this appears to be the only important case in the
+				// current library where the heuristic fails.
+				if doc.pkgName == "os" && tname == "Error" &&
+					name != "NewError" && name != "NewSyscallError" {
+					// not a factory function for os.Error
+					setFunc(doc.funcs, fun) // treat as ordinary function
+					return
+				}
+
+				setFunc(typ.factories, fun)
+				return
+			}
+		}
+	}
+
+	// ordinary function
+	setFunc(doc.funcs, fun)
+}
+
+func (doc *docReader) addDecl(decl ast.Decl) {
+	switch d := decl.(type) {
+	case *ast.GenDecl:
+		if len(d.Specs) > 0 {
+			switch d.Tok {
+			case token.CONST, token.VAR:
+				// constants and variables are always handled as a group
+				doc.addValue(d)
+			case token.TYPE:
+				// types are handled individually
+				for _, spec := range d.Specs {
+					// make a (fake) GenDecl node for this TypeSpec
+					// (we need to do this here - as opposed to just
+					// for printing - so we don't lose the GenDecl
+					// documentation)
+					//
+					// TODO(gri): Consider just collecting the TypeSpec
+					// node (and copy in the GenDecl.doc if there is no
+					// doc in the TypeSpec - this is currently done in
+					// makeTypeDocs below). Simpler data structures, but
+					// would lose GenDecl documentation if the TypeSpec
+					// has documentation as well.
+					doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos})
+					// A new GenDecl node is created, no need to nil out d.Doc.
+				}
+			}
+		}
+	case *ast.FuncDecl:
+		doc.addFunc(d)
+	}
+}
+
+func copyCommentList(list []*ast.Comment) []*ast.Comment {
+	return append([]*ast.Comment(nil), list...)
+}
+
+var (
+	bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
+	bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
+)
+
+// addFile adds the AST for a source file to the docReader.
+// Adding the same AST multiple times is a no-op.
+//
+func (doc *docReader) addFile(src *ast.File) {
+	// add package documentation
+	if src.Doc != nil {
+		doc.addDoc(src.Doc)
+		src.Doc = nil // doc consumed - remove from ast.File node
+	}
+
+	// add all declarations
+	for _, decl := range src.Decls {
+		doc.addDecl(decl)
+	}
+
+	// collect BUG(...) comments
+	for _, c := range src.Comments {
+		text := c.List[0].Text
+		if m := bug_markers.FindStringIndex(text); m != nil {
+			// found a BUG comment; maybe empty
+			if btxt := text[m[1]:]; bug_content.MatchString(btxt) {
+				// non-empty BUG comment; collect comment without BUG prefix
+				list := copyCommentList(c.List)
+				list[0].Text = text[m[1]:]
+				doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
+			}
+		}
+	}
+	src.Comments = nil // consumed unassociated comments - remove from ast.File node
+}
+
+func NewFileDoc(file *ast.File) *PackageDoc {
+	var r docReader
+	r.init(file.Name.Name)
+	r.addFile(file)
+	return r.newDoc("", nil)
+}
+
+func NewPackageDoc(pkg *ast.Package, importpath string) *PackageDoc {
+	var r docReader
+	r.init(pkg.Name)
+	filenames := make([]string, len(pkg.Files))
+	i := 0
+	for filename, f := range pkg.Files {
+		r.addFile(f)
+		filenames[i] = filename
+		i++
+	}
+	return r.newDoc(importpath, filenames)
+}
+
+// ----------------------------------------------------------------------------
+// Conversion to external representation
+
+// ValueDoc is the documentation for a group of declared
+// values, either vars or consts.
+//
+type ValueDoc struct {
+	Doc   string
+	Decl  *ast.GenDecl
+	order int
+}
+
+type sortValueDoc []*ValueDoc
+
+func (p sortValueDoc) Len() int      { return len(p) }
+func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func declName(d *ast.GenDecl) string {
+	if len(d.Specs) != 1 {
+		return ""
+	}
+
+	switch v := d.Specs[0].(type) {
+	case *ast.ValueSpec:
+		return v.Names[0].Name
+	case *ast.TypeSpec:
+		return v.Name.Name
+	}
+
+	return ""
+}
+
+func (p sortValueDoc) Less(i, j int) bool {
+	// sort by name
+	// pull blocks (name = "") up to top
+	// in original order
+	if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
+		return ni < nj
+	}
+	return p[i].order < p[j].order
+}
+
+func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
+	d := make([]*ValueDoc, len(list)) // big enough in any case
+	n := 0
+	for i, decl := range list {
+		if decl.Tok == tok {
+			d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
+			n++
+			decl.Doc = nil // doc consumed - removed from AST
+		}
+	}
+	d = d[0:n]
+	sort.Sort(sortValueDoc(d))
+	return d
+}
+
+// FuncDoc is the documentation for a func declaration,
+// either a top-level function or a method function.
+//
+type FuncDoc struct {
+	Doc  string
+	Recv ast.Expr // TODO(rsc): Would like string here
+	Name string
+	Decl *ast.FuncDecl
+}
+
+type sortFuncDoc []*FuncDoc
+
+func (p sortFuncDoc) Len() int           { return len(p) }
+func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
+func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
+	d := make([]*FuncDoc, len(m))
+	i := 0
+	for _, f := range m {
+		doc := new(FuncDoc)
+		doc.Doc = CommentText(f.Doc)
+		f.Doc = nil // doc consumed - remove from ast.FuncDecl node
+		if f.Recv != nil {
+			doc.Recv = f.Recv.List[0].Type
+		}
+		doc.Name = f.Name.Name
+		doc.Decl = f
+		d[i] = doc
+		i++
+	}
+	sort.Sort(sortFuncDoc(d))
+	return d
+}
+
+// TypeDoc is the documentation for a declared type.
+// Consts and Vars are sorted lists of constants and variables of (mostly) that type.
+// Factories is a sorted list of factory functions that return that type.
+// Methods is a sorted list of method functions on that type.
+type TypeDoc struct {
+	Doc       string
+	Type      *ast.TypeSpec
+	Consts    []*ValueDoc
+	Vars      []*ValueDoc
+	Factories []*FuncDoc
+	Methods   []*FuncDoc
+	Decl      *ast.GenDecl
+	order     int
+}
+
+type sortTypeDoc []*TypeDoc
+
+func (p sortTypeDoc) Len() int      { return len(p) }
+func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p sortTypeDoc) Less(i, j int) bool {
+	// sort by name
+	// pull blocks (name = "") up to top
+	// in original order
+	if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
+		return ni < nj
+	}
+	return p[i].order < p[j].order
+}
+
+// NOTE(rsc): This would appear not to be correct for type ( )
+// blocks, but the doc extractor above has split them into
+// individual declarations.
+func (doc *docReader) makeTypeDocs(m map[string]*typeDoc) []*TypeDoc {
+	d := make([]*TypeDoc, len(m))
+	i := 0
+	for _, old := range m {
+		// all typeDocs should have a declaration associated with
+		// them after processing an entire package - be conservative
+		// and check
+		if decl := old.decl; decl != nil {
+			typespec := decl.Specs[0].(*ast.TypeSpec)
+			t := new(TypeDoc)
+			doc := typespec.Doc
+			typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
+			if doc == nil {
+				// no doc associated with the spec, use the declaration doc, if any
+				doc = decl.Doc
+			}
+			decl.Doc = nil // doc consumed - remove from ast.Decl node
+			t.Doc = CommentText(doc)
+			t.Type = typespec
+			t.Consts = makeValueDocs(old.values, token.CONST)
+			t.Vars = makeValueDocs(old.values, token.VAR)
+			t.Factories = makeFuncDocs(old.factories)
+			t.Methods = makeFuncDocs(old.methods)
+			t.Decl = old.decl
+			t.order = i
+			d[i] = t
+			i++
+		} else {
+			// no corresponding type declaration found - move any associated
+			// values, factory functions, and methods back to the top-level
+			// so that they are not lost (this should only happen if a package
+			// file containing the explicit type declaration is missing or if
+			// an unqualified type name was used after a "." import)
+			// 1) move values
+			doc.values = append(doc.values, old.values...)
+			// 2) move factory functions
+			for name, f := range old.factories {
+				doc.funcs[name] = f
+			}
+			// 3) move methods
+			for name, f := range old.methods {
+				// don't overwrite functions with the same name
+				if _, found := doc.funcs[name]; !found {
+					doc.funcs[name] = f
+				}
+			}
+		}
+	}
+	d = d[0:i] // some types may have been ignored
+	sort.Sort(sortTypeDoc(d))
+	return d
+}
+
+func makeBugDocs(list []*ast.CommentGroup) []string {
+	d := make([]string, len(list))
+	for i, g := range list {
+		d[i] = CommentText(g)
+	}
+	return d
+}
+
+// PackageDoc is the documentation for an entire package.
+//
+type PackageDoc struct {
+	PackageName string
+	ImportPath  string
+	Filenames   []string
+	Doc         string
+	Consts      []*ValueDoc
+	Types       []*TypeDoc
+	Vars        []*ValueDoc
+	Funcs       []*FuncDoc
+	Bugs        []string
+}
+
+// newDoc returns the accumulated documentation for the package.
+//
+func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
+	p := new(PackageDoc)
+	p.PackageName = doc.pkgName
+	p.ImportPath = importpath
+	sort.Strings(filenames)
+	p.Filenames = filenames
+	p.Doc = CommentText(doc.doc)
+	// makeTypeDocs may extend the list of doc.values and
+	// doc.funcs and thus must be called before any other
+	// function consuming those lists
+	p.Types = doc.makeTypeDocs(doc.types)
+	p.Consts = makeValueDocs(doc.values, token.CONST)
+	p.Vars = makeValueDocs(doc.values, token.VAR)
+	p.Funcs = makeFuncDocs(doc.funcs)
+	p.Bugs = makeBugDocs(doc.bugs)
+	return p
+}
+
+// ----------------------------------------------------------------------------
+// Filtering by name
+
+type Filter func(string) bool
+
+func matchFields(fields *ast.FieldList, f Filter) bool {
+	if fields != nil {
+		for _, field := range fields.List {
+			for _, name := range field.Names {
+				if f(name.Name) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func matchDecl(d *ast.GenDecl, f Filter) bool {
+	for _, d := range d.Specs {
+		switch v := d.(type) {
+		case *ast.ValueSpec:
+			for _, name := range v.Names {
+				if f(name.Name) {
+					return true
+				}
+			}
+		case *ast.TypeSpec:
+			if f(v.Name.Name) {
+				return true
+			}
+			switch t := v.Type.(type) {
+			case *ast.StructType:
+				if matchFields(t.Fields, f) {
+					return true
+				}
+			case *ast.InterfaceType:
+				if matchFields(t.Methods, f) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func filterValueDocs(a []*ValueDoc, f Filter) []*ValueDoc {
+	w := 0
+	for _, vd := range a {
+		if matchDecl(vd.Decl, f) {
+			a[w] = vd
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+func filterFuncDocs(a []*FuncDoc, f Filter) []*FuncDoc {
+	w := 0
+	for _, fd := range a {
+		if f(fd.Name) {
+			a[w] = fd
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+func filterTypeDocs(a []*TypeDoc, f Filter) []*TypeDoc {
+	w := 0
+	for _, td := range a {
+		n := 0 // number of matches
+		if matchDecl(td.Decl, f) {
+			n = 1
+		} else {
+			// type name doesn't match, but we may have matching consts, vars, factories or methods
+			td.Consts = filterValueDocs(td.Consts, f)
+			td.Vars = filterValueDocs(td.Vars, f)
+			td.Factories = filterFuncDocs(td.Factories, f)
+			td.Methods = filterFuncDocs(td.Methods, f)
+			n += len(td.Consts) + len(td.Vars) + len(td.Factories) + len(td.Methods)
+		}
+		if n > 0 {
+			a[w] = td
+			w++
+		}
+	}
+	return a[0:w]
+}
+
+// Filter eliminates documentation for names that don't pass through the filter f.
+// TODO: Recognize "Type.Method" as a name.
+//
+func (p *PackageDoc) Filter(f Filter) {
+	p.Consts = filterValueDocs(p.Consts, f)
+	p.Vars = filterValueDocs(p.Vars, f)
+	p.Types = filterTypeDocs(p.Types, f)
+	p.Funcs = filterFuncDocs(p.Funcs, f)
+	p.Doc = "" // don't show top-level package doc
+}
diff --git a/src/pkg/go/parser/Makefile b/src/pkg/go/parser/Makefile
new file mode 100644
index 000000000..d301f41eb
--- /dev/null
+++ b/src/pkg/go/parser/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/parser
+GOFILES=\
+	interface.go\
+	parser.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/parser/interface.go b/src/pkg/go/parser/interface.go
new file mode 100644
index 000000000..4f980fc65
--- /dev/null
+++ b/src/pkg/go/parser/interface.go
@@ -0,0 +1,214 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains the exported entry points for invoking the parser.
+
+package parser
+
+import (
+	"bytes"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// If src != nil, readSource converts src to a []byte if possible;
+// otherwise it returns an error. If src == nil, readSource returns
+// the result of reading the file specified by filename.
+//
+func readSource(filename string, src interface{}) ([]byte, os.Error) {
+	if src != nil {
+		switch s := src.(type) {
+		case string:
+			return []byte(s), nil
+		case []byte:
+			return s, nil
+		case *bytes.Buffer:
+			// is io.Reader, but src is already available in []byte form
+			if s != nil {
+				return s.Bytes(), nil
+			}
+		case io.Reader:
+			var buf bytes.Buffer
+			_, err := io.Copy(&buf, s)
+			if err != nil {
+				return nil, err
+			}
+			return buf.Bytes(), nil
+		default:
+			return nil, os.NewError("invalid source")
+		}
+	}
+
+	return ioutil.ReadFile(filename)
+}
+
+func (p *parser) errors() os.Error {
+	mode := scanner.Sorted
+	if p.mode&SpuriousErrors == 0 {
+		mode = scanner.NoMultiples
+	}
+	return p.GetError(mode)
+}
+
+// ParseExpr parses a Go expression and returns the corresponding
+// AST node. The fset, filename, and src arguments have the same interpretation
+// as for ParseFile. If there is an error, the result expression
+// may be nil or contain a partial AST.
+//
+func ParseExpr(fset *token.FileSet, filename string, src interface{}) (ast.Expr, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	x := p.parseRhs()
+	if p.tok == token.SEMICOLON {
+		p.next() // consume automatically inserted semicolon, if any
+	}
+	p.expect(token.EOF)
+
+	return x, p.errors()
+}
+
+// ParseStmtList parses a list of Go statements and returns the list
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
+// interpretation as for ParseFile. If there is an error, the node
+// list may be nil or contain partial ASTs.
+//
+func ParseStmtList(fset *token.FileSet, filename string, src interface{}) ([]ast.Stmt, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	list := p.parseStmtList()
+	p.expect(token.EOF)
+
+	return list, p.errors()
+}
+
+// ParseDeclList parses a list of Go declarations and returns the list
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
+// interpretation as for ParseFile. If there is an error, the node
+// list may be nil or contain partial ASTs.
+//
+func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast.Decl, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, 0)
+	list := p.parseDeclList()
+	p.expect(token.EOF)
+
+	return list, p.errors()
+}
+
+// ParseFile parses the source code of a single Go source file and returns
+// the corresponding ast.File node. The source code may be provided via
+// the filename of the source file, or via the src parameter.
+//
+// If src != nil, ParseFile parses the source from src and the filename is
+// only used when recording position information. The type of the argument
+// for the src parameter must be string, []byte, or io.Reader.
+//
+// If src == nil, ParseFile parses the file specified by filename.
+//
+// The mode parameter controls the amount of source text parsed and other
+// optional parser functionality. Position information is recorded in the
+// file set fset.
+//
+// If the source couldn't be read, the returned AST is nil and the error
+// indicates the specific failure. If the source was read but syntax
+// errors were found, the result is a partial AST (with ast.BadX nodes
+// representing the fragments of erroneous source code). Multiple errors
+// are returned via a scanner.ErrorList which is sorted by file position.
+//
+func ParseFile(fset *token.FileSet, filename string, src interface{}, mode uint) (*ast.File, os.Error) {
+	data, err := readSource(filename, src)
+	if err != nil {
+		return nil, err
+	}
+
+	var p parser
+	p.init(fset, filename, data, mode)
+	file := p.parseFile() // parseFile reads to EOF
+
+	return file, p.errors()
+}
+
+// ParseFiles calls ParseFile for each file in the filenames list and returns
+// a map of package name -> package AST with all the packages found. The mode
+// bits are passed to ParseFile unchanged. Position information is recorded
+// in the file set fset.
+//
+// Files with parse errors are ignored. In this case the map of packages may
+// be incomplete (missing packages and/or incomplete packages) and the first
+// error encountered is returned.
+//
+func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
+	pkgs = make(map[string]*ast.Package)
+	for _, filename := range filenames {
+		if src, err := ParseFile(fset, filename, nil, mode); err == nil {
+			name := src.Name.Name
+			pkg, found := pkgs[name]
+			if !found {
+				// TODO(gri) Use NewPackage here; reconsider ParseFiles API.
+				pkg = &ast.Package{name, nil, nil, make(map[string]*ast.File)}
+				pkgs[name] = pkg
+			}
+			pkg.Files[filename] = src
+		} else if first == nil {
+			first = err
+		}
+	}
+	return
+}
+
+// ParseDir calls ParseFile for the files in the directory specified by path and
+// returns a map of package name -> package AST with all the packages found. If
+// filter != nil, only the files with os.FileInfo entries passing through the filter
+// are considered. The mode bits are passed to ParseFile unchanged. Position
+// information is recorded in the file set fset.
+//
+// If the directory couldn't be read, a nil map and the respective error are
+// returned. If a parse error occurred, a non-nil but incomplete map and the
+// error are returned.
+//
+func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
+	fd, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	list, err := fd.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	filenames := make([]string, len(list))
+	n := 0
+	for i := 0; i < len(list); i++ {
+		d := &list[i]
+		if filter == nil || filter(d) {
+			filenames[n] = filepath.Join(path, d.Name)
+			n++
+		}
+	}
+	filenames = filenames[0:n]
+
+	return ParseFiles(fset, filenames, mode)
+}
diff --git a/src/pkg/go/parser/parser.go b/src/pkg/go/parser/parser.go
new file mode 100644
index 000000000..9c14d1667
--- /dev/null
+++ b/src/pkg/go/parser/parser.go
@@ -0,0 +1,2161 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package parser implements a parser for Go source files. Input may be
+// provided in a variety of forms (see the various Parse* functions); the
+// output is an abstract syntax tree (AST) representing the Go source. The
+// parser is invoked through one of the Parse* functions.
+//
+package parser
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+)
+
+// The mode parameter to the Parse* functions is a set of flags (or 0).
+// They control the amount of source code parsed and other optional
+// parser functionality.
+//
+const (
+	PackageClauseOnly uint = 1 << iota // parsing stops after package clause
+	ImportsOnly                        // parsing stops after import declarations
+	ParseComments                      // parse comments and add them to AST
+	Trace                              // print a trace of parsed productions
+	DeclarationErrors                  // report declaration errors
+	SpuriousErrors                     // report all (not just the first) errors per line
+)
+
+// The parser structure holds the parser's internal state.
+type parser struct {
+	file *token.File
+	scanner.ErrorVector
+	scanner scanner.Scanner
+
+	// Tracing/debugging
+	mode   uint // parsing mode
+	trace  bool // == (mode & Trace != 0)
+	indent uint // indentation used for tracing output
+
+	// Comments
+	comments    []*ast.CommentGroup
+	leadComment *ast.CommentGroup // last lead comment
+	lineComment *ast.CommentGroup // last line comment
+
+	// Next token
+	pos token.Pos   // token position
+	tok token.Token // one token look-ahead
+	lit string      // token literal
+
+	// Non-syntactic parser control
+	exprLev int // < 0: in control clause, >= 0: in expression
+
+	// Ordinary identifier scopes
+	pkgScope   *ast.Scope        // pkgScope.Outer == nil
+	topScope   *ast.Scope        // top-most scope; may be pkgScope
+	unresolved []*ast.Ident      // unresolved identifiers
+	imports    []*ast.ImportSpec // list of imports
+
+	// Label scope
+	// (maintained by open/close LabelScope)
+	labelScope  *ast.Scope     // label scope for current function
+	targetStack [][]*ast.Ident // stack of unresolved labels
+}
+
+// scannerMode returns the scanner mode bits given the parser's mode bits.
+func scannerMode(mode uint) uint {
+	var m uint = scanner.InsertSemis
+	if mode&ParseComments != 0 {
+		m |= scanner.ScanComments
+	}
+	return m
+}
+
+func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
+	p.file = fset.AddFile(filename, fset.Base(), len(src))
+	p.scanner.Init(p.file, src, p, scannerMode(mode))
+
+	p.mode = mode
+	p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
+
+	p.next()
+
+	// set up the pkgScope here (as opposed to in parseFile) because
+	// there are other parser entry points (ParseExpr, etc.)
+	p.openScope()
+	p.pkgScope = p.topScope
+
+	// for the same reason, set up a label scope
+	p.openLabelScope()
+}
+
+// ----------------------------------------------------------------------------
+// Scoping support
+
+func (p *parser) openScope() {
+	p.topScope = ast.NewScope(p.topScope)
+}
+
+func (p *parser) closeScope() {
+	p.topScope = p.topScope.Outer
+}
+
+func (p *parser) openLabelScope() {
+	p.labelScope = ast.NewScope(p.labelScope)
+	p.targetStack = append(p.targetStack, nil)
+}
+
+func (p *parser) closeLabelScope() {
+	// resolve labels
+	n := len(p.targetStack) - 1
+	scope := p.labelScope
+	for _, ident := range p.targetStack[n] {
+		ident.Obj = scope.Lookup(ident.Name)
+		if ident.Obj == nil && p.mode&DeclarationErrors != 0 {
+			p.error(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name))
+		}
+	}
+	// pop label scope
+	p.targetStack = p.targetStack[0:n]
+	p.labelScope = p.labelScope.Outer
+}
+
+func (p *parser) declare(decl, data interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) {
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		obj := ast.NewObj(kind, ident.Name)
+		// remember the corresponding declaration for redeclaration
+		// errors and global variable resolution/typechecking phase
+		obj.Decl = decl
+		obj.Data = data
+		ident.Obj = obj
+		if ident.Name != "_" {
+			if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 {
+				prevDecl := ""
+				if pos := alt.Pos(); pos.IsValid() {
+					prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.file.Position(pos))
+				}
+				p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl))
+			}
+		}
+	}
+}
+
+func (p *parser) shortVarDecl(idents []*ast.Ident) {
+	// Go spec: A short variable declaration may redeclare variables
+	// provided they were originally declared in the same block with
+	// the same type, and at least one of the non-blank variables is new.
+	n := 0 // number of new variables
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		obj := ast.NewObj(ast.Var, ident.Name)
+		// short var declarations cannot have redeclaration errors
+		// and are not global => no need to remember the respective
+		// declaration
+		ident.Obj = obj
+		if ident.Name != "_" {
+			if alt := p.topScope.Insert(obj); alt != nil {
+				ident.Obj = alt // redeclaration
+			} else {
+				n++ // new declaration
+			}
+		}
+	}
+	if n == 0 && p.mode&DeclarationErrors != 0 {
+		p.error(idents[0].Pos(), "no new variables on left side of :=")
+	}
+}
+
+// The unresolved object is a sentinel to mark identifiers that have been added
+// to the list of unresolved identifiers. The sentinel is only used for verifying
+// internal consistency.
+var unresolved = new(ast.Object)
+
+func (p *parser) resolve(x ast.Expr) {
+	// nothing to do if x is not an identifier or the blank identifier
+	ident, _ := x.(*ast.Ident)
+	if ident == nil {
+		return
+	}
+	assert(ident.Obj == nil, "identifier already declared or resolved")
+	if ident.Name == "_" {
+		return
+	}
+	// try to resolve the identifier
+	for s := p.topScope; s != nil; s = s.Outer {
+		if obj := s.Lookup(ident.Name); obj != nil {
+			ident.Obj = obj
+			return
+		}
+	}
+	// all local scopes are known, so any unresolved identifier
+	// must be found either in the file scope, package scope
+	// (perhaps in another file), or universe scope --- collect
+	// them so that they can be resolved later
+	ident.Obj = unresolved
+	p.unresolved = append(p.unresolved, ident)
+}
+
+// ----------------------------------------------------------------------------
+// Parsing support
+
+func (p *parser) printTrace(a ...interface{}) {
+	const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
+		". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
+	const n = uint(len(dots))
+	pos := p.file.Position(p.pos)
+	fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
+	i := 2 * p.indent
+	for ; i > n; i -= n {
+		fmt.Print(dots)
+	}
+	fmt.Print(dots[0:i])
+	fmt.Println(a...)
+}
+
+func trace(p *parser, msg string) *parser {
+	p.printTrace(msg, "(")
+	p.indent++
+	return p
+}
+
+// Usage pattern: defer un(trace(p, "..."));
+func un(p *parser) {
+	p.indent--
+	p.printTrace(")")
+}
+
+// Advance to the next token.
+func (p *parser) next0() {
+	// Because of one-token look-ahead, print the previous token
+	// when tracing as it provides a more readable output. The
+	// very first token (!p.pos.IsValid()) is not initialized
+	// (it is token.ILLEGAL), so don't print it .
+	if p.trace && p.pos.IsValid() {
+		s := p.tok.String()
+		switch {
+		case p.tok.IsLiteral():
+			p.printTrace(s, p.lit)
+		case p.tok.IsOperator(), p.tok.IsKeyword():
+			p.printTrace("\"" + s + "\"")
+		default:
+			p.printTrace(s)
+		}
+	}
+
+	p.pos, p.tok, p.lit = p.scanner.Scan()
+}
+
+// Consume a comment and return it and the line on which it ends.
+func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
+	// /*-style comments may end on a different line than where they start.
+	// Scan the comment for '\n' chars and adjust endline accordingly.
+	endline = p.file.Line(p.pos)
+	if p.lit[1] == '*' {
+		// don't use range here - no need to decode Unicode code points
+		for i := 0; i < len(p.lit); i++ {
+			if p.lit[i] == '\n' {
+				endline++
+			}
+		}
+	}
+
+	comment = &ast.Comment{p.pos, p.lit}
+	p.next0()
+
+	return
+}
+
+// Consume a group of adjacent comments, add it to the parser's
+// comments list, and return it together with the line at which
+// the last comment in the group ends. An empty line or non-comment
+// token terminates a comment group.
+//
+func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
+	var list []*ast.Comment
+	endline = p.file.Line(p.pos)
+	for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
+		var comment *ast.Comment
+		comment, endline = p.consumeComment()
+		list = append(list, comment)
+	}
+
+	// add comment group to the comments list
+	comments = &ast.CommentGroup{list}
+	p.comments = append(p.comments, comments)
+
+	return
+}
+
+// Advance to the next non-comment token. In the process, collect
+// any comment groups encountered, and remember the last lead and
+// and line comments.
+//
+// A lead comment is a comment group that starts and ends in a
+// line without any other tokens and that is followed by a non-comment
+// token on the line immediately after the comment group.
+//
+// A line comment is a comment group that follows a non-comment
+// token on the same line, and that has no tokens after it on the line
+// where it ends.
+//
+// Lead and line comments may be considered documentation that is
+// stored in the AST.
+//
+func (p *parser) next() {
+	p.leadComment = nil
+	p.lineComment = nil
+	line := p.file.Line(p.pos) // current line
+	p.next0()
+
+	if p.tok == token.COMMENT {
+		var comment *ast.CommentGroup
+		var endline int
+
+		if p.file.Line(p.pos) == line {
+			// The comment is on same line as the previous token; it
+			// cannot be a lead comment but may be a line comment.
+			comment, endline = p.consumeCommentGroup()
+			if p.file.Line(p.pos) != endline {
+				// The next token is on a different line, thus
+				// the last comment group is a line comment.
+				p.lineComment = comment
+			}
+		}
+
+		// consume successor comments, if any
+		endline = -1
+		for p.tok == token.COMMENT {
+			comment, endline = p.consumeCommentGroup()
+		}
+
+		if endline+1 == p.file.Line(p.pos) {
+			// The next token is following on the line immediately after the
+			// comment group, thus the last comment group is a lead comment.
+			p.leadComment = comment
+		}
+	}
+}
+
+func (p *parser) error(pos token.Pos, msg string) {
+	p.Error(p.file.Position(pos), msg)
+}
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
+	msg = "expected " + msg
+	if pos == p.pos {
+		// the error happened at the current position;
+		// make the error message more specific
+		if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
+			msg += ", found newline"
+		} else {
+			msg += ", found '" + p.tok.String() + "'"
+			if p.tok.IsLiteral() {
+				msg += " " + p.lit
+			}
+		}
+	}
+	p.error(pos, msg)
+}
+
+func (p *parser) expect(tok token.Token) token.Pos {
+	pos := p.pos
+	if p.tok != tok {
+		p.errorExpected(pos, "'"+tok.String()+"'")
+	}
+	p.next() // make progress
+	return pos
+}
+
+func (p *parser) expectSemi() {
+	if p.tok != token.RPAREN && p.tok != token.RBRACE {
+		p.expect(token.SEMICOLON)
+	}
+}
+
+func assert(cond bool, msg string) {
+	if !cond {
+		panic("go/parser internal error: " + msg)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Identifiers
+
+func (p *parser) parseIdent() *ast.Ident {
+	pos := p.pos
+	name := "_"
+	if p.tok == token.IDENT {
+		name = p.lit
+		p.next()
+	} else {
+		p.expect(token.IDENT) // use expect() error handling
+	}
+	return &ast.Ident{pos, name, nil}
+}
+
+func (p *parser) parseIdentList() (list []*ast.Ident) {
+	if p.trace {
+		defer un(trace(p, "IdentList"))
+	}
+
+	list = append(list, p.parseIdent())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseIdent())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Common productions
+
+// If lhs is set, result list elements which are identifiers are not resolved.
+func (p *parser) parseExprList(lhs bool) (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ExpressionList"))
+	}
+
+	list = append(list, p.checkExpr(p.parseExpr(lhs)))
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.checkExpr(p.parseExpr(lhs)))
+	}
+
+	return
+}
+
+func (p *parser) parseLhsList() []ast.Expr {
+	list := p.parseExprList(true)
+	switch p.tok {
+	case token.DEFINE:
+		// lhs of a short variable declaration
+		p.shortVarDecl(p.makeIdentList(list))
+	case token.COLON:
+		// lhs of a label declaration or a communication clause of a select
+		// statement (parseLhsList is not called when parsing the case clause
+		// of a switch statement):
+		// - labels are declared by the caller of parseLhsList
+		// - for communication clauses, if there is a stand-alone identifier
+		//   followed by a colon, we have a syntax error; there is no need
+		//   to resolve the identifier in that case
+	default:
+		// identifiers must be declared elsewhere
+		for _, x := range list {
+			p.resolve(x)
+		}
+	}
+	return list
+}
+
+func (p *parser) parseRhsList() []ast.Expr {
+	return p.parseExprList(false)
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+func (p *parser) parseType() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Type"))
+	}
+
+	typ := p.tryType()
+
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		return &ast.BadExpr{pos, p.pos}
+	}
+
+	return typ
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) parseTypeName() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeName"))
+	}
+
+	ident := p.parseIdent()
+	// don't resolve ident yet - it may be a parameter or field name
+
+	if p.tok == token.PERIOD {
+		// ident is a package name
+		p.next()
+		p.resolve(ident)
+		sel := p.parseIdent()
+		return &ast.SelectorExpr{ident, sel}
+	}
+
+	return ident
+}
+
+func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "ArrayType"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	var len ast.Expr
+	if ellipsisOk && p.tok == token.ELLIPSIS {
+		len = &ast.Ellipsis{p.pos, nil}
+		p.next()
+	} else if p.tok != token.RBRACK {
+		len = p.parseRhs()
+	}
+	p.expect(token.RBRACK)
+	elt := p.parseType()
+
+	return &ast.ArrayType{lbrack, len, elt}
+}
+
+func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident {
+	idents := make([]*ast.Ident, len(list))
+	for i, x := range list {
+		ident, isIdent := x.(*ast.Ident)
+		if !isIdent {
+			pos := x.(ast.Expr).Pos()
+			p.errorExpected(pos, "identifier")
+			ident = &ast.Ident{pos, "_", nil}
+		}
+		idents[i] = ident
+	}
+	return idents
+}
+
+func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "FieldDecl"))
+	}
+
+	doc := p.leadComment
+
+	// fields
+	list, typ := p.parseVarList(false)
+
+	// optional tag
+	var tag *ast.BasicLit
+	if p.tok == token.STRING {
+		tag = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	}
+
+	// analyze case
+	var idents []*ast.Ident
+	if typ != nil {
+		// IdentifierList Type
+		idents = p.makeIdentList(list)
+	} else {
+		// ["*"] TypeName (AnonymousField)
+		typ = list[0] // we always have at least one element
+		p.resolve(typ)
+		if n := len(list); n > 1 || !isTypeName(deref(typ)) {
+			pos := typ.Pos()
+			p.errorExpected(pos, "anonymous field")
+			typ = &ast.BadExpr{pos, list[n-1].End()}
+		}
+	}
+
+	p.expectSemi() // call before accessing p.linecomment
+
+	field := &ast.Field{doc, idents, typ, tag, p.lineComment}
+	p.declare(field, nil, scope, ast.Var, idents...)
+
+	return field
+}
+
+func (p *parser) parseStructType() *ast.StructType {
+	if p.trace {
+		defer un(trace(p, "StructType"))
+	}
+
+	pos := p.expect(token.STRUCT)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // struct scope
+	var list []*ast.Field
+	for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN {
+		// a field declaration cannot start with a '(' but we accept
+		// it here for more robust parsing and better error messages
+		// (parseFieldDecl will check and complain if necessary)
+		list = append(list, p.parseFieldDecl(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store struct scope in AST
+	return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parsePointerType() *ast.StarExpr {
+	if p.trace {
+		defer un(trace(p, "PointerType"))
+	}
+
+	star := p.expect(token.MUL)
+	base := p.parseType()
+
+	return &ast.StarExpr{star, base}
+}
+
+func (p *parser) tryVarType(isParam bool) ast.Expr {
+	if isParam && p.tok == token.ELLIPSIS {
+		pos := p.pos
+		p.next()
+		typ := p.tryIdentOrType(isParam) // don't use parseType so we can provide better error message
+		if typ == nil {
+			p.error(pos, "'...' parameter is missing type")
+			typ = &ast.BadExpr{pos, p.pos}
+		}
+		if p.tok != token.RPAREN {
+			p.error(pos, "can use '...' with last parameter type only")
+		}
+		return &ast.Ellipsis{pos, typ}
+	}
+	return p.tryIdentOrType(false)
+}
+
+func (p *parser) parseVarType(isParam bool) ast.Expr {
+	typ := p.tryVarType(isParam)
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		typ = &ast.BadExpr{pos, p.pos}
+	}
+	return typ
+}
+
+func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "VarList"))
+	}
+
+	// a list of identifiers looks like a list of type names
+	for {
+		// parseVarType accepts any type (including parenthesized ones)
+		// even though the syntax does not permit them here: we
+		// accept them all for more robust parsing and complain
+		// afterwards
+		list = append(list, p.parseVarType(isParam))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	// if we had a list of identifiers, it must be followed by a type
+	typ = p.tryVarType(isParam)
+	if typ != nil {
+		p.resolve(typ)
+	}
+
+	return
+}
+
+func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params []*ast.Field) {
+	if p.trace {
+		defer un(trace(p, "ParameterList"))
+	}
+
+	list, typ := p.parseVarList(ellipsisOk)
+	if typ != nil {
+		// IdentifierList Type
+		idents := p.makeIdentList(list)
+		field := &ast.Field{nil, idents, typ, nil, nil}
+		params = append(params, field)
+		// Go spec: The scope of an identifier denoting a function
+		// parameter or result variable is the function body.
+		p.declare(field, nil, scope, ast.Var, idents...)
+		if p.tok == token.COMMA {
+			p.next()
+		}
+
+		for p.tok != token.RPAREN && p.tok != token.EOF {
+			idents := p.parseIdentList()
+			typ := p.parseVarType(ellipsisOk)
+			field := &ast.Field{nil, idents, typ, nil, nil}
+			params = append(params, field)
+			// Go spec: The scope of an identifier denoting a function
+			// parameter or result variable is the function body.
+			p.declare(field, nil, scope, ast.Var, idents...)
+			if p.tok != token.COMMA {
+				break
+			}
+			p.next()
+		}
+
+	} else {
+		// Type { "," Type } (anonymous parameters)
+		params = make([]*ast.Field, len(list))
+		for i, x := range list {
+			p.resolve(x)
+			params[i] = &ast.Field{Type: x}
+		}
+	}
+
+	return
+}
+
+func (p *parser) parseParameters(scope *ast.Scope, ellipsisOk bool) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Parameters"))
+	}
+
+	var params []*ast.Field
+	lparen := p.expect(token.LPAREN)
+	if p.tok != token.RPAREN {
+		params = p.parseParameterList(scope, ellipsisOk)
+	}
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.FieldList{lparen, params, rparen}
+}
+
+func (p *parser) parseResult(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Result"))
+	}
+
+	if p.tok == token.LPAREN {
+		return p.parseParameters(scope, false)
+	}
+
+	typ := p.tryType()
+	if typ != nil {
+		list := make([]*ast.Field, 1)
+		list[0] = &ast.Field{Type: typ}
+		return &ast.FieldList{List: list}
+	}
+
+	return nil
+}
+
+func (p *parser) parseSignature(scope *ast.Scope) (params, results *ast.FieldList) {
+	if p.trace {
+		defer un(trace(p, "Signature"))
+	}
+
+	params = p.parseParameters(scope, true)
+	results = p.parseResult(scope)
+
+	return
+}
+
+func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) {
+	if p.trace {
+		defer un(trace(p, "FuncType"))
+	}
+
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+	params, results := p.parseSignature(scope)
+
+	return &ast.FuncType{pos, params, results}, scope
+}
+
+func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "MethodSpec"))
+	}
+
+	doc := p.leadComment
+	var idents []*ast.Ident
+	var typ ast.Expr
+	x := p.parseTypeName()
+	if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN {
+		// method
+		idents = []*ast.Ident{ident}
+		scope := ast.NewScope(nil) // method scope
+		params, results := p.parseSignature(scope)
+		typ = &ast.FuncType{token.NoPos, params, results}
+	} else {
+		// embedded interface
+		typ = x
+		p.resolve(typ)
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	spec := &ast.Field{doc, idents, typ, nil, p.lineComment}
+	p.declare(spec, nil, scope, ast.Fun, idents...)
+
+	return spec
+}
+
+func (p *parser) parseInterfaceType() *ast.InterfaceType {
+	if p.trace {
+		defer un(trace(p, "InterfaceType"))
+	}
+
+	pos := p.expect(token.INTERFACE)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // interface scope
+	var list []*ast.Field
+	for p.tok == token.IDENT {
+		list = append(list, p.parseMethodSpec(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store interface scope in AST
+	return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parseMapType() *ast.MapType {
+	if p.trace {
+		defer un(trace(p, "MapType"))
+	}
+
+	pos := p.expect(token.MAP)
+	p.expect(token.LBRACK)
+	key := p.parseType()
+	p.expect(token.RBRACK)
+	value := p.parseType()
+
+	return &ast.MapType{pos, key, value}
+}
+
+func (p *parser) parseChanType() *ast.ChanType {
+	if p.trace {
+		defer un(trace(p, "ChanType"))
+	}
+
+	pos := p.pos
+	dir := ast.SEND | ast.RECV
+	if p.tok == token.CHAN {
+		p.next()
+		if p.tok == token.ARROW {
+			p.next()
+			dir = ast.SEND
+		}
+	} else {
+		p.expect(token.ARROW)
+		p.expect(token.CHAN)
+		dir = ast.RECV
+	}
+	value := p.parseType()
+
+	return &ast.ChanType{pos, dir, value}
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) tryIdentOrType(ellipsisOk bool) ast.Expr {
+	switch p.tok {
+	case token.IDENT:
+		return p.parseTypeName()
+	case token.LBRACK:
+		return p.parseArrayType(ellipsisOk)
+	case token.STRUCT:
+		return p.parseStructType()
+	case token.MUL:
+		return p.parsePointerType()
+	case token.FUNC:
+		typ, _ := p.parseFuncType()
+		return typ
+	case token.INTERFACE:
+		return p.parseInterfaceType()
+	case token.MAP:
+		return p.parseMapType()
+	case token.CHAN, token.ARROW:
+		return p.parseChanType()
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		typ := p.parseType()
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, typ, rparen}
+	}
+
+	// no type found
+	return nil
+}
+
+func (p *parser) tryType() ast.Expr {
+	typ := p.tryIdentOrType(false)
+	if typ != nil {
+		p.resolve(typ)
+	}
+	return typ
+}
+
+// ----------------------------------------------------------------------------
+// Blocks
+
+func (p *parser) parseStmtList() (list []ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "StatementList"))
+	}
+
+	for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseStmt())
+	}
+
+	return
+}
+
+func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "Body"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.topScope = scope // open function scope
+	p.openLabelScope()
+	list := p.parseStmtList()
+	p.closeLabelScope()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+func (p *parser) parseBlockStmt() *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "BlockStmt"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.openScope()
+	list := p.parseStmtList()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func (p *parser) parseFuncTypeOrLit() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "FuncTypeOrLit"))
+	}
+
+	typ, scope := p.parseFuncType()
+	if p.tok != token.LBRACE {
+		// function type only
+		return typ
+	}
+
+	p.exprLev++
+	body := p.parseBody(scope)
+	p.exprLev--
+
+	return &ast.FuncLit{typ, body}
+}
+
+// parseOperand may return an expression or a raw type (incl. array
+// types of the form [...]T. Callers must verify the result.
+// If lhs is set and the result is an identifier, it is not resolved.
+//
+func (p *parser) parseOperand(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Operand"))
+	}
+
+	switch p.tok {
+	case token.IDENT:
+		x := p.parseIdent()
+		if !lhs {
+			p.resolve(x)
+		}
+		return x
+
+	case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
+		x := &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+		return x
+
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		p.exprLev++
+		x := p.parseRhsOrType() // types may be parenthesized: (some type)
+		p.exprLev--
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, x, rparen}
+
+	case token.FUNC:
+		return p.parseFuncTypeOrLit()
+
+	default:
+		if typ := p.tryIdentOrType(true); typ != nil {
+			// could be type for composite literal or conversion
+			_, isIdent := typ.(*ast.Ident)
+			assert(!isIdent, "type cannot be identifier")
+			return typ
+		}
+	}
+
+	pos := p.pos
+	p.errorExpected(pos, "operand")
+	p.next() // make progress
+	return &ast.BadExpr{pos, p.pos}
+}
+
+func (p *parser) parseSelector(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Selector"))
+	}
+
+	sel := p.parseIdent()
+
+	return &ast.SelectorExpr{x, sel}
+}
+
+func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeAssertion"))
+	}
+
+	p.expect(token.LPAREN)
+	var typ ast.Expr
+	if p.tok == token.TYPE {
+		// type switch: typ == nil
+		p.next()
+	} else {
+		typ = p.parseType()
+	}
+	p.expect(token.RPAREN)
+
+	return &ast.TypeAssertExpr{x, typ}
+}
+
+func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "IndexOrSlice"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	p.exprLev++
+	var low, high ast.Expr
+	isSlice := false
+	if p.tok != token.COLON {
+		low = p.parseRhs()
+	}
+	if p.tok == token.COLON {
+		isSlice = true
+		p.next()
+		if p.tok != token.RBRACK {
+			high = p.parseRhs()
+		}
+	}
+	p.exprLev--
+	rbrack := p.expect(token.RBRACK)
+
+	if isSlice {
+		return &ast.SliceExpr{x, lbrack, low, high, rbrack}
+	}
+	return &ast.IndexExpr{x, lbrack, low, rbrack}
+}
+
+func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr {
+	if p.trace {
+		defer un(trace(p, "CallOrConversion"))
+	}
+
+	lparen := p.expect(token.LPAREN)
+	p.exprLev++
+	var list []ast.Expr
+	var ellipsis token.Pos
+	for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
+		list = append(list, p.parseRhsOrType()) // builtins may expect a type: make(some type, ...)
+		if p.tok == token.ELLIPSIS {
+			ellipsis = p.pos
+			p.next()
+		}
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+	p.exprLev--
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.CallExpr{fun, lparen, list, ellipsis, rparen}
+}
+
+func (p *parser) parseElement(keyOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Element"))
+	}
+
+	if p.tok == token.LBRACE {
+		return p.parseLiteralValue(nil)
+	}
+
+	x := p.checkExpr(p.parseExpr(keyOk)) // don't resolve if map key
+	if keyOk {
+		if p.tok == token.COLON {
+			colon := p.pos
+			p.next()
+			return &ast.KeyValueExpr{x, colon, p.parseElement(false)}
+		}
+		p.resolve(x) // not a map key
+	}
+
+	return x
+}
+
+func (p *parser) parseElementList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ElementList"))
+	}
+
+	for p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseElement(true))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	return
+}
+
+func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "LiteralValue"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	var elts []ast.Expr
+	p.exprLev++
+	if p.tok != token.RBRACE {
+		elts = p.parseElementList()
+	}
+	p.exprLev--
+	rbrace := p.expect(token.RBRACE)
+	return &ast.CompositeLit{typ, lbrace, elts, rbrace}
+}
+
+// checkExpr checks that x is an expression (and not a type).
+func (p *parser) checkExpr(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.BasicLit:
+	case *ast.FuncLit:
+	case *ast.CompositeLit:
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.SelectorExpr:
+	case *ast.IndexExpr:
+	case *ast.SliceExpr:
+	case *ast.TypeAssertExpr:
+		// If t.Type == nil we have a type assertion of the form
+		// y.(type), which is only allowed in type switch expressions.
+		// It's hard to exclude those but for the case where we are in
+		// a type switch. Instead be lenient and test this in the type
+		// checker.
+	case *ast.CallExpr:
+	case *ast.StarExpr:
+	case *ast.UnaryExpr:
+	case *ast.BinaryExpr:
+	default:
+		// all other nodes are not proper expressions
+		p.errorExpected(x.Pos(), "expression")
+		x = &ast.BadExpr{x.Pos(), x.End()}
+	}
+	return x
+}
+
+// isTypeName returns true iff x is a (qualified) TypeName.
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	default:
+		return false // all other nodes are not type names
+	}
+	return true
+}
+
+// isLiteralType returns true iff x is a legal composite literal type.
+func isLiteralType(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	case *ast.ArrayType:
+	case *ast.StructType:
+	case *ast.MapType:
+	default:
+		return false // all other nodes are not legal composite literal types
+	}
+	return true
+}
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+// If x is of the form (T), unparen returns unparen(T), otherwise it returns x.
+func unparen(x ast.Expr) ast.Expr {
+	if p, isParen := x.(*ast.ParenExpr); isParen {
+		x = unparen(p.X)
+	}
+	return x
+}
+
+// checkExprOrType checks that x is an expression or a type
+// (and not a raw type such as [...]T).
+//
+func (p *parser) checkExprOrType(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.UnaryExpr:
+	case *ast.ArrayType:
+		if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
+			p.error(len.Pos(), "expected array length, found '...'")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	}
+
+	// all other nodes are expressions or types
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parsePrimaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "PrimaryExpr"))
+	}
+
+	x := p.parseOperand(lhs)
+L:
+	for {
+		switch p.tok {
+		case token.PERIOD:
+			p.next()
+			if lhs {
+				p.resolve(x)
+			}
+			switch p.tok {
+			case token.IDENT:
+				x = p.parseSelector(p.checkExpr(x))
+			case token.LPAREN:
+				x = p.parseTypeAssertion(p.checkExpr(x))
+			default:
+				pos := p.pos
+				p.next() // make progress
+				p.errorExpected(pos, "selector or type assertion")
+				x = &ast.BadExpr{pos, p.pos}
+			}
+		case token.LBRACK:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseIndexOrSlice(p.checkExpr(x))
+		case token.LPAREN:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseCallOrConversion(p.checkExprOrType(x))
+		case token.LBRACE:
+			if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) {
+				if lhs {
+					p.resolve(x)
+				}
+				x = p.parseLiteralValue(x)
+			} else {
+				break L
+			}
+		default:
+			break L
+		}
+		lhs = false // no need to try to resolve again
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseUnaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "UnaryExpr"))
+	}
+
+	switch p.tok {
+	case token.ADD, token.SUB, token.NOT, token.XOR, token.AND:
+		pos, op := p.pos, p.tok
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, op, p.checkExpr(x)}
+
+	case token.ARROW:
+		// channel type or receive expression
+		pos := p.pos
+		p.next()
+		if p.tok == token.CHAN {
+			p.next()
+			value := p.parseType()
+			return &ast.ChanType{pos, ast.RECV, value}
+		}
+
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)}
+
+	case token.MUL:
+		// pointer type or unary "*" expression
+		pos := p.pos
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.StarExpr{pos, p.checkExprOrType(x)}
+	}
+
+	return p.parsePrimaryExpr(lhs)
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseBinaryExpr(lhs bool, prec1 int) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "BinaryExpr"))
+	}
+
+	x := p.parseUnaryExpr(lhs)
+	for prec := p.tok.Precedence(); prec >= prec1; prec-- {
+		for p.tok.Precedence() == prec {
+			pos, op := p.pos, p.tok
+			p.next()
+			if lhs {
+				p.resolve(x)
+				lhs = false
+			}
+			y := p.parseBinaryExpr(false, prec+1)
+			x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)}
+		}
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+// The result may be a type or even a raw type ([...]int). Callers must
+// check the result (using checkExpr or checkExprOrType), depending on
+// context.
+func (p *parser) parseExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Expression"))
+	}
+
+	return p.parseBinaryExpr(lhs, token.LowestPrec+1)
+}
+
+func (p *parser) parseRhs() ast.Expr {
+	return p.checkExpr(p.parseExpr(false))
+}
+
+func (p *parser) parseRhsOrType() ast.Expr {
+	return p.checkExprOrType(p.parseExpr(false))
+}
+
+// ----------------------------------------------------------------------------
+// Statements
+
+// Parsing modes for parseSimpleStmt.
+const (
+	basic = iota
+	labelOk
+	rangeOk
+)
+
+// parseSimpleStmt returns true as 2nd result if it parsed the assignment
+// of a range clause (with mode == rangeOk). The returned statement is an
+// assignment with a right-hand side that is a single unary expression of
+// the form "range x". No guarantees are given for the left-hand side.
+func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) {
+	if p.trace {
+		defer un(trace(p, "SimpleStmt"))
+	}
+
+	x := p.parseLhsList()
+
+	switch p.tok {
+	case
+		token.DEFINE, token.ASSIGN, token.ADD_ASSIGN,
+		token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN,
+		token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN,
+		token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN:
+		// assignment statement, possibly part of a range clause
+		pos, tok := p.pos, p.tok
+		p.next()
+		var y []ast.Expr
+		isRange := false
+		if mode == rangeOk && p.tok == token.RANGE && (tok == token.DEFINE || tok == token.ASSIGN) {
+			pos := p.pos
+			p.next()
+			y = []ast.Expr{&ast.UnaryExpr{pos, token.RANGE, p.parseRhs()}}
+			isRange = true
+		} else {
+			y = p.parseRhsList()
+		}
+		return &ast.AssignStmt{x, pos, tok, y}, isRange
+	}
+
+	if len(x) > 1 {
+		p.errorExpected(x[0].Pos(), "1 expression")
+		// continue with first expression
+	}
+
+	switch p.tok {
+	case token.COLON:
+		// labeled statement
+		colon := p.pos
+		p.next()
+		if label, isIdent := x[0].(*ast.Ident); mode == labelOk && isIdent {
+			// Go spec: The scope of a label is the body of the function
+			// in which it is declared and excludes the body of any nested
+			// function.
+			stmt := &ast.LabeledStmt{label, colon, p.parseStmt()}
+			p.declare(stmt, nil, p.labelScope, ast.Lbl, label)
+			return stmt, false
+		}
+		// The label declaration typically starts at x[0].Pos(), but the label
+		// declaration may be erroneous due to a token after that position (and
+		// before the ':'). If SpuriousErrors is not set, the (only) error re-
+		// ported for the line is the illegal label error instead of the token
+		// before the ':' that caused the problem. Thus, use the (latest) colon
+		// position for error reporting.
+		p.error(colon, "illegal label declaration")
+		return &ast.BadStmt{x[0].Pos(), colon + 1}, false
+
+	case token.ARROW:
+		// send statement
+		arrow := p.pos
+		p.next() // consume "<-"
+		y := p.parseRhs()
+		return &ast.SendStmt{x[0], arrow, y}, false
+
+	case token.INC, token.DEC:
+		// increment or decrement
+		s := &ast.IncDecStmt{x[0], p.pos, p.tok}
+		p.next() // consume "++" or "--"
+		return s, false
+	}
+
+	// expression
+	return &ast.ExprStmt{x[0]}, false
+}
+
+func (p *parser) parseCallExpr() *ast.CallExpr {
+	x := p.parseRhsOrType() // could be a conversion: (some type)(x)
+	if call, isCall := x.(*ast.CallExpr); isCall {
+		return call
+	}
+	p.errorExpected(x.Pos(), "function/method call")
+	return nil
+}
+
+func (p *parser) parseGoStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "GoStmt"))
+	}
+
+	pos := p.expect(token.GO)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 2} // len("go")
+	}
+
+	return &ast.GoStmt{pos, call}
+}
+
+func (p *parser) parseDeferStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "DeferStmt"))
+	}
+
+	pos := p.expect(token.DEFER)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 5} // len("defer")
+	}
+
+	return &ast.DeferStmt{pos, call}
+}
+
+func (p *parser) parseReturnStmt() *ast.ReturnStmt {
+	if p.trace {
+		defer un(trace(p, "ReturnStmt"))
+	}
+
+	pos := p.pos
+	p.expect(token.RETURN)
+	var x []ast.Expr
+	if p.tok != token.SEMICOLON && p.tok != token.RBRACE {
+		x = p.parseRhsList()
+	}
+	p.expectSemi()
+
+	return &ast.ReturnStmt{pos, x}
+}
+
+func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
+	if p.trace {
+		defer un(trace(p, "BranchStmt"))
+	}
+
+	pos := p.expect(tok)
+	var label *ast.Ident
+	if tok != token.FALLTHROUGH && p.tok == token.IDENT {
+		label = p.parseIdent()
+		// add to list of unresolved targets
+		n := len(p.targetStack) - 1
+		p.targetStack[n] = append(p.targetStack[n], label)
+	}
+	p.expectSemi()
+
+	return &ast.BranchStmt{pos, tok, label}
+}
+
+func (p *parser) makeExpr(s ast.Stmt) ast.Expr {
+	if s == nil {
+		return nil
+	}
+	if es, isExpr := s.(*ast.ExprStmt); isExpr {
+		return p.checkExpr(es.X)
+	}
+	p.error(s.Pos(), "expected condition, found simple statement")
+	return &ast.BadExpr{s.Pos(), s.End()}
+}
+
+func (p *parser) parseIfStmt() *ast.IfStmt {
+	if p.trace {
+		defer un(trace(p, "IfStmt"))
+	}
+
+	pos := p.expect(token.IF)
+	p.openScope()
+	defer p.closeScope()
+
+	var s ast.Stmt
+	var x ast.Expr
+	{
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok == token.SEMICOLON {
+			p.next()
+			x = p.parseRhs()
+		} else {
+			s, _ = p.parseSimpleStmt(basic)
+			if p.tok == token.SEMICOLON {
+				p.next()
+				x = p.parseRhs()
+			} else {
+				x = p.makeExpr(s)
+				s = nil
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	var else_ ast.Stmt
+	if p.tok == token.ELSE {
+		p.next()
+		else_ = p.parseStmt()
+	} else {
+		p.expectSemi()
+	}
+
+	return &ast.IfStmt{pos, s, x, body, else_}
+}
+
+func (p *parser) parseTypeList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "TypeList"))
+	}
+
+	list = append(list, p.parseType())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseType())
+	}
+
+	return
+}
+
+func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause {
+	if p.trace {
+		defer un(trace(p, "CaseClause"))
+	}
+
+	pos := p.pos
+	var list []ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		if exprSwitch {
+			list = p.parseRhsList()
+		} else {
+			list = p.parseTypeList()
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	p.openScope()
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CaseClause{pos, list, colon, body}
+}
+
+func isExprSwitch(s ast.Stmt) bool {
+	if s == nil {
+		return true
+	}
+	if e, ok := s.(*ast.ExprStmt); ok {
+		if a, ok := e.X.(*ast.TypeAssertExpr); ok {
+			return a.Type != nil // regular type assertion
+		}
+		return true
+	}
+	return false
+}
+
+func (p *parser) parseSwitchStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SwitchStmt"))
+	}
+
+	pos := p.expect(token.SWITCH)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2 ast.Stmt
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2, _ = p.parseSimpleStmt(basic)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.LBRACE {
+				s2, _ = p.parseSimpleStmt(basic)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	exprSwitch := isExprSwitch(s2)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCaseClause(exprSwitch))
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	if exprSwitch {
+		return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body}
+	}
+	// type switch
+	// TODO(gri): do all the checks!
+	return &ast.TypeSwitchStmt{pos, s1, s2, body}
+}
+
+func (p *parser) parseCommClause() *ast.CommClause {
+	if p.trace {
+		defer un(trace(p, "CommClause"))
+	}
+
+	p.openScope()
+	pos := p.pos
+	var comm ast.Stmt
+	if p.tok == token.CASE {
+		p.next()
+		lhs := p.parseLhsList()
+		if p.tok == token.ARROW {
+			// SendStmt
+			if len(lhs) > 1 {
+				p.errorExpected(lhs[0].Pos(), "1 expression")
+				// continue with first expression
+			}
+			arrow := p.pos
+			p.next()
+			rhs := p.parseRhs()
+			comm = &ast.SendStmt{lhs[0], arrow, rhs}
+		} else {
+			// RecvStmt
+			pos := p.pos
+			tok := p.tok
+			var rhs ast.Expr
+			if tok == token.ASSIGN || tok == token.DEFINE {
+				// RecvStmt with assignment
+				if len(lhs) > 2 {
+					p.errorExpected(lhs[0].Pos(), "1 or 2 expressions")
+					// continue with first two expressions
+					lhs = lhs[0:2]
+				}
+				p.next()
+				rhs = p.parseRhs()
+			} else {
+				// rhs must be single receive operation
+				if len(lhs) > 1 {
+					p.errorExpected(lhs[0].Pos(), "1 expression")
+					// continue with first expression
+				}
+				rhs = lhs[0]
+				lhs = nil // there is no lhs
+			}
+			if lhs != nil {
+				comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}}
+			} else {
+				comm = &ast.ExprStmt{rhs}
+			}
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CommClause{pos, comm, colon, body}
+}
+
+func (p *parser) parseSelectStmt() *ast.SelectStmt {
+	if p.trace {
+		defer un(trace(p, "SelectStmt"))
+	}
+
+	pos := p.expect(token.SELECT)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCommClause())
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	return &ast.SelectStmt{pos, body}
+}
+
+func (p *parser) parseForStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "ForStmt"))
+	}
+
+	pos := p.expect(token.FOR)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2, s3 ast.Stmt
+	var isRange bool
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2, isRange = p.parseSimpleStmt(rangeOk)
+		}
+		if !isRange && p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.SEMICOLON {
+				s2, _ = p.parseSimpleStmt(basic)
+			}
+			p.expectSemi()
+			if p.tok != token.LBRACE {
+				s3, _ = p.parseSimpleStmt(basic)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	p.expectSemi()
+
+	if isRange {
+		as := s2.(*ast.AssignStmt)
+		// check lhs
+		var key, value ast.Expr
+		switch len(as.Lhs) {
+		case 2:
+			key, value = as.Lhs[0], as.Lhs[1]
+		case 1:
+			key = as.Lhs[0]
+		default:
+			p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// parseSimpleStmt returned a right-hand side that
+		// is a single unary expression of the form "range x"
+		x := as.Rhs[0].(*ast.UnaryExpr).X
+		return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, x, body}
+	}
+
+	// regular for statement
+	return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body}
+}
+
+func (p *parser) parseStmt() (s ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "Statement"))
+	}
+
+	switch p.tok {
+	case token.CONST, token.TYPE, token.VAR:
+		s = &ast.DeclStmt{p.parseDecl()}
+	case
+		// tokens that may start a top-level expression
+		token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand
+		token.LBRACK, token.STRUCT, // composite type
+		token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators
+		s, _ = p.parseSimpleStmt(labelOk)
+		// because of the required look-ahead, labeled statements are
+		// parsed by parseSimpleStmt - don't expect a semicolon after
+		// them
+		if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt {
+			p.expectSemi()
+		}
+	case token.GO:
+		s = p.parseGoStmt()
+	case token.DEFER:
+		s = p.parseDeferStmt()
+	case token.RETURN:
+		s = p.parseReturnStmt()
+	case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH:
+		s = p.parseBranchStmt(p.tok)
+	case token.LBRACE:
+		s = p.parseBlockStmt()
+		p.expectSemi()
+	case token.IF:
+		s = p.parseIfStmt()
+	case token.SWITCH:
+		s = p.parseSwitchStmt()
+	case token.SELECT:
+		s = p.parseSelectStmt()
+	case token.FOR:
+		s = p.parseForStmt()
+	case token.SEMICOLON:
+		s = &ast.EmptyStmt{p.pos}
+		p.next()
+	case token.RBRACE:
+		// a semicolon may be omitted before a closing "}"
+		s = &ast.EmptyStmt{p.pos}
+	default:
+		// no statement found
+		pos := p.pos
+		p.errorExpected(pos, "statement")
+		p.next() // make progress
+		s = &ast.BadStmt{pos, p.pos}
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+type parseSpecFunction func(p *parser, doc *ast.CommentGroup, iota int) ast.Spec
+
+func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ImportSpec"))
+	}
+
+	var ident *ast.Ident
+	switch p.tok {
+	case token.PERIOD:
+		ident = &ast.Ident{p.pos, ".", nil}
+		p.next()
+	case token.IDENT:
+		ident = p.parseIdent()
+	}
+
+	var path *ast.BasicLit
+	if p.tok == token.STRING {
+		path = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	} else {
+		p.expect(token.STRING) // use expect() error handling
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// collect imports
+	spec := &ast.ImportSpec{doc, ident, path, p.lineComment}
+	p.imports = append(p.imports, spec)
+
+	return spec
+}
+
+func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ConstSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ != nil || p.tok == token.ASSIGN || iota == 0 {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, iota, p.topScope, ast.Con, idents...)
+
+	return spec
+}
+
+func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "TypeSpec"))
+	}
+
+	ident := p.parseIdent()
+
+	// Go spec: The scope of a type identifier declared inside a function begins
+	// at the identifier in the TypeSpec and ends at the end of the innermost
+	// containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.TypeSpec{doc, ident, nil, nil}
+	p.declare(spec, nil, p.topScope, ast.Typ, ident)
+
+	spec.Type = p.parseType()
+	p.expectSemi() // call before accessing p.linecomment
+	spec.Comment = p.lineComment
+
+	return spec
+}
+
+func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "VarSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ == nil || p.tok == token.ASSIGN {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, nil, p.topScope, ast.Var, idents...)
+
+	return spec
+}
+
+func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl {
+	if p.trace {
+		defer un(trace(p, "GenDecl("+keyword.String()+")"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(keyword)
+	var lparen, rparen token.Pos
+	var list []ast.Spec
+	if p.tok == token.LPAREN {
+		lparen = p.pos
+		p.next()
+		for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ {
+			list = append(list, f(p, p.leadComment, iota))
+		}
+		rparen = p.expect(token.RPAREN)
+		p.expectSemi()
+	} else {
+		list = append(list, f(p, nil, 0))
+	}
+
+	return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen}
+}
+
+func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Receiver"))
+	}
+
+	pos := p.pos
+	par := p.parseParameters(scope, false)
+
+	// must have exactly one receiver
+	if par.NumFields() != 1 {
+		p.errorExpected(pos, "exactly one receiver")
+		// TODO determine a better range for BadExpr below
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}}
+		return par
+	}
+
+	// recv type must be of the form ["*"] identifier
+	recv := par.List[0]
+	base := deref(recv.Type)
+	if _, isIdent := base.(*ast.Ident); !isIdent {
+		p.errorExpected(base.Pos(), "(unqualified) identifier")
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}}
+	}
+
+	return par
+}
+
+func (p *parser) parseFuncDecl() *ast.FuncDecl {
+	if p.trace {
+		defer un(trace(p, "FunctionDecl"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+
+	var recv *ast.FieldList
+	if p.tok == token.LPAREN {
+		recv = p.parseReceiver(scope)
+	}
+
+	ident := p.parseIdent()
+
+	params, results := p.parseSignature(scope)
+
+	var body *ast.BlockStmt
+	if p.tok == token.LBRACE {
+		body = p.parseBody(scope)
+	}
+	p.expectSemi()
+
+	decl := &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body}
+	if recv == nil {
+		// Go spec: The scope of an identifier denoting a constant, type,
+		// variable, or function (but not method) declared at top level
+		// (outside any function) is the package block.
+		//
+		// init() functions cannot be referred to and there may
+		// be more than one - don't put them in the pkgScope
+		if ident.Name != "init" {
+			p.declare(decl, nil, p.pkgScope, ast.Fun, ident)
+		}
+	}
+
+	return decl
+}
+
+func (p *parser) parseDecl() ast.Decl {
+	if p.trace {
+		defer un(trace(p, "Declaration"))
+	}
+
+	var f parseSpecFunction
+	switch p.tok {
+	case token.CONST:
+		f = parseConstSpec
+
+	case token.TYPE:
+		f = parseTypeSpec
+
+	case token.VAR:
+		f = parseVarSpec
+
+	case token.FUNC:
+		return p.parseFuncDecl()
+
+	default:
+		pos := p.pos
+		p.errorExpected(pos, "declaration")
+		p.next() // make progress
+		decl := &ast.BadDecl{pos, p.pos}
+		return decl
+	}
+
+	return p.parseGenDecl(p.tok, f)
+}
+
+func (p *parser) parseDeclList() (list []ast.Decl) {
+	if p.trace {
+		defer un(trace(p, "DeclList"))
+	}
+
+	for p.tok != token.EOF {
+		list = append(list, p.parseDecl())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Source files
+
+func (p *parser) parseFile() *ast.File {
+	if p.trace {
+		defer un(trace(p, "File"))
+	}
+
+	// package clause
+	doc := p.leadComment
+	pos := p.expect(token.PACKAGE)
+	// Go spec: The package clause is not a declaration;
+	// the package name does not appear in any scope.
+	ident := p.parseIdent()
+	if ident.Name == "_" {
+		p.error(p.pos, "invalid package name _")
+	}
+	p.expectSemi()
+
+	var decls []ast.Decl
+
+	// Don't bother parsing the rest if we had errors already.
+	// Likely not a Go source file at all.
+
+	if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 {
+		// import decls
+		for p.tok == token.IMPORT {
+			decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec))
+		}
+
+		if p.mode&ImportsOnly == 0 {
+			// rest of package body
+			for p.tok != token.EOF {
+				decls = append(decls, p.parseDecl())
+			}
+		}
+	}
+
+	assert(p.topScope == p.pkgScope, "imbalanced scopes")
+
+	// resolve global identifiers within the same file
+	i := 0
+	for _, ident := range p.unresolved {
+		// i <= index for current ident
+		assert(ident.Obj == unresolved, "object already resolved")
+		ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel
+		if ident.Obj == nil {
+			p.unresolved[i] = ident
+			i++
+		}
+	}
+
+	return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unresolved[0:i], p.comments}
+}
diff --git a/src/pkg/go/parser/parser_test.go b/src/pkg/go/parser/parser_test.go
new file mode 100644
index 000000000..39a78e515
--- /dev/null
+++ b/src/pkg/go/parser/parser_test.go
@@ -0,0 +1,130 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package parser
+
+import (
+	"go/token"
+	"os"
+	"testing"
+)
+
+var fset = token.NewFileSet()
+
+var illegalInputs = []interface{}{
+	nil,
+	3.14,
+	[]byte(nil),
+	"foo!",
+	`package p; func f() { if /* should have condition */ {} };`,
+	`package p; func f() { if ; /* should have condition */ {} };`,
+	`package p; func f() { if f(); /* should have condition */ {} };`,
+	`package p; const c; /* should have constant value */`,
+	`package p; func f() { if _ = range x; true {} };`,
+	`package p; func f() { switch _ = range x; true {} };`,
+	`package p; func f() { for _ = range x ; ; {} };`,
+	`package p; func f() { for ; ; _ = range x {} };`,
+	`package p; func f() { for ; _ = range x ; {} };`,
+	`package p; var a = [1]int; /* illegal expression */`,
+	`package p; var a = [...]int; /* illegal expression */`,
+	`package p; var a = struct{} /* illegal expression */`,
+	`package p; var a = func(); /* illegal expression */`,
+	`package p; var a = interface{} /* illegal expression */`,
+	`package p; var a = []int /* illegal expression */`,
+	`package p; var a = map[int]int /* illegal expression */`,
+	`package p; var a = chan int; /* illegal expression */`,
+	`package p; var a = []int{[]int}; /* illegal expression */`,
+	`package p; var a = ([]int); /* illegal expression */`,
+	`package p; var a = a[[]int:[]int]; /* illegal expression */`,
+	`package p; var a = <- chan int; /* illegal expression */`,
+	`package p; func f() { select { case _ <- chan int: } };`,
+}
+
+func TestParseIllegalInputs(t *testing.T) {
+	for _, src := range illegalInputs {
+		_, err := ParseFile(fset, "", src, 0)
+		if err == nil {
+			t.Errorf("ParseFile(%v) should have failed", src)
+		}
+	}
+}
+
+var validPrograms = []interface{}{
+	"package p\n",
+	`package p;`,
+	`package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`,
+	`package p; func f() { if f(T{}) {} };`,
+	`package p; func f() { _ = (<-chan int)(x) };`,
+	`package p; func f() { _ = (<-chan <-chan int)(x) };`,
+	`package p; func f(func() func() func());`,
+	`package p; func f(...T);`,
+	`package p; func f(float, ...int);`,
+	`package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`,
+	`package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`,
+	`package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`,
+	`package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`,
+	`package p; var a = T{{1, 2}, {3, 4}}`,
+	`package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`,
+	`package p; func f() { select { case x := (<-c): } };`,
+	`package p; func f() { if ; true {} };`,
+	`package p; func f() { switch ; {} };`,
+	`package p; func f() { for _ = range "foo" + "bar" {} };`,
+}
+
+func TestParseValidPrograms(t *testing.T) {
+	for _, src := range validPrograms {
+		_, err := ParseFile(fset, "", src, 0)
+		if err != nil {
+			t.Errorf("ParseFile(%q): %v", src, err)
+		}
+	}
+}
+
+var validFiles = []string{
+	"parser.go",
+	"parser_test.go",
+}
+
+func TestParse3(t *testing.T) {
+	for _, filename := range validFiles {
+		_, err := ParseFile(fset, filename, nil, DeclarationErrors)
+		if err != nil {
+			t.Errorf("ParseFile(%s): %v", filename, err)
+		}
+	}
+}
+
+func nameFilter(filename string) bool {
+	switch filename {
+	case "parser.go":
+	case "interface.go":
+	case "parser_test.go":
+	default:
+		return false
+	}
+	return true
+}
+
+func dirFilter(f *os.FileInfo) bool { return nameFilter(f.Name) }
+
+func TestParse4(t *testing.T) {
+	path := "."
+	pkgs, err := ParseDir(fset, path, dirFilter, 0)
+	if err != nil {
+		t.Fatalf("ParseDir(%s): %v", path, err)
+	}
+	if len(pkgs) != 1 {
+		t.Errorf("incorrect number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs["parser"]
+	if pkg == nil {
+		t.Errorf(`package "parser" not found`)
+		return
+	}
+	for filename := range pkg.Files {
+		if !nameFilter(filename) {
+			t.Errorf("unexpected package file: %s", filename)
+		}
+	}
+}
diff --git a/src/pkg/go/printer/Makefile b/src/pkg/go/printer/Makefile
new file mode 100644
index 000000000..6a71efc93
--- /dev/null
+++ b/src/pkg/go/printer/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/printer
+GOFILES=\
+	printer.go\
+	nodes.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/printer/nodes.go b/src/pkg/go/printer/nodes.go
new file mode 100644
index 000000000..9cd975ec1
--- /dev/null
+++ b/src/pkg/go/printer/nodes.go
@@ -0,0 +1,1510 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements printing of AST nodes; specifically
+// expressions, statements, declarations, and files. It uses
+// the print functionality implemented in printer.go.
+
+package printer
+
+import (
+	"bytes"
+	"go/ast"
+	"go/token"
+)
+
+// Other formatting issues:
+// - better comment formatting for /*-style comments at the end of a line (e.g. a declaration)
+//   when the comment spans multiple lines; if such a comment is just two lines, formatting is
+//   not idempotent
+// - formatting of expression lists
+// - should use blank instead of tab to separate one-line function bodies from
+//   the function header unless there is a group of consecutive one-liners
+
+// ----------------------------------------------------------------------------
+// Common AST nodes.
+
+// Print as many newlines as necessary (but at least min newlines) to get to
+// the current line. ws is printed before the first line break. If newSection
+// is set, the first line break is printed as formfeed. Returns true if any
+// line break was printed; returns false otherwise.
+//
+// TODO(gri): linebreak may add too many lines if the next statement at "line"
+//            is preceded by comments because the computation of n assumes
+//            the current position before the comment and the target position
+//            after the comment. Thus, after interspersing such comments, the
+//            space taken up by them is not considered to reduce the number of
+//            linebreaks. At the moment there is no easy way to know about
+//            future (not yet interspersed) comments in this function.
+//
+func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (printedBreak bool) {
+	n := p.nlines(line-p.pos.Line, min)
+	if n > 0 {
+		p.print(ws)
+		if newSection {
+			p.print(formfeed)
+			n--
+		}
+		for ; n > 0; n-- {
+			p.print(newline)
+		}
+		printedBreak = true
+	}
+	return
+}
+
+// setComment sets g as the next comment if g != nil and if node comments
+// are enabled - this mode is used when printing source code fragments such
+// as exports only. It assumes that there are no other pending comments to
+// intersperse.
+func (p *printer) setComment(g *ast.CommentGroup) {
+	if g == nil || !p.useNodeComments {
+		return
+	}
+	if p.comments == nil {
+		// initialize p.comments lazily
+		p.comments = make([]*ast.CommentGroup, 1)
+	} else if p.cindex < len(p.comments) {
+		// for some reason there are pending comments; this
+		// should never happen - handle gracefully and flush
+		// all comments up to g, ignore anything after that
+		p.flush(p.fset.Position(g.List[0].Pos()), token.ILLEGAL)
+	}
+	p.comments[0] = g
+	p.cindex = 0
+}
+
+type exprListMode uint
+
+const (
+	blankStart exprListMode = 1 << iota // print a blank before a non-empty list
+	blankEnd                            // print a blank after a non-empty list
+	commaSep                            // elements are separated by commas
+	commaTerm                           // list is optionally terminated by a comma
+	noIndent                            // no extra indentation in multi-line lists
+	periodSep                           // elements are separated by periods
+)
+
+// Sets multiLine to true if the identifier list spans multiple lines.
+// If indent is set, a multi-line identifier list is indented after the
+// first linebreak encountered.
+func (p *printer) identList(list []*ast.Ident, indent bool, multiLine *bool) {
+	// convert into an expression list so we can re-use exprList formatting
+	xlist := make([]ast.Expr, len(list))
+	for i, x := range list {
+		xlist[i] = x
+	}
+	mode := commaSep
+	if !indent {
+		mode |= noIndent
+	}
+	p.exprList(token.NoPos, xlist, 1, mode, multiLine, token.NoPos)
+}
+
+// Print a list of expressions. If the list spans multiple
+// source lines, the original line breaks are respected between
+// expressions. Sets multiLine to true if the list spans multiple
+// lines.
+//
+// TODO(gri) Consider rewriting this to be independent of []ast.Expr
+//           so that we can use the algorithm for any kind of list
+//           (e.g., pass list via a channel over which to range).
+func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next0 token.Pos) {
+	if len(list) == 0 {
+		return
+	}
+
+	if mode&blankStart != 0 {
+		p.print(blank)
+	}
+
+	prev := p.fset.Position(prev0)
+	next := p.fset.Position(next0)
+	line := p.fset.Position(list[0].Pos()).Line
+	endLine := p.fset.Position(list[len(list)-1].End()).Line
+
+	if prev.IsValid() && prev.Line == line && line == endLine {
+		// all list entries on a single line
+		for i, x := range list {
+			if i > 0 {
+				if mode&commaSep != 0 {
+					p.print(token.COMMA)
+				}
+				p.print(blank)
+			}
+			p.expr0(x, depth, multiLine)
+		}
+		if mode&blankEnd != 0 {
+			p.print(blank)
+		}
+		return
+	}
+
+	// list entries span multiple lines;
+	// use source code positions to guide line breaks
+
+	// don't add extra indentation if noIndent is set;
+	// i.e., pretend that the first line is already indented
+	ws := ignore
+	if mode&noIndent == 0 {
+		ws = indent
+	}
+
+	// the first linebreak is always a formfeed since this section must not
+	// depend on any previous formatting
+	prevBreak := -1 // index of last expression that was followed by a linebreak
+	if prev.IsValid() && prev.Line < line && p.linebreak(line, 0, ws, true) {
+		ws = ignore
+		*multiLine = true
+		prevBreak = 0
+	}
+
+	// initialize expression/key size: a zero value indicates expr/key doesn't fit on a single line
+	size := 0
+
+	// print all list elements
+	for i, x := range list {
+		prevLine := line
+		line = p.fset.Position(x.Pos()).Line
+
+		// determine if the next linebreak, if any, needs to use formfeed:
+		// in general, use the entire node size to make the decision; for
+		// key:value expressions, use the key size
+		// TODO(gri) for a better result, should probably incorporate both
+		//           the key and the node size into the decision process
+		useFF := true
+
+		// determine element size: all bets are off if we don't have
+		// position information for the previous and next token (likely
+		// generated code - simply ignore the size in this case by setting
+		// it to 0)
+		prevSize := size
+		const infinity = 1e6 // larger than any source line
+		size = p.nodeSize(x, infinity)
+		pair, isPair := x.(*ast.KeyValueExpr)
+		if size <= infinity && prev.IsValid() && next.IsValid() {
+			// x fits on a single line
+			if isPair {
+				size = p.nodeSize(pair.Key, infinity) // size <= infinity
+			}
+		} else {
+			// size too large or we don't have good layout information
+			size = 0
+		}
+
+		// if the previous line and the current line had single-
+		// line-expressions and the key sizes are small or the
+		// the ratio between the key sizes does not exceed a
+		// threshold, align columns and do not use formfeed
+		if prevSize > 0 && size > 0 {
+			const smallSize = 20
+			if prevSize <= smallSize && size <= smallSize {
+				useFF = false
+			} else {
+				const r = 4 // threshold
+				ratio := float64(size) / float64(prevSize)
+				useFF = ratio <= 1/r || r <= ratio
+			}
+		}
+
+		if i > 0 {
+			switch {
+			case mode&commaSep != 0:
+				p.print(token.COMMA)
+			case mode&periodSep != 0:
+				p.print(token.PERIOD)
+			}
+			needsBlank := mode&periodSep == 0 // period-separated list elements don't need a blank
+			if prevLine < line && prevLine > 0 && line > 0 {
+				// lines are broken using newlines so comments remain aligned
+				// unless forceFF is set or there are multiple expressions on
+				// the same line in which case formfeed is used
+				if p.linebreak(line, 0, ws, useFF || prevBreak+1 < i) {
+					ws = ignore
+					*multiLine = true
+					prevBreak = i
+					needsBlank = false // we got a line break instead
+				}
+			}
+			if needsBlank {
+				p.print(blank)
+			}
+		}
+
+		if isPair && size > 0 && len(list) > 1 {
+			// we have a key:value expression that fits onto one line and
+			// is in a list with more then one entry: use a column for the
+			// key such that consecutive entries can align if possible
+			p.expr(pair.Key, multiLine)
+			p.print(pair.Colon, token.COLON, vtab)
+			p.expr(pair.Value, multiLine)
+		} else {
+			p.expr0(x, depth, multiLine)
+		}
+	}
+
+	if mode&commaTerm != 0 && next.IsValid() && p.pos.Line < next.Line {
+		// print a terminating comma if the next token is on a new line
+		p.print(token.COMMA)
+		if ws == ignore && mode&noIndent == 0 {
+			// unindent if we indented
+			p.print(unindent)
+		}
+		p.print(formfeed) // terminating comma needs a line break to look good
+		return
+	}
+
+	if mode&blankEnd != 0 {
+		p.print(blank)
+	}
+
+	if ws == ignore && mode&noIndent == 0 {
+		// unindent if we indented
+		p.print(unindent)
+	}
+}
+
+// Sets multiLine to true if the the parameter list spans multiple lines.
+func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) {
+	p.print(fields.Opening, token.LPAREN)
+	if len(fields.List) > 0 {
+		var prevLine, line int
+		for i, par := range fields.List {
+			if i > 0 {
+				p.print(token.COMMA)
+				if len(par.Names) > 0 {
+					line = p.fset.Position(par.Names[0].Pos()).Line
+				} else {
+					line = p.fset.Position(par.Type.Pos()).Line
+				}
+				if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) {
+					*multiLine = true
+				} else {
+					p.print(blank)
+				}
+			}
+			if len(par.Names) > 0 {
+				p.identList(par.Names, false, multiLine)
+				p.print(blank)
+			}
+			p.expr(par.Type, multiLine)
+			prevLine = p.fset.Position(par.Type.Pos()).Line
+		}
+	}
+	p.print(fields.Closing, token.RPAREN)
+}
+
+// Sets multiLine to true if the signature spans multiple lines.
+func (p *printer) signature(params, result *ast.FieldList, multiLine *bool) {
+	p.parameters(params, multiLine)
+	n := result.NumFields()
+	if n > 0 {
+		p.print(blank)
+		if n == 1 && result.List[0].Names == nil {
+			// single anonymous result; no ()'s
+			p.expr(result.List[0].Type, multiLine)
+			return
+		}
+		p.parameters(result, multiLine)
+	}
+}
+
+func identListSize(list []*ast.Ident, maxSize int) (size int) {
+	for i, x := range list {
+		if i > 0 {
+			size += 2 // ", "
+		}
+		size += len(x.Name)
+		if size >= maxSize {
+			break
+		}
+	}
+	return
+}
+
+func (p *printer) isOneLineFieldList(list []*ast.Field) bool {
+	if len(list) != 1 {
+		return false // allow only one field
+	}
+	f := list[0]
+	if f.Tag != nil || f.Comment != nil {
+		return false // don't allow tags or comments
+	}
+	// only name(s) and type
+	const maxSize = 30 // adjust as appropriate, this is an approximate value
+	namesSize := identListSize(f.Names, maxSize)
+	if namesSize > 0 {
+		namesSize = 1 // blank between names and types
+	}
+	typeSize := p.nodeSize(f.Type, maxSize)
+	return namesSize+typeSize <= maxSize
+}
+
+func (p *printer) setLineComment(text string) {
+	p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{token.NoPos, text}}})
+}
+
+func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) {
+	lbrace := fields.Opening
+	list := fields.List
+	rbrace := fields.Closing
+	srcIsOneLine := lbrace.IsValid() && rbrace.IsValid() && p.fset.Position(lbrace).Line == p.fset.Position(rbrace).Line
+
+	if !isIncomplete && !p.commentBefore(p.fset.Position(rbrace)) && srcIsOneLine {
+		// possibly a one-line struct/interface
+		if len(list) == 0 {
+			// no blank between keyword and {} in this case
+			p.print(lbrace, token.LBRACE, rbrace, token.RBRACE)
+			return
+		} else if isStruct && p.isOneLineFieldList(list) { // for now ignore interfaces
+			// small enough - print on one line
+			// (don't use identList and ignore source line breaks)
+			p.print(lbrace, token.LBRACE, blank)
+			f := list[0]
+			for i, x := range f.Names {
+				if i > 0 {
+					p.print(token.COMMA, blank)
+				}
+				p.expr(x, ignoreMultiLine)
+			}
+			if len(f.Names) > 0 {
+				p.print(blank)
+			}
+			p.expr(f.Type, ignoreMultiLine)
+			p.print(blank, rbrace, token.RBRACE)
+			return
+		}
+	}
+
+	// at least one entry or incomplete
+	p.print(blank, lbrace, token.LBRACE, indent, formfeed)
+	if isStruct {
+
+		sep := vtab
+		if len(list) == 1 {
+			sep = blank
+		}
+		var ml bool
+		for i, f := range list {
+			if i > 0 {
+				p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
+			}
+			ml = false
+			extraTabs := 0
+			p.setComment(f.Doc)
+			if len(f.Names) > 0 {
+				// named fields
+				p.identList(f.Names, false, &ml)
+				p.print(sep)
+				p.expr(f.Type, &ml)
+				extraTabs = 1
+			} else {
+				// anonymous field
+				p.expr(f.Type, &ml)
+				extraTabs = 2
+			}
+			if f.Tag != nil {
+				if len(f.Names) > 0 && sep == vtab {
+					p.print(sep)
+				}
+				p.print(sep)
+				p.expr(f.Tag, &ml)
+				extraTabs = 0
+			}
+			if f.Comment != nil {
+				for ; extraTabs > 0; extraTabs-- {
+					p.print(sep)
+				}
+				p.setComment(f.Comment)
+			}
+		}
+		if isIncomplete {
+			if len(list) > 0 {
+				p.print(formfeed)
+			}
+			p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment
+			p.setLineComment("// contains filtered or unexported fields")
+		}
+
+	} else { // interface
+
+		var ml bool
+		for i, f := range list {
+			if i > 0 {
+				p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
+			}
+			ml = false
+			p.setComment(f.Doc)
+			if ftyp, isFtyp := f.Type.(*ast.FuncType); isFtyp {
+				// method
+				p.expr(f.Names[0], &ml)
+				p.signature(ftyp.Params, ftyp.Results, &ml)
+			} else {
+				// embedded interface
+				p.expr(f.Type, &ml)
+			}
+			p.setComment(f.Comment)
+		}
+		if isIncomplete {
+			if len(list) > 0 {
+				p.print(formfeed)
+			}
+			p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment
+			p.setLineComment("// contains filtered or unexported methods")
+		}
+
+	}
+	p.print(unindent, formfeed, rbrace, token.RBRACE)
+}
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func walkBinary(e *ast.BinaryExpr) (has4, has5 bool, maxProblem int) {
+	switch e.Op.Precedence() {
+	case 4:
+		has4 = true
+	case 5:
+		has5 = true
+	}
+
+	switch l := e.X.(type) {
+	case *ast.BinaryExpr:
+		if l.Op.Precedence() < e.Op.Precedence() {
+			// parens will be inserted.
+			// pretend this is an *ast.ParenExpr and do nothing.
+			break
+		}
+		h4, h5, mp := walkBinary(l)
+		has4 = has4 || h4
+		has5 = has5 || h5
+		if maxProblem < mp {
+			maxProblem = mp
+		}
+	}
+
+	switch r := e.Y.(type) {
+	case *ast.BinaryExpr:
+		if r.Op.Precedence() <= e.Op.Precedence() {
+			// parens will be inserted.
+			// pretend this is an *ast.ParenExpr and do nothing.
+			break
+		}
+		h4, h5, mp := walkBinary(r)
+		has4 = has4 || h4
+		has5 = has5 || h5
+		if maxProblem < mp {
+			maxProblem = mp
+		}
+
+	case *ast.StarExpr:
+		if e.Op == token.QUO { // `*/`
+			maxProblem = 5
+		}
+
+	case *ast.UnaryExpr:
+		switch e.Op.String() + r.Op.String() {
+		case "/*", "&&", "&^":
+			maxProblem = 5
+		case "++", "--":
+			if maxProblem < 4 {
+				maxProblem = 4
+			}
+		}
+	}
+	return
+}
+
+func cutoff(e *ast.BinaryExpr, depth int) int {
+	has4, has5, maxProblem := walkBinary(e)
+	if maxProblem > 0 {
+		return maxProblem + 1
+	}
+	if has4 && has5 {
+		if depth == 1 {
+			return 5
+		}
+		return 4
+	}
+	if depth == 1 {
+		return 6
+	}
+	return 4
+}
+
+func diffPrec(expr ast.Expr, prec int) int {
+	x, ok := expr.(*ast.BinaryExpr)
+	if !ok || prec != x.Op.Precedence() {
+		return 1
+	}
+	return 0
+}
+
+func reduceDepth(depth int) int {
+	depth--
+	if depth < 1 {
+		depth = 1
+	}
+	return depth
+}
+
+// Format the binary expression: decide the cutoff and then format.
+// Let's call depth == 1 Normal mode, and depth > 1 Compact mode.
+// (Algorithm suggestion by Russ Cox.)
+//
+// The precedences are:
+//	5             *  /  %  <<  >>  &  &^
+//	4             +  -  |  ^
+//	3             ==  !=  <  <=  >  >=
+//	2             &&
+//	1             ||
+//
+// The only decision is whether there will be spaces around levels 4 and 5.
+// There are never spaces at level 6 (unary), and always spaces at levels 3 and below.
+//
+// To choose the cutoff, look at the whole expression but excluding primary
+// expressions (function calls, parenthesized exprs), and apply these rules:
+//
+//	1) If there is a binary operator with a right side unary operand
+//	   that would clash without a space, the cutoff must be (in order):
+//
+//		/*	6
+//		&&	6
+//		&^	6
+//		++	5
+//		--	5
+//
+//         (Comparison operators always have spaces around them.)
+//
+//	2) If there is a mix of level 5 and level 4 operators, then the cutoff
+//	   is 5 (use spaces to distinguish precedence) in Normal mode
+//	   and 4 (never use spaces) in Compact mode.
+//
+//	3) If there are no level 4 operators or no level 5 operators, then the
+//	   cutoff is 6 (always use spaces) in Normal mode
+//	   and 4 (never use spaces) in Compact mode.
+//
+// Sets multiLine to true if the binary expression spans multiple lines.
+func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiLine *bool) {
+	prec := x.Op.Precedence()
+	if prec < prec1 {
+		// parenthesis needed
+		// Note: The parser inserts an ast.ParenExpr node; thus this case
+		//       can only occur if the AST is created in a different way.
+		p.print(token.LPAREN)
+		p.expr0(x, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+		p.print(token.RPAREN)
+		return
+	}
+
+	printBlank := prec < cutoff
+
+	ws := indent
+	p.expr1(x.X, prec, depth+diffPrec(x.X, prec), multiLine)
+	if printBlank {
+		p.print(blank)
+	}
+	xline := p.pos.Line // before the operator (it may be on the next line!)
+	yline := p.fset.Position(x.Y.Pos()).Line
+	p.print(x.OpPos, x.Op)
+	if xline != yline && xline > 0 && yline > 0 {
+		// at least one line break, but respect an extra empty line
+		// in the source
+		if p.linebreak(yline, 1, ws, true) {
+			ws = ignore
+			*multiLine = true
+			printBlank = false // no blank after line break
+		}
+	}
+	if printBlank {
+		p.print(blank)
+	}
+	p.expr1(x.Y, prec+1, depth+1, multiLine)
+	if ws == ignore {
+		p.print(unindent)
+	}
+}
+
+func isBinary(expr ast.Expr) bool {
+	_, ok := expr.(*ast.BinaryExpr)
+	return ok
+}
+
+// If the expression contains one or more selector expressions, splits it into
+// two expressions at the rightmost period. Writes entire expr to suffix when
+// selector isn't found. Rewrites AST nodes for calls, index expressions and
+// type assertions, all of which may be found in selector chains, to make them
+// parts of the chain.
+func splitSelector(expr ast.Expr) (body, suffix ast.Expr) {
+	switch x := expr.(type) {
+	case *ast.SelectorExpr:
+		body, suffix = x.X, x.Sel
+		return
+	case *ast.CallExpr:
+		body, suffix = splitSelector(x.Fun)
+		if body != nil {
+			suffix = &ast.CallExpr{suffix, x.Lparen, x.Args, x.Ellipsis, x.Rparen}
+			return
+		}
+	case *ast.IndexExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.IndexExpr{suffix, x.Lbrack, x.Index, x.Rbrack}
+			return
+		}
+	case *ast.SliceExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.SliceExpr{suffix, x.Lbrack, x.Low, x.High, x.Rbrack}
+			return
+		}
+	case *ast.TypeAssertExpr:
+		body, suffix = splitSelector(x.X)
+		if body != nil {
+			suffix = &ast.TypeAssertExpr{suffix, x.Type}
+			return
+		}
+	}
+	suffix = expr
+	return
+}
+
+// Convert an expression into an expression list split at the periods of
+// selector expressions.
+func selectorExprList(expr ast.Expr) (list []ast.Expr) {
+	// split expression
+	for expr != nil {
+		var suffix ast.Expr
+		expr, suffix = splitSelector(expr)
+		list = append(list, suffix)
+	}
+
+	// reverse list
+	for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 {
+		list[i], list[j] = list[j], list[i]
+	}
+
+	return
+}
+
+// Sets multiLine to true if the expression spans multiple lines.
+func (p *printer) expr1(expr ast.Expr, prec1, depth int, multiLine *bool) {
+	p.print(expr.Pos())
+
+	switch x := expr.(type) {
+	case *ast.BadExpr:
+		p.print("BadExpr")
+
+	case *ast.Ident:
+		p.print(x)
+
+	case *ast.BinaryExpr:
+		if depth < 1 {
+			p.internalError("depth < 1:", depth)
+			depth = 1
+		}
+		p.binaryExpr(x, prec1, cutoff(x, depth), depth, multiLine)
+
+	case *ast.KeyValueExpr:
+		p.expr(x.Key, multiLine)
+		p.print(x.Colon, token.COLON, blank)
+		p.expr(x.Value, multiLine)
+
+	case *ast.StarExpr:
+		const prec = token.UnaryPrec
+		if prec < prec1 {
+			// parenthesis needed
+			p.print(token.LPAREN)
+			p.print(token.MUL)
+			p.expr(x.X, multiLine)
+			p.print(token.RPAREN)
+		} else {
+			// no parenthesis needed
+			p.print(token.MUL)
+			p.expr(x.X, multiLine)
+		}
+
+	case *ast.UnaryExpr:
+		const prec = token.UnaryPrec
+		if prec < prec1 {
+			// parenthesis needed
+			p.print(token.LPAREN)
+			p.expr(x, multiLine)
+			p.print(token.RPAREN)
+		} else {
+			// no parenthesis needed
+			p.print(x.Op)
+			if x.Op == token.RANGE {
+				// TODO(gri) Remove this code if it cannot be reached.
+				p.print(blank)
+			}
+			p.expr1(x.X, prec, depth, multiLine)
+		}
+
+	case *ast.BasicLit:
+		p.print(x)
+
+	case *ast.FuncLit:
+		p.expr(x.Type, multiLine)
+		p.funcBody(x.Body, p.distance(x.Type.Pos(), p.pos), true, multiLine)
+
+	case *ast.ParenExpr:
+		if _, hasParens := x.X.(*ast.ParenExpr); hasParens {
+			// don't print parentheses around an already parenthesized expression
+			// TODO(gri) consider making this more general and incorporate precedence levels
+			p.expr0(x.X, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+		} else {
+			p.print(token.LPAREN)
+			p.expr0(x.X, reduceDepth(depth), multiLine) // parentheses undo one level of depth
+			p.print(x.Rparen, token.RPAREN)
+		}
+
+	case *ast.SelectorExpr:
+		parts := selectorExprList(expr)
+		p.exprList(token.NoPos, parts, depth, periodSep, multiLine, token.NoPos)
+
+	case *ast.TypeAssertExpr:
+		p.expr1(x.X, token.HighestPrec, depth, multiLine)
+		p.print(token.PERIOD, token.LPAREN)
+		if x.Type != nil {
+			p.expr(x.Type, multiLine)
+		} else {
+			p.print(token.TYPE)
+		}
+		p.print(token.RPAREN)
+
+	case *ast.IndexExpr:
+		// TODO(gri): should treat[] like parentheses and undo one level of depth
+		p.expr1(x.X, token.HighestPrec, 1, multiLine)
+		p.print(x.Lbrack, token.LBRACK)
+		p.expr0(x.Index, depth+1, multiLine)
+		p.print(x.Rbrack, token.RBRACK)
+
+	case *ast.SliceExpr:
+		// TODO(gri): should treat[] like parentheses and undo one level of depth
+		p.expr1(x.X, token.HighestPrec, 1, multiLine)
+		p.print(x.Lbrack, token.LBRACK)
+		if x.Low != nil {
+			p.expr0(x.Low, depth+1, multiLine)
+		}
+		// blanks around ":" if both sides exist and either side is a binary expression
+		if depth <= 1 && x.Low != nil && x.High != nil && (isBinary(x.Low) || isBinary(x.High)) {
+			p.print(blank, token.COLON, blank)
+		} else {
+			p.print(token.COLON)
+		}
+		if x.High != nil {
+			p.expr0(x.High, depth+1, multiLine)
+		}
+		p.print(x.Rbrack, token.RBRACK)
+
+	case *ast.CallExpr:
+		if len(x.Args) > 1 {
+			depth++
+		}
+		p.expr1(x.Fun, token.HighestPrec, depth, multiLine)
+		p.print(x.Lparen, token.LPAREN)
+		p.exprList(x.Lparen, x.Args, depth, commaSep|commaTerm, multiLine, x.Rparen)
+		if x.Ellipsis.IsValid() {
+			p.print(x.Ellipsis, token.ELLIPSIS)
+		}
+		p.print(x.Rparen, token.RPAREN)
+
+	case *ast.CompositeLit:
+		// composite literal elements that are composite literals themselves may have the type omitted
+		if x.Type != nil {
+			p.expr1(x.Type, token.HighestPrec, depth, multiLine)
+		}
+		p.print(x.Lbrace, token.LBRACE)
+		p.exprList(x.Lbrace, x.Elts, 1, commaSep|commaTerm, multiLine, x.Rbrace)
+		// do not insert extra line breaks because of comments before
+		// the closing '}' as it might break the code if there is no
+		// trailing ','
+		p.print(noExtraLinebreak, x.Rbrace, token.RBRACE, noExtraLinebreak)
+
+	case *ast.Ellipsis:
+		p.print(token.ELLIPSIS)
+		if x.Elt != nil {
+			p.expr(x.Elt, multiLine)
+		}
+
+	case *ast.ArrayType:
+		p.print(token.LBRACK)
+		if x.Len != nil {
+			p.expr(x.Len, multiLine)
+		}
+		p.print(token.RBRACK)
+		p.expr(x.Elt, multiLine)
+
+	case *ast.StructType:
+		p.print(token.STRUCT)
+		p.fieldList(x.Fields, true, x.Incomplete)
+
+	case *ast.FuncType:
+		p.print(token.FUNC)
+		p.signature(x.Params, x.Results, multiLine)
+
+	case *ast.InterfaceType:
+		p.print(token.INTERFACE)
+		p.fieldList(x.Methods, false, x.Incomplete)
+
+	case *ast.MapType:
+		p.print(token.MAP, token.LBRACK)
+		p.expr(x.Key, multiLine)
+		p.print(token.RBRACK)
+		p.expr(x.Value, multiLine)
+
+	case *ast.ChanType:
+		switch x.Dir {
+		case ast.SEND | ast.RECV:
+			p.print(token.CHAN)
+		case ast.RECV:
+			p.print(token.ARROW, token.CHAN)
+		case ast.SEND:
+			p.print(token.CHAN, token.ARROW)
+		}
+		p.print(blank)
+		p.expr(x.Value, multiLine)
+
+	default:
+		panic("unreachable")
+	}
+
+	return
+}
+
+func (p *printer) expr0(x ast.Expr, depth int, multiLine *bool) {
+	p.expr1(x, token.LowestPrec, depth, multiLine)
+}
+
+// Sets multiLine to true if the expression spans multiple lines.
+func (p *printer) expr(x ast.Expr, multiLine *bool) {
+	const depth = 1
+	p.expr1(x, token.LowestPrec, depth, multiLine)
+}
+
+// ----------------------------------------------------------------------------
+// Statements
+
+// Print the statement list indented, but without a newline after the last statement.
+// Extra line breaks between statements in the source are respected but at most one
+// empty line is printed between statements.
+func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) {
+	// TODO(gri): fix _indent code
+	if _indent > 0 {
+		p.print(indent)
+	}
+	var multiLine bool
+	for i, s := range list {
+		// _indent == 0 only for lists of switch/select case clauses;
+		// in those cases each clause is a new section
+		p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, i == 0 || _indent == 0 || multiLine)
+		multiLine = false
+		p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine)
+	}
+	if _indent > 0 {
+		p.print(unindent)
+	}
+}
+
+// block prints an *ast.BlockStmt; it always spans at least two lines.
+func (p *printer) block(s *ast.BlockStmt, indent int) {
+	p.print(s.Pos(), token.LBRACE)
+	p.stmtList(s.List, indent, true)
+	p.linebreak(p.fset.Position(s.Rbrace).Line, 1, ignore, true)
+	p.print(s.Rbrace, token.RBRACE)
+}
+
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.Ident:
+		return true
+	case *ast.SelectorExpr:
+		return isTypeName(t.X)
+	}
+	return false
+}
+
+func stripParens(x ast.Expr) ast.Expr {
+	if px, strip := x.(*ast.ParenExpr); strip {
+		// parentheses must not be stripped if there are any
+		// unparenthesized composite literals starting with
+		// a type name
+		ast.Inspect(px.X, func(node ast.Node) bool {
+			switch x := node.(type) {
+			case *ast.ParenExpr:
+				// parentheses protect enclosed composite literals
+				return false
+			case *ast.CompositeLit:
+				if isTypeName(x.Type) {
+					strip = false // do not strip parentheses
+				}
+				return false
+			}
+			// in all other cases, keep inspecting
+			return true
+		})
+		if strip {
+			return stripParens(px.X)
+		}
+	}
+	return x
+}
+
+func (p *printer) controlClause(isForStmt bool, init ast.Stmt, expr ast.Expr, post ast.Stmt) {
+	p.print(blank)
+	needsBlank := false
+	if init == nil && post == nil {
+		// no semicolons required
+		if expr != nil {
+			p.expr(stripParens(expr), ignoreMultiLine)
+			needsBlank = true
+		}
+	} else {
+		// all semicolons required
+		// (they are not separators, print them explicitly)
+		if init != nil {
+			p.stmt(init, false, ignoreMultiLine)
+		}
+		p.print(token.SEMICOLON, blank)
+		if expr != nil {
+			p.expr(stripParens(expr), ignoreMultiLine)
+			needsBlank = true
+		}
+		if isForStmt {
+			p.print(token.SEMICOLON, blank)
+			needsBlank = false
+			if post != nil {
+				p.stmt(post, false, ignoreMultiLine)
+				needsBlank = true
+			}
+		}
+	}
+	if needsBlank {
+		p.print(blank)
+	}
+}
+
+// Sets multiLine to true if the statements spans multiple lines.
+func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) {
+	p.print(stmt.Pos())
+
+	switch s := stmt.(type) {
+	case *ast.BadStmt:
+		p.print("BadStmt")
+
+	case *ast.DeclStmt:
+		p.decl(s.Decl, multiLine)
+
+	case *ast.EmptyStmt:
+		// nothing to do
+
+	case *ast.LabeledStmt:
+		// a "correcting" unindent immediately following a line break
+		// is applied before the line break if there is no comment
+		// between (see writeWhitespace)
+		p.print(unindent)
+		p.expr(s.Label, multiLine)
+		p.print(s.Colon, token.COLON, indent)
+		if e, isEmpty := s.Stmt.(*ast.EmptyStmt); isEmpty {
+			if !nextIsRBrace {
+				p.print(newline, e.Pos(), token.SEMICOLON)
+				break
+			}
+		} else {
+			p.linebreak(p.fset.Position(s.Stmt.Pos()).Line, 1, ignore, true)
+		}
+		p.stmt(s.Stmt, nextIsRBrace, multiLine)
+
+	case *ast.ExprStmt:
+		const depth = 1
+		p.expr0(s.X, depth, multiLine)
+
+	case *ast.SendStmt:
+		const depth = 1
+		p.expr0(s.Chan, depth, multiLine)
+		p.print(blank, s.Arrow, token.ARROW, blank)
+		p.expr0(s.Value, depth, multiLine)
+
+	case *ast.IncDecStmt:
+		const depth = 1
+		p.expr0(s.X, depth+1, multiLine)
+		p.print(s.TokPos, s.Tok)
+
+	case *ast.AssignStmt:
+		var depth = 1
+		if len(s.Lhs) > 1 && len(s.Rhs) > 1 {
+			depth++
+		}
+		p.exprList(s.Pos(), s.Lhs, depth, commaSep, multiLine, s.TokPos)
+		p.print(blank, s.TokPos, s.Tok)
+		p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, token.NoPos)
+
+	case *ast.GoStmt:
+		p.print(token.GO, blank)
+		p.expr(s.Call, multiLine)
+
+	case *ast.DeferStmt:
+		p.print(token.DEFER, blank)
+		p.expr(s.Call, multiLine)
+
+	case *ast.ReturnStmt:
+		p.print(token.RETURN)
+		if s.Results != nil {
+			p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, token.NoPos)
+		}
+
+	case *ast.BranchStmt:
+		p.print(s.Tok)
+		if s.Label != nil {
+			p.print(blank)
+			p.expr(s.Label, multiLine)
+		}
+
+	case *ast.BlockStmt:
+		p.block(s, 1)
+		*multiLine = true
+
+	case *ast.IfStmt:
+		p.print(token.IF)
+		p.controlClause(false, s.Init, s.Cond, nil)
+		p.block(s.Body, 1)
+		*multiLine = true
+		if s.Else != nil {
+			p.print(blank, token.ELSE, blank)
+			switch s.Else.(type) {
+			case *ast.BlockStmt, *ast.IfStmt:
+				p.stmt(s.Else, nextIsRBrace, ignoreMultiLine)
+			default:
+				p.print(token.LBRACE, indent, formfeed)
+				p.stmt(s.Else, true, ignoreMultiLine)
+				p.print(unindent, formfeed, token.RBRACE)
+			}
+		}
+
+	case *ast.CaseClause:
+		if s.List != nil {
+			p.print(token.CASE)
+			p.exprList(s.Pos(), s.List, 1, blankStart|commaSep, multiLine, s.Colon)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.SwitchStmt:
+		p.print(token.SWITCH)
+		p.controlClause(false, s.Init, s.Tag, nil)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.TypeSwitchStmt:
+		p.print(token.SWITCH)
+		if s.Init != nil {
+			p.print(blank)
+			p.stmt(s.Init, false, ignoreMultiLine)
+			p.print(token.SEMICOLON)
+		}
+		p.print(blank)
+		p.stmt(s.Assign, false, ignoreMultiLine)
+		p.print(blank)
+		p.block(s.Body, 0)
+		*multiLine = true
+
+	case *ast.CommClause:
+		if s.Comm != nil {
+			p.print(token.CASE, blank)
+			p.stmt(s.Comm, false, ignoreMultiLine)
+		} else {
+			p.print(token.DEFAULT)
+		}
+		p.print(s.Colon, token.COLON)
+		p.stmtList(s.Body, 1, nextIsRBrace)
+
+	case *ast.SelectStmt:
+		p.print(token.SELECT, blank)
+		body := s.Body
+		if len(body.List) == 0 && !p.commentBefore(p.fset.Position(body.Rbrace)) {
+			// print empty select statement w/o comments on one line
+			p.print(body.Lbrace, token.LBRACE, body.Rbrace, token.RBRACE)
+		} else {
+			p.block(body, 0)
+			*multiLine = true
+		}
+
+	case *ast.ForStmt:
+		p.print(token.FOR)
+		p.controlClause(true, s.Init, s.Cond, s.Post)
+		p.block(s.Body, 1)
+		*multiLine = true
+
+	case *ast.RangeStmt:
+		p.print(token.FOR, blank)
+		p.expr(s.Key, multiLine)
+		if s.Value != nil {
+			p.print(token.COMMA, blank)
+			p.expr(s.Value, multiLine)
+		}
+		p.print(blank, s.TokPos, s.Tok, blank, token.RANGE, blank)
+		p.expr(stripParens(s.X), multiLine)
+		p.print(blank)
+		p.block(s.Body, 1)
+		*multiLine = true
+
+	default:
+		panic("unreachable")
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+// The keepTypeColumn function determines if the type column of a series of
+// consecutive const or var declarations must be kept, or if initialization
+// values (V) can be placed in the type column (T) instead. The i'th entry
+// in the result slice is true if the type column in spec[i] must be kept.
+//
+// For example, the declaration:
+//
+//	const (
+//		foobar int = 42 // comment
+//		x          = 7  // comment
+//		foo
+//              bar = 991
+//	)
+//
+// leads to the type/values matrix below. A run of value columns (V) can
+// be moved into the type column if there is no type for any of the values
+// in that column (we only move entire columns so that they align properly).
+//
+//	matrix        formatted     result
+//                    matrix
+//	T  V    ->    T  V     ->   true      there is a T and so the type
+//	-  V          -  V          true      column must be kept
+//	-  -          -  -          false
+//	-  V          V  -          false     V is moved into T column
+//
+func keepTypeColumn(specs []ast.Spec) []bool {
+	m := make([]bool, len(specs))
+
+	populate := func(i, j int, keepType bool) {
+		if keepType {
+			for ; i < j; i++ {
+				m[i] = true
+			}
+		}
+	}
+
+	i0 := -1 // if i0 >= 0 we are in a run and i0 is the start of the run
+	var keepType bool
+	for i, s := range specs {
+		t := s.(*ast.ValueSpec)
+		if t.Values != nil {
+			if i0 < 0 {
+				// start of a run of ValueSpecs with non-nil Values
+				i0 = i
+				keepType = false
+			}
+		} else {
+			if i0 >= 0 {
+				// end of a run
+				populate(i0, i, keepType)
+				i0 = -1
+			}
+		}
+		if t.Type != nil {
+			keepType = true
+		}
+	}
+	if i0 >= 0 {
+		// end of a run
+		populate(i0, len(specs), keepType)
+	}
+
+	return m
+}
+
+func (p *printer) valueSpec(s *ast.ValueSpec, keepType, doIndent bool, multiLine *bool) {
+	p.setComment(s.Doc)
+	p.identList(s.Names, doIndent, multiLine) // always present
+	extraTabs := 3
+	if s.Type != nil || keepType {
+		p.print(vtab)
+		extraTabs--
+	}
+	if s.Type != nil {
+		p.expr(s.Type, multiLine)
+	}
+	if s.Values != nil {
+		p.print(vtab, token.ASSIGN)
+		p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
+		extraTabs--
+	}
+	if s.Comment != nil {
+		for ; extraTabs > 0; extraTabs-- {
+			p.print(vtab)
+		}
+		p.setComment(s.Comment)
+	}
+}
+
+// The parameter n is the number of specs in the group. If doIndent is set,
+// multi-line identifier lists in the spec are indented when the first
+// linebreak is encountered.
+// Sets multiLine to true if the spec spans multiple lines.
+//
+func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) {
+	switch s := spec.(type) {
+	case *ast.ImportSpec:
+		p.setComment(s.Doc)
+		if s.Name != nil {
+			p.expr(s.Name, multiLine)
+			p.print(blank)
+		}
+		p.expr(s.Path, multiLine)
+		p.setComment(s.Comment)
+
+	case *ast.ValueSpec:
+		if n != 1 {
+			p.internalError("expected n = 1; got", n)
+		}
+		p.setComment(s.Doc)
+		p.identList(s.Names, doIndent, multiLine) // always present
+		if s.Type != nil {
+			p.print(blank)
+			p.expr(s.Type, multiLine)
+		}
+		if s.Values != nil {
+			p.print(blank, token.ASSIGN)
+			p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
+		}
+		p.setComment(s.Comment)
+
+	case *ast.TypeSpec:
+		p.setComment(s.Doc)
+		p.expr(s.Name, multiLine)
+		if n == 1 {
+			p.print(blank)
+		} else {
+			p.print(vtab)
+		}
+		p.expr(s.Type, multiLine)
+		p.setComment(s.Comment)
+
+	default:
+		panic("unreachable")
+	}
+}
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) {
+	p.setComment(d.Doc)
+	p.print(d.Pos(), d.Tok, blank)
+
+	if d.Lparen.IsValid() {
+		// group of parenthesized declarations
+		p.print(d.Lparen, token.LPAREN)
+		if n := len(d.Specs); n > 0 {
+			p.print(indent, formfeed)
+			if n > 1 && (d.Tok == token.CONST || d.Tok == token.VAR) {
+				// two or more grouped const/var declarations:
+				// determine if the type column must be kept
+				keepType := keepTypeColumn(d.Specs)
+				var ml bool
+				for i, s := range d.Specs {
+					if i > 0 {
+						p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
+					}
+					ml = false
+					p.valueSpec(s.(*ast.ValueSpec), keepType[i], false, &ml)
+				}
+			} else {
+				var ml bool
+				for i, s := range d.Specs {
+					if i > 0 {
+						p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
+					}
+					ml = false
+					p.spec(s, n, false, &ml)
+				}
+			}
+			p.print(unindent, formfeed)
+			*multiLine = true
+		}
+		p.print(d.Rparen, token.RPAREN)
+
+	} else {
+		// single declaration
+		p.spec(d.Specs[0], 1, true, multiLine)
+	}
+}
+
+// nodeSize determines the size of n in chars after formatting.
+// The result is <= maxSize if the node fits on one line with at
+// most maxSize chars and the formatted output doesn't contain
+// any control chars. Otherwise, the result is > maxSize.
+//
+func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) {
+	// nodeSize invokes the printer, which may invoke nodeSize
+	// recursively. For deep composite literal nests, this can
+	// lead to an exponential algorithm. Remember previous
+	// results to prune the recursion (was issue 1628).
+	if size, found := p.nodeSizes[n]; found {
+		return size
+	}
+
+	size = maxSize + 1 // assume n doesn't fit
+	p.nodeSizes[n] = size
+
+	// nodeSize computation must be independent of particular
+	// style so that we always get the same decision; print
+	// in RawFormat
+	cfg := Config{Mode: RawFormat}
+	var buf bytes.Buffer
+	if _, err := cfg.fprint(&buf, p.fset, n, p.nodeSizes); err != nil {
+		return
+	}
+	if buf.Len() <= maxSize {
+		for _, ch := range buf.Bytes() {
+			if ch < ' ' {
+				return
+			}
+		}
+		size = buf.Len() // n fits
+		p.nodeSizes[n] = size
+	}
+	return
+}
+
+func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool {
+	pos1 := b.Pos()
+	pos2 := b.Rbrace
+	if pos1.IsValid() && pos2.IsValid() && p.fset.Position(pos1).Line != p.fset.Position(pos2).Line {
+		// opening and closing brace are on different lines - don't make it a one-liner
+		return false
+	}
+	if len(b.List) > 5 || p.commentBefore(p.fset.Position(pos2)) {
+		// too many statements or there is a comment inside - don't make it a one-liner
+		return false
+	}
+	// otherwise, estimate body size
+	const maxSize = 100
+	bodySize := 0
+	for i, s := range b.List {
+		if i > 0 {
+			bodySize += 2 // space for a semicolon and blank
+		}
+		bodySize += p.nodeSize(s, maxSize)
+	}
+	return headerSize+bodySize <= maxSize
+}
+
+// Sets multiLine to true if the function body spans multiple lines.
+func (p *printer) funcBody(b *ast.BlockStmt, headerSize int, isLit bool, multiLine *bool) {
+	if b == nil {
+		return
+	}
+
+	if p.isOneLineFunc(b, headerSize) {
+		sep := vtab
+		if isLit {
+			sep = blank
+		}
+		p.print(sep, b.Lbrace, token.LBRACE)
+		if len(b.List) > 0 {
+			p.print(blank)
+			for i, s := range b.List {
+				if i > 0 {
+					p.print(token.SEMICOLON, blank)
+				}
+				p.stmt(s, i == len(b.List)-1, ignoreMultiLine)
+			}
+			p.print(blank)
+		}
+		p.print(b.Rbrace, token.RBRACE)
+		return
+	}
+
+	p.print(blank)
+	p.block(b, 1)
+	*multiLine = true
+}
+
+// distance returns the column difference between from and to if both
+// are on the same line; if they are on different lines (or unknown)
+// the result is infinity.
+func (p *printer) distance(from0 token.Pos, to token.Position) int {
+	from := p.fset.Position(from0)
+	if from.IsValid() && to.IsValid() && from.Line == to.Line {
+		return to.Column - from.Column
+	}
+	return infinity
+}
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) funcDecl(d *ast.FuncDecl, multiLine *bool) {
+	p.setComment(d.Doc)
+	p.print(d.Pos(), token.FUNC, blank)
+	if d.Recv != nil {
+		p.parameters(d.Recv, multiLine) // method: print receiver
+		p.print(blank)
+	}
+	p.expr(d.Name, multiLine)
+	p.signature(d.Type.Params, d.Type.Results, multiLine)
+	p.funcBody(d.Body, p.distance(d.Pos(), p.pos), false, multiLine)
+}
+
+// Sets multiLine to true if the declaration spans multiple lines.
+func (p *printer) decl(decl ast.Decl, multiLine *bool) {
+	switch d := decl.(type) {
+	case *ast.BadDecl:
+		p.print(d.Pos(), "BadDecl")
+	case *ast.GenDecl:
+		p.genDecl(d, multiLine)
+	case *ast.FuncDecl:
+		p.funcDecl(d, multiLine)
+	default:
+		panic("unreachable")
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Files
+
+func declToken(decl ast.Decl) (tok token.Token) {
+	tok = token.ILLEGAL
+	switch d := decl.(type) {
+	case *ast.GenDecl:
+		tok = d.Tok
+	case *ast.FuncDecl:
+		tok = token.FUNC
+	}
+	return
+}
+
+func (p *printer) file(src *ast.File) {
+	p.setComment(src.Doc)
+	p.print(src.Pos(), token.PACKAGE, blank)
+	p.expr(src.Name, ignoreMultiLine)
+
+	if len(src.Decls) > 0 {
+		tok := token.ILLEGAL
+		for _, d := range src.Decls {
+			prev := tok
+			tok = declToken(d)
+			// if the declaration token changed (e.g., from CONST to TYPE)
+			// print an empty line between top-level declarations
+			min := 1
+			if prev != tok {
+				min = 2
+			}
+			p.linebreak(p.fset.Position(d.Pos()).Line, min, ignore, false)
+			p.decl(d, ignoreMultiLine)
+		}
+	}
+
+	p.print(newline)
+}
diff --git a/src/pkg/go/printer/performance_test.go b/src/pkg/go/printer/performance_test.go
new file mode 100644
index 000000000..84fb2808e
--- /dev/null
+++ b/src/pkg/go/printer/performance_test.go
@@ -0,0 +1,58 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a simple printer performance benchmark:
+// gotest -bench=BenchmarkPrint 
+
+package printer
+
+import (
+	"bytes"
+	"go/ast"
+	"go/parser"
+	"io"
+	"io/ioutil"
+	"log"
+	"testing"
+)
+
+var testfile *ast.File
+
+func testprint(out io.Writer, file *ast.File) {
+	if _, err := (&Config{TabIndent | UseSpaces, 8}).Fprint(out, fset, file); err != nil {
+		log.Fatalf("print error: %s", err)
+	}
+}
+
+// cannot initialize in init because (printer) Fprint launches goroutines.
+func initialize() {
+	const filename = "testdata/parser.go"
+
+	src, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Fatalf("%s", err)
+	}
+
+	file, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
+	if err != nil {
+		log.Fatalf("%s", err)
+	}
+
+	var buf bytes.Buffer
+	testprint(&buf, file)
+	if !bytes.Equal(buf.Bytes(), src) {
+		log.Fatalf("print error: %s not idempotent", filename)
+	}
+
+	testfile = file
+}
+
+func BenchmarkPrint(b *testing.B) {
+	if testfile == nil {
+		initialize()
+	}
+	for i := 0; i < b.N; i++ {
+		testprint(ioutil.Discard, testfile)
+	}
+}
diff --git a/src/pkg/go/printer/printer.go b/src/pkg/go/printer/printer.go
new file mode 100644
index 000000000..871fefa0c
--- /dev/null
+++ b/src/pkg/go/printer/printer.go
@@ -0,0 +1,1012 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package printer implements printing of AST nodes.
+package printer
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"tabwriter"
+)
+
+const debug = false // enable for debugging
+
+
+type whiteSpace int
+
+const (
+	ignore   = whiteSpace(0)
+	blank    = whiteSpace(' ')
+	vtab     = whiteSpace('\v')
+	newline  = whiteSpace('\n')
+	formfeed = whiteSpace('\f')
+	indent   = whiteSpace('>')
+	unindent = whiteSpace('<')
+)
+
+var (
+	esc       = []byte{tabwriter.Escape}
+	htab      = []byte{'\t'}
+	htabs     = []byte("\t\t\t\t\t\t\t\t")
+	newlines  = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines
+	formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines
+)
+
+// Special positions
+var noPos token.Position // use noPos when a position is needed but not known
+var infinity = 1 << 30
+
+// Use ignoreMultiLine if the multiLine information is not important.
+var ignoreMultiLine = new(bool)
+
+// A pmode value represents the current printer mode.
+type pmode int
+
+const (
+	inLiteral pmode = 1 << iota
+	noExtraLinebreak
+)
+
+type printer struct {
+	// Configuration (does not change after initialization)
+	output io.Writer
+	Config
+	fset   *token.FileSet
+	errors chan os.Error
+
+	// Current state
+	written int         // number of bytes written
+	indent  int         // current indentation
+	mode    pmode       // current printer mode
+	lastTok token.Token // the last token printed (token.ILLEGAL if it's whitespace)
+
+	// Reused buffers
+	wsbuf  []whiteSpace // delayed white space
+	litbuf bytes.Buffer // for creation of escaped literals and comments
+
+	// The (possibly estimated) position in the generated output;
+	// in AST space (i.e., pos is set whenever a token position is
+	// known accurately, and updated dependending on what has been
+	// written).
+	pos token.Position
+
+	// The value of pos immediately after the last item has been
+	// written using writeItem.
+	last token.Position
+
+	// The list of all source comments, in order of appearance.
+	comments        []*ast.CommentGroup // may be nil
+	cindex          int                 // current comment index
+	useNodeComments bool                // if not set, ignore lead and line comments of nodes
+
+	// Cache of already computed node sizes.
+	nodeSizes map[ast.Node]int
+}
+
+func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet, nodeSizes map[ast.Node]int) {
+	p.output = output
+	p.Config = *cfg
+	p.fset = fset
+	p.errors = make(chan os.Error)
+	p.wsbuf = make([]whiteSpace, 0, 16) // whitespace sequences are short
+	p.nodeSizes = nodeSizes
+}
+
+func (p *printer) internalError(msg ...interface{}) {
+	if debug {
+		fmt.Print(p.pos.String() + ": ")
+		fmt.Println(msg...)
+		panic("go/printer")
+	}
+}
+
+// escape escapes string s by bracketing it with tabwriter.Escape.
+// Escaped strings pass through tabwriter unchanged. (Note that
+// valid Go programs cannot contain tabwriter.Escape bytes since
+// they do not appear in legal UTF-8 sequences).
+//
+func (p *printer) escape(s string) string {
+	p.litbuf.Reset()
+	p.litbuf.WriteByte(tabwriter.Escape)
+	p.litbuf.WriteString(s)
+	p.litbuf.WriteByte(tabwriter.Escape)
+	return p.litbuf.String()
+}
+
+// nlines returns the adjusted number of linebreaks given the desired number
+// of breaks n such that min <= result <= max.
+//
+func (p *printer) nlines(n, min int) int {
+	const max = 2 // max. number of newlines
+	switch {
+	case n < min:
+		return min
+	case n > max:
+		return max
+	}
+	return n
+}
+
+// write0 writes raw (uninterpreted) data to p.output and handles errors.
+// write0 does not indent after newlines, and does not HTML-escape or update p.pos.
+//
+func (p *printer) write0(data []byte) {
+	if len(data) > 0 {
+		n, err := p.output.Write(data)
+		p.written += n
+		if err != nil {
+			p.errors <- err
+			runtime.Goexit()
+		}
+	}
+}
+
+// write interprets data and writes it to p.output. It inserts indentation
+// after a line break unless in a tabwriter escape sequence.
+// It updates p.pos as a side-effect.
+//
+func (p *printer) write(data []byte) {
+	i0 := 0
+	for i, b := range data {
+		switch b {
+		case '\n', '\f':
+			// write segment ending in b
+			p.write0(data[i0 : i+1])
+
+			// update p.pos
+			p.pos.Offset += i + 1 - i0
+			p.pos.Line++
+			p.pos.Column = 1
+
+			if p.mode&inLiteral == 0 {
+				// write indentation
+				// use "hard" htabs - indentation columns
+				// must not be discarded by the tabwriter
+				j := p.indent
+				for ; j > len(htabs); j -= len(htabs) {
+					p.write0(htabs)
+				}
+				p.write0(htabs[0:j])
+
+				// update p.pos
+				p.pos.Offset += p.indent
+				p.pos.Column += p.indent
+			}
+
+			// next segment start
+			i0 = i + 1
+
+		case tabwriter.Escape:
+			p.mode ^= inLiteral
+
+			// ignore escape chars introduced by printer - they are
+			// invisible and must not affect p.pos (was issue #1089)
+			p.pos.Offset--
+			p.pos.Column--
+		}
+	}
+
+	// write remaining segment
+	p.write0(data[i0:])
+
+	// update p.pos
+	d := len(data) - i0
+	p.pos.Offset += d
+	p.pos.Column += d
+}
+
+func (p *printer) writeNewlines(n int, useFF bool) {
+	if n > 0 {
+		n = p.nlines(n, 0)
+		if useFF {
+			p.write(formfeeds[0:n])
+		} else {
+			p.write(newlines[0:n])
+		}
+	}
+}
+
+// writeItem writes data at position pos. data is the text corresponding to
+// a single lexical token, but may also be comment text. pos is the actual
+// (or at least very accurately estimated) position of the data in the original
+// source text. writeItem updates p.last to the position immediately following
+// the data.
+//
+func (p *printer) writeItem(pos token.Position, data string) {
+	if pos.IsValid() {
+		// continue with previous position if we don't have a valid pos
+		if p.last.IsValid() && p.last.Filename != pos.Filename {
+			// the file has changed - reset state
+			// (used when printing merged ASTs of different files
+			// e.g., the result of ast.MergePackageFiles)
+			p.indent = 0
+			p.mode = 0
+			p.wsbuf = p.wsbuf[0:0]
+		}
+		p.pos = pos
+	}
+	if debug {
+		// do not update p.pos - use write0
+		_, filename := filepath.Split(pos.Filename)
+		p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column)))
+	}
+	p.write([]byte(data))
+	p.last = p.pos
+}
+
+// writeCommentPrefix writes the whitespace before a comment.
+// If there is any pending whitespace, it consumes as much of
+// it as is likely to help position the comment nicely.
+// pos is the comment position, next the position of the item
+// after all pending comments, prev is the previous comment in
+// a group of comments (or nil), and isKeyword indicates if the
+// next item is a keyword.
+//
+func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, isKeyword bool) {
+	if p.written == 0 {
+		// the comment is the first item to be printed - don't write any whitespace
+		return
+	}
+
+	if pos.IsValid() && pos.Filename != p.last.Filename {
+		// comment in a different file - separate with newlines (writeNewlines will limit the number)
+		p.writeNewlines(10, true)
+		return
+	}
+
+	if pos.Line == p.last.Line && (prev == nil || prev.Text[1] != '/') {
+		// comment on the same line as last item:
+		// separate with at least one separator
+		hasSep := false
+		if prev == nil {
+			// first comment of a comment group
+			j := 0
+			for i, ch := range p.wsbuf {
+				switch ch {
+				case blank:
+					// ignore any blanks before a comment
+					p.wsbuf[i] = ignore
+					continue
+				case vtab:
+					// respect existing tabs - important
+					// for proper formatting of commented structs
+					hasSep = true
+					continue
+				case indent:
+					// apply pending indentation
+					continue
+				}
+				j = i
+				break
+			}
+			p.writeWhitespace(j)
+		}
+		// make sure there is at least one separator
+		if !hasSep {
+			if pos.Line == next.Line {
+				// next item is on the same line as the comment
+				// (which must be a /*-style comment): separate
+				// with a blank instead of a tab
+				p.write([]byte{' '})
+			} else {
+				p.write(htab)
+			}
+		}
+
+	} else {
+		// comment on a different line:
+		// separate with at least one line break
+		if prev == nil {
+			// first comment of a comment group
+			j := 0
+			for i, ch := range p.wsbuf {
+				switch ch {
+				case blank, vtab:
+					// ignore any horizontal whitespace before line breaks
+					p.wsbuf[i] = ignore
+					continue
+				case indent:
+					// apply pending indentation
+					continue
+				case unindent:
+					// if the next token is a keyword, apply the outdent
+					// if it appears that the comment is aligned with the
+					// keyword; otherwise assume the outdent is part of a
+					// closing block and stop (this scenario appears with
+					// comments before a case label where the comments
+					// apply to the next case instead of the current one)
+					if isKeyword && pos.Column == next.Column {
+						continue
+					}
+				case newline, formfeed:
+					// TODO(gri): may want to keep formfeed info in some cases
+					p.wsbuf[i] = ignore
+				}
+				j = i
+				break
+			}
+			p.writeWhitespace(j)
+		}
+		// use formfeeds to break columns before a comment;
+		// this is analogous to using formfeeds to separate
+		// individual lines of /*-style comments - but make
+		// sure there is at least one line break if the previous
+		// comment was a line comment
+		n := pos.Line - p.last.Line // if !pos.IsValid(), pos.Line == 0, and n will be 0
+		if n <= 0 && prev != nil && prev.Text[1] == '/' {
+			n = 1
+		}
+		p.writeNewlines(n, true)
+	}
+}
+
+// TODO(gri): It should be possible to convert the code below from using
+//            []byte to string and in the process eliminate some conversions.
+
+// Split comment text into lines
+func split(text []byte) [][]byte {
+	// count lines (comment text never ends in a newline)
+	n := 1
+	for _, c := range text {
+		if c == '\n' {
+			n++
+		}
+	}
+
+	// split
+	lines := make([][]byte, n)
+	n = 0
+	i := 0
+	for j, c := range text {
+		if c == '\n' {
+			lines[n] = text[i:j] // exclude newline
+			i = j + 1            // discard newline
+			n++
+		}
+	}
+	lines[n] = text[i:]
+
+	return lines
+}
+
+func isBlank(s []byte) bool {
+	for _, b := range s {
+		if b > ' ' {
+			return false
+		}
+	}
+	return true
+}
+
+func commonPrefix(a, b []byte) []byte {
+	i := 0
+	for i < len(a) && i < len(b) && a[i] == b[i] && (a[i] <= ' ' || a[i] == '*') {
+		i++
+	}
+	return a[0:i]
+}
+
+func stripCommonPrefix(lines [][]byte) {
+	if len(lines) < 2 {
+		return // at most one line - nothing to do
+	}
+	// len(lines) >= 2
+
+	// The heuristic in this function tries to handle a few
+	// common patterns of /*-style comments: Comments where
+	// the opening /* and closing */ are aligned and the
+	// rest of the comment text is aligned and indented with
+	// blanks or tabs, cases with a vertical "line of stars"
+	// on the left, and cases where the closing */ is on the
+	// same line as the last comment text.
+
+	// Compute maximum common white prefix of all but the first,
+	// last, and blank lines, and replace blank lines with empty
+	// lines (the first line starts with /* and has no prefix).
+	// In case of two-line comments, consider the last line for
+	// the prefix computation since otherwise the prefix would
+	// be empty.
+	//
+	// Note that the first and last line are never empty (they
+	// contain the opening /* and closing */ respectively) and
+	// thus they can be ignored by the blank line check.
+	var prefix []byte
+	if len(lines) > 2 {
+		for i, line := range lines[1 : len(lines)-1] {
+			switch {
+			case isBlank(line):
+				lines[1+i] = nil // range starts at line 1
+			case prefix == nil:
+				prefix = commonPrefix(line, line)
+			default:
+				prefix = commonPrefix(prefix, line)
+			}
+		}
+	} else { // len(lines) == 2
+		line := lines[1]
+		prefix = commonPrefix(line, line)
+	}
+
+	/*
+	 * Check for vertical "line of stars" and correct prefix accordingly.
+	 */
+	lineOfStars := false
+	if i := bytes.Index(prefix, []byte{'*'}); i >= 0 {
+		// Line of stars present.
+		if i > 0 && prefix[i-1] == ' ' {
+			i-- // remove trailing blank from prefix so stars remain aligned
+		}
+		prefix = prefix[0:i]
+		lineOfStars = true
+	} else {
+		// No line of stars present.
+		// Determine the white space on the first line after the /*
+		// and before the beginning of the comment text, assume two
+		// blanks instead of the /* unless the first character after
+		// the /* is a tab. If the first comment line is empty but
+		// for the opening /*, assume up to 3 blanks or a tab. This
+		// whitespace may be found as suffix in the common prefix.
+		first := lines[0]
+		if isBlank(first[2:]) {
+			// no comment text on the first line:
+			// reduce prefix by up to 3 blanks or a tab
+			// if present - this keeps comment text indented
+			// relative to the /* and */'s if it was indented
+			// in the first place
+			i := len(prefix)
+			for n := 0; n < 3 && i > 0 && prefix[i-1] == ' '; n++ {
+				i--
+			}
+			if i == len(prefix) && i > 0 && prefix[i-1] == '\t' {
+				i--
+			}
+			prefix = prefix[0:i]
+		} else {
+			// comment text on the first line
+			suffix := make([]byte, len(first))
+			n := 2 // start after opening /*
+			for n < len(first) && first[n] <= ' ' {
+				suffix[n] = first[n]
+				n++
+			}
+			if n > 2 && suffix[2] == '\t' {
+				// assume the '\t' compensates for the /*
+				suffix = suffix[2:n]
+			} else {
+				// otherwise assume two blanks
+				suffix[0], suffix[1] = ' ', ' '
+				suffix = suffix[0:n]
+			}
+			// Shorten the computed common prefix by the length of
+			// suffix, if it is found as suffix of the prefix.
+			if bytes.HasSuffix(prefix, suffix) {
+				prefix = prefix[0 : len(prefix)-len(suffix)]
+			}
+		}
+	}
+
+	// Handle last line: If it only contains a closing */, align it
+	// with the opening /*, otherwise align the text with the other
+	// lines.
+	last := lines[len(lines)-1]
+	closing := []byte("*/")
+	i := bytes.Index(last, closing)
+	if isBlank(last[0:i]) {
+		// last line only contains closing */
+		var sep []byte
+		if lineOfStars {
+			// insert an aligning blank
+			sep = []byte{' '}
+		}
+		lines[len(lines)-1] = bytes.Join([][]byte{prefix, closing}, sep)
+	} else {
+		// last line contains more comment text - assume
+		// it is aligned like the other lines
+		prefix = commonPrefix(prefix, last)
+	}
+
+	// Remove the common prefix from all but the first and empty lines.
+	for i, line := range lines[1:] {
+		if len(line) != 0 {
+			lines[1+i] = line[len(prefix):] // range starts at line 1
+		}
+	}
+}
+
+func (p *printer) writeComment(comment *ast.Comment) {
+	text := comment.Text
+
+	// shortcut common case of //-style comments
+	if text[1] == '/' {
+		p.writeItem(p.fset.Position(comment.Pos()), p.escape(text))
+		return
+	}
+
+	// for /*-style comments, print line by line and let the
+	// write function take care of the proper indentation
+	lines := split([]byte(text))
+	stripCommonPrefix(lines)
+
+	// write comment lines, separated by formfeed,
+	// without a line break after the last line
+	linebreak := formfeeds[0:1]
+	pos := p.fset.Position(comment.Pos())
+	for i, line := range lines {
+		if i > 0 {
+			p.write(linebreak)
+			pos = p.pos
+		}
+		if len(line) > 0 {
+			p.writeItem(pos, p.escape(string(line)))
+		}
+	}
+}
+
+// writeCommentSuffix writes a line break after a comment if indicated
+// and processes any leftover indentation information. If a line break
+// is needed, the kind of break (newline vs formfeed) depends on the
+// pending whitespace. writeCommentSuffix returns true if a pending
+// formfeed was dropped from the whitespace buffer.
+//
+func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) {
+	for i, ch := range p.wsbuf {
+		switch ch {
+		case blank, vtab:
+			// ignore trailing whitespace
+			p.wsbuf[i] = ignore
+		case indent, unindent:
+			// don't lose indentation information
+		case newline, formfeed:
+			// if we need a line break, keep exactly one
+			// but remember if we dropped any formfeeds
+			if needsLinebreak {
+				needsLinebreak = false
+			} else {
+				if ch == formfeed {
+					droppedFF = true
+				}
+				p.wsbuf[i] = ignore
+			}
+		}
+	}
+	p.writeWhitespace(len(p.wsbuf))
+
+	// make sure we have a line break
+	if needsLinebreak {
+		p.write([]byte{'\n'})
+	}
+
+	return
+}
+
+// intersperseComments consumes all comments that appear before the next token
+// tok and prints it together with the buffered whitespace (i.e., the whitespace
+// that needs to be written before the next token). A heuristic is used to mix
+// the comments and whitespace. intersperseComments returns true if a pending
+// formfeed was dropped from the whitespace buffer.
+//
+func (p *printer) intersperseComments(next token.Position, tok token.Token) (droppedFF bool) {
+	var last *ast.Comment
+	for ; p.commentBefore(next); p.cindex++ {
+		for _, c := range p.comments[p.cindex].List {
+			p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last, tok.IsKeyword())
+			p.writeComment(c)
+			last = c
+		}
+	}
+
+	if last != nil {
+		if last.Text[1] == '*' && p.fset.Position(last.Pos()).Line == next.Line {
+			// the last comment is a /*-style comment and the next item
+			// follows on the same line: separate with an extra blank
+			p.write([]byte{' '})
+		}
+		// ensure that there is a line break after a //-style comment,
+		// before a closing '}' unless explicitly disabled, or at eof
+		needsLinebreak :=
+			last.Text[1] == '/' ||
+				tok == token.RBRACE && p.mode&noExtraLinebreak == 0 ||
+				tok == token.EOF
+		return p.writeCommentSuffix(needsLinebreak)
+	}
+
+	// no comment was written - we should never reach here since
+	// intersperseComments should not be called in that case
+	p.internalError("intersperseComments called without pending comments")
+	return false
+}
+
+// whiteWhitespace writes the first n whitespace entries.
+func (p *printer) writeWhitespace(n int) {
+	// write entries
+	var data [1]byte
+	for i := 0; i < n; i++ {
+		switch ch := p.wsbuf[i]; ch {
+		case ignore:
+			// ignore!
+		case indent:
+			p.indent++
+		case unindent:
+			p.indent--
+			if p.indent < 0 {
+				p.internalError("negative indentation:", p.indent)
+				p.indent = 0
+			}
+		case newline, formfeed:
+			// A line break immediately followed by a "correcting"
+			// unindent is swapped with the unindent - this permits
+			// proper label positioning. If a comment is between
+			// the line break and the label, the unindent is not
+			// part of the comment whitespace prefix and the comment
+			// will be positioned correctly indented.
+			if i+1 < n && p.wsbuf[i+1] == unindent {
+				// Use a formfeed to terminate the current section.
+				// Otherwise, a long label name on the next line leading
+				// to a wide column may increase the indentation column
+				// of lines before the label; effectively leading to wrong
+				// indentation.
+				p.wsbuf[i], p.wsbuf[i+1] = unindent, formfeed
+				i-- // do it again
+				continue
+			}
+			fallthrough
+		default:
+			data[0] = byte(ch)
+			p.write(data[0:])
+		}
+	}
+
+	// shift remaining entries down
+	i := 0
+	for ; n < len(p.wsbuf); n++ {
+		p.wsbuf[i] = p.wsbuf[n]
+		i++
+	}
+	p.wsbuf = p.wsbuf[0:i]
+}
+
+// ----------------------------------------------------------------------------
+// Printing interface
+
+
+func mayCombine(prev token.Token, next byte) (b bool) {
+	switch prev {
+	case token.INT:
+		b = next == '.' // 1.
+	case token.ADD:
+		b = next == '+' // ++
+	case token.SUB:
+		b = next == '-' // --
+	case token.QUO:
+		b = next == '*' // /*
+	case token.LSS:
+		b = next == '-' || next == '<' // <- or <<
+	case token.AND:
+		b = next == '&' || next == '^' // && or &^
+	}
+	return
+}
+
+// print prints a list of "items" (roughly corresponding to syntactic
+// tokens, but also including whitespace and formatting information).
+// It is the only print function that should be called directly from
+// any of the AST printing functions in nodes.go.
+//
+// Whitespace is accumulated until a non-whitespace token appears. Any
+// comments that need to appear before that token are printed first,
+// taking into account the amount and structure of any pending white-
+// space for best comment placement. Then, any leftover whitespace is
+// printed, followed by the actual token.
+//
+func (p *printer) print(args ...interface{}) {
+	for _, f := range args {
+		next := p.pos // estimated position of next item
+		var data string
+		var tok token.Token
+
+		switch x := f.(type) {
+		case pmode:
+			// toggle printer mode
+			p.mode ^= x
+		case whiteSpace:
+			if x == ignore {
+				// don't add ignore's to the buffer; they
+				// may screw up "correcting" unindents (see
+				// LabeledStmt)
+				break
+			}
+			i := len(p.wsbuf)
+			if i == cap(p.wsbuf) {
+				// Whitespace sequences are very short so this should
+				// never happen. Handle gracefully (but possibly with
+				// bad comment placement) if it does happen.
+				p.writeWhitespace(i)
+				i = 0
+			}
+			p.wsbuf = p.wsbuf[0 : i+1]
+			p.wsbuf[i] = x
+		case *ast.Ident:
+			data = x.Name
+			tok = token.IDENT
+		case *ast.BasicLit:
+			data = p.escape(x.Value)
+			tok = x.Kind
+		case token.Token:
+			s := x.String()
+			if mayCombine(p.lastTok, s[0]) {
+				// the previous and the current token must be
+				// separated by a blank otherwise they combine
+				// into a different incorrect token sequence
+				// (except for token.INT followed by a '.' this
+				// should never happen because it is taken care
+				// of via binary expression formatting)
+				if len(p.wsbuf) != 0 {
+					p.internalError("whitespace buffer not empty")
+				}
+				p.wsbuf = p.wsbuf[0:1]
+				p.wsbuf[0] = ' '
+			}
+			data = s
+			tok = x
+		case token.Pos:
+			if x.IsValid() {
+				next = p.fset.Position(x) // accurate position of next item
+			}
+			tok = p.lastTok
+		default:
+			fmt.Fprintf(os.Stderr, "print: unsupported argument type %T\n", f)
+			panic("go/printer type")
+		}
+		p.lastTok = tok
+		p.pos = next
+
+		if data != "" {
+			droppedFF := p.flush(next, tok)
+
+			// intersperse extra newlines if present in the source
+			// (don't do this in flush as it will cause extra newlines
+			// at the end of a file) - use formfeeds if we dropped one
+			// before
+			p.writeNewlines(next.Line-p.pos.Line, droppedFF)
+
+			p.writeItem(next, data)
+		}
+	}
+}
+
+// commentBefore returns true iff the current comment occurs
+// before the next position in the source code.
+//
+func (p *printer) commentBefore(next token.Position) bool {
+	return p.cindex < len(p.comments) && p.fset.Position(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset
+}
+
+// Flush prints any pending comments and whitespace occurring
+// textually before the position of the next token tok. Flush
+// returns true if a pending formfeed character was dropped
+// from the whitespace buffer as a result of interspersing
+// comments.
+//
+func (p *printer) flush(next token.Position, tok token.Token) (droppedFF bool) {
+	if p.commentBefore(next) {
+		// if there are comments before the next item, intersperse them
+		droppedFF = p.intersperseComments(next, tok)
+	} else {
+		// otherwise, write any leftover whitespace
+		p.writeWhitespace(len(p.wsbuf))
+	}
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Trimmer
+
+// A trimmer is an io.Writer filter for stripping tabwriter.Escape
+// characters, trailing blanks and tabs, and for converting formfeed
+// and vtab characters into newlines and htabs (in case no tabwriter
+// is used). Text bracketed by tabwriter.Escape characters is passed
+// through unchanged.
+//
+type trimmer struct {
+	output io.Writer
+	state  int
+	space  bytes.Buffer
+}
+
+// trimmer is implemented as a state machine.
+// It can be in one of the following states:
+const (
+	inSpace  = iota // inside space
+	inEscape        // inside text bracketed by tabwriter.Escapes
+	inText          // inside text
+)
+
+// Design note: It is tempting to eliminate extra blanks occurring in
+//              whitespace in this function as it could simplify some
+//              of the blanks logic in the node printing functions.
+//              However, this would mess up any formatting done by
+//              the tabwriter.
+
+func (p *trimmer) Write(data []byte) (n int, err os.Error) {
+	// invariants:
+	// p.state == inSpace:
+	//	p.space is unwritten
+	// p.state == inEscape, inText:
+	//	data[m:n] is unwritten
+	m := 0
+	var b byte
+	for n, b = range data {
+		if b == '\v' {
+			b = '\t' // convert to htab
+		}
+		switch p.state {
+		case inSpace:
+			switch b {
+			case '\t', ' ':
+				p.space.WriteByte(b) // WriteByte returns no errors
+			case '\n', '\f':
+				p.space.Reset()                        // discard trailing space
+				_, err = p.output.Write(newlines[0:1]) // write newline
+			case tabwriter.Escape:
+				_, err = p.output.Write(p.space.Bytes())
+				p.state = inEscape
+				m = n + 1 // +1: skip tabwriter.Escape
+			default:
+				_, err = p.output.Write(p.space.Bytes())
+				p.state = inText
+				m = n
+			}
+		case inEscape:
+			if b == tabwriter.Escape {
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.Reset()
+			}
+		case inText:
+			switch b {
+			case '\t', ' ':
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.Reset()
+				p.space.WriteByte(b) // WriteByte returns no errors
+			case '\n', '\f':
+				_, err = p.output.Write(data[m:n])
+				p.state = inSpace
+				p.space.Reset()
+				_, err = p.output.Write(newlines[0:1]) // write newline
+			case tabwriter.Escape:
+				_, err = p.output.Write(data[m:n])
+				p.state = inEscape
+				m = n + 1 // +1: skip tabwriter.Escape
+			}
+		default:
+			panic("unreachable")
+		}
+		if err != nil {
+			return
+		}
+	}
+	n = len(data)
+
+	switch p.state {
+	case inEscape, inText:
+		_, err = p.output.Write(data[m:n])
+		p.state = inSpace
+		p.space.Reset()
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Public interface
+
+// General printing is controlled with these Config.Mode flags.
+const (
+	RawFormat uint = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored
+	TabIndent                  // use tabs for indentation independent of UseSpaces
+	UseSpaces                  // use spaces instead of tabs for alignment
+)
+
+// A Config node controls the output of Fprint.
+type Config struct {
+	Mode     uint // default: 0
+	Tabwidth int  // default: 8
+}
+
+// fprint implements Fprint and takes a nodesSizes map for setting up the printer state.
+func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{}, nodeSizes map[ast.Node]int) (int, os.Error) {
+	// redirect output through a trimmer to eliminate trailing whitespace
+	// (Input to a tabwriter must be untrimmed since trailing tabs provide
+	// formatting information. The tabwriter could provide trimming
+	// functionality but no tabwriter is used when RawFormat is set.)
+	output = &trimmer{output: output}
+
+	// setup tabwriter if needed and redirect output
+	var tw *tabwriter.Writer
+	if cfg.Mode&RawFormat == 0 {
+		minwidth := cfg.Tabwidth
+
+		padchar := byte('\t')
+		if cfg.Mode&UseSpaces != 0 {
+			padchar = ' '
+		}
+
+		twmode := tabwriter.DiscardEmptyColumns
+		if cfg.Mode&TabIndent != 0 {
+			minwidth = 0
+			twmode |= tabwriter.TabIndent
+		}
+
+		tw = tabwriter.NewWriter(output, minwidth, cfg.Tabwidth, 1, padchar, twmode)
+		output = tw
+	}
+
+	// setup printer and print node
+	var p printer
+	p.init(output, cfg, fset, nodeSizes)
+	go func() {
+		switch n := node.(type) {
+		case ast.Expr:
+			p.useNodeComments = true
+			p.expr(n, ignoreMultiLine)
+		case ast.Stmt:
+			p.useNodeComments = true
+			// A labeled statement will un-indent to position the
+			// label. Set indent to 1 so we don't get indent "underflow".
+			if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt {
+				p.indent = 1
+			}
+			p.stmt(n, false, ignoreMultiLine)
+		case ast.Decl:
+			p.useNodeComments = true
+			p.decl(n, ignoreMultiLine)
+		case ast.Spec:
+			p.useNodeComments = true
+			p.spec(n, 1, false, ignoreMultiLine)
+		case *ast.File:
+			p.comments = n.Comments
+			p.useNodeComments = n.Comments == nil
+			p.file(n)
+		default:
+			p.errors <- fmt.Errorf("printer.Fprint: unsupported node type %T", n)
+			runtime.Goexit()
+		}
+		p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF)
+		p.errors <- nil // no errors
+	}()
+	err := <-p.errors // wait for completion of goroutine
+
+	// flush tabwriter, if any
+	if tw != nil {
+		tw.Flush() // ignore errors
+	}
+
+	return p.written, err
+}
+
+// Fprint "pretty-prints" an AST node to output and returns the number
+// of bytes written and an error (if any) for a given configuration cfg.
+// Position information is interpreted relative to the file set fset.
+// The node type must be *ast.File, or assignment-compatible to ast.Expr,
+// ast.Decl, ast.Spec, or ast.Stmt.
+//
+func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) (int, os.Error) {
+	return cfg.fprint(output, fset, node, make(map[ast.Node]int))
+}
+
+// Fprint "pretty-prints" an AST node to output.
+// It calls Config.Fprint with default settings.
+//
+func Fprint(output io.Writer, fset *token.FileSet, node interface{}) os.Error {
+	_, err := (&Config{Tabwidth: 8}).Fprint(output, fset, node) // don't care about number of bytes written
+	return err
+}
diff --git a/src/pkg/go/printer/printer_test.go b/src/pkg/go/printer/printer_test.go
new file mode 100644
index 000000000..ff2d906b5
--- /dev/null
+++ b/src/pkg/go/printer/printer_test.go
@@ -0,0 +1,194 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package printer
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+const (
+	dataDir  = "testdata"
+	tabwidth = 8
+)
+
+var update = flag.Bool("update", false, "update golden files")
+
+var fset = token.NewFileSet()
+
+func lineString(text []byte, i int) string {
+	i0 := i
+	for i < len(text) && text[i] != '\n' {
+		i++
+	}
+	return string(text[i0:i])
+}
+
+type checkMode uint
+
+const (
+	export checkMode = 1 << iota
+	rawFormat
+)
+
+func runcheck(t *testing.T, source, golden string, mode checkMode) {
+	// parse source
+	prog, err := parser.ParseFile(fset, source, nil, parser.ParseComments)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// filter exports if necessary
+	if mode&export != 0 {
+		ast.FileExports(prog) // ignore result
+		prog.Comments = nil   // don't print comments that are not in AST
+	}
+
+	// determine printer configuration
+	cfg := Config{Tabwidth: tabwidth}
+	if mode&rawFormat != 0 {
+		cfg.Mode |= RawFormat
+	}
+
+	// format source
+	var buf bytes.Buffer
+	if _, err := cfg.Fprint(&buf, fset, prog); err != nil {
+		t.Error(err)
+	}
+	res := buf.Bytes()
+
+	// update golden files if necessary
+	if *update {
+		if err := ioutil.WriteFile(golden, res, 0644); err != nil {
+			t.Error(err)
+		}
+		return
+	}
+
+	// get golden
+	gld, err := ioutil.ReadFile(golden)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// compare lengths
+	if len(res) != len(gld) {
+		t.Errorf("len = %d, expected %d (= len(%s))", len(res), len(gld), golden)
+	}
+
+	// compare contents
+	for i, line, offs := 0, 1, 0; i < len(res) && i < len(gld); i++ {
+		ch := res[i]
+		if ch != gld[i] {
+			t.Errorf("%s:%d:%d: %s", source, line, i-offs+1, lineString(res, offs))
+			t.Errorf("%s:%d:%d: %s", golden, line, i-offs+1, lineString(gld, offs))
+			t.Error()
+			return
+		}
+		if ch == '\n' {
+			line++
+			offs = i + 1
+		}
+	}
+}
+
+func check(t *testing.T, source, golden string, mode checkMode) {
+	// start a timer to produce a time-out signal
+	tc := make(chan int)
+	go func() {
+		time.Sleep(10e9) // plenty of a safety margin, even for very slow machines
+		tc <- 0
+	}()
+
+	// run the test
+	cc := make(chan int)
+	go func() {
+		runcheck(t, source, golden, mode)
+		cc <- 0
+	}()
+
+	// wait for the first finisher
+	select {
+	case <-tc:
+		// test running past time out
+		t.Errorf("%s: running too slowly", source)
+	case <-cc:
+		// test finished within alloted time margin
+	}
+}
+
+type entry struct {
+	source, golden string
+	mode           checkMode
+}
+
+// Use gotest -update to create/update the respective golden files.
+var data = []entry{
+	{"empty.input", "empty.golden", 0},
+	{"comments.input", "comments.golden", 0},
+	{"comments.input", "comments.x", export},
+	{"linebreaks.input", "linebreaks.golden", 0},
+	{"expressions.input", "expressions.golden", 0},
+	{"expressions.input", "expressions.raw", rawFormat},
+	{"declarations.input", "declarations.golden", 0},
+	{"statements.input", "statements.golden", 0},
+	{"slow.input", "slow.golden", 0},
+}
+
+func TestFiles(t *testing.T) {
+	for i, e := range data {
+		source := filepath.Join(dataDir, e.source)
+		golden := filepath.Join(dataDir, e.golden)
+		check(t, source, golden, e.mode)
+		// TODO(gri) check that golden is idempotent
+		//check(t, golden, golden, e.mode)
+		if testing.Short() && i >= 3 {
+			break
+		}
+	}
+}
+
+// TestLineComments, using a simple test case, checks that consequtive line
+// comments are properly terminated with a newline even if the AST position
+// information is incorrect.
+//
+func TestLineComments(t *testing.T) {
+	const src = `// comment 1
+	// comment 2
+	// comment 3
+	package main
+	`
+
+	fset := token.NewFileSet()
+	ast1, err1 := parser.ParseFile(fset, "", src, parser.ParseComments)
+	if err1 != nil {
+		panic(err1)
+	}
+
+	var buf bytes.Buffer
+	fset = token.NewFileSet() // use the wrong file set
+	Fprint(&buf, fset, ast1)
+
+	nlines := 0
+	for _, ch := range buf.Bytes() {
+		if ch == '\n' {
+			nlines++
+		}
+	}
+
+	const expected = 3
+	if nlines < expected {
+		t.Errorf("got %d, expected %d\n", nlines, expected)
+	}
+}
diff --git a/src/pkg/go/printer/testdata/comments.golden b/src/pkg/go/printer/testdata/comments.golden
new file mode 100644
index 000000000..7b332252c
--- /dev/null
+++ b/src/pkg/go/printer/testdata/comments.golden
@@ -0,0 +1,473 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+import "fmt"	// fmt
+
+const c0 = 0	// zero
+const (
+	c1	= iota	// c1
+	c2		// c2
+)
+
+// Alignment of comments in declarations>
+const (
+	_	T	= iota	// comment
+	_			// comment
+	_			// comment
+	_	= iota + 10
+	_	// comments
+
+	_		= 10	// comment
+	_	T	= 20	// comment
+)
+
+const (
+	_____	= iota	// foo
+	_		// bar
+	_	= 0	// bal
+	_		// bat
+)
+
+const (
+	_	T	= iota	// comment
+	_			// comment
+	_			// comment
+	_	= iota + 10
+	_		// comment
+	_		= 10
+	_		= 20	// comment
+	_	T	= 0	// comment
+)
+
+// The SZ struct; it is empty.
+type SZ struct{}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	int
+	x, y, z	int	// 3 unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C	float	// 3 exported fields
+	D, b, c	int	// 2 unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C	float	// 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface{}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	f(x int) int	// unexported method
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float	// exported methods
+	g(x int) int		// unexported method
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float	// exported method
+	G(x float) float	// exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1	int	// line comment for F1
+	// lead comment for F2
+	F2	int	// line comment for F2
+	f3	int	// f3 is not exported
+}
+
+// This comment group should be separated
+// with a newline from the next comment
+// group.
+
+// This comment should NOT be associated with the next declaration.
+
+var x int	// x
+var ()
+
+// This comment SHOULD be associated with the next declaration.
+func f0() {
+	const pi = 3.14	// pi
+	var s1 struct{}	/* an empty struct */	/* foo */
+	// a struct constructor
+	// --------------------
+	var s2 struct{} = struct{}{}
+	x := pi
+}
+//
+// NO SPACE HERE
+//
+func f1() {
+	f0()
+	/* 1 */
+	// 2
+	/* 3 */
+	/* 4 */
+	f0()
+}
+
+func _() {
+	// this comment should be properly indented
+}
+
+func _(x int) int {
+	if x < 0 {	// the tab printed before this comment's // must not affect the remaining lines
+		return -x	// this statement should be properly indented
+	}
+	if x < 0 {	/* the tab printed before this comment's /* must not affect the remaining lines */
+		return -x	// this statement should be properly indented
+	}
+	return x
+}
+
+func typeswitch(x interface{}) {
+	switch v := x.(type) {
+	case bool, int, float:
+	case string:
+	default:
+	}
+
+	switch x.(type) {
+	}
+
+	switch v0, ok := x.(int); v := x.(type) {
+	}
+
+	switch v0, ok := x.(int); x.(type) {
+	case byte:	// this comment should be on the same line as the keyword
+		// this comment should be normally indented
+		_ = 0
+	case bool, int, float:
+		// this comment should be indented
+	case string:
+	default:
+		// this comment should be indented
+	}
+	// this comment should not be indented
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line */
+}
+
+/*
+ * line
+ * of
+ * stars
+ */
+
+/* another line
+ * of
+ * stars */
+
+/*	and another line
+ *	of
+ *	stars */
+
+/* a line of
+ * stars */
+
+/*	and another line of
+ *	stars */
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/*
+aligned in middle
+here
+        not here
+*/
+
+/*
+blank line in middle:
+
+with no leading spaces on blank line.
+*/
+
+/*
+   aligned in middle
+   here
+           not here
+*/
+
+/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+
+func _() {
+	/*
+	 * line
+	 * of
+	 * stars
+	 */
+
+	/*
+		aligned in middle
+		here
+			not here
+	*/
+
+	/*
+		blank line in middle:
+
+		with no leading spaces on blank line.
+	*/
+}
+
+// Some interesting interspersed comments
+func _( /* this */ x /* is */ /* an */ int) {
+}
+
+func _( /* no params */ )	{}
+
+func _() {
+	f( /* no args */ )
+}
+
+func ( /* comment1 */ T /* comment2 */ ) _()	{}
+
+func _() { /* one-line functions with comments are formatted as multi-line functions */
+}
+
+func _() {
+	_ = 0
+	/* closing curly brace should be on new line */
+}
+
+func _() {
+	_ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */ }
+}
+
+// Comments immediately adjacent to punctuation (for which the go/printer
+// may only have estimated position information) must remain after the punctuation.
+func _() {
+	_ = T{
+		1,	// comment after comma
+		2,	/* comment after comma */
+		3,	// comment after comma
+	}
+	_ = T{
+		1,	// comment after comma
+		2,	/* comment after comma */
+		3,	// comment after comma
+	}
+	_ = T{
+		/* comment before literal */ 1,
+		2,	/* comment before comma - ok to move after comma */
+		3,	/* comment before comma - ok to move after comma */
+	}
+
+	for i = 0;	// comment after semicolon
+	i < 9;		/* comment after semicolon */
+	i++ {		// comment after opening curly brace
+	}
+
+	// TODO(gri) the last comment in this example should be aligned */
+	for i = 0;	// comment after semicolon
+	i < 9;		/* comment before semicolon - ok to move after semicolon */
+	i++ /* comment before opening curly brace */ {
+	}
+}
+
+// Line comments with tabs
+func _() {
+	var finput *bufio.Reader	// input file
+	var stderr *bufio.Writer
+	var ftable *bufio.Writer	// y.go file
+	var foutput *bufio.Writer	// y.output file
+
+	var oflag string	// -o [y.go]		- y.go file
+	var vflag string	// -v [y.output]	- y.output file
+	var lflag bool		// -l			- disable line directives
+}
+
+/* This comment is the last entry in this file. It must be printed and should be followed by a newline */
diff --git a/src/pkg/go/printer/testdata/comments.input b/src/pkg/go/printer/testdata/comments.input
new file mode 100644
index 000000000..2a9a86b68
--- /dev/null
+++ b/src/pkg/go/printer/testdata/comments.input
@@ -0,0 +1,483 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+import "fmt"  // fmt
+
+const c0 = 0  // zero
+const (
+	c1 = iota  // c1
+	c2  // c2
+)
+
+// Alignment of comments in declarations>
+const (
+	_ T = iota  // comment
+	_  // comment
+	_  // comment
+	_ = iota+10
+	_  // comments
+
+	_ = 10  // comment
+	_ T = 20  // comment
+)
+
+const (
+	_____ = iota // foo
+	_ // bar
+	_  = 0    // bal
+	_ // bat
+)
+
+const (
+	_ T = iota // comment
+	_ // comment
+	_ // comment
+	_ = iota + 10
+	_ // comment
+	_ = 10
+	_ = 20 // comment
+	_ T = 0 // comment
+)
+
+// The SZ struct; it is empty.
+type SZ struct {}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	int
+	x, y, z int  // 3 unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C float  // 3 exported fields
+	D, b, c int  // 2 unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C float  // 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface {}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	f(x int) int  // unexported method
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float  // exported methods
+	g(x int) int  // unexported method
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float  // exported method
+	G(x float) float  // exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1 int // line comment for F1
+	// lead comment for F2
+	F2 int // line comment for F2
+	f3 int // f3 is not exported
+}
+
+// This comment group should be separated
+// with a newline from the next comment
+// group.
+
+// This comment should NOT be associated with the next declaration.
+
+var x int  // x
+var ()
+
+
+// This comment SHOULD be associated with the next declaration.
+func f0() {
+	const pi = 3.14  // pi
+	var s1 struct {}  /* an empty struct */ /* foo */
+	// a struct constructor
+	// --------------------
+	var s2 struct {} = struct {}{}
+	x := pi
+}
+//
+// NO SPACE HERE
+//
+func f1() {
+	f0()
+	/* 1 */
+	// 2
+	/* 3 */
+	/* 4 */
+	f0()
+}
+
+
+func _() {
+	// this comment should be properly indented
+}
+
+
+func _(x int) int {
+	if x < 0 {  // the tab printed before this comment's // must not affect the remaining lines
+		return -x  // this statement should be properly indented
+	}
+	if x < 0 {  /* the tab printed before this comment's /* must not affect the remaining lines */
+		return -x  // this statement should be properly indented
+	}
+	return x
+}
+
+
+func typeswitch(x interface{}) {
+	switch v := x.(type) {
+	case bool, int, float:
+	case string:
+	default:
+	}
+
+	switch x.(type) {
+	}
+
+	switch v0, ok := x.(int); v := x.(type) {
+	}
+
+	switch v0, ok := x.(int); x.(type) {
+	case byte:  // this comment should be on the same line as the keyword
+		// this comment should be normally indented
+		_ = 0
+	case bool, int, float:
+		// this comment should be indented
+	case string:
+	default:
+		// this comment should be indented
+	}
+	// this comment should not be indented
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line
+	   */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line
+		*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned		line
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line
+	   */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned		line
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line
+		*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned		line
+		aligned line */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line
+	   */
+}
+
+func _() {
+	/* freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line
+		*/
+}
+
+func _() {
+	/*	freestanding comment
+		aligned line */
+}
+
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	*/
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line
+	   */
+}
+
+func _() {
+	/*
+	   freestanding comment
+	   aligned line */
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+	*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line
+		*/
+}
+
+func _() {
+	/*
+		freestanding comment
+		aligned line */
+}
+
+/*
+ * line
+ * of
+ * stars
+ */
+
+/* another line
+ * of
+ * stars */
+
+/*	and another line
+ *	of
+ *	stars */
+
+/* a line of
+ * stars */
+
+/*	and another line of
+ *	stars */
+
+/* a line of stars
+*/
+
+/*	and another line of
+*/
+
+/* a line of stars
+ */
+
+/*	and another line of
+ */
+
+/*
+aligned in middle
+here
+        not here
+*/
+
+/*
+blank line in middle:
+
+with no leading spaces on blank line.
+*/
+
+/*
+   aligned in middle
+   here
+           not here
+*/
+
+/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+
+func _() {
+	/*
+	 * line
+	 * of
+	 * stars
+	 */
+
+	/*
+	aligned in middle
+	here
+		not here
+	*/
+
+	/*
+	blank line in middle:
+
+	with no leading spaces on blank line.
+*/
+}
+
+
+// Some interesting interspersed comments
+func _(/* this */x/* is *//* an */ int) {
+}
+
+func _(/* no params */) {}
+
+func _() {
+	f(/* no args */)
+}
+
+func (/* comment1 */ T /* comment2 */) _() {}
+
+func _() { /* one-line functions with comments are formatted as multi-line functions */ }
+
+func _() {
+	_ = 0
+	/* closing curly brace should be on new line */ }
+
+func _() {
+	_ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */}
+}
+
+
+// Comments immediately adjacent to punctuation (for which the go/printer
+// may only have estimated position information) must remain after the punctuation.
+func _() {
+	_ = T{
+		1,    // comment after comma
+		2,    /* comment after comma */
+		3  ,  // comment after comma
+	}
+	_ = T{
+		1  ,// comment after comma
+		2  ,/* comment after comma */
+		3,// comment after comma
+	}
+	_ = T{
+		/* comment before literal */1,
+		2/* comment before comma - ok to move after comma */,
+		3  /* comment before comma - ok to move after comma */  ,
+	}
+
+	for
+		i=0;// comment after semicolon
+		i<9;/* comment after semicolon */
+		i++{// comment after opening curly brace
+	}
+
+	// TODO(gri) the last comment in this example should be aligned */
+	for
+		i=0;// comment after semicolon
+		i<9/* comment before semicolon - ok to move after semicolon */;
+		i++ /* comment before opening curly brace */ {
+	}
+}
+
+
+// Line comments with tabs
+func _() {
+var	finput		*bufio.Reader			// input file
+var	stderr		*bufio.Writer
+var	ftable		*bufio.Writer			// y.go file
+var	foutput		*bufio.Writer			// y.output file
+
+var	oflag		string				// -o [y.go]		- y.go file
+var	vflag		string				// -v [y.output]	- y.output file
+var	lflag		bool				// -l			- disable line directives
+}
+
+
+/* This comment is the last entry in this file. It must be printed and should be followed by a newline */
diff --git a/src/pkg/go/printer/testdata/comments.x b/src/pkg/go/printer/testdata/comments.x
new file mode 100644
index 000000000..ae7729286
--- /dev/null
+++ b/src/pkg/go/printer/testdata/comments.x
@@ -0,0 +1,56 @@
+// This is a package for testing comment placement by go/printer.
+//
+package main
+
+// The SZ struct; it is empty.
+type SZ struct{}
+
+// The S0 struct; no field is exported.
+type S0 struct {
+	// contains filtered or unexported fields
+}
+
+// The S1 struct; some fields are not exported.
+type S1 struct {
+	S0
+	A, B, C	float	// 3 exported fields
+	D	int	// 2 unexported fields
+	// contains filtered or unexported fields
+}
+
+// The S2 struct; all fields are exported.
+type S2 struct {
+	S1
+	A, B, C	float	// 3 exported fields
+}
+
+// The IZ interface; it is empty.
+type SZ interface{}
+
+// The I0 interface; no method is exported.
+type I0 interface {
+	// contains filtered or unexported methods
+}
+
+// The I1 interface; some methods are not exported.
+type I1 interface {
+	I0
+	F(x float) float	// exported methods
+	// contains filtered or unexported methods
+}
+
+// The I2 interface; all methods are exported.
+type I2 interface {
+	I0
+	F(x float) float	// exported method
+	G(x float) float	// exported method
+}
+
+// The S3 struct; all comments except for the last one must appear in the export.
+type S3 struct {
+	// lead comment for F1
+	F1	int	// line comment for F1
+	// lead comment for F2
+	F2	int	// line comment for F2
+	// contains filtered or unexported fields
+}
diff --git a/src/pkg/go/printer/testdata/declarations.golden b/src/pkg/go/printer/testdata/declarations.golden
new file mode 100644
index 000000000..970533e8c
--- /dev/null
+++ b/src/pkg/go/printer/testdata/declarations.golden
@@ -0,0 +1,747 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package imports
+
+import "io"
+
+import (
+	_ "io"
+)
+
+import _ "io"
+
+import (
+	"io"
+	"io"
+	"io"
+)
+
+import (
+	"io"
+	aLongRename "io"
+
+	b "io"
+)
+
+import (
+	"unrenamed"
+	renamed "renameMe"
+	. "io"
+	_ "io"
+	"io"
+	. "os"
+)
+
+// no newlines between consecutive single imports, but
+// respect extra line breaks in the source (at most one empty line)
+import _ "io"
+import _ "io"
+import _ "io"
+
+import _ "os"
+import _ "os"
+import _ "os"
+
+import _ "fmt"
+import _ "fmt"
+import _ "fmt"
+
+import "foo"	// a comment
+import "bar"	// a comment
+
+import (
+	_ "foo"
+	// a comment
+	"bar"
+	"foo"	// a comment
+	"bar"	// a comment
+)
+
+// comments + renames
+import (
+	"unrenamed"	// a comment
+	renamed "renameMe"
+	. "io"		/* a comment */
+	_ "io/ioutil"	// a comment
+	"io"		// testing alignment
+	. "os"
+	// a comment
+)
+
+// a case that caused problems in the past (comment placement)
+import (
+	. "fmt"
+	"io"
+	"malloc"	// for the malloc count test only
+	"math"
+	"strings"
+	"testing"
+)
+
+// more import examples
+import (
+	"xxx"
+	"much longer name"	// comment
+	"short name"		// comment
+)
+
+import (
+	_ "xxx"
+	"much longer name"	// comment
+)
+
+import (
+	mymath "math"
+	"/foo/bar/long_package_path"	// a comment
+)
+
+import (
+	"package_a"	// comment
+	"package_b"
+	my_better_c "package_c"	// comment
+	"package_d"		// comment
+	my_e "package_e"	// comment
+
+	"package_a"	// comment
+	"package_bb"
+	"package_ccc"	// comment
+	"package_dddd"	// comment
+)
+
+// at least one empty line between declarations of different kind
+import _ "io"
+
+var _ int
+
+// printing of constant literals
+const (
+	_	= "foobar"
+	_	= "a۰۱۸"
+	_	= "foo६४"
+	_	= "bar９８７６"
+	_	= 0
+	_	= 1
+	_	= 123456789012345678890
+	_	= 01234567
+	_	= 0xcafebabe
+	_	= 0.
+	_	= .0
+	_	= 3.14159265
+	_	= 1e0
+	_	= 1e+100
+	_	= 1e-100
+	_	= 2.71828e-1000
+	_	= 0i
+	_	= 1i
+	_	= 012345678901234567889i
+	_	= 123456789012345678890i
+	_	= 0.i
+	_	= .0i
+	_	= 3.14159265i
+	_	= 1e0i
+	_	= 1e+100i
+	_	= 1e-100i
+	_	= 2.71828e-1000i
+	_	= 'a'
+	_	= '\000'
+	_	= '\xFF'
+	_	= '\uff16'
+	_	= '\U0000ff16'
+	_	= `foobar`
+	_	= `foo
+---
+---
+bar`
+)
+
+func _() {
+	type _ int
+	type _ *int
+	type _ []int
+	type _ map[string]int
+	type _ chan int
+	type _ func() int
+
+	var _ int
+	var _ *int
+	var _ []int
+	var _ map[string]int
+	var _ chan int
+	var _ func() int
+
+	type _ struct{}
+	type _ *struct{}
+	type _ []struct{}
+	type _ map[string]struct{}
+	type _ chan struct{}
+	type _ func() struct{}
+
+	type _ interface{}
+	type _ *interface{}
+	type _ []interface{}
+	type _ map[string]interface{}
+	type _ chan interface{}
+	type _ func() interface{}
+
+	var _ struct{}
+	var _ *struct{}
+	var _ []struct{}
+	var _ map[string]struct{}
+	var _ chan struct{}
+	var _ func() struct{}
+
+	var _ interface{}
+	var _ *interface{}
+	var _ []interface{}
+	var _ map[string]interface{}
+	var _ chan interface{}
+	var _ func() interface{}
+}
+
+// don't lose blank lines in grouped declarations
+const (
+	_	int	= 0
+	_	float	= 1
+
+	_	string	= "foo"
+
+	_	= iota
+	_
+
+	// a comment
+	_
+
+	_
+)
+
+type (
+	_	int
+	_	struct{}
+
+	_	interface{}
+
+	// a comment
+	_	map[string]int
+)
+
+var (
+	_	int	= 0
+	_	float	= 1
+
+	_	string	= "foo"
+
+	_	bool
+
+	// a comment
+	_	bool
+)
+
+// don't lose blank lines in this struct
+type _ struct {
+	String	struct {
+		Str, Len int
+	}
+	Slice	struct {
+		Array, Len, Cap int
+	}
+	Eface	struct {
+		Typ, Ptr int
+	}
+
+	UncommonType	struct {
+		Name, PkgPath int
+	}
+	CommonType	struct {
+		Size, Hash, Alg, Align, FieldAlign, String, UncommonType int
+	}
+	Type	struct {
+		Typ, Ptr int
+	}
+	StructField	struct {
+		Name, PkgPath, Typ, Tag, Offset int
+	}
+	StructType	struct {
+		Fields int
+	}
+	PtrType	struct {
+		Elem int
+	}
+	SliceType	struct {
+		Elem int
+	}
+	ArrayType	struct {
+		Elem, Len int
+	}
+
+	Stktop	struct {
+		Stackguard, Stackbase, Gobuf int
+	}
+	Gobuf	struct {
+		Sp, Pc, G int
+	}
+	G	struct {
+		Stackbase, Sched, Status, Alllink int
+	}
+}
+
+// no tabs for single or ungrouped decls
+func _() {
+	const xxxxxx = 0
+	type x int
+	var xxx int
+	var yyyy float = 3.14
+	var zzzzz = "bar"
+
+	const (
+		xxxxxx = 0
+	)
+	type (
+		x int
+	)
+	var (
+		xxx int
+	)
+	var (
+		yyyy float = 3.14
+	)
+	var (
+		zzzzz = "bar"
+	)
+}
+
+// tabs for multiple or grouped decls
+func _() {
+	// no entry has a type
+	const (
+		zzzzzz	= 1
+		z	= 2
+		zzz	= 3
+	)
+	// some entries have a type
+	const (
+		xxxxxx			= 1
+		x			= 2
+		xxx			= 3
+		yyyyyyyy	float	= iota
+		yyyy			= "bar"
+		yyy
+		yy	= 2
+	)
+}
+
+func _() {
+	// no entry has a type
+	var (
+		zzzzzz	= 1
+		z	= 2
+		zzz	= 3
+	)
+	// no entry has a value
+	var (
+		_	int
+		_	float
+		_	string
+
+		_	int	// comment
+		_	float	// comment
+		_	string	// comment
+	)
+	// some entries have a type
+	var (
+		xxxxxx		int
+		x		float
+		xxx		string
+		yyyyyyyy	int	= 1234
+		y		float	= 3.14
+		yyyy			= "bar"
+		yyy		string	= "foo"
+	)
+	// mixed entries - all comments should be aligned
+	var (
+		a, b, c			int
+		x			= 10
+		d			int			// comment
+		y				= 20		// comment
+		f, ff, fff, ffff	int	= 0, 1, 2, 3	// comment
+	)
+	// respect original line breaks
+	var _ = []T{
+		T{0x20, "Telugu"},
+	}
+	var _ = []T{
+		// respect original line breaks
+		T{0x20, "Telugu"},
+	}
+}
+
+func _() {
+	type (
+		xxxxxx	int
+		x	float
+		xxx	string
+		xxxxx	[]x
+		xx	struct{}
+		xxxxxxx	struct {
+			_, _	int
+			_	float
+		}
+		xxxx	chan<- string
+	)
+}
+
+// alignment of "=" in consecutive lines (extended example from issue 1414)
+const (
+	umax	uint	= ^uint(0)		// maximum value for a uint
+	bpu		= 1 << (5 + umax>>63)	// bits per uint
+	foo
+	bar	= -1
+)
+
+// typical enum
+const (
+	a	MyType	= iota
+	abcd
+	b
+	c
+	def
+)
+
+// excerpt from godoc.go
+var (
+	goroot		= flag.String("goroot", runtime.GOROOT(), "Go root directory")
+	testDir		= flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)")
+	pkgPath		= flag.String("path", "", "additional package directories (colon-separated)")
+	filter		= flag.String("filter", "", "filter file containing permitted package directory paths")
+	filterMin	= flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0")
+	filterDelay	delayTime	// actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially
+)
+
+// formatting of structs
+type _ struct{}
+
+type _ struct { /* this comment should be visible */
+}
+
+type _ struct {
+	// this comment should be visible and properly indented
+}
+
+type _ struct {	// this comment must not change indentation
+	f			int
+	f, ff, fff, ffff	int
+}
+
+type _ struct {
+	string
+}
+
+type _ struct {
+	string	// comment
+}
+
+type _ struct {
+	string "tag"
+}
+
+type _ struct {
+	string "tag"	// comment
+}
+
+type _ struct {
+	f int
+}
+
+type _ struct {
+	f int	// comment
+}
+
+type _ struct {
+	f int "tag"
+}
+
+type _ struct {
+	f int "tag"	// comment
+}
+
+type _ struct {
+	bool
+	a, b, c			int
+	int			"tag"
+	ES				// comment
+	float			"tag"	// comment
+	f			int	// comment
+	f, ff, fff, ffff	int	// comment
+	g			float	"tag"
+	h			float	"tag"	// comment
+}
+
+type _ struct {
+	a, b,
+	c, d	int	// this line should be indented
+	u, v, w, x	float	// this line should be indented
+	p, q,
+	r, s	float	// this line should be indented
+}
+
+// difficult cases
+type _ struct {
+	bool		// comment
+	text	[]byte	// comment
+}
+
+// formatting of interfaces
+type EI interface{}
+
+type _ interface {
+	EI
+}
+
+type _ interface {
+	f()
+	fffff()
+}
+
+type _ interface {
+	EI
+	f()
+	fffffg()
+}
+
+type _ interface {	// this comment must not change indentation
+	EI				// here's a comment
+	f()				// no blank between identifier and ()
+	fffff()				// no blank between identifier and ()
+	gggggggggggg(x, y, z int)	// hurray
+}
+
+// formatting of variable declarations
+func _() {
+	type day struct {
+		n		int
+		short, long	string
+	}
+	var (
+		Sunday		= day{0, "SUN", "Sunday"}
+		Monday		= day{1, "MON", "Monday"}
+		Tuesday		= day{2, "TUE", "Tuesday"}
+		Wednesday	= day{3, "WED", "Wednesday"}
+		Thursday	= day{4, "THU", "Thursday"}
+		Friday		= day{5, "FRI", "Friday"}
+		Saturday	= day{6, "SAT", "Saturday"}
+	)
+}
+
+// formatting of multi-line variable declarations
+var a1, b1, c1 int	// all on one line
+
+var a2, b2,
+	c2 int	// this line should be indented
+
+var (
+	a3, b3,
+	c3, d3	int	// this line should be indented
+	a4, b4, c4	int	// this line should be indented
+)
+
+func _() {
+	var privateKey2 = &Block{Type:	"RSA PRIVATE KEY",
+		Headers:	map[string]string{},
+		Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
+			0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
+			0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
+			0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
+		},
+	}
+}
+
+func _() {
+	var Universe = Scope{
+		Names: map[string]*Ident{
+			// basic types
+			"bool":		nil,
+			"byte":		nil,
+			"int8":		nil,
+			"int16":	nil,
+			"int32":	nil,
+			"int64":	nil,
+			"uint8":	nil,
+			"uint16":	nil,
+			"uint32":	nil,
+			"uint64":	nil,
+			"float32":	nil,
+			"float64":	nil,
+			"string":	nil,
+
+			// convenience types
+			"int":		nil,
+			"uint":		nil,
+			"uintptr":	nil,
+			"float":	nil,
+
+			// constants
+			"false":	nil,
+			"true":		nil,
+			"iota":		nil,
+			"nil":		nil,
+
+			// functions
+			"cap":		nil,
+			"len":		nil,
+			"new":		nil,
+			"make":		nil,
+			"panic":	nil,
+			"panicln":	nil,
+			"print":	nil,
+			"println":	nil,
+		},
+	}
+}
+
+// alignment of map composite entries
+var _ = map[int]int{
+	// small key sizes: always align even if size ratios are large
+	a:			a,
+	abcdefghabcdefgh:	a,
+	ab:			a,
+	abc:			a,
+	abcdefgabcdefg:		a,
+	abcd:			a,
+	abcde:			a,
+	abcdef:			a,
+
+	// mixed key sizes: align when key sizes change within accepted ratio
+	abcdefgh:		a,
+	abcdefghabcdefg:	a,
+	abcdefghij:		a,
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij:	a,	// outlier - do not align with previous line
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij:		a,	// align with previous line
+
+	ab:	a,	// do not align with previous line
+	abcde:	a,	// align with previous line
+}
+
+func _() {
+	var _ = T{
+		a,	// must introduce trailing comma
+	}
+}
+
+// formatting of function results
+func _() func()				{}
+func _() func(int)			{ return nil }
+func _() func(int) int			{ return nil }
+func _() func(int) func(int) func()	{ return nil }
+
+// formatting of consecutive single-line functions
+func _()	{}
+func _()	{}
+func _()	{}
+
+func _()	{}	// an empty line before this function
+func _()	{}
+func _()	{}
+
+func _()		{ f(1, 2, 3) }
+func _(x int) int	{ y := x; return y + 1 }
+func _() int		{ type T struct{}; var x T; return x }
+
+// these must remain multi-line since they are multi-line in the source
+func _() {
+	f(1, 2, 3)
+}
+func _(x int) int {
+	y := x
+	return y + 1
+}
+func _() int {
+	type T struct{}
+	var x T
+	return x
+}
+
+// making function declarations safe for new semicolon rules
+func _() { /* multi-line func because of comment */
+}
+
+func _() {
+	/* multi-line func because block is on multiple lines */
+}
+
+// ellipsis parameters
+func _(...int)
+func _(...*int)
+func _(...[]int)
+func _(...struct{})
+func _(bool, ...interface{})
+func _(bool, ...func())
+func _(bool, ...func(...int))
+func _(bool, ...map[string]int)
+func _(bool, ...chan int)
+
+func _(b bool, x ...int)
+func _(b bool, x ...*int)
+func _(b bool, x ...[]int)
+func _(b bool, x ...struct{})
+func _(x ...interface{})
+func _(x ...func())
+func _(x ...func(...int))
+func _(x ...map[string]int)
+func _(x ...chan int)
+
+// these parameter lists must remain multi-line since they are multi-line in the source
+func _(bool,
+int) {
+}
+func _(x bool,
+y int) {
+}
+func _(x,
+y bool) {
+}
+func _(bool,	// comment
+int) {
+}
+func _(x bool,	// comment
+y int) {
+}
+func _(x,	// comment
+y bool) {
+}
+func _(bool,	// comment
+// comment
+int) {
+}
+func _(x bool,	// comment
+// comment
+y int) {
+}
+func _(x,	// comment
+// comment
+y bool) {
+}
+func _(bool,
+// comment
+int) {
+}
+func _(x bool,
+// comment
+y int) {
+}
+func _(x,
+// comment
+y bool) {
+}
+func _(x,	// comment
+y,	// comment
+z bool) {
+}
+func _(x,	// comment
+y,	// comment
+z bool) {
+}
+func _(x int,	// comment
+y float,	// comment
+z bool) {
+}
diff --git a/src/pkg/go/printer/testdata/declarations.input b/src/pkg/go/printer/testdata/declarations.input
new file mode 100644
index 000000000..c6134096b
--- /dev/null
+++ b/src/pkg/go/printer/testdata/declarations.input
@@ -0,0 +1,757 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package imports
+
+import "io"
+
+import (
+	_ "io"
+)
+
+import _ "io"
+
+import (
+	"io"
+	"io"
+	"io"
+)
+
+import (
+	"io"
+	aLongRename "io"
+
+	b "io"
+)
+
+import (
+       "unrenamed"
+       renamed "renameMe"
+       . "io"
+       _ "io"
+       "io"
+       . "os"
+)
+
+// no newlines between consecutive single imports, but
+// respect extra line breaks in the source (at most one empty line)
+import _ "io"
+import _ "io"
+import _ "io"
+
+import _ "os"
+import _ "os"
+import _ "os"
+
+
+import _ "fmt"
+import _ "fmt"
+import _ "fmt"
+
+import "foo"  // a comment
+import "bar"  // a comment
+
+import (
+	_ "foo"
+	// a comment
+	"bar"
+	"foo"  // a comment
+	"bar"  // a comment
+)
+
+// comments + renames
+import (
+       "unrenamed" // a comment
+       renamed "renameMe"
+       . "io" /* a comment */
+       _ "io/ioutil" // a comment
+       "io" // testing alignment
+       . "os"
+       // a comment
+)
+
+// a case that caused problems in the past (comment placement)
+import (
+	. "fmt"
+	"io"
+	"malloc"	// for the malloc count test only
+	"math"
+	"strings"
+	"testing"
+)
+
+// more import examples
+import (
+	"xxx"
+	"much longer name" // comment
+	"short name" // comment
+)
+
+import (
+	_ "xxx"
+	"much longer name" // comment
+)
+
+import (
+	mymath "math"
+	"/foo/bar/long_package_path" // a comment
+)
+
+import (
+	"package_a" // comment
+	"package_b"
+	my_better_c "package_c" // comment
+	"package_d" // comment
+	my_e "package_e" // comment
+
+	"package_a"    // comment
+	"package_bb"
+	"package_ccc"  // comment
+	"package_dddd" // comment
+)
+
+// at least one empty line between declarations of different kind
+import _ "io"
+var _ int
+
+
+// printing of constant literals
+const (
+	_ = "foobar"
+	_ = "a۰۱۸"
+	_ = "foo६४"
+	_ = "bar９８７６"
+	_ = 0
+	_ = 1
+	_ = 123456789012345678890
+	_ = 01234567
+	_ = 0xcafebabe
+	_ = 0.
+	_ = .0
+	_ = 3.14159265
+	_ = 1e0
+	_ = 1e+100
+	_ = 1e-100
+	_ = 2.71828e-1000
+	_ = 0i
+	_ = 1i
+	_ = 012345678901234567889i
+	_ = 123456789012345678890i
+	_ = 0.i
+	_ = .0i
+	_ = 3.14159265i
+	_ = 1e0i
+	_ = 1e+100i
+	_ = 1e-100i
+	_ = 2.71828e-1000i
+	_ = 'a'
+	_ = '\000'
+	_ = '\xFF'
+	_ = '\uff16'
+	_ = '\U0000ff16'
+	_ = `foobar`
+	_ = `foo
+---
+---
+bar`
+)
+
+
+func _() {
+	type _ int
+	type _ *int
+	type _ []int
+	type _ map[string]int
+	type _ chan int
+	type _ func() int
+
+	var _ int
+	var _ *int
+	var _ []int
+	var _ map[string]int
+	var _ chan int
+	var _ func() int
+
+	type _ struct{}
+	type _ *struct{}
+	type _ []struct{}
+	type _ map[string]struct{}
+	type _ chan struct{}
+	type _ func() struct{}
+
+	type _ interface{}
+	type _ *interface{}
+	type _ []interface{}
+	type _ map[string]interface{}
+	type _ chan interface{}
+	type _ func() interface{}
+
+	var _ struct{}
+	var _ *struct{}
+	var _ []struct{}
+	var _ map[string]struct{}
+	var _ chan struct{}
+	var _ func() struct{}
+
+	var _ interface{}
+	var _ *interface{}
+	var _ []interface{}
+	var _ map[string]interface{}
+	var _ chan interface{}
+	var _ func() interface{}
+}
+
+
+// don't lose blank lines in grouped declarations
+const (
+	_ int = 0
+	_ float = 1
+
+	_ string = "foo"
+
+	_ = iota
+	_
+	
+	// a comment
+	_
+
+	_
+)
+
+
+type (
+	_ int
+	_ struct {}
+	
+	_ interface{}
+	
+	// a comment
+	_ map[string]int
+)
+
+
+var (
+	_ int = 0
+	_ float = 1
+
+	_ string = "foo"
+
+	_ bool
+	
+	// a comment
+	_ bool
+)
+
+
+// don't lose blank lines in this struct
+type _ struct {
+	String struct {
+		Str, Len int
+	}
+	Slice struct {
+		Array, Len, Cap int
+	}
+	Eface struct {
+		Typ, Ptr int
+	}
+
+	UncommonType struct {
+		Name, PkgPath int
+	}
+	CommonType struct {
+		Size, Hash, Alg, Align, FieldAlign, String, UncommonType int
+	}
+	Type struct {
+		Typ, Ptr int
+	}
+	StructField struct {
+		Name, PkgPath, Typ, Tag, Offset int
+	}
+	StructType struct {
+		Fields int
+	}
+	PtrType struct {
+		Elem int
+	}
+	SliceType struct {
+		Elem int
+	}
+	ArrayType struct {
+		Elem, Len int
+	}
+
+	Stktop struct {
+		Stackguard, Stackbase, Gobuf int
+	}
+	Gobuf struct {
+		Sp, Pc, G int
+	}
+	G struct {
+		Stackbase, Sched, Status, Alllink int
+	}
+}
+
+
+// no tabs for single or ungrouped decls
+func _() {
+	const xxxxxx = 0
+	type x int
+	var xxx int
+	var yyyy float = 3.14
+	var zzzzz = "bar"
+
+	const (
+		xxxxxx = 0
+	)
+	type (
+		x int
+	)
+	var (
+		xxx int
+	)
+	var (
+		yyyy float = 3.14
+	)
+	var (
+		zzzzz = "bar"
+	)
+}
+
+// tabs for multiple or grouped decls
+func _() {
+	// no entry has a type
+	const (
+		zzzzzz = 1
+		z = 2
+		zzz = 3
+	)
+	// some entries have a type
+	const (
+		xxxxxx = 1
+		x = 2
+		xxx = 3
+		yyyyyyyy float = iota
+		yyyy = "bar"
+		yyy
+		yy = 2
+	)
+}
+
+func _() {
+	// no entry has a type
+	var (
+		zzzzzz = 1
+		z = 2
+		zzz = 3
+	)
+	// no entry has a value
+	var (
+		_ int
+		_ float
+		_ string
+
+		_ int  // comment
+		_ float  // comment
+		_ string  // comment
+	)
+	// some entries have a type
+	var (
+		xxxxxx int
+		x float
+		xxx string
+		yyyyyyyy int = 1234
+		y float = 3.14
+		yyyy = "bar"
+		yyy string = "foo"
+	)
+	// mixed entries - all comments should be aligned
+	var (
+		a, b, c int
+		x = 10
+		d int  // comment
+		y = 20  // comment
+		f, ff, fff, ffff int = 0, 1, 2, 3  // comment
+	)
+	// respect original line breaks
+	var _ = []T {
+		T{0x20,	"Telugu"},
+	}
+	var _ = []T {
+		// respect original line breaks
+		T{0x20,	"Telugu"},
+	}
+}
+
+func _() {
+	type (
+		xxxxxx int
+		x float
+		xxx string
+		xxxxx []x
+		xx struct{}
+		xxxxxxx struct {
+			_, _ int
+			_ float
+		}
+		xxxx chan<- string
+	)
+}
+
+// alignment of "=" in consecutive lines (extended example from issue 1414)
+const (
+	umax uint                  = ^uint(0) // maximum value for a uint
+	bpu  = 1 << (5 + umax>>63)            // bits per uint
+	foo
+	bar  = -1
+)
+
+// typical enum
+const (
+	a MyType = iota
+	abcd
+	b
+	c
+	def
+)
+
+// excerpt from godoc.go
+var (
+	goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory")
+	testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)")
+	pkgPath = flag.String("path", "", "additional package directories (colon-separated)")
+	filter = flag.String("filter", "", "filter file containing permitted package directory paths")
+	filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0")
+	filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially
+)
+
+
+// formatting of structs
+type _ struct{}
+
+type _ struct{ /* this comment should be visible */ }
+
+type _ struct{
+	// this comment should be visible and properly indented
+}
+
+type _ struct {  // this comment must not change indentation
+	f int
+	f, ff, fff, ffff int
+}
+
+type _ struct {
+	string
+}
+
+type _ struct {
+	string  // comment
+}
+
+type _ struct {
+	string "tag"
+}
+
+type _ struct {
+	string "tag"  // comment
+}
+
+type _ struct {
+	f int
+}
+
+type _ struct {
+	f int  // comment
+}
+
+type _ struct {
+	f int "tag"
+}
+
+type _ struct {
+	f int "tag"  // comment
+}
+
+type _ struct {
+	bool
+	a, b, c int
+	int "tag"
+	ES // comment
+	float "tag"  // comment
+	f int  // comment
+	f, ff, fff, ffff int  // comment
+	g float "tag"
+	h float "tag"  // comment
+}
+
+type _ struct { a, b,
+c, d int  // this line should be indented
+u, v, w, x float // this line should be indented
+p, q,
+r, s float // this line should be indented
+}
+
+
+// difficult cases
+type _ struct {
+	bool  // comment
+	text []byte  // comment
+}
+
+
+// formatting of interfaces
+type EI interface{}
+
+type _ interface {
+	EI
+}
+
+type _ interface {
+	f()
+	fffff()
+}
+
+type _ interface {
+	EI
+	f()
+	fffffg()
+}
+
+type _ interface {  // this comment must not change indentation
+	EI  // here's a comment
+	f()  // no blank between identifier and ()
+	fffff()  // no blank between identifier and ()
+	gggggggggggg(x, y, z int) ()  // hurray
+}
+
+
+// formatting of variable declarations
+func _() {
+	type day struct { n int; short, long string }
+	var (
+		Sunday = day{ 0, "SUN", "Sunday" }
+		Monday = day{ 1, "MON", "Monday" }
+		Tuesday = day{ 2, "TUE", "Tuesday" }
+		Wednesday = day{ 3, "WED", "Wednesday" }
+		Thursday = day{ 4, "THU", "Thursday" }
+		Friday = day{ 5, "FRI", "Friday" }
+		Saturday = day{ 6, "SAT", "Saturday" }
+	)
+}
+
+
+// formatting of multi-line variable declarations
+var a1, b1, c1 int  // all on one line
+
+var a2, b2,
+c2 int  // this line should be indented
+
+var (a3, b3,
+c3, d3 int  // this line should be indented
+a4, b4, c4 int  // this line should be indented
+)
+
+
+func _() {
+	var privateKey2 = &Block{Type: "RSA PRIVATE KEY",
+					Headers: map[string]string{},
+					Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
+			0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
+			0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
+			0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
+		},
+	}
+}
+
+
+func _() {
+	var Universe = Scope {
+		Names: map[string]*Ident {
+			// basic types
+			"bool": nil,
+			"byte": nil,
+			"int8": nil,
+			"int16": nil,
+			"int32": nil,
+			"int64": nil,
+			"uint8": nil,
+			"uint16": nil,
+			"uint32": nil,
+			"uint64": nil,
+			"float32": nil,
+			"float64": nil,
+			"string": nil,
+
+			// convenience types
+			"int": nil,
+			"uint": nil,
+			"uintptr": nil,
+			"float": nil,
+
+			// constants
+			"false": nil,
+			"true": nil,
+			"iota": nil,
+			"nil": nil,
+
+			// functions
+			"cap": nil,
+			"len": nil,
+			"new": nil,
+			"make": nil,
+			"panic": nil,
+			"panicln": nil,
+			"print": nil,
+			"println": nil,
+		},
+	}
+}
+
+
+// alignment of map composite entries
+var _ = map[int]int{
+	// small key sizes: always align even if size ratios are large
+	a: a,
+	abcdefghabcdefgh: a,
+	ab: a,
+	abc: a,
+	abcdefgabcdefg: a,
+	abcd: a,
+	abcde: a,
+	abcdef: a,
+
+	// mixed key sizes: align when key sizes change within accepted ratio
+	abcdefgh: a,
+	abcdefghabcdefg: a,
+	abcdefghij: a,
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij: a, // outlier - do not align with previous line
+	abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij: a, // align with previous line
+
+	ab: a, // do not align with previous line
+	abcde: a, // align with previous line
+}
+
+
+func _() {
+	var _ = T{
+		a,	// must introduce trailing comma
+	}
+}
+
+
+// formatting of function results
+func _() func() {}
+func _() func(int) { return nil }
+func _() func(int) int { return nil }
+func _() func(int) func(int) func() { return nil }
+
+
+// formatting of consecutive single-line functions
+func _() {}
+func _() {}
+func _() {}
+
+func _() {}  // an empty line before this function
+func _() {}
+func _() {}
+
+func _() { f(1, 2, 3) }
+func _(x int) int { y := x; return y+1 }
+func _() int { type T struct{}; var x T; return x }
+
+// these must remain multi-line since they are multi-line in the source
+func _() {
+	f(1, 2, 3)
+}
+func _(x int) int {
+	y := x; return y+1
+}
+func _() int {
+	type T struct{}; var x T; return x
+}
+
+
+// making function declarations safe for new semicolon rules
+func _() { /* multi-line func because of comment */ }
+
+func _() {
+/* multi-line func because block is on multiple lines */ }
+
+
+// ellipsis parameters
+func _(...int)
+func _(...*int)
+func _(...[]int)
+func _(...struct{})
+func _(bool, ...interface{})
+func _(bool, ...func())
+func _(bool, ...func(...int))
+func _(bool, ...map[string]int)
+func _(bool, ...chan int)
+
+func _(b bool, x ...int)
+func _(b bool, x ...*int)
+func _(b bool, x ...[]int)
+func _(b bool, x ...struct{})
+func _(x ...interface{})
+func _(x ...func())
+func _(x ...func(...int))
+func _(x ...map[string]int)
+func _(x ...chan int)
+
+
+// these parameter lists must remain multi-line since they are multi-line in the source
+func _(bool,
+int) {
+}
+func _(x bool,
+y int) {
+}
+func _(x,
+y bool) {
+}
+func _(bool, // comment
+int) {
+}
+func _(x bool, // comment
+y int) {
+}
+func _(x, // comment
+y bool) {
+}
+func _(bool, // comment
+// comment
+int) {
+}
+func _(x bool, // comment
+// comment
+y int) {
+}
+func _(x, // comment
+// comment
+y bool) {
+}
+func _(bool,
+// comment
+int) {
+}
+func _(x bool,
+// comment
+y int) {
+}
+func _(x,
+// comment
+y bool) {
+}
+func _(x, // comment
+y,// comment
+z bool) {
+}
+func _(x, // comment
+	y,// comment
+	z bool) {
+}
+func _(x int,	// comment
+	y float,	// comment
+	z bool) {
+}
diff --git a/src/pkg/go/printer/testdata/empty.golden b/src/pkg/go/printer/testdata/empty.golden
new file mode 100644
index 000000000..a055f4758
--- /dev/null
+++ b/src/pkg/go/printer/testdata/empty.golden
@@ -0,0 +1,5 @@
+// a comment at the beginning of the file
+
+package empty
+
+// a comment at the end of the file
diff --git a/src/pkg/go/printer/testdata/empty.input b/src/pkg/go/printer/testdata/empty.input
new file mode 100644
index 000000000..a055f4758
--- /dev/null
+++ b/src/pkg/go/printer/testdata/empty.input
@@ -0,0 +1,5 @@
+// a comment at the beginning of the file
+
+package empty
+
+// a comment at the end of the file
diff --git a/src/pkg/go/printer/testdata/expressions.golden b/src/pkg/go/printer/testdata/expressions.golden
new file mode 100644
index 000000000..d0cf24ad6
--- /dev/null
+++ b/src/pkg/go/printer/testdata/expressions.golden
@@ -0,0 +1,627 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expressions
+
+type T struct {
+	x, y, z int
+}
+
+var (
+	a, b, c, d, e						int
+	under_bar						int
+	longIdentifier1, longIdentifier2, longIdentifier3	int
+	t0, t1, t2						T
+	s							string
+	p							*int
+)
+
+func _() {
+	// no spaces around simple or parenthesized expressions
+	_ = (a + 0)
+	_ = a + b
+	_ = a + b + c
+	_ = a + b - c
+	_ = a - b - c
+	_ = a + (b * c)
+	_ = a + (b / c)
+	_ = a - (b % c)
+	_ = 1 + a
+	_ = a + 1
+	_ = a + b + 1
+	_ = s[a]
+	_ = s[a:]
+	_ = s[:b]
+	_ = s[1:2]
+	_ = s[a:b]
+	_ = s[0:len(s)]
+	_ = s[0] << 1
+	_ = (s[0] << 1) & 0xf
+	_ = s[0]<<2 | s[1]>>4
+	_ = "foo" + s
+	_ = s + "foo"
+	_ = 'a' + 'b'
+	_ = len(s) / 2
+	_ = len(t0.x) / a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[:b-c]
+	_ = s[a+b:]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2*longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x*t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a + b + c) * 2
+	_ = a - b + c - d + (a + b + c) + d&e
+	_ = under_bar - 1
+	_ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+
+	// the parser does not restrict expressions that may appear as statements
+	true
+	42
+	"foo"
+	x
+	(x)
+	a + b
+	a + b + c
+	a + (b * c)
+	a + (b / c)
+	1 + a
+	a + 1
+	s[a]
+	x << 1
+	(s[0] << 1) & 0xf
+	"foo" + s
+	x == y
+	x < y || z > 42
+}
+
+func _() {
+	_ = a + b
+	_ = a + b + c
+	_ = a + b*c
+	_ = a + (b * c)
+	_ = (a + b) * c
+	_ = a + (b * c * d)
+	_ = a + (b*c + d)
+
+	_ = 1 << x
+	_ = -1 << x
+	_ = 1<>4
+
+	b.buf = b.buf[0 : b.off+m+n]
+	b.buf = b.buf[0 : b.off+m*n]
+	f(b.buf[0 : b.off+m+n])
+
+	signed += ' ' * 8
+	tw.octal(header[148:155], chksum)
+
+	_ = x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2) >> w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1 << z) | (x0 >> (uint(w) - z))
+	x1 = x1<>(uint(w)-z)
+
+	_ = buf[0 : len(buf)+1]
+	_ = buf[0 : n+1]
+
+	a, b = b, a
+	a = b + c
+	a = b*c + d
+	_ = a*b + c
+	_ = a - b - c
+	_ = a - (b - c)
+	_ = a - b*c
+	_ = a - (b * c)
+	_ = a * b / c
+	_ = a / *b
+	_ = x[a|^b]
+	_ = x[a / *b]
+	_ = a & ^b
+	_ = a + +b
+	_ = a - -b
+	_ = x[a*-b]
+	_ = x[a + +b]
+	_ = x ^ y ^ z
+	_ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	_ = len(longVariableName) * 2
+
+	_ = token(matchType + xlength<> 4
+	_ = "foo"+s
+	_ = s+"foo"
+	_ = 'a'+'b'
+	_ = len(s)/2
+	_ = len(t0.x)/a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[: b-c]
+	_ = s[a+b :]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2 * longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x * t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a+b+c)*2
+	_ = a - b + c - d + (a+b+c) + d&e
+	_ = under_bar-1
+	_ = Open(dpath + "/file", O_WRONLY | O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+
+	// the parser does not restrict expressions that may appear as statements
+	true
+	42
+	"foo"
+	x
+	(x)
+	a+b
+	a+b+c
+	a+(b*c)
+	a+(b/c)
+	1+a
+	a+1
+	s[a]
+	x<<1
+	(s[0]<<1)&0xf
+	"foo"+s
+	x == y
+	x < y || z > 42
+}
+
+
+func _() {
+	_ = a+b
+	_ = a+b+c
+	_ = a+b*c
+	_ = a+(b*c)
+	_ = (a+b)*c
+	_ = a+(b*c*d)
+	_ = a+(b*c+d)
+
+	_ = 1<>4
+
+	b.buf = b.buf[0:b.off+m+n]
+	b.buf = b.buf[0:b.off+m*n]
+	f(b.buf[0:b.off+m+n])
+
+	signed += ' '*8
+	tw.octal(header[148:155], chksum)
+
+	_ = x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2)>>w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1<>(uint(w)-z))
+	x1 = x1<>(uint(w)-z)
+
+	_ = buf[0:len(buf)+1]
+	_ = buf[0:n+1]
+
+	a,b = b,a
+	a = b+c
+	a = b*c+d
+	_ = a*b+c
+	_ = a-b-c
+	_ = a-(b-c)
+	_ = a-b*c
+	_ = a-(b*c)
+	_ = a*b/c
+	_ = a/ *b
+	_ = x[a|^b]
+	_ = x[a/ *b]
+	_ = a& ^b
+	_ = a+ +b
+	_ = a- -b
+	_ = x[a*-b]
+	_ = x[a+ +b]
+	_ = x^y^z
+	_ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	_ = len(longVariableName)*2
+
+	_ = token(matchType + xlength<>4
+	_ = "foo" + s
+	_ = s + "foo"
+	_ = 'a' + 'b'
+	_ = len(s) / 2
+	_ = len(t0.x) / a
+
+	// spaces around expressions of different precedence or expressions containing spaces
+	_ = a + -b
+	_ = a - ^b
+	_ = a / *p
+	_ = a + b*c
+	_ = 1 + b*c
+	_ = a + 2*c
+	_ = a + c*2
+	_ = 1 + 2*3
+	_ = s[1 : 2*3]
+	_ = s[a : b-c]
+	_ = s[0:]
+	_ = s[a+b]
+	_ = s[:b-c]
+	_ = s[a+b:]
+	_ = a[a< b
+	_ = a >= b
+	_ = a < b
+	_ = a <= b
+	_ = a < b && c > d
+	_ = a < b || c > d
+
+	// spaces around "long" operands
+	_ = a + longIdentifier1
+	_ = longIdentifier1 + a
+	_ = longIdentifier1 + longIdentifier2*longIdentifier3
+	_ = s + "a longer string"
+
+	// some selected cases
+	_ = a + t0.x
+	_ = a + t0.x + t1.x*t2.x
+	_ = a + b + c + d + e + 2*3
+	_ = a + b + c + 2*3 + d + e
+	_ = (a + b + c) * 2
+	_ = a - b + c - d + (a + b + c) + d&e
+	_ = under_bar - 1
+	_ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
+	_ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx)
+
+	// the parser does not restrict expressions that may appear as statements
+	true
+	42
+	"foo"
+	x
+	(x)
+	a + b
+	a + b + c
+	a + (b * c)
+	a + (b / c)
+	1 + a
+	a + 1
+	s[a]
+	x << 1
+	(s[0] << 1) & 0xf
+	"foo" + s
+	x == y
+	x < y || z > 42
+}
+
+func _() {
+	_ = a + b
+	_ = a + b + c
+	_ = a + b*c
+	_ = a + (b * c)
+	_ = (a + b) * c
+	_ = a + (b * c * d)
+	_ = a + (b*c + d)
+
+	_ = 1 << x
+	_ = -1 << x
+	_ = 1<>4
+
+	b.buf = b.buf[0 : b.off+m+n]
+	b.buf = b.buf[0 : b.off+m*n]
+	f(b.buf[0 : b.off+m+n])
+
+	signed += ' ' * 8
+	tw.octal(header[148:155], chksum)
+
+	_ = x > 0 && i >= 0
+
+	x1, x0 := x>>w2, x&m2
+	z0 = t1<>w2) >> w2
+	q1, r1 := x1/d1, x1%d1
+	r1 = r1*b2 | x0>>w2
+	x1 = (x1 << z) | (x0 >> (uint(w) - z))
+	x1 = x1<>(uint(w)-z)
+
+	_ = buf[0 : len(buf)+1]
+	_ = buf[0 : n+1]
+
+	a, b = b, a
+	a = b + c
+	a = b*c + d
+	_ = a*b + c
+	_ = a - b - c
+	_ = a - (b - c)
+	_ = a - b*c
+	_ = a - (b * c)
+	_ = a * b / c
+	_ = a / *b
+	_ = x[a|^b]
+	_ = x[a / *b]
+	_ = a & ^b
+	_ = a + +b
+	_ = a - -b
+	_ = x[a*-b]
+	_ = x[a + +b]
+	_ = x ^ y ^ z
+	_ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF]
+	_ = len(longVariableName) * 2
+
+	_ = token(matchType + xlength< /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	&writerTest{
+		file:	"testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name:		"tmp/16gig.txt",
+					Mode:		0640,
+					Uid:		73025,
+					Gid:		5000,
+					Size:		16 << 30,
+					Mtime:		1254699560,
+					Typeflag:	'0',
+					Uname:		"dsymonds",
+					Gname:		"eng",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+type untarTest struct {
+	file	string
+	headers	[]*Header
+}
+
+var untarTests = []*untarTest{
+	&untarTest{
+		file:	"testdata/gnu.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244428340,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244436044,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+			},
+		},
+	},
+	&untarTest{
+		file:	"testdata/star.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244592783,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+				Atime:		1244592783,
+				Ctime:		1244592783,
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0640,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244592783,
+				Typeflag:	'0',
+				Uname:		"dsymonds",
+				Gname:		"eng",
+				Atime:		1244592783,
+				Ctime:		1244592783,
+			},
+		},
+	},
+	&untarTest{
+		file:	"testdata/v7.tar",
+		headers: []*Header{
+			&Header{
+				Name:		"small.txt",
+				Mode:		0444,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		5,
+				Mtime:		1244593104,
+				Typeflag:	'\x00',
+			},
+			&Header{
+				Name:		"small2.txt",
+				Mode:		0444,
+				Uid:		73025,
+				Gid:		5000,
+				Size:		11,
+				Mtime:		1244593104,
+				Typeflag:	'\x00',
+			},
+		},
+	},
+}
+
+var facts = map[int]string{
+	0:	"1",
+	1:	"1",
+	2:	"2",
+	10:	"3628800",
+	20:	"2432902008176640000",
+	100: "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000",
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		// TODO(gri): the 2nd string of this string list should not be indented
+		"usage: godoc package [name ...]\n"+
+			"	godoc -http=:6060\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func TestReader(t *testing.T) {
+testLoop:
+	for i, test := range untarTests {
+		f, err := os.Open(test.file, os.O_RDONLY, 0444)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+		tr := NewReader(f)
+		for j, header := range test.headers {
+			hdr, err := tr.Next()
+			if err != nil || hdr == nil {
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
+				f.Close()
+				continue testLoop
+			}
+			if !reflect.DeepEqual(hdr, header) {
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
+					i, j, *hdr, *header)
+			}
+		}
+		hdr, err := tr.Next()
+		if hdr != nil || err != nil {
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err)
+		}
+		f.Close()
+	}
+}
+
+// There should be exactly one linebreak after this comment.
diff --git a/src/pkg/go/printer/testdata/linebreaks.input b/src/pkg/go/printer/testdata/linebreaks.input
new file mode 100644
index 000000000..457b491e6
--- /dev/null
+++ b/src/pkg/go/printer/testdata/linebreaks.input
@@ -0,0 +1,223 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package linebreaks
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type writerTestEntry struct {
+	header *Header
+	contents string
+}
+
+type writerTest struct {
+	file string  // filename of expected output
+	entries []*writerTestEntry
+}
+
+var writerTests = []*writerTest{
+	&writerTest{
+		file: "testdata/writer.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name: "small.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 5,
+					Mtime: 1246508266,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				contents: "Kilts",
+			},
+			&writerTestEntry{
+				header: &Header{
+					Name: "small2.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 11,
+					Mtime: 1245217492,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				contents: "Google.com\n",
+			},
+		},
+	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	&writerTest{
+		file: "testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			&writerTestEntry{
+				header: &Header{
+					Name: "tmp/16gig.txt",
+					Mode: 0640,
+					Uid: 73025,
+					Gid: 5000,
+					Size: 16 << 30,
+					Mtime: 1254699560,
+					Typeflag: '0',
+					Uname: "dsymonds",
+					Gname: "eng",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+type untarTest struct {
+	file string
+	headers []*Header
+}
+
+var untarTests = []*untarTest{
+	&untarTest{
+		file: "testdata/gnu.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244428340,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244436044,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+			},
+		},
+	},
+	&untarTest{
+		file: "testdata/star.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244592783,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+				Atime: 1244592783,
+				Ctime: 1244592783,
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0640,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244592783,
+				Typeflag: '0',
+				Uname: "dsymonds",
+				Gname: "eng",
+				Atime: 1244592783,
+				Ctime: 1244592783,
+			},
+		},
+	},
+	&untarTest{
+		file: "testdata/v7.tar",
+		headers: []*Header{
+			&Header{
+				Name: "small.txt",
+				Mode: 0444,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 5,
+				Mtime: 1244593104,
+				Typeflag: '\x00',
+			},
+			&Header{
+				Name: "small2.txt",
+				Mode: 0444,
+				Uid: 73025,
+				Gid: 5000,
+				Size: 11,
+				Mtime: 1244593104,
+				Typeflag: '\x00',
+			},
+		},
+	},
+}
+
+var facts = map[int] string {
+	0: "1",
+	1: "1",
+	2: "2",
+	10: "3628800",
+	20: "2432902008176640000",
+	100: "933262154439441526816992388562667004907159682643816214685929" +
+		"638952175999932299156089414639761565182862536979208272237582" +
+		"51185210916864000000000000000000000000",
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		// TODO(gri): the 2nd string of this string list should not be indented
+		"usage: godoc package [name ...]\n" +
+		"	godoc -http=:6060\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+func TestReader(t *testing.T) {
+testLoop:
+	for i, test := range untarTests {
+		f, err := os.Open(test.file, os.O_RDONLY, 0444)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+		tr := NewReader(f)
+		for j, header := range test.headers {
+			hdr, err := tr.Next()
+			if err != nil || hdr == nil {
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
+				f.Close()
+				continue testLoop
+			}
+			if !reflect.DeepEqual(hdr, header) {
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
+					 i, j, *hdr, *header)
+			}
+		}
+		hdr, err := tr.Next()
+		if hdr != nil || err != nil {
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err)
+		}
+		f.Close()
+	}
+}
+
+// There should be exactly one linebreak after this comment.
diff --git a/src/pkg/go/printer/testdata/parser.go b/src/pkg/go/printer/testdata/parser.go
new file mode 100644
index 000000000..2d27af499
--- /dev/null
+++ b/src/pkg/go/printer/testdata/parser.go
@@ -0,0 +1,2153 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package parser implements a parser for Go source files. Input may be
+// provided in a variety of forms (see the various Parse* functions); the
+// output is an abstract syntax tree (AST) representing the Go source. The
+// parser is invoked through one of the Parse* functions.
+//
+package parser
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+)
+
+// The mode parameter to the Parse* functions is a set of flags (or 0).
+// They control the amount of source code parsed and other optional
+// parser functionality.
+//
+const (
+	PackageClauseOnly uint = 1 << iota // parsing stops after package clause
+	ImportsOnly                        // parsing stops after import declarations
+	ParseComments                      // parse comments and add them to AST
+	Trace                              // print a trace of parsed productions
+	DeclarationErrors                  // report declaration errors
+)
+
+// The parser structure holds the parser's internal state.
+type parser struct {
+	file *token.File
+	scanner.ErrorVector
+	scanner scanner.Scanner
+
+	// Tracing/debugging
+	mode   uint // parsing mode
+	trace  bool // == (mode & Trace != 0)
+	indent uint // indentation used for tracing output
+
+	// Comments
+	comments    []*ast.CommentGroup
+	leadComment *ast.CommentGroup // last lead comment
+	lineComment *ast.CommentGroup // last line comment
+
+	// Next token
+	pos token.Pos   // token position
+	tok token.Token // one token look-ahead
+	lit string      // token literal
+
+	// Non-syntactic parser control
+	exprLev int // < 0: in control clause, >= 0: in expression
+
+	// Ordinary identifer scopes
+	pkgScope   *ast.Scope        // pkgScope.Outer == nil
+	topScope   *ast.Scope        // top-most scope; may be pkgScope
+	unresolved []*ast.Ident      // unresolved identifiers
+	imports    []*ast.ImportSpec // list of imports
+
+	// Label scope
+	// (maintained by open/close LabelScope)
+	labelScope  *ast.Scope     // label scope for current function
+	targetStack [][]*ast.Ident // stack of unresolved labels
+}
+
+// scannerMode returns the scanner mode bits given the parser's mode bits.
+func scannerMode(mode uint) uint {
+	var m uint = scanner.InsertSemis
+	if mode&ParseComments != 0 {
+		m |= scanner.ScanComments
+	}
+	return m
+}
+
+func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
+	p.file = fset.AddFile(filename, fset.Base(), len(src))
+	p.scanner.Init(p.file, src, p, scannerMode(mode))
+
+	p.mode = mode
+	p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
+
+	p.next()
+
+	// set up the pkgScope here (as opposed to in parseFile) because
+	// there are other parser entry points (ParseExpr, etc.)
+	p.openScope()
+	p.pkgScope = p.topScope
+
+	// for the same reason, set up a label scope
+	p.openLabelScope()
+}
+
+// ----------------------------------------------------------------------------
+// Scoping support
+
+func (p *parser) openScope() {
+	p.topScope = ast.NewScope(p.topScope)
+}
+
+func (p *parser) closeScope() {
+	p.topScope = p.topScope.Outer
+}
+
+func (p *parser) openLabelScope() {
+	p.labelScope = ast.NewScope(p.labelScope)
+	p.targetStack = append(p.targetStack, nil)
+}
+
+func (p *parser) closeLabelScope() {
+	// resolve labels
+	n := len(p.targetStack) - 1
+	scope := p.labelScope
+	for _, ident := range p.targetStack[n] {
+		ident.Obj = scope.Lookup(ident.Name)
+		if ident.Obj == nil && p.mode&DeclarationErrors != 0 {
+			p.error(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name))
+		}
+	}
+	// pop label scope
+	p.targetStack = p.targetStack[0:n]
+	p.labelScope = p.labelScope.Outer
+}
+
+func (p *parser) declare(decl interface{}, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) {
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		if ident.Name != "_" {
+			obj := ast.NewObj(kind, ident.Name)
+			// remember the corresponding declaration for redeclaration
+			// errors and global variable resolution/typechecking phase
+			obj.Decl = decl
+			if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 {
+				prevDecl := ""
+				if pos := alt.Pos(); pos.IsValid() {
+					prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.file.Position(pos))
+				}
+				p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl))
+			}
+			ident.Obj = obj
+		}
+	}
+}
+
+func (p *parser) shortVarDecl(idents []*ast.Ident) {
+	// Go spec: A short variable declaration may redeclare variables
+	// provided they were originally declared in the same block with
+	// the same type, and at least one of the non-blank variables is new.
+	n := 0 // number of new variables
+	for _, ident := range idents {
+		assert(ident.Obj == nil, "identifier already declared or resolved")
+		if ident.Name != "_" {
+			obj := ast.NewObj(ast.Var, ident.Name)
+			// short var declarations cannot have redeclaration errors
+			// and are not global => no need to remember the respective
+			// declaration
+			alt := p.topScope.Insert(obj)
+			if alt == nil {
+				n++ // new declaration
+				alt = obj
+			}
+			ident.Obj = alt
+		}
+	}
+	if n == 0 && p.mode&DeclarationErrors != 0 {
+		p.error(idents[0].Pos(), "no new variables on left side of :=")
+	}
+}
+
+// The unresolved object is a sentinel to mark identifiers that have been added
+// to the list of unresolved identifiers. The sentinel is only used for verifying
+// internal consistency.
+var unresolved = new(ast.Object)
+
+func (p *parser) resolve(x ast.Expr) {
+	// nothing to do if x is not an identifier or the blank identifier
+	ident, _ := x.(*ast.Ident)
+	if ident == nil {
+		return
+	}
+	assert(ident.Obj == nil, "identifier already declared or resolved")
+	if ident.Name == "_" {
+		return
+	}
+	// try to resolve the identifier
+	for s := p.topScope; s != nil; s = s.Outer {
+		if obj := s.Lookup(ident.Name); obj != nil {
+			ident.Obj = obj
+			return
+		}
+	}
+	// all local scopes are known, so any unresolved identifier
+	// must be found either in the file scope, package scope
+	// (perhaps in another file), or universe scope --- collect
+	// them so that they can be resolved later
+	ident.Obj = unresolved
+	p.unresolved = append(p.unresolved, ident)
+}
+
+// ----------------------------------------------------------------------------
+// Parsing support
+
+func (p *parser) printTrace(a ...interface{}) {
+	const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
+		". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
+	const n = uint(len(dots))
+	pos := p.file.Position(p.pos)
+	fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
+	i := 2 * p.indent
+	for ; i > n; i -= n {
+		fmt.Print(dots)
+	}
+	fmt.Print(dots[0:i])
+	fmt.Println(a...)
+}
+
+func trace(p *parser, msg string) *parser {
+	p.printTrace(msg, "(")
+	p.indent++
+	return p
+}
+
+// Usage pattern: defer un(trace(p, "..."));
+func un(p *parser) {
+	p.indent--
+	p.printTrace(")")
+}
+
+// Advance to the next token.
+func (p *parser) next0() {
+	// Because of one-token look-ahead, print the previous token
+	// when tracing as it provides a more readable output. The
+	// very first token (!p.pos.IsValid()) is not initialized
+	// (it is token.ILLEGAL), so don't print it .
+	if p.trace && p.pos.IsValid() {
+		s := p.tok.String()
+		switch {
+		case p.tok.IsLiteral():
+			p.printTrace(s, p.lit)
+		case p.tok.IsOperator(), p.tok.IsKeyword():
+			p.printTrace("\"" + s + "\"")
+		default:
+			p.printTrace(s)
+		}
+	}
+
+	p.pos, p.tok, p.lit = p.scanner.Scan()
+}
+
+// Consume a comment and return it and the line on which it ends.
+func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
+	// /*-style comments may end on a different line than where they start.
+	// Scan the comment for '\n' chars and adjust endline accordingly.
+	endline = p.file.Line(p.pos)
+	if p.lit[1] == '*' {
+		// don't use range here - no need to decode Unicode code points
+		for i := 0; i < len(p.lit); i++ {
+			if p.lit[i] == '\n' {
+				endline++
+			}
+		}
+	}
+
+	comment = &ast.Comment{p.pos, p.lit}
+	p.next0()
+
+	return
+}
+
+// Consume a group of adjacent comments, add it to the parser's
+// comments list, and return it together with the line at which
+// the last comment in the group ends. An empty line or non-comment
+// token terminates a comment group.
+//
+func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
+	var list []*ast.Comment
+	endline = p.file.Line(p.pos)
+	for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
+		var comment *ast.Comment
+		comment, endline = p.consumeComment()
+		list = append(list, comment)
+	}
+
+	// add comment group to the comments list
+	comments = &ast.CommentGroup{list}
+	p.comments = append(p.comments, comments)
+
+	return
+}
+
+// Advance to the next non-comment token. In the process, collect
+// any comment groups encountered, and remember the last lead and
+// and line comments.
+//
+// A lead comment is a comment group that starts and ends in a
+// line without any other tokens and that is followed by a non-comment
+// token on the line immediately after the comment group.
+//
+// A line comment is a comment group that follows a non-comment
+// token on the same line, and that has no tokens after it on the line
+// where it ends.
+//
+// Lead and line comments may be considered documentation that is
+// stored in the AST.
+//
+func (p *parser) next() {
+	p.leadComment = nil
+	p.lineComment = nil
+	line := p.file.Line(p.pos) // current line
+	p.next0()
+
+	if p.tok == token.COMMENT {
+		var comment *ast.CommentGroup
+		var endline int
+
+		if p.file.Line(p.pos) == line {
+			// The comment is on same line as the previous token; it
+			// cannot be a lead comment but may be a line comment.
+			comment, endline = p.consumeCommentGroup()
+			if p.file.Line(p.pos) != endline {
+				// The next token is on a different line, thus
+				// the last comment group is a line comment.
+				p.lineComment = comment
+			}
+		}
+
+		// consume successor comments, if any
+		endline = -1
+		for p.tok == token.COMMENT {
+			comment, endline = p.consumeCommentGroup()
+		}
+
+		if endline+1 == p.file.Line(p.pos) {
+			// The next token is following on the line immediately after the
+			// comment group, thus the last comment group is a lead comment.
+			p.leadComment = comment
+		}
+	}
+}
+
+func (p *parser) error(pos token.Pos, msg string) {
+	p.Error(p.file.Position(pos), msg)
+}
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
+	msg = "expected " + msg
+	if pos == p.pos {
+		// the error happened at the current position;
+		// make the error message more specific
+		if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
+			msg += ", found newline"
+		} else {
+			msg += ", found '" + p.tok.String() + "'"
+			if p.tok.IsLiteral() {
+				msg += " " + p.lit
+			}
+		}
+	}
+	p.error(pos, msg)
+}
+
+func (p *parser) expect(tok token.Token) token.Pos {
+	pos := p.pos
+	if p.tok != tok {
+		p.errorExpected(pos, "'"+tok.String()+"'")
+	}
+	p.next() // make progress
+	return pos
+}
+
+func (p *parser) expectSemi() {
+	if p.tok != token.RPAREN && p.tok != token.RBRACE {
+		p.expect(token.SEMICOLON)
+	}
+}
+
+func assert(cond bool, msg string) {
+	if !cond {
+		panic("go/parser internal error: " + msg)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Identifiers
+
+func (p *parser) parseIdent() *ast.Ident {
+	pos := p.pos
+	name := "_"
+	if p.tok == token.IDENT {
+		name = p.lit
+		p.next()
+	} else {
+		p.expect(token.IDENT) // use expect() error handling
+	}
+	return &ast.Ident{pos, name, nil}
+}
+
+func (p *parser) parseIdentList() (list []*ast.Ident) {
+	if p.trace {
+		defer un(trace(p, "IdentList"))
+	}
+
+	list = append(list, p.parseIdent())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseIdent())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Common productions
+
+// If lhs is set, result list elements which are identifiers are not resolved.
+func (p *parser) parseExprList(lhs bool) (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ExpressionList"))
+	}
+
+	list = append(list, p.parseExpr(lhs))
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseExpr(lhs))
+	}
+
+	return
+}
+
+func (p *parser) parseLhsList() []ast.Expr {
+	list := p.parseExprList(true)
+	switch p.tok {
+	case token.DEFINE:
+		// lhs of a short variable declaration
+		p.shortVarDecl(p.makeIdentList(list))
+	case token.COLON:
+		// lhs of a label declaration or a communication clause of a select
+		// statement (parseLhsList is not called when parsing the case clause
+		// of a switch statement):
+		// - labels are declared by the caller of parseLhsList
+		// - for communication clauses, if there is a stand-alone identifier
+		//   followed by a colon, we have a syntax error; there is no need
+		//   to resolve the identifier in that case
+	default:
+		// identifiers must be declared elsewhere
+		for _, x := range list {
+			p.resolve(x)
+		}
+	}
+	return list
+}
+
+func (p *parser) parseRhsList() []ast.Expr {
+	return p.parseExprList(false)
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+func (p *parser) parseType() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Type"))
+	}
+
+	typ := p.tryType()
+
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		return &ast.BadExpr{pos, p.pos}
+	}
+
+	return typ
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) parseTypeName() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeName"))
+	}
+
+	ident := p.parseIdent()
+	// don't resolve ident yet - it may be a parameter or field name
+
+	if p.tok == token.PERIOD {
+		// ident is a package name
+		p.next()
+		p.resolve(ident)
+		sel := p.parseIdent()
+		return &ast.SelectorExpr{ident, sel}
+	}
+
+	return ident
+}
+
+func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "ArrayType"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	var len ast.Expr
+	if ellipsisOk && p.tok == token.ELLIPSIS {
+		len = &ast.Ellipsis{p.pos, nil}
+		p.next()
+	} else if p.tok != token.RBRACK {
+		len = p.parseRhs()
+	}
+	p.expect(token.RBRACK)
+	elt := p.parseType()
+
+	return &ast.ArrayType{lbrack, len, elt}
+}
+
+func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident {
+	idents := make([]*ast.Ident, len(list))
+	for i, x := range list {
+		ident, isIdent := x.(*ast.Ident)
+		if !isIdent {
+			pos := x.(ast.Expr).Pos()
+			p.errorExpected(pos, "identifier")
+			ident = &ast.Ident{pos, "_", nil}
+		}
+		idents[i] = ident
+	}
+	return idents
+}
+
+func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "FieldDecl"))
+	}
+
+	doc := p.leadComment
+
+	// fields
+	list, typ := p.parseVarList(false)
+
+	// optional tag
+	var tag *ast.BasicLit
+	if p.tok == token.STRING {
+		tag = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	}
+
+	// analyze case
+	var idents []*ast.Ident
+	if typ != nil {
+		// IdentifierList Type
+		idents = p.makeIdentList(list)
+	} else {
+		// ["*"] TypeName (AnonymousField)
+		typ = list[0] // we always have at least one element
+		p.resolve(typ)
+		if n := len(list); n > 1 || !isTypeName(deref(typ)) {
+			pos := typ.Pos()
+			p.errorExpected(pos, "anonymous field")
+			typ = &ast.BadExpr{pos, list[n-1].End()}
+		}
+	}
+
+	p.expectSemi() // call before accessing p.linecomment
+
+	field := &ast.Field{doc, idents, typ, tag, p.lineComment}
+	p.declare(field, scope, ast.Var, idents...)
+
+	return field
+}
+
+func (p *parser) parseStructType() *ast.StructType {
+	if p.trace {
+		defer un(trace(p, "StructType"))
+	}
+
+	pos := p.expect(token.STRUCT)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // struct scope
+	var list []*ast.Field
+	for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN {
+		// a field declaration cannot start with a '(' but we accept
+		// it here for more robust parsing and better error messages
+		// (parseFieldDecl will check and complain if necessary)
+		list = append(list, p.parseFieldDecl(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store struct scope in AST
+	return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parsePointerType() *ast.StarExpr {
+	if p.trace {
+		defer un(trace(p, "PointerType"))
+	}
+
+	star := p.expect(token.MUL)
+	base := p.parseType()
+
+	return &ast.StarExpr{star, base}
+}
+
+func (p *parser) tryVarType(isParam bool) ast.Expr {
+	if isParam && p.tok == token.ELLIPSIS {
+		pos := p.pos
+		p.next()
+		typ := p.tryIdentOrType(isParam) // don't use parseType so we can provide better error message
+		if typ == nil {
+			p.error(pos, "'...' parameter is missing type")
+			typ = &ast.BadExpr{pos, p.pos}
+		}
+		if p.tok != token.RPAREN {
+			p.error(pos, "can use '...' with last parameter type only")
+		}
+		return &ast.Ellipsis{pos, typ}
+	}
+	return p.tryIdentOrType(false)
+}
+
+func (p *parser) parseVarType(isParam bool) ast.Expr {
+	typ := p.tryVarType(isParam)
+	if typ == nil {
+		pos := p.pos
+		p.errorExpected(pos, "type")
+		p.next() // make progress
+		typ = &ast.BadExpr{pos, p.pos}
+	}
+	return typ
+}
+
+func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "VarList"))
+	}
+
+	// a list of identifiers looks like a list of type names
+	for {
+		// parseVarType accepts any type (including parenthesized ones)
+		// even though the syntax does not permit them here: we
+		// accept them all for more robust parsing and complain
+		// afterwards
+		list = append(list, p.parseVarType(isParam))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	// if we had a list of identifiers, it must be followed by a type
+	typ = p.tryVarType(isParam)
+	if typ != nil {
+		p.resolve(typ)
+	}
+
+	return
+}
+
+func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params []*ast.Field) {
+	if p.trace {
+		defer un(trace(p, "ParameterList"))
+	}
+
+	list, typ := p.parseVarList(ellipsisOk)
+	if typ != nil {
+		// IdentifierList Type
+		idents := p.makeIdentList(list)
+		field := &ast.Field{nil, idents, typ, nil, nil}
+		params = append(params, field)
+		// Go spec: The scope of an identifier denoting a function
+		// parameter or result variable is the function body.
+		p.declare(field, scope, ast.Var, idents...)
+		if p.tok == token.COMMA {
+			p.next()
+		}
+
+		for p.tok != token.RPAREN && p.tok != token.EOF {
+			idents := p.parseIdentList()
+			typ := p.parseVarType(ellipsisOk)
+			field := &ast.Field{nil, idents, typ, nil, nil}
+			params = append(params, field)
+			// Go spec: The scope of an identifier denoting a function
+			// parameter or result variable is the function body.
+			p.declare(field, scope, ast.Var, idents...)
+			if p.tok != token.COMMA {
+				break
+			}
+			p.next()
+		}
+
+	} else {
+		// Type { "," Type } (anonymous parameters)
+		params = make([]*ast.Field, len(list))
+		for i, x := range list {
+			p.resolve(x)
+			params[i] = &ast.Field{Type: x}
+		}
+	}
+
+	return
+}
+
+func (p *parser) parseParameters(scope *ast.Scope, ellipsisOk bool) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Parameters"))
+	}
+
+	var params []*ast.Field
+	lparen := p.expect(token.LPAREN)
+	if p.tok != token.RPAREN {
+		params = p.parseParameterList(scope, ellipsisOk)
+	}
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.FieldList{lparen, params, rparen}
+}
+
+func (p *parser) parseResult(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Result"))
+	}
+
+	if p.tok == token.LPAREN {
+		return p.parseParameters(scope, false)
+	}
+
+	typ := p.tryType()
+	if typ != nil {
+		list := make([]*ast.Field, 1)
+		list[0] = &ast.Field{Type: typ}
+		return &ast.FieldList{List: list}
+	}
+
+	return nil
+}
+
+func (p *parser) parseSignature(scope *ast.Scope) (params, results *ast.FieldList) {
+	if p.trace {
+		defer un(trace(p, "Signature"))
+	}
+
+	params = p.parseParameters(scope, true)
+	results = p.parseResult(scope)
+
+	return
+}
+
+func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) {
+	if p.trace {
+		defer un(trace(p, "FuncType"))
+	}
+
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+	params, results := p.parseSignature(scope)
+
+	return &ast.FuncType{pos, params, results}, scope
+}
+
+func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field {
+	if p.trace {
+		defer un(trace(p, "MethodSpec"))
+	}
+
+	doc := p.leadComment
+	var idents []*ast.Ident
+	var typ ast.Expr
+	x := p.parseTypeName()
+	if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN {
+		// method
+		idents = []*ast.Ident{ident}
+		scope := ast.NewScope(nil) // method scope
+		params, results := p.parseSignature(scope)
+		typ = &ast.FuncType{token.NoPos, params, results}
+	} else {
+		// embedded interface
+		typ = x
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	spec := &ast.Field{doc, idents, typ, nil, p.lineComment}
+	p.declare(spec, scope, ast.Fun, idents...)
+
+	return spec
+}
+
+func (p *parser) parseInterfaceType() *ast.InterfaceType {
+	if p.trace {
+		defer un(trace(p, "InterfaceType"))
+	}
+
+	pos := p.expect(token.INTERFACE)
+	lbrace := p.expect(token.LBRACE)
+	scope := ast.NewScope(nil) // interface scope
+	var list []*ast.Field
+	for p.tok == token.IDENT {
+		list = append(list, p.parseMethodSpec(scope))
+	}
+	rbrace := p.expect(token.RBRACE)
+
+	// TODO(gri): store interface scope in AST
+	return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false}
+}
+
+func (p *parser) parseMapType() *ast.MapType {
+	if p.trace {
+		defer un(trace(p, "MapType"))
+	}
+
+	pos := p.expect(token.MAP)
+	p.expect(token.LBRACK)
+	key := p.parseType()
+	p.expect(token.RBRACK)
+	value := p.parseType()
+
+	return &ast.MapType{pos, key, value}
+}
+
+func (p *parser) parseChanType() *ast.ChanType {
+	if p.trace {
+		defer un(trace(p, "ChanType"))
+	}
+
+	pos := p.pos
+	dir := ast.SEND | ast.RECV
+	if p.tok == token.CHAN {
+		p.next()
+		if p.tok == token.ARROW {
+			p.next()
+			dir = ast.SEND
+		}
+	} else {
+		p.expect(token.ARROW)
+		p.expect(token.CHAN)
+		dir = ast.RECV
+	}
+	value := p.parseType()
+
+	return &ast.ChanType{pos, dir, value}
+}
+
+// If the result is an identifier, it is not resolved.
+func (p *parser) tryIdentOrType(ellipsisOk bool) ast.Expr {
+	switch p.tok {
+	case token.IDENT:
+		return p.parseTypeName()
+	case token.LBRACK:
+		return p.parseArrayType(ellipsisOk)
+	case token.STRUCT:
+		return p.parseStructType()
+	case token.MUL:
+		return p.parsePointerType()
+	case token.FUNC:
+		typ, _ := p.parseFuncType()
+		return typ
+	case token.INTERFACE:
+		return p.parseInterfaceType()
+	case token.MAP:
+		return p.parseMapType()
+	case token.CHAN, token.ARROW:
+		return p.parseChanType()
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		typ := p.parseType()
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, typ, rparen}
+	}
+
+	// no type found
+	return nil
+}
+
+func (p *parser) tryType() ast.Expr {
+	typ := p.tryIdentOrType(false)
+	if typ != nil {
+		p.resolve(typ)
+	}
+	return typ
+}
+
+// ----------------------------------------------------------------------------
+// Blocks
+
+func (p *parser) parseStmtList() (list []ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "StatementList"))
+	}
+
+	for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseStmt())
+	}
+
+	return
+}
+
+func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "Body"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.topScope = scope // open function scope
+	p.openLabelScope()
+	list := p.parseStmtList()
+	p.closeLabelScope()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+func (p *parser) parseBlockStmt() *ast.BlockStmt {
+	if p.trace {
+		defer un(trace(p, "BlockStmt"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	p.openScope()
+	list := p.parseStmtList()
+	p.closeScope()
+	rbrace := p.expect(token.RBRACE)
+
+	return &ast.BlockStmt{lbrace, list, rbrace}
+}
+
+// ----------------------------------------------------------------------------
+// Expressions
+
+func (p *parser) parseFuncTypeOrLit() ast.Expr {
+	if p.trace {
+		defer un(trace(p, "FuncTypeOrLit"))
+	}
+
+	typ, scope := p.parseFuncType()
+	if p.tok != token.LBRACE {
+		// function type only
+		return typ
+	}
+
+	p.exprLev++
+	body := p.parseBody(scope)
+	p.exprLev--
+
+	return &ast.FuncLit{typ, body}
+}
+
+// parseOperand may return an expression or a raw type (incl. array
+// types of the form [...]T. Callers must verify the result.
+// If lhs is set and the result is an identifier, it is not resolved.
+//
+func (p *parser) parseOperand(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Operand"))
+	}
+
+	switch p.tok {
+	case token.IDENT:
+		x := p.parseIdent()
+		if !lhs {
+			p.resolve(x)
+		}
+		return x
+
+	case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
+		x := &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+		return x
+
+	case token.LPAREN:
+		lparen := p.pos
+		p.next()
+		p.exprLev++
+		x := p.parseRhs()
+		p.exprLev--
+		rparen := p.expect(token.RPAREN)
+		return &ast.ParenExpr{lparen, x, rparen}
+
+	case token.FUNC:
+		return p.parseFuncTypeOrLit()
+
+	default:
+		if typ := p.tryIdentOrType(true); typ != nil {
+			// could be type for composite literal or conversion
+			_, isIdent := typ.(*ast.Ident)
+			assert(!isIdent, "type cannot be identifier")
+			return typ
+		}
+	}
+
+	pos := p.pos
+	p.errorExpected(pos, "operand")
+	p.next() // make progress
+	return &ast.BadExpr{pos, p.pos}
+}
+
+func (p *parser) parseSelector(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Selector"))
+	}
+
+	sel := p.parseIdent()
+
+	return &ast.SelectorExpr{x, sel}
+}
+
+func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "TypeAssertion"))
+	}
+
+	p.expect(token.LPAREN)
+	var typ ast.Expr
+	if p.tok == token.TYPE {
+		// type switch: typ == nil
+		p.next()
+	} else {
+		typ = p.parseType()
+	}
+	p.expect(token.RPAREN)
+
+	return &ast.TypeAssertExpr{x, typ}
+}
+
+func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "IndexOrSlice"))
+	}
+
+	lbrack := p.expect(token.LBRACK)
+	p.exprLev++
+	var low, high ast.Expr
+	isSlice := false
+	if p.tok != token.COLON {
+		low = p.parseRhs()
+	}
+	if p.tok == token.COLON {
+		isSlice = true
+		p.next()
+		if p.tok != token.RBRACK {
+			high = p.parseRhs()
+		}
+	}
+	p.exprLev--
+	rbrack := p.expect(token.RBRACK)
+
+	if isSlice {
+		return &ast.SliceExpr{x, lbrack, low, high, rbrack}
+	}
+	return &ast.IndexExpr{x, lbrack, low, rbrack}
+}
+
+func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr {
+	if p.trace {
+		defer un(trace(p, "CallOrConversion"))
+	}
+
+	lparen := p.expect(token.LPAREN)
+	p.exprLev++
+	var list []ast.Expr
+	var ellipsis token.Pos
+	for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
+		list = append(list, p.parseRhs())
+		if p.tok == token.ELLIPSIS {
+			ellipsis = p.pos
+			p.next()
+		}
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+	p.exprLev--
+	rparen := p.expect(token.RPAREN)
+
+	return &ast.CallExpr{fun, lparen, list, ellipsis, rparen}
+}
+
+func (p *parser) parseElement(keyOk bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Element"))
+	}
+
+	if p.tok == token.LBRACE {
+		return p.parseLiteralValue(nil)
+	}
+
+	x := p.parseExpr(keyOk) // don't resolve if map key
+	if keyOk {
+		if p.tok == token.COLON {
+			colon := p.pos
+			p.next()
+			return &ast.KeyValueExpr{x, colon, p.parseElement(false)}
+		}
+		p.resolve(x) // not a map key
+	}
+
+	return x
+}
+
+func (p *parser) parseElementList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "ElementList"))
+	}
+
+	for p.tok != token.RBRACE && p.tok != token.EOF {
+		list = append(list, p.parseElement(true))
+		if p.tok != token.COMMA {
+			break
+		}
+		p.next()
+	}
+
+	return
+}
+
+func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "LiteralValue"))
+	}
+
+	lbrace := p.expect(token.LBRACE)
+	var elts []ast.Expr
+	p.exprLev++
+	if p.tok != token.RBRACE {
+		elts = p.parseElementList()
+	}
+	p.exprLev--
+	rbrace := p.expect(token.RBRACE)
+	return &ast.CompositeLit{typ, lbrace, elts, rbrace}
+}
+
+// checkExpr checks that x is an expression (and not a type).
+func (p *parser) checkExpr(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.BasicLit:
+	case *ast.FuncLit:
+	case *ast.CompositeLit:
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.SelectorExpr:
+	case *ast.IndexExpr:
+	case *ast.SliceExpr:
+	case *ast.TypeAssertExpr:
+		if t.Type == nil {
+			// the form X.(type) is only allowed in type switch expressions
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.CallExpr:
+	case *ast.StarExpr:
+	case *ast.UnaryExpr:
+		if t.Op == token.RANGE {
+			// the range operator is only allowed at the top of a for statement
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.BinaryExpr:
+	default:
+		// all other nodes are not proper expressions
+		p.errorExpected(x.Pos(), "expression")
+		x = &ast.BadExpr{x.Pos(), x.End()}
+	}
+	return x
+}
+
+// isTypeName returns true iff x is a (qualified) TypeName.
+func isTypeName(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	default:
+		return false // all other nodes are not type names
+	}
+	return true
+}
+
+// isLiteralType returns true iff x is a legal composite literal type.
+func isLiteralType(x ast.Expr) bool {
+	switch t := x.(type) {
+	case *ast.BadExpr:
+	case *ast.Ident:
+	case *ast.SelectorExpr:
+		_, isIdent := t.X.(*ast.Ident)
+		return isIdent
+	case *ast.ArrayType:
+	case *ast.StructType:
+	case *ast.MapType:
+	default:
+		return false // all other nodes are not legal composite literal types
+	}
+	return true
+}
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+// If x is of the form (T), unparen returns unparen(T), otherwise it returns x.
+func unparen(x ast.Expr) ast.Expr {
+	if p, isParen := x.(*ast.ParenExpr); isParen {
+		x = unparen(p.X)
+	}
+	return x
+}
+
+// checkExprOrType checks that x is an expression or a type
+// (and not a raw type such as [...]T).
+//
+func (p *parser) checkExprOrType(x ast.Expr) ast.Expr {
+	switch t := unparen(x).(type) {
+	case *ast.ParenExpr:
+		panic("unreachable")
+	case *ast.UnaryExpr:
+		if t.Op == token.RANGE {
+			// the range operator is only allowed at the top of a for statement
+			p.errorExpected(x.Pos(), "expression")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	case *ast.ArrayType:
+		if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
+			p.error(len.Pos(), "expected array length, found '...'")
+			x = &ast.BadExpr{x.Pos(), x.End()}
+		}
+	}
+
+	// all other nodes are expressions or types
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parsePrimaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "PrimaryExpr"))
+	}
+
+	x := p.parseOperand(lhs)
+L:
+	for {
+		switch p.tok {
+		case token.PERIOD:
+			p.next()
+			if lhs {
+				p.resolve(x)
+			}
+			switch p.tok {
+			case token.IDENT:
+				x = p.parseSelector(p.checkExpr(x))
+			case token.LPAREN:
+				x = p.parseTypeAssertion(p.checkExpr(x))
+			default:
+				pos := p.pos
+				p.next() // make progress
+				p.errorExpected(pos, "selector or type assertion")
+				x = &ast.BadExpr{pos, p.pos}
+			}
+		case token.LBRACK:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseIndexOrSlice(p.checkExpr(x))
+		case token.LPAREN:
+			if lhs {
+				p.resolve(x)
+			}
+			x = p.parseCallOrConversion(p.checkExprOrType(x))
+		case token.LBRACE:
+			if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) {
+				if lhs {
+					p.resolve(x)
+				}
+				x = p.parseLiteralValue(x)
+			} else {
+				break L
+			}
+		default:
+			break L
+		}
+		lhs = false // no need to try to resolve again
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseUnaryExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "UnaryExpr"))
+	}
+
+	switch p.tok {
+	case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.RANGE:
+		pos, op := p.pos, p.tok
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, op, p.checkExpr(x)}
+
+	case token.ARROW:
+		// channel type or receive expression
+		pos := p.pos
+		p.next()
+		if p.tok == token.CHAN {
+			p.next()
+			value := p.parseType()
+			return &ast.ChanType{pos, ast.RECV, value}
+		}
+
+		x := p.parseUnaryExpr(false)
+		return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)}
+
+	case token.MUL:
+		// pointer type or unary "*" expression
+		pos := p.pos
+		p.next()
+		x := p.parseUnaryExpr(false)
+		return &ast.StarExpr{pos, p.checkExprOrType(x)}
+	}
+
+	return p.parsePrimaryExpr(lhs)
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+func (p *parser) parseBinaryExpr(lhs bool, prec1 int) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "BinaryExpr"))
+	}
+
+	x := p.parseUnaryExpr(lhs)
+	for prec := p.tok.Precedence(); prec >= prec1; prec-- {
+		for p.tok.Precedence() == prec {
+			pos, op := p.pos, p.tok
+			p.next()
+			if lhs {
+				p.resolve(x)
+				lhs = false
+			}
+			y := p.parseBinaryExpr(false, prec+1)
+			x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)}
+		}
+	}
+
+	return x
+}
+
+// If lhs is set and the result is an identifier, it is not resolved.
+// TODO(gri): parseExpr may return a type or even a raw type ([..]int) -
+//            should reject when a type/raw type is obviously not allowed
+func (p *parser) parseExpr(lhs bool) ast.Expr {
+	if p.trace {
+		defer un(trace(p, "Expression"))
+	}
+
+	return p.parseBinaryExpr(lhs, token.LowestPrec+1)
+}
+
+func (p *parser) parseRhs() ast.Expr {
+	return p.parseExpr(false)
+}
+
+// ----------------------------------------------------------------------------
+// Statements
+
+func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SimpleStmt"))
+	}
+
+	x := p.parseLhsList()
+
+	switch p.tok {
+	case
+		token.DEFINE, token.ASSIGN, token.ADD_ASSIGN,
+		token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN,
+		token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN,
+		token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN:
+		// assignment statement
+		pos, tok := p.pos, p.tok
+		p.next()
+		y := p.parseRhsList()
+		return &ast.AssignStmt{x, pos, tok, y}
+	}
+
+	if len(x) > 1 {
+		p.errorExpected(x[0].Pos(), "1 expression")
+		// continue with first expression
+	}
+
+	switch p.tok {
+	case token.COLON:
+		// labeled statement
+		colon := p.pos
+		p.next()
+		if label, isIdent := x[0].(*ast.Ident); labelOk && isIdent {
+			// Go spec: The scope of a label is the body of the function
+			// in which it is declared and excludes the body of any nested
+			// function.
+			stmt := &ast.LabeledStmt{label, colon, p.parseStmt()}
+			p.declare(stmt, p.labelScope, ast.Lbl, label)
+			return stmt
+		}
+		p.error(x[0].Pos(), "illegal label declaration")
+		return &ast.BadStmt{x[0].Pos(), colon + 1}
+
+	case token.ARROW:
+		// send statement
+		arrow := p.pos
+		p.next() // consume "<-"
+		y := p.parseRhs()
+		return &ast.SendStmt{x[0], arrow, y}
+
+	case token.INC, token.DEC:
+		// increment or decrement
+		s := &ast.IncDecStmt{x[0], p.pos, p.tok}
+		p.next() // consume "++" or "--"
+		return s
+	}
+
+	// expression
+	return &ast.ExprStmt{x[0]}
+}
+
+func (p *parser) parseCallExpr() *ast.CallExpr {
+	x := p.parseRhs()
+	if call, isCall := x.(*ast.CallExpr); isCall {
+		return call
+	}
+	p.errorExpected(x.Pos(), "function/method call")
+	return nil
+}
+
+func (p *parser) parseGoStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "GoStmt"))
+	}
+
+	pos := p.expect(token.GO)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 2} // len("go")
+	}
+
+	return &ast.GoStmt{pos, call}
+}
+
+func (p *parser) parseDeferStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "DeferStmt"))
+	}
+
+	pos := p.expect(token.DEFER)
+	call := p.parseCallExpr()
+	p.expectSemi()
+	if call == nil {
+		return &ast.BadStmt{pos, pos + 5} // len("defer")
+	}
+
+	return &ast.DeferStmt{pos, call}
+}
+
+func (p *parser) parseReturnStmt() *ast.ReturnStmt {
+	if p.trace {
+		defer un(trace(p, "ReturnStmt"))
+	}
+
+	pos := p.pos
+	p.expect(token.RETURN)
+	var x []ast.Expr
+	if p.tok != token.SEMICOLON && p.tok != token.RBRACE {
+		x = p.parseRhsList()
+	}
+	p.expectSemi()
+
+	return &ast.ReturnStmt{pos, x}
+}
+
+func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
+	if p.trace {
+		defer un(trace(p, "BranchStmt"))
+	}
+
+	pos := p.expect(tok)
+	var label *ast.Ident
+	if tok != token.FALLTHROUGH && p.tok == token.IDENT {
+		label = p.parseIdent()
+		// add to list of unresolved targets
+		n := len(p.targetStack) - 1
+		p.targetStack[n] = append(p.targetStack[n], label)
+	}
+	p.expectSemi()
+
+	return &ast.BranchStmt{pos, tok, label}
+}
+
+func (p *parser) makeExpr(s ast.Stmt) ast.Expr {
+	if s == nil {
+		return nil
+	}
+	if es, isExpr := s.(*ast.ExprStmt); isExpr {
+		return p.checkExpr(es.X)
+	}
+	p.error(s.Pos(), "expected condition, found simple statement")
+	return &ast.BadExpr{s.Pos(), s.End()}
+}
+
+func (p *parser) parseIfStmt() *ast.IfStmt {
+	if p.trace {
+		defer un(trace(p, "IfStmt"))
+	}
+
+	pos := p.expect(token.IF)
+	p.openScope()
+	defer p.closeScope()
+
+	var s ast.Stmt
+	var x ast.Expr
+	{
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok == token.SEMICOLON {
+			p.next()
+			x = p.parseRhs()
+		} else {
+			s = p.parseSimpleStmt(false)
+			if p.tok == token.SEMICOLON {
+				p.next()
+				x = p.parseRhs()
+			} else {
+				x = p.makeExpr(s)
+				s = nil
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	var else_ ast.Stmt
+	if p.tok == token.ELSE {
+		p.next()
+		else_ = p.parseStmt()
+	} else {
+		p.expectSemi()
+	}
+
+	return &ast.IfStmt{pos, s, x, body, else_}
+}
+
+func (p *parser) parseTypeList() (list []ast.Expr) {
+	if p.trace {
+		defer un(trace(p, "TypeList"))
+	}
+
+	list = append(list, p.parseType())
+	for p.tok == token.COMMA {
+		p.next()
+		list = append(list, p.parseType())
+	}
+
+	return
+}
+
+func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause {
+	if p.trace {
+		defer un(trace(p, "CaseClause"))
+	}
+
+	pos := p.pos
+	var list []ast.Expr
+	if p.tok == token.CASE {
+		p.next()
+		if exprSwitch {
+			list = p.parseRhsList()
+		} else {
+			list = p.parseTypeList()
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	p.openScope()
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CaseClause{pos, list, colon, body}
+}
+
+func isExprSwitch(s ast.Stmt) bool {
+	if s == nil {
+		return true
+	}
+	if e, ok := s.(*ast.ExprStmt); ok {
+		if a, ok := e.X.(*ast.TypeAssertExpr); ok {
+			return a.Type != nil // regular type assertion
+		}
+		return true
+	}
+	return false
+}
+
+func (p *parser) parseSwitchStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "SwitchStmt"))
+	}
+
+	pos := p.expect(token.SWITCH)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2 ast.Stmt
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2 = p.parseSimpleStmt(false)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.LBRACE {
+				s2 = p.parseSimpleStmt(false)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	exprSwitch := isExprSwitch(s2)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCaseClause(exprSwitch))
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	if exprSwitch {
+		return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body}
+	}
+	// type switch
+	// TODO(gri): do all the checks!
+	return &ast.TypeSwitchStmt{pos, s1, s2, body}
+}
+
+func (p *parser) parseCommClause() *ast.CommClause {
+	if p.trace {
+		defer un(trace(p, "CommClause"))
+	}
+
+	p.openScope()
+	pos := p.pos
+	var comm ast.Stmt
+	if p.tok == token.CASE {
+		p.next()
+		lhs := p.parseLhsList()
+		if p.tok == token.ARROW {
+			// SendStmt
+			if len(lhs) > 1 {
+				p.errorExpected(lhs[0].Pos(), "1 expression")
+				// continue with first expression
+			}
+			arrow := p.pos
+			p.next()
+			rhs := p.parseRhs()
+			comm = &ast.SendStmt{lhs[0], arrow, rhs}
+		} else {
+			// RecvStmt
+			pos := p.pos
+			tok := p.tok
+			var rhs ast.Expr
+			if tok == token.ASSIGN || tok == token.DEFINE {
+				// RecvStmt with assignment
+				if len(lhs) > 2 {
+					p.errorExpected(lhs[0].Pos(), "1 or 2 expressions")
+					// continue with first two expressions
+					lhs = lhs[0:2]
+				}
+				p.next()
+				rhs = p.parseRhs()
+			} else {
+				// rhs must be single receive operation
+				if len(lhs) > 1 {
+					p.errorExpected(lhs[0].Pos(), "1 expression")
+					// continue with first expression
+				}
+				rhs = lhs[0]
+				lhs = nil // there is no lhs
+			}
+			if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW {
+				p.errorExpected(rhs.Pos(), "send or receive operation")
+				rhs = &ast.BadExpr{rhs.Pos(), rhs.End()}
+			}
+			if lhs != nil {
+				comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}}
+			} else {
+				comm = &ast.ExprStmt{rhs}
+			}
+		}
+	} else {
+		p.expect(token.DEFAULT)
+	}
+
+	colon := p.expect(token.COLON)
+	body := p.parseStmtList()
+	p.closeScope()
+
+	return &ast.CommClause{pos, comm, colon, body}
+}
+
+func (p *parser) parseSelectStmt() *ast.SelectStmt {
+	if p.trace {
+		defer un(trace(p, "SelectStmt"))
+	}
+
+	pos := p.expect(token.SELECT)
+	lbrace := p.expect(token.LBRACE)
+	var list []ast.Stmt
+	for p.tok == token.CASE || p.tok == token.DEFAULT {
+		list = append(list, p.parseCommClause())
+	}
+	rbrace := p.expect(token.RBRACE)
+	p.expectSemi()
+	body := &ast.BlockStmt{lbrace, list, rbrace}
+
+	return &ast.SelectStmt{pos, body}
+}
+
+func (p *parser) parseForStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "ForStmt"))
+	}
+
+	pos := p.expect(token.FOR)
+	p.openScope()
+	defer p.closeScope()
+
+	var s1, s2, s3 ast.Stmt
+	if p.tok != token.LBRACE {
+		prevLev := p.exprLev
+		p.exprLev = -1
+		if p.tok != token.SEMICOLON {
+			s2 = p.parseSimpleStmt(false)
+		}
+		if p.tok == token.SEMICOLON {
+			p.next()
+			s1 = s2
+			s2 = nil
+			if p.tok != token.SEMICOLON {
+				s2 = p.parseSimpleStmt(false)
+			}
+			p.expectSemi()
+			if p.tok != token.LBRACE {
+				s3 = p.parseSimpleStmt(false)
+			}
+		}
+		p.exprLev = prevLev
+	}
+
+	body := p.parseBlockStmt()
+	p.expectSemi()
+
+	if as, isAssign := s2.(*ast.AssignStmt); isAssign {
+		// possibly a for statement with a range clause; check assignment operator
+		if as.Tok != token.ASSIGN && as.Tok != token.DEFINE {
+			p.errorExpected(as.TokPos, "'=' or ':='")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// check lhs
+		var key, value ast.Expr
+		switch len(as.Lhs) {
+		case 2:
+			key, value = as.Lhs[0], as.Lhs[1]
+		case 1:
+			key = as.Lhs[0]
+		default:
+			p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		// check rhs
+		if len(as.Rhs) != 1 {
+			p.errorExpected(as.Rhs[0].Pos(), "1 expression")
+			return &ast.BadStmt{pos, body.End()}
+		}
+		if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE {
+			// rhs is range expression
+			// (any short variable declaration was handled by parseSimpleStat above)
+			return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, rhs.X, body}
+		}
+		p.errorExpected(s2.Pos(), "range clause")
+		return &ast.BadStmt{pos, body.End()}
+	}
+
+	// regular for statement
+	return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body}
+}
+
+func (p *parser) parseStmt() (s ast.Stmt) {
+	if p.trace {
+		defer un(trace(p, "Statement"))
+	}
+
+	switch p.tok {
+	case token.CONST, token.TYPE, token.VAR:
+		s = &ast.DeclStmt{p.parseDecl()}
+	case
+		// tokens that may start a top-level expression
+		token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand
+		token.LBRACK, token.STRUCT, // composite type
+		token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators
+		s = p.parseSimpleStmt(true)
+		// because of the required look-ahead, labeled statements are
+		// parsed by parseSimpleStmt - don't expect a semicolon after
+		// them
+		if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt {
+			p.expectSemi()
+		}
+	case token.GO:
+		s = p.parseGoStmt()
+	case token.DEFER:
+		s = p.parseDeferStmt()
+	case token.RETURN:
+		s = p.parseReturnStmt()
+	case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH:
+		s = p.parseBranchStmt(p.tok)
+	case token.LBRACE:
+		s = p.parseBlockStmt()
+		p.expectSemi()
+	case token.IF:
+		s = p.parseIfStmt()
+	case token.SWITCH:
+		s = p.parseSwitchStmt()
+	case token.SELECT:
+		s = p.parseSelectStmt()
+	case token.FOR:
+		s = p.parseForStmt()
+	case token.SEMICOLON:
+		s = &ast.EmptyStmt{p.pos}
+		p.next()
+	case token.RBRACE:
+		// a semicolon may be omitted before a closing "}"
+		s = &ast.EmptyStmt{p.pos}
+	default:
+		// no statement found
+		pos := p.pos
+		p.errorExpected(pos, "statement")
+		p.next() // make progress
+		s = &ast.BadStmt{pos, p.pos}
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+type parseSpecFunction func(p *parser, doc *ast.CommentGroup, iota int) ast.Spec
+
+func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ImportSpec"))
+	}
+
+	var ident *ast.Ident
+	switch p.tok {
+	case token.PERIOD:
+		ident = &ast.Ident{p.pos, ".", nil}
+		p.next()
+	case token.IDENT:
+		ident = p.parseIdent()
+	}
+
+	var path *ast.BasicLit
+	if p.tok == token.STRING {
+		path = &ast.BasicLit{p.pos, p.tok, p.lit}
+		p.next()
+	} else {
+		p.expect(token.STRING) // use expect() error handling
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// collect imports
+	spec := &ast.ImportSpec{doc, ident, path, p.lineComment}
+	p.imports = append(p.imports, spec)
+
+	return spec
+}
+
+func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "ConstSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ != nil || p.tok == token.ASSIGN || iota == 0 {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, p.topScope, ast.Con, idents...)
+
+	return spec
+}
+
+func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "TypeSpec"))
+	}
+
+	ident := p.parseIdent()
+
+	// Go spec: The scope of a type identifier declared inside a function begins
+	// at the identifier in the TypeSpec and ends at the end of the innermost
+	// containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.TypeSpec{doc, ident, nil, nil}
+	p.declare(spec, p.topScope, ast.Typ, ident)
+
+	spec.Type = p.parseType()
+	p.expectSemi() // call before accessing p.linecomment
+	spec.Comment = p.lineComment
+
+	return spec
+}
+
+func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec {
+	if p.trace {
+		defer un(trace(p, "VarSpec"))
+	}
+
+	idents := p.parseIdentList()
+	typ := p.tryType()
+	var values []ast.Expr
+	if typ == nil || p.tok == token.ASSIGN {
+		p.expect(token.ASSIGN)
+		values = p.parseRhsList()
+	}
+	p.expectSemi() // call before accessing p.linecomment
+
+	// Go spec: The scope of a constant or variable identifier declared inside
+	// a function begins at the end of the ConstSpec or VarSpec and ends at
+	// the end of the innermost containing block.
+	// (Global identifiers are resolved in a separate phase after parsing.)
+	spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment}
+	p.declare(spec, p.topScope, ast.Var, idents...)
+
+	return spec
+}
+
+func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl {
+	if p.trace {
+		defer un(trace(p, "GenDecl("+keyword.String()+")"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(keyword)
+	var lparen, rparen token.Pos
+	var list []ast.Spec
+	if p.tok == token.LPAREN {
+		lparen = p.pos
+		p.next()
+		for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ {
+			list = append(list, f(p, p.leadComment, iota))
+		}
+		rparen = p.expect(token.RPAREN)
+		p.expectSemi()
+	} else {
+		list = append(list, f(p, nil, 0))
+	}
+
+	return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen}
+}
+
+func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList {
+	if p.trace {
+		defer un(trace(p, "Receiver"))
+	}
+
+	pos := p.pos
+	par := p.parseParameters(scope, false)
+
+	// must have exactly one receiver
+	if par.NumFields() != 1 {
+		p.errorExpected(pos, "exactly one receiver")
+		// TODO determine a better range for BadExpr below
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}}
+		return par
+	}
+
+	// recv type must be of the form ["*"] identifier
+	recv := par.List[0]
+	base := deref(recv.Type)
+	if _, isIdent := base.(*ast.Ident); !isIdent {
+		p.errorExpected(base.Pos(), "(unqualified) identifier")
+		par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}}
+	}
+
+	return par
+}
+
+func (p *parser) parseFuncDecl() *ast.FuncDecl {
+	if p.trace {
+		defer un(trace(p, "FunctionDecl"))
+	}
+
+	doc := p.leadComment
+	pos := p.expect(token.FUNC)
+	scope := ast.NewScope(p.topScope) // function scope
+
+	var recv *ast.FieldList
+	if p.tok == token.LPAREN {
+		recv = p.parseReceiver(scope)
+	}
+
+	ident := p.parseIdent()
+
+	params, results := p.parseSignature(scope)
+
+	var body *ast.BlockStmt
+	if p.tok == token.LBRACE {
+		body = p.parseBody(scope)
+	}
+	p.expectSemi()
+
+	decl := &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body}
+	if recv == nil {
+		// Go spec: The scope of an identifier denoting a constant, type,
+		// variable, or function (but not method) declared at top level
+		// (outside any function) is the package block.
+		//
+		// init() functions cannot be referred to and there may
+		// be more than one - don't put them in the pkgScope
+		if ident.Name != "init" {
+			p.declare(decl, p.pkgScope, ast.Fun, ident)
+		}
+	}
+
+	return decl
+}
+
+func (p *parser) parseDecl() ast.Decl {
+	if p.trace {
+		defer un(trace(p, "Declaration"))
+	}
+
+	var f parseSpecFunction
+	switch p.tok {
+	case token.CONST:
+		f = parseConstSpec
+
+	case token.TYPE:
+		f = parseTypeSpec
+
+	case token.VAR:
+		f = parseVarSpec
+
+	case token.FUNC:
+		return p.parseFuncDecl()
+
+	default:
+		pos := p.pos
+		p.errorExpected(pos, "declaration")
+		p.next() // make progress
+		decl := &ast.BadDecl{pos, p.pos}
+		return decl
+	}
+
+	return p.parseGenDecl(p.tok, f)
+}
+
+func (p *parser) parseDeclList() (list []ast.Decl) {
+	if p.trace {
+		defer un(trace(p, "DeclList"))
+	}
+
+	for p.tok != token.EOF {
+		list = append(list, p.parseDecl())
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Source files
+
+func (p *parser) parseFile() *ast.File {
+	if p.trace {
+		defer un(trace(p, "File"))
+	}
+
+	// package clause
+	doc := p.leadComment
+	pos := p.expect(token.PACKAGE)
+	// Go spec: The package clause is not a declaration;
+	// the package name does not appear in any scope.
+	ident := p.parseIdent()
+	if ident.Name == "_" {
+		p.error(p.pos, "invalid package name _")
+	}
+	p.expectSemi()
+
+	var decls []ast.Decl
+
+	// Don't bother parsing the rest if we had errors already.
+	// Likely not a Go source file at all.
+
+	if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 {
+		// import decls
+		for p.tok == token.IMPORT {
+			decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec))
+		}
+
+		if p.mode&ImportsOnly == 0 {
+			// rest of package body
+			for p.tok != token.EOF {
+				decls = append(decls, p.parseDecl())
+			}
+		}
+	}
+
+	assert(p.topScope == p.pkgScope, "imbalanced scopes")
+
+	// resolve global identifiers within the same file
+	i := 0
+	for _, ident := range p.unresolved {
+		// i <= index for current ident
+		assert(ident.Obj == unresolved, "object already resolved")
+		ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel
+		if ident.Obj == nil {
+			p.unresolved[i] = ident
+			i++
+		}
+	}
+
+	// TODO(gri): store p.imports in AST
+	return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unresolved[0:i], p.comments}
+}
diff --git a/src/pkg/go/printer/testdata/slow.golden b/src/pkg/go/printer/testdata/slow.golden
new file mode 100644
index 000000000..43a15cb1d
--- /dev/null
+++ b/src/pkg/go/printer/testdata/slow.golden
@@ -0,0 +1,85 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deepequal_test
+
+import (
+	"testing"
+	"google3/spam/archer/frontend/deepequal"
+)
+
+func TestTwoNilValues(t *testing.T) {
+	if err := deepequal.Check(nil, nil); err != nil {
+		t.Errorf("expected nil, saw %v", err)
+	}
+}
+
+type Foo struct {
+	bar	*Bar
+	bang	*Bar
+}
+
+type Bar struct {
+	baz	*Baz
+	foo	[]*Foo
+}
+
+type Baz struct {
+	entries		map[int]interface{}
+	whatever	string
+}
+
+func newFoo() *Foo {
+	return &Foo{bar: &Bar{baz: &Baz{
+		entries: map[int]interface{}{
+			42:	&Foo{},
+			21:	&Bar{},
+			11:	&Baz{whatever: "it's just a test"}}}},
+		bang: &Bar{foo: []*Foo{
+			&Foo{bar: &Bar{baz: &Baz{
+				entries: map[int]interface{}{
+					43:	&Foo{},
+					22:	&Bar{},
+					13:	&Baz{whatever: "this is nuts"}}}},
+				bang: &Bar{foo: []*Foo{
+					&Foo{bar: &Bar{baz: &Baz{
+						entries: map[int]interface{}{
+							61:	&Foo{},
+							71:	&Bar{},
+							11:	&Baz{whatever: "no, it's Go"}}}},
+						bang: &Bar{foo: []*Foo{
+							&Foo{bar: &Bar{baz: &Baz{
+								entries: map[int]interface{}{
+									0:	&Foo{},
+									-2:	&Bar{},
+									-11:	&Baz{whatever: "we need to go deeper"}}}},
+								bang: &Bar{foo: []*Foo{
+									&Foo{bar: &Bar{baz: &Baz{
+										entries: map[int]interface{}{
+											-2:	&Foo{},
+											-5:	&Bar{},
+											-7:	&Baz{whatever: "are you serious?"}}}},
+										bang:	&Bar{foo: []*Foo{}}},
+									&Foo{bar: &Bar{baz: &Baz{
+										entries: map[int]interface{}{
+											-100:	&Foo{},
+											50:	&Bar{},
+											20:	&Baz{whatever: "na, not really ..."}}}},
+										bang:	&Bar{foo: []*Foo{}}}}}}}}},
+					&Foo{bar: &Bar{baz: &Baz{
+						entries: map[int]interface{}{
+							2:	&Foo{},
+							1:	&Bar{},
+							-1:	&Baz{whatever: "... it's just a test."}}}},
+						bang:	&Bar{foo: []*Foo{}}}}}}}}}
+}
+
+func TestElaborate(t *testing.T) {
+	a := newFoo()
+	b := newFoo()
+
+	if err := deepequal.Check(a, b); err != nil {
+		t.Errorf("expected nil, saw %v", err)
+	}
+}
diff --git a/src/pkg/go/printer/testdata/slow.input b/src/pkg/go/printer/testdata/slow.input
new file mode 100644
index 000000000..0e5a23d88
--- /dev/null
+++ b/src/pkg/go/printer/testdata/slow.input
@@ -0,0 +1,85 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deepequal_test
+
+import (
+        "testing"
+        "google3/spam/archer/frontend/deepequal"
+)
+
+func TestTwoNilValues(t *testing.T) {
+        if err := deepequal.Check(nil, nil); err != nil {
+                t.Errorf("expected nil, saw %v", err)
+        }
+}
+
+type Foo struct {
+        bar *Bar
+        bang *Bar
+}
+
+type Bar struct {
+        baz *Baz
+        foo []*Foo
+}
+
+type Baz struct {
+        entries  map[int]interface{}
+        whatever string
+}
+
+func newFoo() (*Foo) {
+return &Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+42: &Foo{},
+21: &Bar{},
+11: &Baz{ whatever: "it's just a test" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+43: &Foo{},
+22: &Bar{},
+13: &Baz{ whatever: "this is nuts" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+61: &Foo{},
+71: &Bar{},
+11: &Baz{ whatever: "no, it's Go" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+0: &Foo{},
+-2: &Bar{},
+-11: &Baz{ whatever: "we need to go deeper" }}}},
+        bang: &Bar{foo: []*Foo{
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+-2: &Foo{},
+-5: &Bar{},
+-7: &Baz{ whatever: "are you serious?" }}}},
+        bang: &Bar{foo: []*Foo{}}},
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+-100: &Foo{},
+50: &Bar{},
+20: &Baz{ whatever: "na, not really ..." }}}},
+        bang: &Bar{foo: []*Foo{}}}}}}}}},
+&Foo{bar: &Bar{ baz: &Baz{
+entries: map[int]interface{}{
+2: &Foo{},
+1: &Bar{},
+-1: &Baz{ whatever: "... it's just a test." }}}},
+        bang: &Bar{foo: []*Foo{}}}}}}}}}
+}
+
+func TestElaborate(t *testing.T) {
+        a := newFoo()
+        b := newFoo()
+
+        if err := deepequal.Check(a, b); err != nil {
+                t.Errorf("expected nil, saw %v", err)
+        }
+}
diff --git a/src/pkg/go/printer/testdata/statements.golden b/src/pkg/go/printer/testdata/statements.golden
new file mode 100644
index 000000000..a6d85107f
--- /dev/null
+++ b/src/pkg/go/printer/testdata/statements.golden
@@ -0,0 +1,412 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package statements
+
+var expr bool
+
+func use(x interface{})	{}
+
+// Formatting of if-statement headers.
+func _() {
+	if true {
+	}
+	if true {
+	}	// no semicolon printed
+	if expr {
+	}
+	if expr {
+	}	// no semicolon printed
+	if expr {
+	}	// no parens printed
+	if expr {
+	}	// no semicolon and parens printed
+	if x := expr; true {
+		use(x)
+	}
+	if x := expr; expr {
+		use(x)
+	}
+}
+
+// Formatting of switch-statement headers.
+func _() {
+	switch {
+	}
+	switch {
+	}	// no semicolon printed
+	switch expr {
+	}
+	switch expr {
+	}	// no semicolon printed
+	switch expr {
+	}	// no parens printed
+	switch expr {
+	}	// no semicolon and parens printed
+	switch x := expr; {
+	default:
+		use(
+			x)
+	}
+	switch x := expr; expr {
+	default:
+		use(x)
+	}
+}
+
+// Formatting of switch statement bodies.
+func _() {
+	switch {
+	}
+
+	switch x := 0; x {
+	case 1:
+		use(x)
+		use(x)	// followed by an empty line
+
+	case 2:	// followed by an empty line
+
+		use(x)	// followed by an empty line
+
+	case 3:	// no empty lines
+		use(x)
+		use(x)
+	}
+
+	switch x {
+	case 0:
+		use(x)
+	case 1:	// this comment should have no effect on the previous or next line
+		use(x)
+	}
+
+	switch x := 0; x {
+	case 1:
+		x = 0
+		// this comment should be indented
+	case 2:
+		x = 0
+	// this comment should not be indented, it is aligned with the next case
+	case 3:
+		x = 0
+		/* indented comment
+		   aligned
+		   aligned
+		*/
+		// bla
+		/* and more */
+	case 4:
+		x = 0
+	/* not indented comment
+	   aligned
+	   aligned
+	*/
+	// bla
+	/* and more */
+	case 5:
+	}
+}
+
+// Formatting of selected select statements.
+func _() {
+	select {}
+	select { /* this comment should not be tab-aligned because the closing } is on the same line */
+	}
+	select {	/* this comment should be tab-aligned */
+	}
+	select {	// this comment should be tab-aligned
+	}
+	select {
+	case <-c:
+	}
+}
+
+// Formatting of for-statement headers.
+func _() {
+	for {
+	}
+	for expr {
+	}
+	for expr {
+	}	// no parens printed
+	for {
+	}	// no semicolons printed
+	for x := expr; ; {
+		use(x)
+	}
+	for expr {
+	}	// no semicolons printed
+	for expr {
+	}	// no semicolons and parens printed
+	for ; ; expr = false {
+	}
+	for x := expr; expr; {
+		use(x)
+	}
+	for x := expr; ; expr = false {
+		use(x)
+	}
+	for ; expr; expr = false {
+	}
+	for x := expr; expr; expr = false {
+		use(x)
+	}
+	for x := range []int{} {
+		use(x)
+	}
+	for x := range []int{} {
+		use(x)
+	}	// no parens printed
+}
+
+// Don't remove mandatory parentheses around composite literals in control clauses.
+func _() {
+	// strip parentheses - no composite literals or composite literals don't start with a type name
+	if x {
+	}
+	if x {
+	}
+	if []T{} {
+	}
+	if []T{} {
+	}
+	if []T{} {
+	}
+
+	for x {
+	}
+	for x {
+	}
+	for []T{} {
+	}
+	for []T{} {
+	}
+	for []T{} {
+	}
+
+	switch x {
+	}
+	switch x {
+	}
+	switch []T{} {
+	}
+	switch []T{} {
+	}
+
+	for _ = range []T{T{42}} {
+	}
+
+	// leave parentheses - composite literals start with a type name
+	if (T{}) {
+	}
+	if (T{}) {
+	}
+	if (T{}) {
+	}
+
+	for (T{}) {
+	}
+	for (T{}) {
+	}
+	for (T{}) {
+	}
+
+	switch (T{}) {
+	}
+	switch (T{}) {
+	}
+
+	for _ = range (T1{T{42}}) {
+	}
+
+	if x == (T{42}[0]) {
+	}
+	if (x == T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == (T{42}[0]) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if (x == a+b*T{42}[0]) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if x == a+(b * (T{42}[0])) {
+	}
+	if x == a+b*(T{42}[0]) {
+	}
+	if (a + b*(T{42}[0])) == x {
+	}
+	if (a + b*(T{42}[0])) == x {
+	}
+
+	if struct{ x bool }{false}.x {
+	}
+	if (struct{ x bool }{false}.x) == false {
+	}
+	if struct{ x bool }{false}.x == false {
+	}
+}
+
+// Extra empty lines inside functions. Do respect source code line
+// breaks between statement boundaries but print at most one empty
+// line at a time.
+func _() {
+
+	const _ = 0
+
+	const _ = 1
+	type _ int
+	type _ float
+
+	var _ = 0
+	var x = 1
+
+	// Each use(x) call below should have at most one empty line before and after.
+	// Known bug: The first use call may have more than one empty line before
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+	use(x)
+
+	if x < x {
+
+		use(x)
+
+	} else {
+
+		use(x)
+
+	}
+}
+
+// Formatting around labels.
+func _() {
+L:
+}
+
+func _() {
+	// this comment should be indented
+L:	// no semicolon needed
+}
+
+func _() {
+	switch 0 {
+	case 0:
+	L0:
+		;	// semicolon required
+	case 1:
+	L1:
+		;	// semicolon required
+	default:
+	L2:	// no semicolon needed
+	}
+}
+
+func _() {
+	f()
+L1:
+	f()
+L2:
+	;
+L3:
+}
+
+func _() {
+	// this comment should be indented
+L:
+}
+
+func _() {
+L:
+	_ = 0
+}
+
+func _() {
+	// this comment should be indented
+L:
+	_ = 0
+}
+
+func _() {
+	for {
+	L1:
+		_ = 0
+	L2:
+		_ = 0
+	}
+}
+
+func _() {
+	// this comment should be indented
+	for {
+	L1:
+		_ = 0
+	L2:
+		_ = 0
+	}
+}
+
+func _() {
+	if true {
+		_ = 0
+	}
+	_ = 0	// the indentation here should not be affected by the long label name
+AnOverlongLabel:
+	_ = 0
+
+	if true {
+		_ = 0
+	}
+	_ = 0
+
+L:
+	_ = 0
+}
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	MoreCode()
+}
+
+func _() {
+	for {
+		goto L
+	}
+L:	// A comment on the same line as the label, followed by a single empty line.
+	// Known bug: There may be more than one empty line before MoreCode()
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+	MoreCode()
+}
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
+
+func _() {
+	for {
+		goto AVeryLongLabelThatShouldNotAffectFormatting
+	}
+AVeryLongLabelThatShouldNotAffectFormatting:
+	// There should be a single empty line after this comment.
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
diff --git a/src/pkg/go/printer/testdata/statements.input b/src/pkg/go/printer/testdata/statements.input
new file mode 100644
index 000000000..86a753c5a
--- /dev/null
+++ b/src/pkg/go/printer/testdata/statements.input
@@ -0,0 +1,351 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package statements
+
+var expr bool
+
+func use(x interface{}) {}
+
+// Formatting of if-statement headers.
+func _() {
+	if true {}
+	if; true {}  // no semicolon printed
+	if expr{}
+	if;expr{}  // no semicolon printed
+	if (expr){}  // no parens printed
+	if;((expr)){}  // no semicolon and parens printed
+	if x:=expr;true{
+	use(x)}
+	if x:=expr; expr {use(x)}
+}
+
+
+// Formatting of switch-statement headers.
+func _() {
+	switch {}
+	switch;{}  // no semicolon printed
+	switch expr {}
+	switch;expr{}  // no semicolon printed
+	switch (expr) {}  // no parens printed
+	switch;((expr)){}  // no semicolon and parens printed
+	switch x := expr; { default:use(
+x)
+	}
+	switch x := expr; expr {default:use(x)}
+}
+
+
+// Formatting of switch statement bodies.
+func _() {
+	switch {
+	}
+
+	switch x := 0; x {
+	case 1:
+		use(x)
+		use(x)  // followed by an empty line
+
+	case 2:  // followed by an empty line
+
+		use(x)  // followed by an empty line
+
+	case 3:  // no empty lines
+		use(x)
+		use(x)
+	}
+
+	switch x {
+	case 0:
+		use(x)
+	case 1:  // this comment should have no effect on the previous or next line
+		use(x)
+	}
+
+	switch x := 0; x {
+	case 1:
+		x = 0
+		// this comment should be indented
+	case 2:
+		x = 0
+	// this comment should not be indented, it is aligned with the next case
+	case 3:
+		x = 0
+		/* indented comment
+		   aligned
+		   aligned
+		*/
+		// bla
+		/* and more */
+	case 4:
+		x = 0
+	/* not indented comment
+	   aligned
+	   aligned
+	*/
+	// bla
+	/* and more */
+	case 5:
+	}
+}
+
+
+// Formatting of selected select statements.
+func _() {
+	select {
+	}
+	select { /* this comment should not be tab-aligned because the closing } is on the same line */ }
+	select { /* this comment should be tab-aligned */
+	}
+	select { // this comment should be tab-aligned
+	}
+	select { case <-c: }
+}
+
+
+// Formatting of for-statement headers.
+func _() {
+	for{}
+	for expr {}
+	for (expr) {}  // no parens printed
+	for;;{}  // no semicolons printed
+	for x :=expr;; {use( x)}
+	for; expr;{}  // no semicolons printed
+	for; ((expr));{}  // no semicolons and parens printed
+	for; ; expr = false {}
+	for x :=expr; expr; {use(x)}
+	for x := expr;; expr=false {use(x)}
+	for;expr;expr =false {
+	}
+	for x := expr;expr;expr = false { use(x) }
+	for x := range []int{} { use(x) }
+	for x := range (([]int{})) { use(x) }  // no parens printed
+}
+
+
+// Don't remove mandatory parentheses around composite literals in control clauses.
+func _() {
+	// strip parentheses - no composite literals or composite literals don't start with a type name
+	if (x) {}
+	if (((x))) {}
+	if ([]T{}) {}
+	if (([]T{})) {}
+	if ; (((([]T{})))) {}
+
+	for (x) {}
+	for (((x))) {}
+	for ([]T{}) {}
+	for (([]T{})) {}
+	for ; (((([]T{})))) ; {}
+
+	switch (x) {}
+	switch (((x))) {}
+	switch ([]T{}) {}
+	switch ; (((([]T{})))) {}
+
+	for _ = range ((([]T{T{42}}))) {}
+
+	// leave parentheses - composite literals start with a type name
+	if (T{}) {}
+	if ((T{})) {}
+	if ; ((((T{})))) {}
+
+	for (T{}) {}
+	for ((T{})) {}
+	for ; ((((T{})))) ; {}
+
+	switch (T{}) {}
+	switch ; ((((T{})))) {}
+
+	for _ = range (((T1{T{42}}))) {}
+
+	if x == (T{42}[0]) {}
+	if (x == T{42}[0]) {}
+	if (x == (T{42}[0])) {}
+	if (x == (((T{42}[0])))) {}
+	if (((x == (T{42}[0])))) {}
+	if x == a + b*(T{42}[0]) {}
+	if (x == a + b*T{42}[0]) {}
+	if (x == a + b*(T{42}[0])) {}
+	if (x == a + ((b * (T{42}[0])))) {}
+	if (((x == a + b * (T{42}[0])))) {}
+	if (((a + b * (T{42}[0])) == x)) {}
+	if (((a + b * (T{42}[0])))) == x {}
+
+	if (struct{x bool}{false}.x) {}
+	if (struct{x bool}{false}.x) == false {}
+	if (struct{x bool}{false}.x == false) {}
+}
+
+
+// Extra empty lines inside functions. Do respect source code line
+// breaks between statement boundaries but print at most one empty
+// line at a time.
+func _() {
+
+	const _ = 0
+
+	const _ = 1
+	type _ int
+	type _ float
+
+	var _ = 0
+	var x = 1
+
+	// Each use(x) call below should have at most one empty line before and after.
+	// Known bug: The first use call may have more than one empty line before
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+
+	use(x)
+
+	if x < x {
+
+		use(x)
+
+	} else {
+
+		use(x)
+
+	}
+}
+
+
+// Formatting around labels.
+func _() {
+	L:
+}
+
+
+func _() {
+	// this comment should be indented
+	L: ;  // no semicolon needed
+}
+
+
+func _() {
+	switch 0 {
+	case 0:
+		L0: ;  // semicolon required
+	case 1:
+		L1: ;  // semicolon required
+	default:
+		L2: ;  // no semicolon needed
+	}
+}
+
+
+func _() {
+	f()
+L1:
+	f()
+L2:
+	;
+L3:
+}
+
+
+func _() {
+	// this comment should be indented
+	L:
+}
+
+
+func _() {
+	L: _ = 0
+}
+
+
+func _() {
+	// this comment should be indented
+	L: _ = 0
+}
+
+
+func _() {
+	for {
+	L1: _ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+		// this comment should be indented
+	for {
+	L1: _ = 0
+	L2:
+		_ = 0
+	}
+}
+
+
+func _() {
+	if true {
+		_ = 0
+	}
+	_ = 0  // the indentation here should not be affected by the long label name
+AnOverlongLabel:
+	_ = 0
+	
+	if true {
+		_ = 0
+	}
+	_ = 0
+
+L:	_ = 0
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:	// A comment on the same line as the label, followed by a single empty line.
+	// Known bug: There may be more than one empty line before MoreCode()
+	//            (see go/printer/nodes.go, func linebreak).
+
+
+
+
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto L
+	}
+L:
+
+
+
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
+
+
+func _() {
+	for {
+		goto AVeryLongLabelThatShouldNotAffectFormatting
+	}
+AVeryLongLabelThatShouldNotAffectFormatting:
+	// There should be a single empty line after this comment.
+
+	// There should be a single empty line before this comment.
+	MoreCode()
+}
diff --git a/src/pkg/go/scanner/Makefile b/src/pkg/go/scanner/Makefile
new file mode 100644
index 000000000..453faac00
--- /dev/null
+++ b/src/pkg/go/scanner/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/scanner
+GOFILES=\
+	errors.go\
+	scanner.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/scanner/errors.go b/src/pkg/go/scanner/errors.go
new file mode 100644
index 000000000..a0927e416
--- /dev/null
+++ b/src/pkg/go/scanner/errors.go
@@ -0,0 +1,168 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"fmt"
+	"go/token"
+	"io"
+	"os"
+	"sort"
+)
+
+// An implementation of an ErrorHandler may be provided to the Scanner.
+// If a syntax error is encountered and a handler was installed, Error
+// is called with a position and an error message. The position points
+// to the beginning of the offending token.
+//
+type ErrorHandler interface {
+	Error(pos token.Position, msg string)
+}
+
+// ErrorVector implements the ErrorHandler interface. It maintains a list
+// of errors which can be retrieved with GetErrorList and GetError. The
+// zero value for an ErrorVector is an empty ErrorVector ready to use.
+//
+// A common usage pattern is to embed an ErrorVector alongside a
+// scanner in a data structure that uses the scanner. By passing a
+// reference to an ErrorVector to the scanner's Init call, default
+// error handling is obtained.
+//
+type ErrorVector struct {
+	errors []*Error
+}
+
+// Reset resets an ErrorVector to no errors.
+func (h *ErrorVector) Reset() { h.errors = h.errors[:0] }
+
+// ErrorCount returns the number of errors collected.
+func (h *ErrorVector) ErrorCount() int { return len(h.errors) }
+
+// Within ErrorVector, an error is represented by an Error node. The
+// position Pos, if valid, points to the beginning of the offending
+// token, and the error condition is described by Msg.
+//
+type Error struct {
+	Pos token.Position
+	Msg string
+}
+
+func (e *Error) String() string {
+	if e.Pos.Filename != "" || e.Pos.IsValid() {
+		// don't print ""
+		// TODO(gri) reconsider the semantics of Position.IsValid
+		return e.Pos.String() + ": " + e.Msg
+	}
+	return e.Msg
+}
+
+// An ErrorList is a (possibly sorted) list of Errors.
+type ErrorList []*Error
+
+// ErrorList implements the sort Interface.
+func (p ErrorList) Len() int      { return len(p) }
+func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p ErrorList) Less(i, j int) bool {
+	e := &p[i].Pos
+	f := &p[j].Pos
+	// Note that it is not sufficient to simply compare file offsets because
+	// the offsets do not reflect modified line information (through //line
+	// comments).
+	if e.Filename < f.Filename {
+		return true
+	}
+	if e.Filename == f.Filename {
+		if e.Line < f.Line {
+			return true
+		}
+		if e.Line == f.Line {
+			return e.Column < f.Column
+		}
+	}
+	return false
+}
+
+func (p ErrorList) String() string {
+	switch len(p) {
+	case 0:
+		return "unspecified error"
+	case 1:
+		return p[0].String()
+	}
+	return fmt.Sprintf("%s (and %d more errors)", p[0].String(), len(p)-1)
+}
+
+// These constants control the construction of the ErrorList
+// returned by GetErrors.
+//
+const (
+	Raw         = iota // leave error list unchanged
+	Sorted             // sort error list by file, line, and column number
+	NoMultiples        // sort error list and leave only the first error per line
+)
+
+// GetErrorList returns the list of errors collected by an ErrorVector.
+// The construction of the ErrorList returned is controlled by the mode
+// parameter. If there are no errors, the result is nil.
+//
+func (h *ErrorVector) GetErrorList(mode int) ErrorList {
+	if len(h.errors) == 0 {
+		return nil
+	}
+
+	list := make(ErrorList, len(h.errors))
+	copy(list, h.errors)
+
+	if mode >= Sorted {
+		sort.Sort(list)
+	}
+
+	if mode >= NoMultiples {
+		var last token.Position // initial last.Line is != any legal error line
+		i := 0
+		for _, e := range list {
+			if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
+				last = e.Pos
+				list[i] = e
+				i++
+			}
+		}
+		list = list[0:i]
+	}
+
+	return list
+}
+
+// GetError is like GetErrorList, but it returns an os.Error instead
+// so that a nil result can be assigned to an os.Error variable and
+// remains nil.
+//
+func (h *ErrorVector) GetError(mode int) os.Error {
+	if len(h.errors) == 0 {
+		return nil
+	}
+
+	return h.GetErrorList(mode)
+}
+
+// ErrorVector implements the ErrorHandler interface.
+func (h *ErrorVector) Error(pos token.Position, msg string) {
+	h.errors = append(h.errors, &Error{pos, msg})
+}
+
+// PrintError is a utility function that prints a list of errors to w,
+// one error per line, if the err parameter is an ErrorList. Otherwise
+// it prints the err string.
+//
+func PrintError(w io.Writer, err os.Error) {
+	if list, ok := err.(ErrorList); ok {
+		for _, e := range list {
+			fmt.Fprintf(w, "%s\n", e)
+		}
+	} else {
+		fmt.Fprintf(w, "%s\n", err)
+	}
+}
diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go
new file mode 100644
index 000000000..7f3dd2373
--- /dev/null
+++ b/src/pkg/go/scanner/scanner.go
@@ -0,0 +1,670 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package scanner implements a scanner for Go source text. Takes a []byte as
+// source which can then be tokenized through repeated calls to the Scan
+// function. Typical use:
+//
+//	var s Scanner
+//	fset := token.NewFileSet()  // position information is relative to fset
+//      file := fset.AddFile(filename, fset.Base(), len(src))  // register file
+//	s.Init(file, src, nil /* no error handler */, 0)
+//	for {
+//		pos, tok, lit := s.Scan()
+//		if tok == token.EOF {
+//			break
+//		}
+//		// do something here with pos, tok, and lit
+//	}
+//
+package scanner
+
+import (
+	"bytes"
+	"fmt"
+	"go/token"
+	"path/filepath"
+	"strconv"
+	"unicode"
+	"utf8"
+)
+
+// A Scanner holds the scanner's internal state while processing
+// a given text.  It can be allocated as part of another data
+// structure but must be initialized via Init before use.
+//
+type Scanner struct {
+	// immutable state
+	file *token.File  // source file handle
+	dir  string       // directory portion of file.Name()
+	src  []byte       // source
+	err  ErrorHandler // error reporting; or nil
+	mode uint         // scanning mode
+
+	// scanning state
+	ch         int  // current character
+	offset     int  // character offset
+	rdOffset   int  // reading offset (position after current character)
+	lineOffset int  // current line offset
+	insertSemi bool // insert a semicolon before next newline
+
+	// public state - ok to modify
+	ErrorCount int // number of errors encountered
+}
+
+// Read the next Unicode char into S.ch.
+// S.ch < 0 means end-of-file.
+//
+func (S *Scanner) next() {
+	if S.rdOffset < len(S.src) {
+		S.offset = S.rdOffset
+		if S.ch == '\n' {
+			S.lineOffset = S.offset
+			S.file.AddLine(S.offset)
+		}
+		r, w := int(S.src[S.rdOffset]), 1
+		switch {
+		case r == 0:
+			S.error(S.offset, "illegal character NUL")
+		case r >= 0x80:
+			// not ASCII
+			r, w = utf8.DecodeRune(S.src[S.rdOffset:])
+			if r == utf8.RuneError && w == 1 {
+				S.error(S.offset, "illegal UTF-8 encoding")
+			}
+		}
+		S.rdOffset += w
+		S.ch = r
+	} else {
+		S.offset = len(S.src)
+		if S.ch == '\n' {
+			S.lineOffset = S.offset
+			S.file.AddLine(S.offset)
+		}
+		S.ch = -1 // eof
+	}
+}
+
+// The mode parameter to the Init function is a set of flags (or 0).
+// They control scanner behavior.
+//
+const (
+	ScanComments      = 1 << iota // return comments as COMMENT tokens
+	AllowIllegalChars             // do not report an error for illegal chars
+	InsertSemis                   // automatically insert semicolons
+)
+
+// Init prepares the scanner S to tokenize the text src by setting the
+// scanner at the beginning of src. The scanner uses the file set file
+// for position information and it adds line information for each line.
+// It is ok to re-use the same file when re-scanning the same file as
+// line information which is already present is ignored. Init causes a
+// panic if the file size does not match the src size.
+//
+// Calls to Scan will use the error handler err if they encounter a
+// syntax error and err is not nil. Also, for each error encountered,
+// the Scanner field ErrorCount is incremented by one. The mode parameter
+// determines how comments, illegal characters, and semicolons are handled.
+//
+// Note that Init may call err if there is an error in the first character
+// of the file.
+//
+func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) {
+	// Explicitly initialize all fields since a scanner may be reused.
+	if file.Size() != len(src) {
+		panic("file size does not match src len")
+	}
+	S.file = file
+	S.dir, _ = filepath.Split(file.Name())
+	S.src = src
+	S.err = err
+	S.mode = mode
+
+	S.ch = ' '
+	S.offset = 0
+	S.rdOffset = 0
+	S.lineOffset = 0
+	S.insertSemi = false
+	S.ErrorCount = 0
+
+	S.next()
+}
+
+func (S *Scanner) error(offs int, msg string) {
+	if S.err != nil {
+		S.err.Error(S.file.Position(S.file.Pos(offs)), msg)
+	}
+	S.ErrorCount++
+}
+
+var prefix = []byte("//line ")
+
+func (S *Scanner) interpretLineComment(text []byte) {
+	if bytes.HasPrefix(text, prefix) {
+		// get filename and line number, if any
+		if i := bytes.LastIndex(text, []byte{':'}); i > 0 {
+			if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
+				// valid //line filename:line comment;
+				filename := filepath.Clean(string(text[len(prefix):i]))
+				if !filepath.IsAbs(filename) {
+					// make filename relative to current directory
+					filename = filepath.Join(S.dir, filename)
+				}
+				// update scanner position
+				S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line
+			}
+		}
+	}
+}
+
+func (S *Scanner) scanComment() {
+	// initial '/' already consumed; S.ch == '/' || S.ch == '*'
+	offs := S.offset - 1 // position of initial '/'
+
+	if S.ch == '/' {
+		//-style comment
+		S.next()
+		for S.ch != '\n' && S.ch >= 0 {
+			S.next()
+		}
+		if offs == S.lineOffset {
+			// comment starts at the beginning of the current line
+			S.interpretLineComment(S.src[offs:S.offset])
+		}
+		return
+	}
+
+	/*-style comment */
+	S.next()
+	for S.ch >= 0 {
+		ch := S.ch
+		S.next()
+		if ch == '*' && S.ch == '/' {
+			S.next()
+			return
+		}
+	}
+
+	S.error(offs, "comment not terminated")
+}
+
+func (S *Scanner) findLineEnd() bool {
+	// initial '/' already consumed
+
+	defer func(offs int) {
+		// reset scanner state to where it was upon calling findLineEnd
+		S.ch = '/'
+		S.offset = offs
+		S.rdOffset = offs + 1
+		S.next() // consume initial '/' again
+	}(S.offset - 1)
+
+	// read ahead until a newline, EOF, or non-comment token is found
+	for S.ch == '/' || S.ch == '*' {
+		if S.ch == '/' {
+			//-style comment always contains a newline
+			return true
+		}
+		/*-style comment: look for newline */
+		S.next()
+		for S.ch >= 0 {
+			ch := S.ch
+			if ch == '\n' {
+				return true
+			}
+			S.next()
+			if ch == '*' && S.ch == '/' {
+				S.next()
+				break
+			}
+		}
+		S.skipWhitespace() // S.insertSemi is set
+		if S.ch < 0 || S.ch == '\n' {
+			return true
+		}
+		if S.ch != '/' {
+			// non-comment token
+			return false
+		}
+		S.next() // consume '/'
+	}
+
+	return false
+}
+
+func isLetter(ch int) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
+}
+
+func isDigit(ch int) bool {
+	return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
+}
+
+func (S *Scanner) scanIdentifier() token.Token {
+	offs := S.offset
+	for isLetter(S.ch) || isDigit(S.ch) {
+		S.next()
+	}
+	return token.Lookup(S.src[offs:S.offset])
+}
+
+func digitVal(ch int) int {
+	switch {
+	case '0' <= ch && ch <= '9':
+		return ch - '0'
+	case 'a' <= ch && ch <= 'f':
+		return ch - 'a' + 10
+	case 'A' <= ch && ch <= 'F':
+		return ch - 'A' + 10
+	}
+	return 16 // larger than any legal digit val
+}
+
+func (S *Scanner) scanMantissa(base int) {
+	for digitVal(S.ch) < base {
+		S.next()
+	}
+}
+
+func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
+	// digitVal(S.ch) < 10
+	tok := token.INT
+
+	if seenDecimalPoint {
+		tok = token.FLOAT
+		S.scanMantissa(10)
+		goto exponent
+	}
+
+	if S.ch == '0' {
+		// int or float
+		offs := S.offset
+		S.next()
+		if S.ch == 'x' || S.ch == 'X' {
+			// hexadecimal int
+			S.next()
+			S.scanMantissa(16)
+			if S.offset-offs <= 2 {
+				// only scanned "0x" or "0X"
+				S.error(offs, "illegal hexadecimal number")
+			}
+		} else {
+			// octal int or float
+			seenDecimalDigit := false
+			S.scanMantissa(8)
+			if S.ch == '8' || S.ch == '9' {
+				// illegal octal int or float
+				seenDecimalDigit = true
+				S.scanMantissa(10)
+			}
+			if S.ch == '.' || S.ch == 'e' || S.ch == 'E' || S.ch == 'i' {
+				goto fraction
+			}
+			// octal int
+			if seenDecimalDigit {
+				S.error(offs, "illegal octal number")
+			}
+		}
+		goto exit
+	}
+
+	// decimal int or float
+	S.scanMantissa(10)
+
+fraction:
+	if S.ch == '.' {
+		tok = token.FLOAT
+		S.next()
+		S.scanMantissa(10)
+	}
+
+exponent:
+	if S.ch == 'e' || S.ch == 'E' {
+		tok = token.FLOAT
+		S.next()
+		if S.ch == '-' || S.ch == '+' {
+			S.next()
+		}
+		S.scanMantissa(10)
+	}
+
+	if S.ch == 'i' {
+		tok = token.IMAG
+		S.next()
+	}
+
+exit:
+	return tok
+}
+
+func (S *Scanner) scanEscape(quote int) {
+	offs := S.offset
+
+	var i, base, max uint32
+	switch S.ch {
+	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
+		S.next()
+		return
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		i, base, max = 3, 8, 255
+	case 'x':
+		S.next()
+		i, base, max = 2, 16, 255
+	case 'u':
+		S.next()
+		i, base, max = 4, 16, unicode.MaxRune
+	case 'U':
+		S.next()
+		i, base, max = 8, 16, unicode.MaxRune
+	default:
+		S.next() // always make progress
+		S.error(offs, "unknown escape sequence")
+		return
+	}
+
+	var x uint32
+	for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+		d := uint32(digitVal(S.ch))
+		if d >= base {
+			S.error(S.offset, "illegal character in escape sequence")
+			break
+		}
+		x = x*base + d
+		S.next()
+	}
+	// in case of an error, consume remaining chars
+	for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+		S.next()
+	}
+	if x > max || 0xd800 <= x && x < 0xe000 {
+		S.error(offs, "escape sequence is invalid Unicode code point")
+	}
+}
+
+func (S *Scanner) scanChar() {
+	// '\'' opening already consumed
+	offs := S.offset - 1
+
+	n := 0
+	for S.ch != '\'' {
+		ch := S.ch
+		n++
+		S.next()
+		if ch == '\n' || ch < 0 {
+			S.error(offs, "character literal not terminated")
+			n = 1
+			break
+		}
+		if ch == '\\' {
+			S.scanEscape('\'')
+		}
+	}
+
+	S.next()
+
+	if n != 1 {
+		S.error(offs, "illegal character literal")
+	}
+}
+
+func (S *Scanner) scanString() {
+	// '"' opening already consumed
+	offs := S.offset - 1
+
+	for S.ch != '"' {
+		ch := S.ch
+		S.next()
+		if ch == '\n' || ch < 0 {
+			S.error(offs, "string not terminated")
+			break
+		}
+		if ch == '\\' {
+			S.scanEscape('"')
+		}
+	}
+
+	S.next()
+}
+
+func (S *Scanner) scanRawString() {
+	// '`' opening already consumed
+	offs := S.offset - 1
+
+	for S.ch != '`' {
+		ch := S.ch
+		S.next()
+		if ch < 0 {
+			S.error(offs, "string not terminated")
+			break
+		}
+	}
+
+	S.next()
+}
+
+func (S *Scanner) skipWhitespace() {
+	for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' && !S.insertSemi || S.ch == '\r' {
+		S.next()
+	}
+}
+
+// Helper functions for scanning multi-byte tokens such as >> += >>= .
+// Different routines recognize different length tok_i based on matches
+// of ch_i. If a token ends in '=', the result is tok1 or tok3
+// respectively. Otherwise, the result is tok0 if there was no other
+// matching character, or tok2 if the matching character was ch2.
+
+func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	return tok0
+}
+
+func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	if S.ch == ch2 {
+		S.next()
+		return tok2
+	}
+	return tok0
+}
+
+func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
+	if S.ch == '=' {
+		S.next()
+		return tok1
+	}
+	if S.ch == ch2 {
+		S.next()
+		if S.ch == '=' {
+			S.next()
+			return tok3
+		}
+		return tok2
+	}
+	return tok0
+}
+
+// Scan scans the next token and returns the token position,
+// the token, and the literal string corresponding to the
+// token. The source end is indicated by token.EOF.
+//
+// If the returned token is token.SEMICOLON, the corresponding
+// literal string is ";" if the semicolon was present in the source,
+// and "\n" if the semicolon was inserted because of a newline or
+// at EOF.
+//
+// For more tolerant parsing, Scan will return a valid token if
+// possible even if a syntax error was encountered. Thus, even
+// if the resulting token sequence contains no illegal tokens,
+// a client may not assume that no error occurred. Instead it
+// must check the scanner's ErrorCount or the number of calls
+// of the error handler, if there was one installed.
+//
+// Scan adds line information to the file added to the file
+// set with Init. Token positions are relative to that file
+// and thus relative to the file set.
+//
+func (S *Scanner) Scan() (token.Pos, token.Token, string) {
+scanAgain:
+	S.skipWhitespace()
+
+	// current token start
+	insertSemi := false
+	offs := S.offset
+	tok := token.ILLEGAL
+
+	// determine token value
+	switch ch := S.ch; {
+	case isLetter(ch):
+		tok = S.scanIdentifier()
+		switch tok {
+		case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
+			insertSemi = true
+		}
+	case digitVal(ch) < 10:
+		insertSemi = true
+		tok = S.scanNumber(false)
+	default:
+		S.next() // always make progress
+		switch ch {
+		case -1:
+			if S.insertSemi {
+				S.insertSemi = false // EOF consumed
+				return S.file.Pos(offs), token.SEMICOLON, "\n"
+			}
+			tok = token.EOF
+		case '\n':
+			// we only reach here if S.insertSemi was
+			// set in the first place and exited early
+			// from S.skipWhitespace()
+			S.insertSemi = false // newline consumed
+			return S.file.Pos(offs), token.SEMICOLON, "\n"
+		case '"':
+			insertSemi = true
+			tok = token.STRING
+			S.scanString()
+		case '\'':
+			insertSemi = true
+			tok = token.CHAR
+			S.scanChar()
+		case '`':
+			insertSemi = true
+			tok = token.STRING
+			S.scanRawString()
+		case ':':
+			tok = S.switch2(token.COLON, token.DEFINE)
+		case '.':
+			if digitVal(S.ch) < 10 {
+				insertSemi = true
+				tok = S.scanNumber(true)
+			} else if S.ch == '.' {
+				S.next()
+				if S.ch == '.' {
+					S.next()
+					tok = token.ELLIPSIS
+				}
+			} else {
+				tok = token.PERIOD
+			}
+		case ',':
+			tok = token.COMMA
+		case ';':
+			tok = token.SEMICOLON
+		case '(':
+			tok = token.LPAREN
+		case ')':
+			insertSemi = true
+			tok = token.RPAREN
+		case '[':
+			tok = token.LBRACK
+		case ']':
+			insertSemi = true
+			tok = token.RBRACK
+		case '{':
+			tok = token.LBRACE
+		case '}':
+			insertSemi = true
+			tok = token.RBRACE
+		case '+':
+			tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
+			if tok == token.INC {
+				insertSemi = true
+			}
+		case '-':
+			tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
+			if tok == token.DEC {
+				insertSemi = true
+			}
+		case '*':
+			tok = S.switch2(token.MUL, token.MUL_ASSIGN)
+		case '/':
+			if S.ch == '/' || S.ch == '*' {
+				// comment
+				if S.insertSemi && S.findLineEnd() {
+					// reset position to the beginning of the comment
+					S.ch = '/'
+					S.offset = offs
+					S.rdOffset = offs + 1
+					S.insertSemi = false // newline consumed
+					return S.file.Pos(offs), token.SEMICOLON, "\n"
+				}
+				S.scanComment()
+				if S.mode&ScanComments == 0 {
+					// skip comment
+					S.insertSemi = false // newline consumed
+					goto scanAgain
+				}
+				tok = token.COMMENT
+			} else {
+				tok = S.switch2(token.QUO, token.QUO_ASSIGN)
+			}
+		case '%':
+			tok = S.switch2(token.REM, token.REM_ASSIGN)
+		case '^':
+			tok = S.switch2(token.XOR, token.XOR_ASSIGN)
+		case '<':
+			if S.ch == '-' {
+				S.next()
+				tok = token.ARROW
+			} else {
+				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
+			}
+		case '>':
+			tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+		case '=':
+			tok = S.switch2(token.ASSIGN, token.EQL)
+		case '!':
+			tok = S.switch2(token.NOT, token.NEQ)
+		case '&':
+			if S.ch == '^' {
+				S.next()
+				tok = S.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
+			} else {
+				tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
+			}
+		case '|':
+			tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+		default:
+			if S.mode&AllowIllegalChars == 0 {
+				S.error(offs, fmt.Sprintf("illegal character %#U", ch))
+			}
+			insertSemi = S.insertSemi // preserve insertSemi info
+		}
+	}
+
+	if S.mode&InsertSemis != 0 {
+		S.insertSemi = insertSemi
+	}
+
+	// TODO(gri): The scanner API should change such that the literal string
+	//            is only valid if an actual literal was scanned. This will
+	//            permit a more efficient implementation.
+	return S.file.Pos(offs), tok, string(S.src[offs:S.offset])
+}
diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go
new file mode 100644
index 000000000..eb9e1cb81
--- /dev/null
+++ b/src/pkg/go/scanner/scanner_test.go
@@ -0,0 +1,672 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"go/token"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+var fset = token.NewFileSet()
+
+const /* class */ (
+	special = iota
+	literal
+	operator
+	keyword
+)
+
+func tokenclass(tok token.Token) int {
+	switch {
+	case tok.IsLiteral():
+		return literal
+	case tok.IsOperator():
+		return operator
+	case tok.IsKeyword():
+		return keyword
+	}
+	return special
+}
+
+type elt struct {
+	tok   token.Token
+	lit   string
+	class int
+}
+
+var tokens = [...]elt{
+	// Special tokens
+	{token.COMMENT, "/* a comment */", special},
+	{token.COMMENT, "// a comment \n", special},
+
+	// Identifiers and basic type literals
+	{token.IDENT, "foobar", literal},
+	{token.IDENT, "a۰۱۸", literal},
+	{token.IDENT, "foo६४", literal},
+	{token.IDENT, "bar９８７６", literal},
+	{token.INT, "0", literal},
+	{token.INT, "1", literal},
+	{token.INT, "123456789012345678890", literal},
+	{token.INT, "01234567", literal},
+	{token.INT, "0xcafebabe", literal},
+	{token.FLOAT, "0.", literal},
+	{token.FLOAT, ".0", literal},
+	{token.FLOAT, "3.14159265", literal},
+	{token.FLOAT, "1e0", literal},
+	{token.FLOAT, "1e+100", literal},
+	{token.FLOAT, "1e-100", literal},
+	{token.FLOAT, "2.71828e-1000", literal},
+	{token.IMAG, "0i", literal},
+	{token.IMAG, "1i", literal},
+	{token.IMAG, "012345678901234567889i", literal},
+	{token.IMAG, "123456789012345678890i", literal},
+	{token.IMAG, "0.i", literal},
+	{token.IMAG, ".0i", literal},
+	{token.IMAG, "3.14159265i", literal},
+	{token.IMAG, "1e0i", literal},
+	{token.IMAG, "1e+100i", literal},
+	{token.IMAG, "1e-100i", literal},
+	{token.IMAG, "2.71828e-1000i", literal},
+	{token.CHAR, "'a'", literal},
+	{token.CHAR, "'\\000'", literal},
+	{token.CHAR, "'\\xFF'", literal},
+	{token.CHAR, "'\\uff16'", literal},
+	{token.CHAR, "'\\U0000ff16'", literal},
+	{token.STRING, "`foobar`", literal},
+	{token.STRING, "`" + `foo
+	                        bar` +
+		"`",
+		literal,
+	},
+
+	// Operators and delimiters
+	{token.ADD, "+", operator},
+	{token.SUB, "-", operator},
+	{token.MUL, "*", operator},
+	{token.QUO, "/", operator},
+	{token.REM, "%", operator},
+
+	{token.AND, "&", operator},
+	{token.OR, "|", operator},
+	{token.XOR, "^", operator},
+	{token.SHL, "<<", operator},
+	{token.SHR, ">>", operator},
+	{token.AND_NOT, "&^", operator},
+
+	{token.ADD_ASSIGN, "+=", operator},
+	{token.SUB_ASSIGN, "-=", operator},
+	{token.MUL_ASSIGN, "*=", operator},
+	{token.QUO_ASSIGN, "/=", operator},
+	{token.REM_ASSIGN, "%=", operator},
+
+	{token.AND_ASSIGN, "&=", operator},
+	{token.OR_ASSIGN, "|=", operator},
+	{token.XOR_ASSIGN, "^=", operator},
+	{token.SHL_ASSIGN, "<<=", operator},
+	{token.SHR_ASSIGN, ">>=", operator},
+	{token.AND_NOT_ASSIGN, "&^=", operator},
+
+	{token.LAND, "&&", operator},
+	{token.LOR, "||", operator},
+	{token.ARROW, "<-", operator},
+	{token.INC, "++", operator},
+	{token.DEC, "--", operator},
+
+	{token.EQL, "==", operator},
+	{token.LSS, "<", operator},
+	{token.GTR, ">", operator},
+	{token.ASSIGN, "=", operator},
+	{token.NOT, "!", operator},
+
+	{token.NEQ, "!=", operator},
+	{token.LEQ, "<=", operator},
+	{token.GEQ, ">=", operator},
+	{token.DEFINE, ":=", operator},
+	{token.ELLIPSIS, "...", operator},
+
+	{token.LPAREN, "(", operator},
+	{token.LBRACK, "[", operator},
+	{token.LBRACE, "{", operator},
+	{token.COMMA, ",", operator},
+	{token.PERIOD, ".", operator},
+
+	{token.RPAREN, ")", operator},
+	{token.RBRACK, "]", operator},
+	{token.RBRACE, "}", operator},
+	{token.SEMICOLON, ";", operator},
+	{token.COLON, ":", operator},
+
+	// Keywords
+	{token.BREAK, "break", keyword},
+	{token.CASE, "case", keyword},
+	{token.CHAN, "chan", keyword},
+	{token.CONST, "const", keyword},
+	{token.CONTINUE, "continue", keyword},
+
+	{token.DEFAULT, "default", keyword},
+	{token.DEFER, "defer", keyword},
+	{token.ELSE, "else", keyword},
+	{token.FALLTHROUGH, "fallthrough", keyword},
+	{token.FOR, "for", keyword},
+
+	{token.FUNC, "func", keyword},
+	{token.GO, "go", keyword},
+	{token.GOTO, "goto", keyword},
+	{token.IF, "if", keyword},
+	{token.IMPORT, "import", keyword},
+
+	{token.INTERFACE, "interface", keyword},
+	{token.MAP, "map", keyword},
+	{token.PACKAGE, "package", keyword},
+	{token.RANGE, "range", keyword},
+	{token.RETURN, "return", keyword},
+
+	{token.SELECT, "select", keyword},
+	{token.STRUCT, "struct", keyword},
+	{token.SWITCH, "switch", keyword},
+	{token.TYPE, "type", keyword},
+	{token.VAR, "var", keyword},
+}
+
+const whitespace = "  \t  \n\n\n" // to separate tokens
+
+type testErrorHandler struct {
+	t *testing.T
+}
+
+func (h *testErrorHandler) Error(pos token.Position, msg string) {
+	h.t.Errorf("Error() called (msg = %s)", msg)
+}
+
+func newlineCount(s string) int {
+	n := 0
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\n' {
+			n++
+		}
+	}
+	return n
+}
+
+func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
+	pos := fset.Position(p)
+	if pos.Filename != expected.Filename {
+		t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
+	}
+	if pos.Offset != expected.Offset {
+		t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
+	}
+	if pos.Line != expected.Line {
+		t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
+	}
+	if pos.Column != expected.Column {
+		t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
+	}
+}
+
+// Verify that calling Scan() provides the correct results.
+func TestScan(t *testing.T) {
+	// make source
+	var src string
+	for _, e := range tokens {
+		src += e.lit + whitespace
+	}
+	src_linecount := newlineCount(src)
+	whitespace_linecount := newlineCount(whitespace)
+
+	// verify scan
+	var s Scanner
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
+	index := 0
+	epos := token.Position{"", 0, 1, 1} // expected position
+	for {
+		pos, tok, lit := s.Scan()
+		e := elt{token.EOF, "", special}
+		if index < len(tokens) {
+			e = tokens[index]
+		}
+		if tok == token.EOF {
+			lit = ""
+			epos.Line = src_linecount
+			epos.Column = 2
+		}
+		checkPos(t, lit, pos, epos)
+		if tok != e.tok {
+			t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
+		}
+		if e.tok.IsLiteral() && lit != e.lit {
+			t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
+		}
+		if tokenclass(tok) != e.class {
+			t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
+		}
+		epos.Offset += len(lit) + len(whitespace)
+		epos.Line += newlineCount(lit) + whitespace_linecount
+		if tok == token.COMMENT && lit[1] == '/' {
+			// correct for unaccounted '/n' in //-style comment
+			epos.Offset++
+			epos.Line++
+		}
+		index++
+		if tok == token.EOF {
+			break
+		}
+	}
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+func checkSemi(t *testing.T, line string, mode uint) {
+	var S Scanner
+	file := fset.AddFile("TestSemis", fset.Base(), len(line))
+	S.Init(file, []byte(line), nil, mode)
+	pos, tok, lit := S.Scan()
+	for tok != token.EOF {
+		if tok == token.ILLEGAL {
+			// the illegal token literal indicates what
+			// kind of semicolon literal to expect
+			semiLit := "\n"
+			if lit[0] == '#' {
+				semiLit = ";"
+			}
+			// next token must be a semicolon
+			semiPos := file.Position(pos)
+			semiPos.Offset++
+			semiPos.Column++
+			pos, tok, lit = S.Scan()
+			if tok == token.SEMICOLON {
+				if lit != semiLit {
+					t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
+				}
+				checkPos(t, line, pos, semiPos)
+			} else {
+				t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
+			}
+		} else if tok == token.SEMICOLON {
+			t.Errorf("bad token for %q: got ;, expected no ;", line)
+		}
+		pos, tok, lit = S.Scan()
+	}
+}
+
+var lines = []string{
+	// # indicates a semicolon present in the source
+	// $ indicates an automatically inserted semicolon
+	"",
+	"#;",
+	"foo$\n",
+	"123$\n",
+	"1.2$\n",
+	"'x'$\n",
+	`"x"` + "$\n",
+	"`x`$\n",
+
+	"+\n",
+	"-\n",
+	"*\n",
+	"/\n",
+	"%\n",
+
+	"&\n",
+	"|\n",
+	"^\n",
+	"<<\n",
+	">>\n",
+	"&^\n",
+
+	"+=\n",
+	"-=\n",
+	"*=\n",
+	"/=\n",
+	"%=\n",
+
+	"&=\n",
+	"|=\n",
+	"^=\n",
+	"<<=\n",
+	">>=\n",
+	"&^=\n",
+
+	"&&\n",
+	"||\n",
+	"<-\n",
+	"++$\n",
+	"--$\n",
+
+	"==\n",
+	"<\n",
+	">\n",
+	"=\n",
+	"!\n",
+
+	"!=\n",
+	"<=\n",
+	">=\n",
+	":=\n",
+	"...\n",
+
+	"(\n",
+	"[\n",
+	"{\n",
+	",\n",
+	".\n",
+
+	")$\n",
+	"]$\n",
+	"}$\n",
+	"#;\n",
+	":\n",
+
+	"break$\n",
+	"case\n",
+	"chan\n",
+	"const\n",
+	"continue$\n",
+
+	"default\n",
+	"defer\n",
+	"else\n",
+	"fallthrough$\n",
+	"for\n",
+
+	"func\n",
+	"go\n",
+	"goto\n",
+	"if\n",
+	"import\n",
+
+	"interface\n",
+	"map\n",
+	"package\n",
+	"range\n",
+	"return$\n",
+
+	"select\n",
+	"struct\n",
+	"switch\n",
+	"type\n",
+	"var\n",
+
+	"foo$//comment\n",
+	"foo$//comment",
+	"foo$/*comment*/\n",
+	"foo$/*\n*/",
+	"foo$/*comment*/    \n",
+	"foo$/*\n*/    ",
+
+	"foo    $// comment\n",
+	"foo    $// comment",
+	"foo    $/*comment*/\n",
+	"foo    $/*\n*/",
+	"foo    $/*  */ /* \n */ bar$/**/\n",
+	"foo    $/*0*/ /*1*/ /*2*/\n",
+
+	"foo    $/*comment*/    \n",
+	"foo    $/*0*/ /*1*/ /*2*/    \n",
+	"foo	$/**/ /*-------------*/       /*----\n*/bar       $/*  \n*/baa$\n",
+	"foo    $/* an EOF terminates a line */",
+	"foo    $/* an EOF terminates a line */ /*",
+	"foo    $/* an EOF terminates a line */ //",
+
+	"package main$\n\nfunc main() {\n\tif {\n\t\treturn /* */ }$\n}$\n",
+	"package main$",
+}
+
+func TestSemis(t *testing.T) {
+	for _, line := range lines {
+		checkSemi(t, line, AllowIllegalChars|InsertSemis)
+		checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
+
+		// if the input ended in newlines, the input must tokenize the
+		// same with or without those newlines
+		for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
+			checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
+			checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
+		}
+	}
+}
+
+type segment struct {
+	srcline  string // a line of source text
+	filename string // filename for current token
+	line     int    // line number for current token
+}
+
+var segments = []segment{
+	// exactly one token per line since the test consumes one token per segment
+	{"  line1", filepath.Join("dir", "TestLineComments"), 1},
+	{"\nline2", filepath.Join("dir", "TestLineComments"), 2},
+	{"\nline3  //line File1.go:100", filepath.Join("dir", "TestLineComments"), 3}, // bad line comment, ignored
+	{"\nline4", filepath.Join("dir", "TestLineComments"), 4},
+	{"\n//line File1.go:100\n  line100", filepath.Join("dir", "File1.go"), 100},
+	{"\n//line File2.go:200\n  line200", filepath.Join("dir", "File2.go"), 200},
+	{"\n//line :1\n  line1", "dir", 1},
+	{"\n//line foo:42\n  line42", filepath.Join("dir", "foo"), 42},
+	{"\n //line foo:42\n  line44", filepath.Join("dir", "foo"), 44},           // bad line comment, ignored
+	{"\n//line foo 42\n  line46", filepath.Join("dir", "foo"), 46},            // bad line comment, ignored
+	{"\n//line foo:42 extra text\n  line48", filepath.Join("dir", "foo"), 48}, // bad line comment, ignored
+	{"\n//line ./foo:42\n  line42", filepath.Join("dir", "foo"), 42},
+	{"\n//line a/b/c/File1.go:100\n  line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100},
+}
+
+var unixsegments = []segment{
+	{"\n//line /bar:42\n  line42", "/bar", 42},
+}
+
+var winsegments = []segment{
+	{"\n//line c:\\bar:42\n  line42", "c:\\bar", 42},
+	{"\n//line c:\\dir\\File1.go:100\n  line100", "c:\\dir\\File1.go", 100},
+}
+
+// Verify that comments of the form "//line filename:line" are interpreted correctly.
+func TestLineComments(t *testing.T) {
+	segs := segments
+	if runtime.GOOS == "windows" {
+		segs = append(segs, winsegments...)
+	} else {
+		segs = append(segs, unixsegments...)
+	}
+
+	// make source
+	var src string
+	for _, e := range segs {
+		src += e.srcline
+	}
+
+	// verify scan
+	var S Scanner
+	file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
+	S.Init(file, []byte(src), nil, 0)
+	for _, s := range segs {
+		p, _, lit := S.Scan()
+		pos := file.Position(p)
+		checkPos(t, lit, p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+	}
+
+	if S.ErrorCount != 0 {
+		t.Errorf("found %d errors", S.ErrorCount)
+	}
+}
+
+// Verify that initializing the same scanner more then once works correctly.
+func TestInit(t *testing.T) {
+	var s Scanner
+
+	// 1st init
+	src1 := "if true { }"
+	f1 := fset.AddFile("src1", fset.Base(), len(src1))
+	s.Init(f1, []byte(src1), nil, 0)
+	if f1.Size() != len(src1) {
+		t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
+	}
+	s.Scan()              // if
+	s.Scan()              // true
+	_, tok, _ := s.Scan() // {
+	if tok != token.LBRACE {
+		t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
+	}
+
+	// 2nd init
+	src2 := "go true { ]"
+	f2 := fset.AddFile("src2", fset.Base(), len(src2))
+	s.Init(f2, []byte(src2), nil, 0)
+	if f2.Size() != len(src2) {
+		t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
+	}
+	_, tok, _ = s.Scan() // go
+	if tok != token.GO {
+		t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
+	}
+
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+func TestIllegalChars(t *testing.T) {
+	var s Scanner
+
+	const src = "*?*$*@*"
+	file := fset.AddFile("", fset.Base(), len(src))
+	s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
+	for offs, ch := range src {
+		pos, tok, lit := s.Scan()
+		if poffs := file.Offset(pos); poffs != offs {
+			t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
+		}
+		if tok == token.ILLEGAL && lit != string(ch) {
+			t.Errorf("bad token: got %s, expected %s", lit, string(ch))
+		}
+	}
+
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
+	}
+}
+
+func TestStdErrorHander(t *testing.T) {
+	const src = "@\n" + // illegal character, cause an error
+		"@ @\n" + // two errors on the same line
+		"//line File2:20\n" +
+		"@\n" + // different file, but same line
+		"//line File2:1\n" +
+		"@ @\n" + // same file, decreasing line number
+		"//line File1:1\n" +
+		"@ @ @" // original file, line 1 again
+
+	v := new(ErrorVector)
+	var s Scanner
+	s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
+	for {
+		if _, tok, _ := s.Scan(); tok == token.EOF {
+			break
+		}
+	}
+
+	list := v.GetErrorList(Raw)
+	if len(list) != 9 {
+		t.Errorf("found %d raw errors, expected 9", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	list = v.GetErrorList(Sorted)
+	if len(list) != 9 {
+		t.Errorf("found %d sorted errors, expected 9", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	list = v.GetErrorList(NoMultiples)
+	if len(list) != 4 {
+		t.Errorf("found %d one-per-line errors, expected 4", len(list))
+		PrintError(os.Stderr, list)
+	}
+
+	if v.ErrorCount() != s.ErrorCount {
+		t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
+	}
+}
+
+type errorCollector struct {
+	cnt int            // number of errors encountered
+	msg string         // last error message encountered
+	pos token.Position // last error position encountered
+}
+
+func (h *errorCollector) Error(pos token.Position, msg string) {
+	h.cnt++
+	h.msg = msg
+	h.pos = pos
+}
+
+func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
+	var s Scanner
+	var h errorCollector
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
+	_, tok0, _ := s.Scan()
+	_, tok1, _ := s.Scan()
+	if tok0 != tok {
+		t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+	}
+	if tok1 != token.EOF {
+		t.Errorf("%q: got %s, expected EOF", src, tok1)
+	}
+	cnt := 0
+	if err != "" {
+		cnt = 1
+	}
+	if h.cnt != cnt {
+		t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+	}
+	if h.msg != err {
+		t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+	}
+	if h.pos.Offset != pos {
+		t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
+	}
+}
+
+var errors = []struct {
+	src string
+	tok token.Token
+	pos int
+	err string
+}{
+	{"\a", token.ILLEGAL, 0, "illegal character U+0007"},
+	{`#`, token.ILLEGAL, 0, "illegal character U+0023 '#'"},
+	{`…`, token.ILLEGAL, 0, "illegal character U+2026 '…'"},
+	{`' '`, token.CHAR, 0, ""},
+	{`''`, token.CHAR, 0, "illegal character literal"},
+	{`'\8'`, token.CHAR, 2, "unknown escape sequence"},
+	{`'\08'`, token.CHAR, 3, "illegal character in escape sequence"},
+	{`'\x0g'`, token.CHAR, 4, "illegal character in escape sequence"},
+	{`'\Uffffffff'`, token.CHAR, 2, "escape sequence is invalid Unicode code point"},
+	{`'`, token.CHAR, 0, "character literal not terminated"},
+	{`""`, token.STRING, 0, ""},
+	{`"`, token.STRING, 0, "string not terminated"},
+	{"``", token.STRING, 0, ""},
+	{"`", token.STRING, 0, "string not terminated"},
+	{"/**/", token.COMMENT, 0, ""},
+	{"/*", token.COMMENT, 0, "comment not terminated"},
+	{"077", token.INT, 0, ""},
+	{"078.", token.FLOAT, 0, ""},
+	{"07801234567.", token.FLOAT, 0, ""},
+	{"078e0", token.FLOAT, 0, ""},
+	{"078", token.INT, 0, "illegal octal number"},
+	{"07800000009", token.INT, 0, "illegal octal number"},
+	{"0x", token.INT, 0, "illegal hexadecimal number"},
+	{"0X", token.INT, 0, "illegal hexadecimal number"},
+	{"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"},
+	{"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"},
+}
+
+func TestScanErrors(t *testing.T) {
+	for _, e := range errors {
+		checkError(t, e.src, e.tok, e.pos, e.err)
+	}
+}
diff --git a/src/pkg/go/token/Makefile b/src/pkg/go/token/Makefile
new file mode 100644
index 000000000..4a4e64dc8
--- /dev/null
+++ b/src/pkg/go/token/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/token
+GOFILES=\
+	position.go\
+	token.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/token/position.go b/src/pkg/go/token/position.go
new file mode 100644
index 000000000..c559e19f8
--- /dev/null
+++ b/src/pkg/go/token/position.go
@@ -0,0 +1,424 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO(gri) consider making this a separate package outside the go directory.
+
+package token
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+)
+
+// Position describes an arbitrary source position
+// including the file, line, and column location.
+// A Position is valid if the line number is > 0.
+//
+type Position struct {
+	Filename string // filename, if any
+	Offset   int    // offset, starting at 0
+	Line     int    // line number, starting at 1
+	Column   int    // column number, starting at 1 (character count)
+}
+
+// IsValid returns true if the position is valid.
+func (pos *Position) IsValid() bool { return pos.Line > 0 }
+
+// String returns a string in one of several forms:
+//
+//	file:line:column    valid position with file name
+//	line:column         valid position without file name
+//	file                invalid position with file name
+//	-                   invalid position without file name
+//
+func (pos Position) String() string {
+	s := pos.Filename
+	if pos.IsValid() {
+		if s != "" {
+			s += ":"
+		}
+		s += fmt.Sprintf("%d:%d", pos.Line, pos.Column)
+	}
+	if s == "" {
+		s = "-"
+	}
+	return s
+}
+
+// Pos is a compact encoding of a source position within a file set.
+// It can be converted into a Position for a more convenient, but much
+// larger, representation.
+//
+// The Pos value for a given file is a number in the range [base, base+size],
+// where base and size are specified when adding the file to the file set via
+// AddFile.
+//
+// To create the Pos value for a specific source offset, first add
+// the respective file to the current file set (via FileSet.AddFile)
+// and then call File.Pos(offset) for that file. Given a Pos value p
+// for a specific file set fset, the corresponding Position value is
+// obtained by calling fset.Position(p).
+//
+// Pos values can be compared directly with the usual comparison operators:
+// If two Pos values p and q are in the same file, comparing p and q is
+// equivalent to comparing the respective source file offsets. If p and q
+// are in different files, p < q is true if the file implied by p was added
+// to the respective file set before the file implied by q.
+//
+type Pos int
+
+// The zero value for Pos is NoPos; there is no file and line information
+// associated with it, and NoPos().IsValid() is false. NoPos is always
+// smaller than any other Pos value. The corresponding Position value
+// for NoPos is the zero value for Position.
+// 
+const NoPos Pos = 0
+
+// IsValid returns true if the position is valid.
+func (p Pos) IsValid() bool {
+	return p != NoPos
+}
+
+func searchFiles(a []*File, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1
+}
+
+func (s *FileSet) file(p Pos) *File {
+	if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size {
+		return f
+	}
+	if i := searchFiles(s.files, int(p)); i >= 0 {
+		f := s.files[i]
+		// f.base <= int(p) by definition of searchFiles
+		if int(p) <= f.base+f.size {
+			s.last = f
+			return f
+		}
+	}
+	return nil
+}
+
+// File returns the file which contains the position p.
+// If no such file is found (for instance for p == NoPos),
+// the result is nil.
+//
+func (s *FileSet) File(p Pos) (f *File) {
+	if p != NoPos {
+		s.mutex.RLock()
+		f = s.file(p)
+		s.mutex.RUnlock()
+	}
+	return
+}
+
+func (f *File) position(p Pos) (pos Position) {
+	offset := int(p) - f.base
+	pos.Offset = offset
+	pos.Filename, pos.Line, pos.Column = f.info(offset)
+	return
+}
+
+// Position converts a Pos in the fileset into a general Position.
+func (s *FileSet) Position(p Pos) (pos Position) {
+	if p != NoPos {
+		// TODO(gri) consider optimizing the case where p
+		//           is in the last file added, or perhaps
+		//           looked at - will eliminate one level
+		//           of search
+		s.mutex.RLock()
+		if f := s.file(p); f != nil {
+			pos = f.position(p)
+		}
+		s.mutex.RUnlock()
+	}
+	return
+}
+
+type lineInfo struct {
+	offset   int
+	filename string
+	line     int
+}
+
+// AddLineInfo adds alternative file and line number information for
+// a given file offset. The offset must be larger than the offset for
+// the previously added alternative line info and smaller than the
+// file size; otherwise the information is ignored.
+//
+// AddLineInfo is typically used to register alternative position
+// information for //line filename:line comments in source files.
+//
+func (f *File) AddLineInfo(offset int, filename string, line int) {
+	f.set.mutex.Lock()
+	if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset < f.size {
+		f.infos = append(f.infos, lineInfo{offset, filename, line})
+	}
+	f.set.mutex.Unlock()
+}
+
+// A File is a handle for a file belonging to a FileSet.
+// A File has a name, size, and line offset table.
+//
+type File struct {
+	set  *FileSet
+	name string // file name as provided to AddFile
+	base int    // Pos value range for this file is [base...base+size]
+	size int    // file size as provided to AddFile
+
+	// lines and infos are protected by set.mutex
+	lines []int
+	infos []lineInfo
+}
+
+// Name returns the file name of file f as registered with AddFile.
+func (f *File) Name() string {
+	return f.name
+}
+
+// Base returns the base offset of file f as registered with AddFile.
+func (f *File) Base() int {
+	return f.base
+}
+
+// Size returns the size of file f as registered with AddFile.
+func (f *File) Size() int {
+	return f.size
+}
+
+// LineCount returns the number of lines in file f.
+func (f *File) LineCount() int {
+	f.set.mutex.RLock()
+	n := len(f.lines)
+	f.set.mutex.RUnlock()
+	return n
+}
+
+// AddLine adds the line offset for a new line.
+// The line offset must be larger than the offset for the previous line
+// and smaller than the file size; otherwise the line offset is ignored.
+//
+func (f *File) AddLine(offset int) {
+	f.set.mutex.Lock()
+	if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size {
+		f.lines = append(f.lines, offset)
+	}
+	f.set.mutex.Unlock()
+}
+
+// SetLines sets the line offsets for a file and returns true if successful.
+// The line offsets are the offsets of the first character of each line;
+// for instance for the content "ab\nc\n" the line offsets are {0, 3}.
+// An empty file has an empty line offset table.
+// Each line offset must be larger than the offset for the previous line
+// and smaller than the file size; otherwise SetLines fails and returns
+// false.
+//
+func (f *File) SetLines(lines []int) bool {
+	// verify validity of lines table
+	size := f.size
+	for i, offset := range lines {
+		if i > 0 && offset <= lines[i-1] || size <= offset {
+			return false
+		}
+	}
+
+	// set lines table
+	f.set.mutex.Lock()
+	f.lines = lines
+	f.set.mutex.Unlock()
+	return true
+}
+
+// SetLinesForContent sets the line offsets for the given file content.
+func (f *File) SetLinesForContent(content []byte) {
+	var lines []int
+	line := 0
+	for offset, b := range content {
+		if line >= 0 {
+			lines = append(lines, line)
+		}
+		line = -1
+		if b == '\n' {
+			line = offset + 1
+		}
+	}
+
+	// set lines table
+	f.set.mutex.Lock()
+	f.lines = lines
+	f.set.mutex.Unlock()
+}
+
+// Pos returns the Pos value for the given file offset;
+// the offset must be <= f.Size().
+// f.Pos(f.Offset(p)) == p.
+//
+func (f *File) Pos(offset int) Pos {
+	if offset > f.size {
+		panic("illegal file offset")
+	}
+	return Pos(f.base + offset)
+}
+
+// Offset returns the offset for the given file position p;
+// p must be a valid Pos value in that file.
+// f.Offset(f.Pos(offset)) == offset.
+//
+func (f *File) Offset(p Pos) int {
+	if int(p) < f.base || int(p) > f.base+f.size {
+		panic("illegal Pos value")
+	}
+	return int(p) - f.base
+}
+
+// Line returns the line number for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Line(p Pos) int {
+	// TODO(gri) this can be implemented much more efficiently
+	return f.Position(p).Line
+}
+
+// Position returns the Position value for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Position(p Pos) (pos Position) {
+	if p != NoPos {
+		if int(p) < f.base || int(p) > f.base+f.size {
+			panic("illegal Pos value")
+		}
+		pos = f.position(p)
+	}
+	return
+}
+
+func searchInts(a []int, x int) int {
+	// This function body is a manually inlined version of:
+	//
+	//   return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1
+	//
+	// With better compiler optimizations, this may not be needed in the
+	// future, but at the moment this change improves the go/printer
+	// benchmark performance by ~30%. This has a direct impact on the
+	// speed of gofmt and thus seems worthwhile (2011-04-29).
+	i, j := 0, len(a)
+	for i < j {
+		h := i + (j-i)/2 // avoid overflow when computing h
+		// i ≤ h < j
+		if a[h] <= x {
+			i = h + 1
+		} else {
+			j = h
+		}
+	}
+	return i - 1
+}
+
+func searchLineInfos(a []lineInfo, x int) int {
+	return sort.Search(len(a), func(i int) bool { return a[i].offset > x }) - 1
+}
+
+// info returns the file name, line, and column number for a file offset.
+func (f *File) info(offset int) (filename string, line, column int) {
+	filename = f.name
+	if i := searchInts(f.lines, offset); i >= 0 {
+		line, column = i+1, offset-f.lines[i]+1
+	}
+	if len(f.infos) > 0 {
+		// almost no files have extra line infos
+		if i := searchLineInfos(f.infos, offset); i >= 0 {
+			alt := &f.infos[i]
+			filename = alt.filename
+			if i := searchInts(f.lines, alt.offset); i >= 0 {
+				line += alt.line - i - 1
+			}
+		}
+	}
+	return
+}
+
+// A FileSet represents a set of source files.
+// Methods of file sets are synchronized; multiple goroutines
+// may invoke them concurrently.
+//
+type FileSet struct {
+	mutex sync.RWMutex // protects the file set
+	base  int          // base offset for the next file
+	files []*File      // list of files in the order added to the set
+	last  *File        // cache of last file looked up
+}
+
+// NewFileSet creates a new file set.
+func NewFileSet() *FileSet {
+	s := new(FileSet)
+	s.base = 1 // 0 == NoPos
+	return s
+}
+
+// Base returns the minimum base offset that must be provided to
+// AddFile when adding the next file.
+//
+func (s *FileSet) Base() int {
+	s.mutex.RLock()
+	b := s.base
+	s.mutex.RUnlock()
+	return b
+
+}
+
+// AddFile adds a new file with a given filename, base offset, and file size
+// to the file set s and returns the file. Multiple files may have the same
+// name. The base offset must not be smaller than the FileSet's Base(), and
+// size must not be negative.
+//
+// Adding the file will set the file set's Base() value to base + size + 1
+// as the minimum base value for the next file. The following relationship
+// exists between a Pos value p for a given file offset offs:
+//
+//	int(p) = base + offs
+//
+// with offs in the range [0, size] and thus p in the range [base, base+size].
+// For convenience, File.Pos may be used to create file-specific position
+// values from a file offset.
+//
+func (s *FileSet) AddFile(filename string, base, size int) *File {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if base < s.base || size < 0 {
+		panic("illegal base or size")
+	}
+	// base >= s.base && size >= 0
+	f := &File{s, filename, base, size, []int{0}, nil}
+	base += size + 1 // +1 because EOF also has a position
+	if base < 0 {
+		panic("token.Pos offset overflow (> 2G of source code in file set)")
+	}
+	// add the file to the file set
+	s.base = base
+	s.files = append(s.files, f)
+	s.last = f
+	return f
+}
+
+// Files returns the files added to the file set.
+func (s *FileSet) Files() <-chan *File {
+	ch := make(chan *File)
+	go func() {
+		for i := 0; ; i++ {
+			var f *File
+			s.mutex.RLock()
+			if i < len(s.files) {
+				f = s.files[i]
+			}
+			s.mutex.RUnlock()
+			if f == nil {
+				break
+			}
+			ch <- f
+		}
+		close(ch)
+	}()
+	return ch
+}
diff --git a/src/pkg/go/token/position_test.go b/src/pkg/go/token/position_test.go
new file mode 100644
index 000000000..30bec5991
--- /dev/null
+++ b/src/pkg/go/token/position_test.go
@@ -0,0 +1,180 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+import (
+	"fmt"
+	"testing"
+)
+
+func checkPos(t *testing.T, msg string, p, q Position) {
+	if p.Filename != q.Filename {
+		t.Errorf("%s: expected filename = %q; got %q", msg, q.Filename, p.Filename)
+	}
+	if p.Offset != q.Offset {
+		t.Errorf("%s: expected offset = %d; got %d", msg, q.Offset, p.Offset)
+	}
+	if p.Line != q.Line {
+		t.Errorf("%s: expected line = %d; got %d", msg, q.Line, p.Line)
+	}
+	if p.Column != q.Column {
+		t.Errorf("%s: expected column = %d; got %d", msg, q.Column, p.Column)
+	}
+}
+
+func TestNoPos(t *testing.T) {
+	if NoPos.IsValid() {
+		t.Errorf("NoPos should not be valid")
+	}
+	var fset *FileSet
+	checkPos(t, "nil NoPos", fset.Position(NoPos), Position{})
+	fset = NewFileSet()
+	checkPos(t, "fset NoPos", fset.Position(NoPos), Position{})
+}
+
+var tests = []struct {
+	filename string
+	source   []byte // may be nil
+	size     int
+	lines    []int
+}{
+	{"a", []byte{}, 0, []int{}},
+	{"b", []byte("01234"), 5, []int{0}},
+	{"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
+	{"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
+	{"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
+	{"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}},
+	{"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}},
+	{"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}},
+}
+
+func linecol(lines []int, offs int) (int, int) {
+	prevLineOffs := 0
+	for line, lineOffs := range lines {
+		if offs < lineOffs {
+			return line, offs - prevLineOffs + 1
+		}
+		prevLineOffs = lineOffs
+	}
+	return len(lines), offs - prevLineOffs + 1
+}
+
+func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
+	for offs := 0; offs < f.Size(); offs++ {
+		p := f.Pos(offs)
+		offs2 := f.Offset(p)
+		if offs2 != offs {
+			t.Errorf("%s, Offset: expected offset %d; got %d", f.Name(), offs, offs2)
+		}
+		line, col := linecol(lines, offs)
+		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+		checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
+		checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
+	}
+}
+
+func makeTestSource(size int, lines []int) []byte {
+	src := make([]byte, size)
+	for _, offs := range lines {
+		if offs > 0 {
+			src[offs-1] = '\n'
+		}
+	}
+	return src
+}
+
+func TestPositions(t *testing.T) {
+	const delta = 7 // a non-zero base offset increment
+	fset := NewFileSet()
+	for _, test := range tests {
+		// verify consistency of test case
+		if test.source != nil && len(test.source) != test.size {
+			t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source))
+		}
+
+		// add file and verify name and size
+		f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
+		if f.Name() != test.filename {
+			t.Errorf("expected filename %q; got %q", test.filename, f.Name())
+		}
+		if f.Size() != test.size {
+			t.Errorf("%s: expected file size %d; got %d", f.Name(), test.size, f.Size())
+		}
+		if fset.File(f.Pos(0)) != f {
+			t.Errorf("%s: f.Pos(0) was not found in f", f.Name())
+		}
+
+		// add lines individually and verify all positions
+		for i, offset := range test.lines {
+			f.AddLine(offset)
+			if f.LineCount() != i+1 {
+				t.Errorf("%s, AddLine: expected line count %d; got %d", f.Name(), i+1, f.LineCount())
+			}
+			// adding the same offset again should be ignored
+			f.AddLine(offset)
+			if f.LineCount() != i+1 {
+				t.Errorf("%s, AddLine: expected unchanged line count %d; got %d", f.Name(), i+1, f.LineCount())
+			}
+			verifyPositions(t, fset, f, test.lines[0:i+1])
+		}
+
+		// add lines with SetLines and verify all positions
+		if ok := f.SetLines(test.lines); !ok {
+			t.Errorf("%s: SetLines failed", f.Name())
+		}
+		if f.LineCount() != len(test.lines) {
+			t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+		}
+		verifyPositions(t, fset, f, test.lines)
+
+		// add lines with SetLinesForContent and verify all positions
+		src := test.source
+		if src == nil {
+			// no test source available - create one from scratch
+			src = makeTestSource(test.size, test.lines)
+		}
+		f.SetLinesForContent(src)
+		if f.LineCount() != len(test.lines) {
+			t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+		}
+		verifyPositions(t, fset, f, test.lines)
+	}
+}
+
+func TestLineInfo(t *testing.T) {
+	fset := NewFileSet()
+	f := fset.AddFile("foo", fset.Base(), 500)
+	lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401}
+	// add lines individually and provide alternative line information
+	for _, offs := range lines {
+		f.AddLine(offs)
+		f.AddLineInfo(offs, "bar", 42)
+	}
+	// verify positions for all offsets
+	for offs := 0; offs <= f.Size(); offs++ {
+		p := f.Pos(offs)
+		_, col := linecol(lines, offs)
+		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+		checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
+		checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
+	}
+}
+
+func TestFiles(t *testing.T) {
+	fset := NewFileSet()
+	for i, test := range tests {
+		fset.AddFile(test.filename, fset.Base(), test.size)
+		j := 0
+		for g := range fset.Files() {
+			if g.Name() != tests[j].filename {
+				t.Errorf("expected filename = %s; got %s", tests[j].filename, g.Name())
+			}
+			j++
+		}
+		if j != i+1 {
+			t.Errorf("expected %d files; got %d", i+1, j)
+		}
+	}
+}
diff --git a/src/pkg/go/token/token.go b/src/pkg/go/token/token.go
new file mode 100644
index 000000000..557374052
--- /dev/null
+++ b/src/pkg/go/token/token.go
@@ -0,0 +1,310 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package token defines constants representing the lexical tokens of the Go
+// programming language and basic operations on tokens (printing, predicates).
+//
+package token
+
+import "strconv"
+
+// Token is the set of lexical tokens of the Go programming language.
+type Token int
+
+// The list of tokens.
+const (
+	// Special tokens
+	ILLEGAL Token = iota
+	EOF
+	COMMENT
+
+	literal_beg
+	// Identifiers and basic type literals
+	// (these tokens stand for classes of literals)
+	IDENT  // main
+	INT    // 12345
+	FLOAT  // 123.45
+	IMAG   // 123.45i
+	CHAR   // 'a'
+	STRING // "abc"
+	literal_end
+
+	operator_beg
+	// Operators and delimiters
+	ADD // +
+	SUB // -
+	MUL // *
+	QUO // /
+	REM // %
+
+	AND     // &
+	OR      // |
+	XOR     // ^
+	SHL     // <<
+	SHR     // >>
+	AND_NOT // &^
+
+	ADD_ASSIGN // +=
+	SUB_ASSIGN // -=
+	MUL_ASSIGN // *=
+	QUO_ASSIGN // /=
+	REM_ASSIGN // %=
+
+	AND_ASSIGN     // &=
+	OR_ASSIGN      // |=
+	XOR_ASSIGN     // ^=
+	SHL_ASSIGN     // <<=
+	SHR_ASSIGN     // >>=
+	AND_NOT_ASSIGN // &^=
+
+	LAND  // &&
+	LOR   // ||
+	ARROW // <-
+	INC   // ++
+	DEC   // --
+
+	EQL    // ==
+	LSS    // <
+	GTR    // >
+	ASSIGN // =
+	NOT    // !
+
+	NEQ      // !=
+	LEQ      // <=
+	GEQ      // >=
+	DEFINE   // :=
+	ELLIPSIS // ...
+
+	LPAREN // (
+	LBRACK // [
+	LBRACE // {
+	COMMA  // ,
+	PERIOD // .
+
+	RPAREN    // )
+	RBRACK    // ]
+	RBRACE    // }
+	SEMICOLON // ;
+	COLON     // :
+	operator_end
+
+	keyword_beg
+	// Keywords
+	BREAK
+	CASE
+	CHAN
+	CONST
+	CONTINUE
+
+	DEFAULT
+	DEFER
+	ELSE
+	FALLTHROUGH
+	FOR
+
+	FUNC
+	GO
+	GOTO
+	IF
+	IMPORT
+
+	INTERFACE
+	MAP
+	PACKAGE
+	RANGE
+	RETURN
+
+	SELECT
+	STRUCT
+	SWITCH
+	TYPE
+	VAR
+	keyword_end
+)
+
+var tokens = [...]string{
+	ILLEGAL: "ILLEGAL",
+
+	EOF:     "EOF",
+	COMMENT: "COMMENT",
+
+	IDENT:  "IDENT",
+	INT:    "INT",
+	FLOAT:  "FLOAT",
+	IMAG:   "IMAG",
+	CHAR:   "CHAR",
+	STRING: "STRING",
+
+	ADD: "+",
+	SUB: "-",
+	MUL: "*",
+	QUO: "/",
+	REM: "%",
+
+	AND:     "&",
+	OR:      "|",
+	XOR:     "^",
+	SHL:     "<<",
+	SHR:     ">>",
+	AND_NOT: "&^",
+
+	ADD_ASSIGN: "+=",
+	SUB_ASSIGN: "-=",
+	MUL_ASSIGN: "*=",
+	QUO_ASSIGN: "/=",
+	REM_ASSIGN: "%=",
+
+	AND_ASSIGN:     "&=",
+	OR_ASSIGN:      "|=",
+	XOR_ASSIGN:     "^=",
+	SHL_ASSIGN:     "<<=",
+	SHR_ASSIGN:     ">>=",
+	AND_NOT_ASSIGN: "&^=",
+
+	LAND:  "&&",
+	LOR:   "||",
+	ARROW: "<-",
+	INC:   "++",
+	DEC:   "--",
+
+	EQL:    "==",
+	LSS:    "<",
+	GTR:    ">",
+	ASSIGN: "=",
+	NOT:    "!",
+
+	NEQ:      "!=",
+	LEQ:      "<=",
+	GEQ:      ">=",
+	DEFINE:   ":=",
+	ELLIPSIS: "...",
+
+	LPAREN: "(",
+	LBRACK: "[",
+	LBRACE: "{",
+	COMMA:  ",",
+	PERIOD: ".",
+
+	RPAREN:    ")",
+	RBRACK:    "]",
+	RBRACE:    "}",
+	SEMICOLON: ";",
+	COLON:     ":",
+
+	BREAK:    "break",
+	CASE:     "case",
+	CHAN:     "chan",
+	CONST:    "const",
+	CONTINUE: "continue",
+
+	DEFAULT:     "default",
+	DEFER:       "defer",
+	ELSE:        "else",
+	FALLTHROUGH: "fallthrough",
+	FOR:         "for",
+
+	FUNC:   "func",
+	GO:     "go",
+	GOTO:   "goto",
+	IF:     "if",
+	IMPORT: "import",
+
+	INTERFACE: "interface",
+	MAP:       "map",
+	PACKAGE:   "package",
+	RANGE:     "range",
+	RETURN:    "return",
+
+	SELECT: "select",
+	STRUCT: "struct",
+	SWITCH: "switch",
+	TYPE:   "type",
+	VAR:    "var",
+}
+
+// String returns the string corresponding to the token tok.
+// For operators, delimiters, and keywords the string is the actual
+// token character sequence (e.g., for the token ADD, the string is
+// "+"). For all other tokens the string corresponds to the token
+// constant name (e.g. for the token IDENT, the string is "IDENT").
+//
+func (tok Token) String() string {
+	s := ""
+	if 0 <= tok && tok < Token(len(tokens)) {
+		s = tokens[tok]
+	}
+	if s == "" {
+		s = "token(" + strconv.Itoa(int(tok)) + ")"
+	}
+	return s
+}
+
+// A set of constants for precedence-based expression parsing.
+// Non-operators have lowest precedence, followed by operators
+// starting with precedence 1 up to unary operators. The highest
+// precedence corresponds serves as "catch-all" precedence for
+// selector, indexing, and other operator and delimiter tokens.
+//
+const (
+	LowestPrec  = 0 // non-operators
+	UnaryPrec   = 6
+	HighestPrec = 7
+)
+
+// Precedence returns the operator precedence of the binary
+// operator op. If op is not a binary operator, the result
+// is LowestPrecedence.
+//
+func (op Token) Precedence() int {
+	switch op {
+	case LOR:
+		return 1
+	case LAND:
+		return 2
+	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
+		return 3
+	case ADD, SUB, OR, XOR:
+		return 4
+	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
+		return 5
+	}
+	return LowestPrec
+}
+
+var keywords map[string]Token
+
+func init() {
+	keywords = make(map[string]Token)
+	for i := keyword_beg + 1; i < keyword_end; i++ {
+		keywords[tokens[i]] = i
+	}
+}
+
+// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
+//
+func Lookup(ident []byte) Token {
+	// TODO Maps with []byte key are illegal because []byte does not
+	//      support == . Should find a more efficient solution eventually.
+	if tok, is_keyword := keywords[string(ident)]; is_keyword {
+		return tok
+	}
+	return IDENT
+}
+
+// Predicates
+
+// IsLiteral returns true for tokens corresponding to identifiers
+// and basic type literals; returns false otherwise.
+//
+func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
+
+// IsOperator returns true for tokens corresponding to operators and
+// delimiters; returns false otherwise.
+//
+func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
+
+// IsKeyword returns true for tokens corresponding to keywords;
+// returns false otherwise.
+//
+func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
diff --git a/src/pkg/go/typechecker/Makefile b/src/pkg/go/typechecker/Makefile
new file mode 100644
index 000000000..83af3ef4e
--- /dev/null
+++ b/src/pkg/go/typechecker/Makefile
@@ -0,0 +1,14 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/typechecker
+GOFILES=\
+	scope.go\
+	type.go\
+	typechecker.go\
+	universe.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/typechecker/scope.go b/src/pkg/go/typechecker/scope.go
new file mode 100644
index 000000000..d73d1a450
--- /dev/null
+++ b/src/pkg/go/typechecker/scope.go
@@ -0,0 +1,69 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// DEPRECATED FILE - WILL GO AWAY EVENTUALLY.
+//
+// Scope handling is now done in go/parser.
+// The functionality here is only present to
+// keep the typechecker running for now.
+
+package typechecker
+
+import "go/ast"
+
+func (tc *typechecker) openScope() *ast.Scope {
+	tc.topScope = ast.NewScope(tc.topScope)
+	return tc.topScope
+}
+
+func (tc *typechecker) closeScope() {
+	tc.topScope = tc.topScope.Outer
+}
+
+// declInScope declares an object of a given kind and name in scope and sets the object's Decl and N fields.
+// It returns the newly allocated object. If an object with the same name already exists in scope, an error
+// is reported and the object is not inserted.
+func (tc *typechecker) declInScope(scope *ast.Scope, kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object {
+	obj := ast.NewObj(kind, name.Name)
+	obj.Decl = decl
+	//obj.N = n
+	name.Obj = obj
+	if name.Name != "_" {
+		if alt := scope.Insert(obj); alt != nil {
+			tc.Errorf(name.Pos(), "%s already declared at %s", name.Name, tc.fset.Position(alt.Pos()).String())
+		}
+	}
+	return obj
+}
+
+// decl is the same as declInScope(tc.topScope, ...)
+func (tc *typechecker) decl(kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object {
+	return tc.declInScope(tc.topScope, kind, name, decl, n)
+}
+
+// find returns the object with the given name if visible in the current scope hierarchy.
+// If no such object is found, an error is reported and a bad object is returned instead.
+func (tc *typechecker) find(name *ast.Ident) (obj *ast.Object) {
+	for s := tc.topScope; s != nil && obj == nil; s = s.Outer {
+		obj = s.Lookup(name.Name)
+	}
+	if obj == nil {
+		tc.Errorf(name.Pos(), "%s not declared", name.Name)
+		obj = ast.NewObj(ast.Bad, name.Name)
+	}
+	name.Obj = obj
+	return
+}
+
+// findField returns the object with the given name if visible in the type's scope.
+// If no such object is found, an error is reported and a bad object is returned instead.
+func (tc *typechecker) findField(typ *Type, name *ast.Ident) (obj *ast.Object) {
+	// TODO(gri) This is simplistic at the moment and ignores anonymous fields.
+	obj = typ.Scope.Lookup(name.Name)
+	if obj == nil {
+		tc.Errorf(name.Pos(), "%s not declared", name.Name)
+		obj = ast.NewObj(ast.Bad, name.Name)
+	}
+	return
+}
diff --git a/src/pkg/go/typechecker/testdata/test0.src b/src/pkg/go/typechecker/testdata/test0.src
new file mode 100644
index 000000000..4e317f214
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test0.src
@@ -0,0 +1,94 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// type declarations
+
+package P0
+
+type (
+	B bool
+	I int32
+	A [10]P
+	T struct {
+		x, y P
+	}
+	P *T
+	R *R
+	F func(A) I
+	Y interface {
+		f(A) I
+	}
+	S []P
+	M map[I]F
+	C chan<- I
+)
+
+type (
+	a/* ERROR "illegal cycle" */ a
+	a/* ERROR "already declared" */ int
+
+	b/* ERROR "illegal cycle" */ c
+	c d
+	d e
+	e b /* ERROR "not a type" */
+
+	t *t
+
+	U V
+	V W
+	W *U
+
+	P1 *S2
+	P2 P1
+
+	S1 struct {
+		a, b, c int
+		u, v, a/* ERROR "already declared" */ float
+	}
+	S2/* ERROR "illegal cycle" */ struct {
+		x S2
+	}
+
+	L1 []L1
+	L2 []int
+
+	A1 [10]int
+	A2/* ERROR "illegal cycle" */ [10]A2
+	A3/* ERROR "illegal cycle" */ [10]struct {
+		x A4
+	}
+	A4 [10]A3
+
+	F1 func()
+	F2 func(x, y, z float)
+	F3 func(x, y, x /* ERROR "already declared" */ float)
+	F4 func() (x, y, x /* ERROR "already declared" */ float)
+	F5 func(x int) (x /* ERROR "already declared" */ float)
+
+	I1 interface{}
+	I2 interface {
+		m1()
+	}
+	I3 interface {
+		m1()
+		m1 /* ERROR "already declared" */ ()
+	}
+	I4 interface {
+		m1(x, y, x /* ERROR "already declared" */ float)
+		m2() (x, y, x /* ERROR "already declared" */ float)
+		m3(x int) (x /* ERROR "already declared" */ float)
+	}
+	I5 interface {
+		m1(I5)
+	}
+
+	C1 chan int
+	C2 <-chan int
+	C3 chan<- C3
+
+	M1 map[Last]string
+	M2 map[string]M2
+
+	Last int
+)
diff --git a/src/pkg/go/typechecker/testdata/test1.src b/src/pkg/go/typechecker/testdata/test1.src
new file mode 100644
index 000000000..b5531fb9f
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test1.src
@@ -0,0 +1,13 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// const and var declarations
+
+package P1
+
+const (
+	c1 = 0
+	c2     int = 0
+	c3, c4 = 0
+)
diff --git a/src/pkg/go/typechecker/testdata/test3.src b/src/pkg/go/typechecker/testdata/test3.src
new file mode 100644
index 000000000..2e1a9fa8f
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test3.src
@@ -0,0 +1,41 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package P3
+
+// function and method signatures
+
+func _()                                        {}
+func _()                                        {}
+func _(x, x /* ERROR "already declared" */ int) {}
+
+func f()                                 {}
+func f /* ERROR "already declared" */ () {}
+
+func (*foo /* ERROR "invalid receiver" */ ) m() {}
+func (bar /* ERROR "not a type" */ ) m()        {}
+
+func f1(x, _, _ int) (_, _ float)                              {}
+func f2(x, y, x /* ERROR "already declared" */ int)            {}
+func f3(x, y int) (a, b, x /* ERROR "already declared" */ int) {}
+
+func (x *T) m1()                                 {}
+func (x *T) m1 /* ERROR "already declared" */ () {}
+func (x T) m1 /* ERROR "already declared" */ ()  {}
+func (T) m1 /* ERROR "already declared" */ ()    {}
+
+func (x *T) m2(u, x /* ERROR "already declared" */ int)               {}
+func (x *T) m3(a, b, c int) (u, x /* ERROR "already declared" */ int) {}
+// The following are disabled for now because the typechecker
+// in in the process of being rewritten and cannot handle them
+// at the moment
+//func (T) _(x, x /* "already declared" */ int)                   {}
+//func (T) _() (x, x /* "already declared" */ int)                {}
+
+//func (PT) _() {}
+
+var bar int
+
+type T struct{}
+type PT (T)
diff --git a/src/pkg/go/typechecker/testdata/test4.src b/src/pkg/go/typechecker/testdata/test4.src
new file mode 100644
index 000000000..94d3558f9
--- /dev/null
+++ b/src/pkg/go/typechecker/testdata/test4.src
@@ -0,0 +1,11 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Constant declarations
+
+package P4
+
+const (
+	c0 = 0
+)
diff --git a/src/pkg/go/typechecker/type.go b/src/pkg/go/typechecker/type.go
new file mode 100644
index 000000000..1b88eb54b
--- /dev/null
+++ b/src/pkg/go/typechecker/type.go
@@ -0,0 +1,118 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package typechecker
+
+import "go/ast"
+
+// A Type represents a Go type.
+type Type struct {
+	Form     Form
+	Obj      *ast.Object // corresponding type name, or nil
+	Scope    *ast.Scope  // fields and methods, always present
+	N        uint        // basic type id, array length, number of function results, or channel direction
+	Key, Elt *Type       // map key and array, pointer, slice, map or channel element
+	Params   *ast.Scope  // function (receiver, input and result) parameters, tuple expressions (results of function calls), or nil
+	Expr     ast.Expr    // corresponding AST expression
+}
+
+// NewType creates a new type of a given form.
+func NewType(form Form) *Type {
+	return &Type{Form: form, Scope: ast.NewScope(nil)}
+}
+
+// Form describes the form of a type.
+type Form int
+
+// The list of possible type forms.
+const (
+	BadType    Form = iota // for error handling
+	Unresolved             // type not fully setup
+	Basic
+	Array
+	Struct
+	Pointer
+	Function
+	Method
+	Interface
+	Slice
+	Map
+	Channel
+	Tuple
+)
+
+var formStrings = [...]string{
+	BadType:    "badType",
+	Unresolved: "unresolved",
+	Basic:      "basic",
+	Array:      "array",
+	Struct:     "struct",
+	Pointer:    "pointer",
+	Function:   "function",
+	Method:     "method",
+	Interface:  "interface",
+	Slice:      "slice",
+	Map:        "map",
+	Channel:    "channel",
+	Tuple:      "tuple",
+}
+
+func (form Form) String() string { return formStrings[form] }
+
+// The list of basic type id's.
+const (
+	Bool = iota
+	Byte
+	Uint
+	Int
+	Float
+	Complex
+	Uintptr
+	String
+
+	Uint8
+	Uint16
+	Uint32
+	Uint64
+
+	Int8
+	Int16
+	Int32
+	Int64
+
+	Float32
+	Float64
+
+	Complex64
+	Complex128
+
+	// TODO(gri) ideal types are missing
+)
+
+var BasicTypes = map[uint]string{
+	Bool:    "bool",
+	Byte:    "byte",
+	Uint:    "uint",
+	Int:     "int",
+	Float:   "float",
+	Complex: "complex",
+	Uintptr: "uintptr",
+	String:  "string",
+
+	Uint8:  "uint8",
+	Uint16: "uint16",
+	Uint32: "uint32",
+	Uint64: "uint64",
+
+	Int8:  "int8",
+	Int16: "int16",
+	Int32: "int32",
+	Int64: "int64",
+
+	Float32: "float32",
+	Float64: "float64",
+
+	Complex64:  "complex64",
+	Complex128: "complex128",
+}
diff --git a/src/pkg/go/typechecker/typechecker.go b/src/pkg/go/typechecker/typechecker.go
new file mode 100644
index 000000000..24480165b
--- /dev/null
+++ b/src/pkg/go/typechecker/typechecker.go
@@ -0,0 +1,468 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// DEPRECATED PACKAGE - SEE go/types INSTEAD.
+// This package implements typechecking of a Go AST.
+// The result of the typecheck is an augmented AST
+// with object and type information for each identifier.
+//
+package typechecker
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+	"go/scanner"
+	"os"
+)
+
+// TODO(gri) don't report errors for objects/types that are marked as bad.
+
+
+const debug = true // set for debugging output
+
+// An importer takes an import path and returns the data describing the
+// respective package's exported interface. The data format is TBD.
+//
+type Importer func(path string) ([]byte, os.Error)
+
+// CheckPackage typechecks a package and augments the AST by setting
+// *ast.Object, *ast.Type, and *ast.Scope fields accordingly. If an
+// importer is provided, it is used to handle imports, otherwise they
+// are ignored (likely leading to typechecking errors).
+//
+// If errors are reported, the AST may be incompletely augmented (fields
+// may be nil) or contain incomplete object, type, or scope information.
+//
+func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error {
+	var tc typechecker
+	tc.fset = fset
+	tc.importer = importer
+	tc.checkPackage(pkg)
+	return tc.GetError(scanner.Sorted)
+}
+
+// CheckFile typechecks a single file, but otherwise behaves like
+// CheckPackage. If the complete package consists of more than just
+// one file, the file may not typecheck without errors.
+//
+func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error {
+	// create a single-file dummy package
+	pkg := &ast.Package{file.Name.Name, nil, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}}
+	return CheckPackage(fset, pkg, importer)
+}
+
+// ----------------------------------------------------------------------------
+// Typechecker state
+
+type typechecker struct {
+	fset *token.FileSet
+	scanner.ErrorVector
+	importer Importer
+	globals  []*ast.Object        // list of global objects
+	topScope *ast.Scope           // current top-most scope
+	cyclemap map[*ast.Object]bool // for cycle detection
+	iota     int                  // current value of iota
+}
+
+func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) {
+	tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...))
+}
+
+func assert(pred bool) {
+	if !pred {
+		panic("internal error")
+	}
+}
+
+/*
+Typechecking is done in several phases:
+
+phase 1: declare all global objects; also collect all function and method declarations
+	- all objects have kind, name, decl fields; the decl field permits
+	  quick lookup of an object's declaration
+	- constant objects have an iota value
+	- type objects have unresolved types with empty scopes, all others have nil types
+	- report global double declarations
+
+phase 2: bind methods to their receiver base types
+	- receiver base types must be declared in the package, thus for
+	  each method a corresponding (unresolved) type must exist
+	- report method double declarations and errors with base types
+
+phase 3: resolve all global objects
+	- sequentially iterate through all objects in the global scope
+	- resolve types for all unresolved types and assign types to
+	  all attached methods
+	- assign types to all other objects, possibly by evaluating
+	  constant and initializer expressions
+	- resolution may recurse; a cyclemap is used to detect cycles
+	- report global typing errors
+
+phase 4: sequentially typecheck function and method bodies
+	- all global objects are declared and have types and values;
+	  all methods have types
+	- sequentially process statements in each body; any object
+	  referred to must be fully defined at this point
+	- report local typing errors
+*/
+
+func (tc *typechecker) checkPackage(pkg *ast.Package) {
+	// setup package scope
+	tc.topScope = Universe
+	tc.openScope()
+	defer tc.closeScope()
+
+	// TODO(gri) there's no file scope at the moment since we ignore imports
+
+	// phase 1: declare all global objects; also collect all function and method declarations
+	var funcs []*ast.FuncDecl
+	for _, file := range pkg.Files {
+		for _, decl := range file.Decls {
+			tc.declGlobal(decl)
+			if f, isFunc := decl.(*ast.FuncDecl); isFunc {
+				funcs = append(funcs, f)
+			}
+		}
+	}
+
+	// phase 2: bind methods to their receiver base types
+	for _, m := range funcs {
+		if m.Recv != nil {
+			tc.bindMethod(m)
+		}
+	}
+
+	// phase 3: resolve all global objects
+	tc.cyclemap = make(map[*ast.Object]bool)
+	for _, obj := range tc.globals {
+		tc.resolve(obj)
+	}
+	assert(len(tc.cyclemap) == 0)
+
+	// 4: sequentially typecheck function and method bodies
+	for _, f := range funcs {
+		ftype, _ := f.Name.Obj.Type.(*Type)
+		tc.checkBlock(f.Body.List, ftype)
+	}
+
+	pkg.Scope = tc.topScope
+}
+
+func (tc *typechecker) declGlobal(global ast.Decl) {
+	switch d := global.(type) {
+	case *ast.BadDecl:
+		// ignore
+
+	case *ast.GenDecl:
+		iota := 0
+		var prev *ast.ValueSpec
+		for _, spec := range d.Specs {
+			switch s := spec.(type) {
+			case *ast.ImportSpec:
+				// TODO(gri) imports go into file scope
+			case *ast.ValueSpec:
+				switch d.Tok {
+				case token.CONST:
+					if s.Values == nil {
+						// create a new spec with type and values from the previous one
+						if prev != nil {
+							s = &ast.ValueSpec{s.Doc, s.Names, prev.Type, prev.Values, s.Comment}
+						} else {
+							// TODO(gri) this should probably go into the const decl code
+							tc.Errorf(s.Pos(), "missing initializer for const %s", s.Names[0].Name)
+						}
+					}
+					for _, name := range s.Names {
+						tc.globals = append(tc.globals, tc.decl(ast.Con, name, s, iota))
+					}
+				case token.VAR:
+					for _, name := range s.Names {
+						tc.globals = append(tc.globals, tc.decl(ast.Var, name, s, 0))
+					}
+				default:
+					panic("unreachable")
+				}
+				prev = s
+				iota++
+			case *ast.TypeSpec:
+				obj := tc.decl(ast.Typ, s.Name, s, 0)
+				tc.globals = append(tc.globals, obj)
+				// give all type objects an unresolved type so
+				// that we can collect methods in the type scope
+				typ := NewType(Unresolved)
+				obj.Type = typ
+				typ.Obj = obj
+			default:
+				panic("unreachable")
+			}
+		}
+
+	case *ast.FuncDecl:
+		if d.Recv == nil {
+			tc.globals = append(tc.globals, tc.decl(ast.Fun, d.Name, d, 0))
+		}
+
+	default:
+		panic("unreachable")
+	}
+}
+
+// If x is of the form *T, deref returns T, otherwise it returns x.
+func deref(x ast.Expr) ast.Expr {
+	if p, isPtr := x.(*ast.StarExpr); isPtr {
+		x = p.X
+	}
+	return x
+}
+
+func (tc *typechecker) bindMethod(method *ast.FuncDecl) {
+	// a method is declared in the receiver base type's scope
+	var scope *ast.Scope
+	base := deref(method.Recv.List[0].Type)
+	if name, isIdent := base.(*ast.Ident); isIdent {
+		// if base is not an *ast.Ident, we had a syntax
+		// error and the parser reported an error already
+		obj := tc.topScope.Lookup(name.Name)
+		if obj == nil {
+			tc.Errorf(name.Pos(), "invalid receiver: %s is not declared in this package", name.Name)
+		} else if obj.Kind != ast.Typ {
+			tc.Errorf(name.Pos(), "invalid receiver: %s is not a type", name.Name)
+		} else {
+			typ := obj.Type.(*Type)
+			assert(typ.Form == Unresolved)
+			scope = typ.Scope
+		}
+	}
+	if scope == nil {
+		// no receiver type found; use a dummy scope
+		// (we still want to type-check the method
+		// body, so make sure there is a name object
+		// and type)
+		// TODO(gri) should we record the scope so
+		// that we don't lose the receiver for type-
+		// checking of the method body?
+		scope = ast.NewScope(nil)
+	}
+	tc.declInScope(scope, ast.Fun, method.Name, method, 0)
+}
+
+func (tc *typechecker) resolve(obj *ast.Object) {
+	// check for declaration cycles
+	if tc.cyclemap[obj] {
+		tc.Errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name)
+		obj.Kind = ast.Bad
+		return
+	}
+	tc.cyclemap[obj] = true
+	defer func() {
+		tc.cyclemap[obj] = false, false
+	}()
+
+	// resolve non-type objects
+	typ, _ := obj.Type.(*Type)
+	if typ == nil {
+		switch obj.Kind {
+		case ast.Bad:
+			// ignore
+
+		case ast.Con:
+			tc.declConst(obj)
+
+		case ast.Var:
+			tc.declVar(obj)
+			obj.Type = tc.typeFor(nil, obj.Decl.(*ast.ValueSpec).Type, false)
+
+		case ast.Fun:
+			obj.Type = NewType(Function)
+			t := obj.Decl.(*ast.FuncDecl).Type
+			tc.declSignature(obj.Type.(*Type), nil, t.Params, t.Results)
+
+		default:
+			// type objects have non-nil types when resolve is called
+			if debug {
+				fmt.Printf("kind = %s\n", obj.Kind)
+			}
+			panic("unreachable")
+		}
+		return
+	}
+
+	// resolve type objects
+	if typ.Form == Unresolved {
+		tc.typeFor(typ, typ.Obj.Decl.(*ast.TypeSpec).Type, false)
+
+		// provide types for all methods
+		for _, obj := range typ.Scope.Objects {
+			if obj.Kind == ast.Fun {
+				assert(obj.Type == nil)
+				obj.Type = NewType(Method)
+				f := obj.Decl.(*ast.FuncDecl)
+				t := f.Type
+				tc.declSignature(obj.Type.(*Type), f.Recv, t.Params, t.Results)
+			}
+		}
+	}
+}
+
+func (tc *typechecker) checkBlock(body []ast.Stmt, ftype *Type) {
+	tc.openScope()
+	defer tc.closeScope()
+
+	// inject function/method parameters into block scope, if any
+	if ftype != nil {
+		for _, par := range ftype.Params.Objects {
+			if par.Name != "_" {
+				alt := tc.topScope.Insert(par)
+				assert(alt == nil) // ftype has no double declarations
+			}
+		}
+	}
+
+	for _, stmt := range body {
+		tc.checkStmt(stmt)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+// unparen removes parentheses around x, if any.
+func unparen(x ast.Expr) ast.Expr {
+	if ux, hasParens := x.(*ast.ParenExpr); hasParens {
+		return unparen(ux.X)
+	}
+	return x
+}
+
+func (tc *typechecker) declFields(scope *ast.Scope, fields *ast.FieldList, ref bool) (n uint) {
+	if fields != nil {
+		for _, f := range fields.List {
+			typ := tc.typeFor(nil, f.Type, ref)
+			for _, name := range f.Names {
+				fld := tc.declInScope(scope, ast.Var, name, f, 0)
+				fld.Type = typ
+				n++
+			}
+		}
+	}
+	return n
+}
+
+func (tc *typechecker) declSignature(typ *Type, recv, params, results *ast.FieldList) {
+	assert((typ.Form == Method) == (recv != nil))
+	typ.Params = ast.NewScope(nil)
+	tc.declFields(typ.Params, recv, true)
+	tc.declFields(typ.Params, params, true)
+	typ.N = tc.declFields(typ.Params, results, true)
+}
+
+func (tc *typechecker) typeFor(def *Type, x ast.Expr, ref bool) (typ *Type) {
+	x = unparen(x)
+
+	// type name
+	if t, isIdent := x.(*ast.Ident); isIdent {
+		obj := tc.find(t)
+
+		if obj.Kind != ast.Typ {
+			tc.Errorf(t.Pos(), "%s is not a type", t.Name)
+			if def == nil {
+				typ = NewType(BadType)
+			} else {
+				typ = def
+				typ.Form = BadType
+			}
+			typ.Expr = x
+			return
+		}
+
+		if !ref {
+			tc.resolve(obj) // check for cycles even if type resolved
+		}
+		typ = obj.Type.(*Type)
+
+		if def != nil {
+			// new type declaration: copy type structure
+			def.Form = typ.Form
+			def.N = typ.N
+			def.Key, def.Elt = typ.Key, typ.Elt
+			def.Params = typ.Params
+			def.Expr = x
+			typ = def
+		}
+		return
+	}
+
+	// type literal
+	typ = def
+	if typ == nil {
+		typ = NewType(BadType)
+	}
+	typ.Expr = x
+
+	switch t := x.(type) {
+	case *ast.SelectorExpr:
+		if debug {
+			fmt.Println("qualified identifier unimplemented")
+		}
+		typ.Form = BadType
+
+	case *ast.StarExpr:
+		typ.Form = Pointer
+		typ.Elt = tc.typeFor(nil, t.X, true)
+
+	case *ast.ArrayType:
+		if t.Len != nil {
+			typ.Form = Array
+			// TODO(gri) compute the real length
+			// (this may call resolve recursively)
+			(*typ).N = 42
+		} else {
+			typ.Form = Slice
+		}
+		typ.Elt = tc.typeFor(nil, t.Elt, t.Len == nil)
+
+	case *ast.StructType:
+		typ.Form = Struct
+		tc.declFields(typ.Scope, t.Fields, false)
+
+	case *ast.FuncType:
+		typ.Form = Function
+		tc.declSignature(typ, nil, t.Params, t.Results)
+
+	case *ast.InterfaceType:
+		typ.Form = Interface
+		tc.declFields(typ.Scope, t.Methods, true)
+
+	case *ast.MapType:
+		typ.Form = Map
+		typ.Key = tc.typeFor(nil, t.Key, true)
+		typ.Elt = tc.typeFor(nil, t.Value, true)
+
+	case *ast.ChanType:
+		typ.Form = Channel
+		typ.N = uint(t.Dir)
+		typ.Elt = tc.typeFor(nil, t.Value, true)
+
+	default:
+		if debug {
+			fmt.Printf("x is %T\n", x)
+		}
+		panic("unreachable")
+	}
+
+	return
+}
+
+// ----------------------------------------------------------------------------
+// TODO(gri) implement these place holders
+
+func (tc *typechecker) declConst(*ast.Object) {
+}
+
+func (tc *typechecker) declVar(*ast.Object) {
+}
+
+func (tc *typechecker) checkStmt(ast.Stmt) {
+}
diff --git a/src/pkg/go/typechecker/typechecker_test.go b/src/pkg/go/typechecker/typechecker_test.go
new file mode 100644
index 000000000..4bad4499a
--- /dev/null
+++ b/src/pkg/go/typechecker/typechecker_test.go
@@ -0,0 +1,163 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a simple typechecker test harness. Packages found
+// in the testDir directory are typechecked. Error messages reported by
+// the typechecker are compared against the error messages expected for
+// the test files.
+//
+// Expected errors are indicated in the test files by putting a comment
+// of the form /* ERROR "rx" */ immediately following an offending token.
+// The harness will verify that an error matching the regular expression
+// rx is reported at that source position. Consecutive comments may be
+// used to indicate multiple errors for the same token position.
+//
+// For instance, the following test file indicates that a "not declared"
+// error should be reported for the undeclared variable x:
+//
+//	package P0
+//	func f() {
+//		_ = x /* ERROR "not declared" */ + 1
+//	}
+// 
+// If the -pkg flag is set, only packages with package names matching
+// the regular expression provided via the flag value are tested.
+
+package typechecker
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/scanner"
+	"go/token"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const testDir = "./testdata" // location of test packages
+
+var fset = token.NewFileSet()
+
+var (
+	pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name")
+	trace  = flag.Bool("trace", false, "print package names")
+)
+
+// ERROR comments must be of the form /* ERROR "rx" */ and rx is
+// a regular expression that matches the expected error message.
+var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`)
+
+// expectedErrors collects the regular expressions of ERROR comments
+// found in the package files of pkg and returns them in sorted order
+// (by filename and position).
+func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) {
+	// scan all package files
+	for filename := range pkg.Files {
+		src, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Fatalf("expectedErrors(%s): %v", pkg.Name, err)
+		}
+
+		var s scanner.Scanner
+		file := fset.AddFile(filename, fset.Base(), len(src))
+		s.Init(file, src, nil, scanner.ScanComments)
+		var prev token.Pos // position of last non-comment token
+	loop:
+		for {
+			pos, tok, lit := s.Scan()
+			switch tok {
+			case token.EOF:
+				break loop
+			case token.COMMENT:
+				s := errRx.FindStringSubmatch(lit)
+				if len(s) == 2 {
+					list = append(list, &scanner.Error{fset.Position(prev), string(s[1])})
+				}
+			default:
+				prev = pos
+			}
+		}
+	}
+	sort.Sort(list) // multiple files may not be sorted
+	return
+}
+
+func testFilter(f *os.FileInfo) bool {
+	return strings.HasSuffix(f.Name, ".src") && f.Name[0] != '.'
+}
+
+func checkError(t *testing.T, expected, found *scanner.Error) {
+	rx, err := regexp.Compile(expected.Msg)
+	if err != nil {
+		t.Errorf("%s: %v", expected.Pos, err)
+		return
+	}
+
+	match := rx.MatchString(found.Msg)
+
+	if expected.Pos.Offset != found.Pos.Offset {
+		if match {
+			t.Errorf("%s: expected error should have been at %s", expected.Pos, found.Pos)
+		} else {
+			t.Errorf("%s: error matching %q expected", expected.Pos, expected.Msg)
+			return
+		}
+	}
+
+	if !match {
+		t.Errorf("%s: %q does not match %q", expected.Pos, expected.Msg, found.Msg)
+	}
+}
+
+func TestTypeCheck(t *testing.T) {
+	flag.Parse()
+	pkgRx, err := regexp.Compile(*pkgPat)
+	if err != nil {
+		t.Fatalf("illegal flag value %q: %s", *pkgPat, err)
+	}
+
+	pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0)
+	if err != nil {
+		scanner.PrintError(os.Stderr, err)
+		t.Fatalf("packages in %s contain syntax errors", testDir)
+	}
+
+	for _, pkg := range pkgs {
+		if !pkgRx.MatchString(pkg.Name) {
+			continue // only test selected packages
+		}
+
+		if *trace {
+			fmt.Println(pkg.Name)
+		}
+
+		xlist := expectedErrors(t, pkg)
+		err := CheckPackage(fset, pkg, nil)
+		if err != nil {
+			if elist, ok := err.(scanner.ErrorList); ok {
+				// verify that errors match
+				for i := 0; i < len(xlist) && i < len(elist); i++ {
+					checkError(t, xlist[i], elist[i])
+				}
+				// the correct number or errors must have been found
+				if len(xlist) != len(elist) {
+					fmt.Fprintf(os.Stderr, "%s\n", pkg.Name)
+					scanner.PrintError(os.Stderr, elist)
+					fmt.Fprintln(os.Stderr)
+					t.Errorf("TypeCheck(%s): %d errors expected but %d reported", pkg.Name, len(xlist), len(elist))
+				}
+			} else {
+				t.Errorf("TypeCheck(%s): %v", pkg.Name, err)
+			}
+		} else if len(xlist) > 0 {
+			t.Errorf("TypeCheck(%s): %d errors expected but 0 reported", pkg.Name, len(xlist))
+		}
+	}
+}
diff --git a/src/pkg/go/typechecker/universe.go b/src/pkg/go/typechecker/universe.go
new file mode 100644
index 000000000..81c14a05e
--- /dev/null
+++ b/src/pkg/go/typechecker/universe.go
@@ -0,0 +1,36 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package typechecker
+
+import "go/ast"
+
+// TODO(gri) should this be in package ast?
+
+// The Universe scope contains all predeclared identifiers.
+var Universe *ast.Scope
+
+func def(obj *ast.Object) {
+	alt := Universe.Insert(obj)
+	if alt != nil {
+		panic("object declared twice")
+	}
+}
+
+func init() {
+	Universe = ast.NewScope(nil)
+
+	// basic types
+	for n, name := range BasicTypes {
+		typ := NewType(Basic)
+		typ.N = n
+		obj := ast.NewObj(ast.Typ, name)
+		obj.Type = typ
+		typ.Obj = obj
+		def(obj)
+	}
+
+	// built-in functions
+	// TODO(gri) implement this
+}
diff --git a/src/pkg/go/types/Makefile b/src/pkg/go/types/Makefile
new file mode 100644
index 000000000..4ca707c73
--- /dev/null
+++ b/src/pkg/go/types/Makefile
@@ -0,0 +1,16 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/types
+GOFILES=\
+	check.go\
+	const.go\
+	exportdata.go\
+	gcimporter.go\
+	types.go\
+	universe.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/types/check.go b/src/pkg/go/types/check.go
new file mode 100644
index 000000000..87e3e93da
--- /dev/null
+++ b/src/pkg/go/types/check.go
@@ -0,0 +1,226 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements the Check function, which typechecks a package.
+
+package types
+
+import (
+	"fmt"
+	"go/ast"
+	"go/scanner"
+	"go/token"
+	"os"
+	"strconv"
+)
+
+const debug = false
+
+type checker struct {
+	fset *token.FileSet
+	scanner.ErrorVector
+	types map[ast.Expr]Type
+}
+
+func (c *checker) errorf(pos token.Pos, format string, args ...interface{}) string {
+	msg := fmt.Sprintf(format, args...)
+	c.Error(c.fset.Position(pos), msg)
+	return msg
+}
+
+// collectFields collects struct fields tok = token.STRUCT), interface methods
+// (tok = token.INTERFACE), and function arguments/results (tok = token.FUNC).
+func (c *checker) collectFields(tok token.Token, list *ast.FieldList, cycleOk bool) (fields ObjList, tags []string, isVariadic bool) {
+	if list != nil {
+		for _, field := range list.List {
+			ftype := field.Type
+			if t, ok := ftype.(*ast.Ellipsis); ok {
+				ftype = t.Elt
+				isVariadic = true
+			}
+			typ := c.makeType(ftype, cycleOk)
+			tag := ""
+			if field.Tag != nil {
+				assert(field.Tag.Kind == token.STRING)
+				tag, _ = strconv.Unquote(field.Tag.Value)
+			}
+			if len(field.Names) > 0 {
+				// named fields
+				for _, name := range field.Names {
+					obj := name.Obj
+					obj.Type = typ
+					fields = append(fields, obj)
+					if tok == token.STRUCT {
+						tags = append(tags, tag)
+					}
+				}
+			} else {
+				// anonymous field
+				switch tok {
+				case token.STRUCT:
+					tags = append(tags, tag)
+					fallthrough
+				case token.FUNC:
+					obj := ast.NewObj(ast.Var, "")
+					obj.Type = typ
+					fields = append(fields, obj)
+				case token.INTERFACE:
+					utyp := Underlying(typ)
+					if typ, ok := utyp.(*Interface); ok {
+						// TODO(gri) This is not good enough. Check for double declarations!
+						fields = append(fields, typ.Methods...)
+					} else if _, ok := utyp.(*Bad); !ok {
+						// if utyp is Bad, don't complain (the root cause was reported before)
+						c.errorf(ftype.Pos(), "interface contains embedded non-interface type")
+					}
+				default:
+					panic("unreachable")
+				}
+			}
+		}
+	}
+	return
+}
+
+// makeType makes a new type for an AST type specification x or returns
+// the type referred to by a type name x. If cycleOk is set, a type may
+// refer to itself directly or indirectly; otherwise cycles are errors.
+//
+func (c *checker) makeType(x ast.Expr, cycleOk bool) (typ Type) {
+	if debug {
+		fmt.Printf("makeType (cycleOk = %v)\n", cycleOk)
+		ast.Print(c.fset, x)
+		defer func() {
+			fmt.Printf("-> %T %v\n\n", typ, typ)
+		}()
+	}
+
+	switch t := x.(type) {
+	case *ast.BadExpr:
+		return &Bad{}
+
+	case *ast.Ident:
+		// type name
+		obj := t.Obj
+		if obj == nil {
+			// unresolved identifier (error has been reported before)
+			return &Bad{Msg: "unresolved identifier"}
+		}
+		if obj.Kind != ast.Typ {
+			msg := c.errorf(t.Pos(), "%s is not a type", t.Name)
+			return &Bad{Msg: msg}
+		}
+		c.checkObj(obj, cycleOk)
+		if !cycleOk && obj.Type.(*Name).Underlying == nil {
+			// TODO(gri) Enable this message again once its position
+			// is independent of the underlying map implementation.
+			// msg := c.errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name)
+			msg := "illegal cycle"
+			return &Bad{Msg: msg}
+		}
+		return obj.Type.(Type)
+
+	case *ast.ParenExpr:
+		return c.makeType(t.X, cycleOk)
+
+	case *ast.SelectorExpr:
+		// qualified identifier
+		// TODO (gri) eventually, this code belongs to expression
+		//            type checking - here for the time being
+		if ident, ok := t.X.(*ast.Ident); ok {
+			if obj := ident.Obj; obj != nil {
+				if obj.Kind != ast.Pkg {
+					msg := c.errorf(ident.Pos(), "%s is not a package", obj.Name)
+					return &Bad{Msg: msg}
+				}
+				// TODO(gri) we have a package name but don't
+				// have the mapping from package name to package
+				// scope anymore (created in ast.NewPackage).
+				return &Bad{} // for now
+			}
+		}
+		// TODO(gri) can this really happen (the parser should have excluded this)?
+		msg := c.errorf(t.Pos(), "expected qualified identifier")
+		return &Bad{Msg: msg}
+
+	case *ast.StarExpr:
+		return &Pointer{Base: c.makeType(t.X, true)}
+
+	case *ast.ArrayType:
+		if t.Len != nil {
+			// TODO(gri) compute length
+			return &Array{Elt: c.makeType(t.Elt, cycleOk)}
+		}
+		return &Slice{Elt: c.makeType(t.Elt, true)}
+
+	case *ast.StructType:
+		fields, tags, _ := c.collectFields(token.STRUCT, t.Fields, cycleOk)
+		return &Struct{Fields: fields, Tags: tags}
+
+	case *ast.FuncType:
+		params, _, _ := c.collectFields(token.FUNC, t.Params, true)
+		results, _, isVariadic := c.collectFields(token.FUNC, t.Results, true)
+		return &Func{Recv: nil, Params: params, Results: results, IsVariadic: isVariadic}
+
+	case *ast.InterfaceType:
+		methods, _, _ := c.collectFields(token.INTERFACE, t.Methods, cycleOk)
+		methods.Sort()
+		return &Interface{Methods: methods}
+
+	case *ast.MapType:
+		return &Map{Key: c.makeType(t.Key, true), Elt: c.makeType(t.Key, true)}
+
+	case *ast.ChanType:
+		return &Chan{Dir: t.Dir, Elt: c.makeType(t.Value, true)}
+	}
+
+	panic(fmt.Sprintf("unreachable (%T)", x))
+}
+
+// checkObj type checks an object.
+func (c *checker) checkObj(obj *ast.Object, ref bool) {
+	if obj.Type != nil {
+		// object has already been type checked
+		return
+	}
+
+	switch obj.Kind {
+	case ast.Bad:
+		// ignore
+
+	case ast.Con:
+		// TODO(gri) complete this
+
+	case ast.Typ:
+		typ := &Name{Obj: obj}
+		obj.Type = typ // "mark" object so recursion terminates
+		typ.Underlying = Underlying(c.makeType(obj.Decl.(*ast.TypeSpec).Type, ref))
+
+	case ast.Var:
+		// TODO(gri) complete this
+
+	case ast.Fun:
+		// TODO(gri) complete this
+
+	default:
+		panic("unreachable")
+	}
+}
+
+// Check typechecks a package.
+// It augments the AST by assigning types to all ast.Objects and returns a map
+// of types for all expression nodes in statements, and a scanner.ErrorList if
+// there are errors.
+//
+func Check(fset *token.FileSet, pkg *ast.Package) (types map[ast.Expr]Type, err os.Error) {
+	var c checker
+	c.fset = fset
+	c.types = make(map[ast.Expr]Type)
+
+	for _, obj := range pkg.Scope.Objects {
+		c.checkObj(obj, false)
+	}
+
+	return c.types, c.GetError(scanner.NoMultiples)
+}
diff --git a/src/pkg/go/types/check_test.go b/src/pkg/go/types/check_test.go
new file mode 100644
index 000000000..8be653fcb
--- /dev/null
+++ b/src/pkg/go/types/check_test.go
@@ -0,0 +1,215 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements a typechecker test harness. The packages specified
+// in tests are typechecked. Error messages reported by the typechecker are
+// compared against the error messages expected in the test files.
+//
+// Expected errors are indicated in the test files by putting a comment
+// of the form /* ERROR "rx" */ immediately following an offending token.
+// The harness will verify that an error matching the regular expression
+// rx is reported at that source position. Consecutive comments may be
+// used to indicate multiple errors for the same token position.
+//
+// For instance, the following test file indicates that a "not declared"
+// error should be reported for the undeclared variable x:
+//
+//	package p
+//	func f() {
+//		_ = x /* ERROR "not declared" */ + 1
+//	}
+
+package types
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/scanner"
+	"go/token"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"testing"
+)
+
+// The test filenames do not end in .go so that they are invisible
+// to gofmt since they contain comments that must not change their
+// positions relative to surrounding tokens.
+
+var tests = []struct {
+	name  string
+	files []string
+}{
+	{"test0", []string{"testdata/test0.src"}},
+}
+
+var fset = token.NewFileSet()
+
+// TODO(gri) This functionality should be in token.Fileset.
+func getFile(filename string) *token.File {
+	for f := range fset.Files() {
+		if f.Name() == filename {
+			return f
+		}
+	}
+	return nil
+}
+
+// TODO(gri) This functionality should be in token.Fileset.
+func getPos(filename string, offset int) token.Pos {
+	if f := getFile(filename); f != nil {
+		return f.Pos(offset)
+	}
+	return token.NoPos
+}
+
+// TODO(gri) Need to revisit parser interface. We should be able to use parser.ParseFiles
+//           or a similar function instead.
+func parseFiles(t *testing.T, testname string, filenames []string) (map[string]*ast.File, os.Error) {
+	files := make(map[string]*ast.File)
+	var errors scanner.ErrorList
+	for _, filename := range filenames {
+		if _, exists := files[filename]; exists {
+			t.Fatalf("%s: duplicate file %s", testname, filename)
+		}
+		file, err := parser.ParseFile(fset, filename, nil, parser.DeclarationErrors)
+		if file == nil {
+			t.Fatalf("%s: could not parse file %s", testname, filename)
+		}
+		files[filename] = file
+		if err != nil {
+			// if the parser returns a non-scanner.ErrorList error
+			// the file couldn't be read in the first place and
+			// file == nil; in that case we shouldn't reach here
+			errors = append(errors, err.(scanner.ErrorList)...)
+		}
+
+	}
+	return files, errors
+}
+
+// ERROR comments must be of the form /* ERROR "rx" */ and rx is
+// a regular expression that matches the expected error message.
+//
+var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`)
+
+// expectedErrors collects the regular expressions of ERROR comments found
+// in files and returns them as a map of error positions to error messages.
+//
+func expectedErrors(t *testing.T, testname string, files map[string]*ast.File) map[token.Pos]string {
+	errors := make(map[token.Pos]string)
+	for filename := range files {
+		src, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Fatalf("%s: could not read %s", testname, filename)
+		}
+
+		var s scanner.Scanner
+		// file was parsed already - do not add it again to the file
+		// set otherwise the position information returned here will
+		// not match the position information collected by the parser
+		s.Init(getFile(filename), src, nil, scanner.ScanComments)
+		var prev token.Pos // position of last non-comment token
+
+	scanFile:
+		for {
+			pos, tok, lit := s.Scan()
+			switch tok {
+			case token.EOF:
+				break scanFile
+			case token.COMMENT:
+				s := errRx.FindStringSubmatch(lit)
+				if len(s) == 2 {
+					errors[prev] = string(s[1])
+				}
+			default:
+				prev = pos
+			}
+		}
+	}
+	return errors
+}
+
+func eliminate(t *testing.T, expected map[token.Pos]string, errors os.Error) {
+	if errors == nil {
+		return
+	}
+	for _, error := range errors.(scanner.ErrorList) {
+		// error.Pos is a token.Position, but we want
+		// a token.Pos so we can do a map lookup
+		// TODO(gri) Need to move scanner.Errors over
+		//           to use token.Pos and file set info.
+		pos := getPos(error.Pos.Filename, error.Pos.Offset)
+		if msg, found := expected[pos]; found {
+			// we expect a message at pos; check if it matches
+			rx, err := regexp.Compile(msg)
+			if err != nil {
+				t.Errorf("%s: %v", error.Pos, err)
+				continue
+			}
+			if match := rx.MatchString(error.Msg); !match {
+				t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg)
+				continue
+			}
+			// we have a match - eliminate this error
+			expected[pos] = "", false
+		} else {
+			// To keep in mind when analyzing failed test output:
+			// If the same error position occurs multiple times in errors,
+			// this message will be triggered (because the first error at
+			// the position removes this position from the expected errors).
+			t.Errorf("%s: no (multiple?) error expected, but found: %s", error.Pos, error.Msg)
+		}
+	}
+}
+
+func check(t *testing.T, testname string, testfiles []string) {
+	// TODO(gri) Eventually all these different phases should be
+	//           subsumed into a single function call that takes
+	//           a set of files and creates a fully resolved and
+	//           type-checked AST.
+
+	files, err := parseFiles(t, testname, testfiles)
+
+	// we are expecting the following errors
+	// (collect these after parsing the files so that
+	// they are found in the file set)
+	errors := expectedErrors(t, testname, files)
+
+	// verify errors returned by the parser
+	eliminate(t, errors, err)
+
+	// verify errors returned after resolving identifiers
+	pkg, err := ast.NewPackage(fset, files, GcImporter, Universe)
+	eliminate(t, errors, err)
+
+	// verify errors returned by the typechecker
+	_, err = Check(fset, pkg)
+	eliminate(t, errors, err)
+
+	// there should be no expected errors left
+	if len(errors) > 0 {
+		t.Errorf("%s: %d errors not reported:", testname, len(errors))
+		for pos, msg := range errors {
+			t.Errorf("%s: %s\n", fset.Position(pos), msg)
+		}
+	}
+}
+
+func TestCheck(t *testing.T) {
+	// For easy debugging w/o changing the testing code,
+	// if there is a local test file, only test that file.
+	const testfile = "test.go"
+	if fi, err := os.Stat(testfile); err == nil && fi.IsRegular() {
+		fmt.Printf("WARNING: Testing only %s (remove it to run all tests)\n", testfile)
+		check(t, testfile, []string{testfile})
+		return
+	}
+
+	// Otherwise, run all the tests.
+	for _, test := range tests {
+		check(t, test.name, test.files)
+	}
+}
diff --git a/src/pkg/go/types/const.go b/src/pkg/go/types/const.go
new file mode 100644
index 000000000..1ef95d9f9
--- /dev/null
+++ b/src/pkg/go/types/const.go
@@ -0,0 +1,332 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements operations on ideal constants.
+
+package types
+
+import (
+	"big"
+	"go/token"
+	"strconv"
+)
+
+// TODO(gri) Consider changing the API so Const is an interface
+//           and operations on consts don't have to type switch.
+
+// A Const implements an ideal constant Value.
+// The zero value z for a Const is not a valid constant value.
+type Const struct {
+	// representation of constant values:
+	// ideal bool     ->  bool
+	// ideal int      ->  *big.Int
+	// ideal float    ->  *big.Rat
+	// ideal complex  ->  cmplx
+	// ideal string   ->  string
+	val interface{}
+}
+
+// Representation of complex values.
+type cmplx struct {
+	re, im *big.Rat
+}
+
+func assert(cond bool) {
+	if !cond {
+		panic("go/types internal error: assertion failed")
+	}
+}
+
+// MakeConst makes an ideal constant from a literal
+// token and the corresponding literal string.
+func MakeConst(tok token.Token, lit string) Const {
+	switch tok {
+	case token.INT:
+		var x big.Int
+		_, ok := x.SetString(lit, 0)
+		assert(ok)
+		return Const{&x}
+	case token.FLOAT:
+		var y big.Rat
+		_, ok := y.SetString(lit)
+		assert(ok)
+		return Const{&y}
+	case token.IMAG:
+		assert(lit[len(lit)-1] == 'i')
+		var im big.Rat
+		_, ok := im.SetString(lit[0 : len(lit)-1])
+		assert(ok)
+		return Const{cmplx{big.NewRat(0, 1), &im}}
+	case token.CHAR:
+		assert(lit[0] == '\'' && lit[len(lit)-1] == '\'')
+		code, _, _, err := strconv.UnquoteChar(lit[1:len(lit)-1], '\'')
+		assert(err == nil)
+		return Const{big.NewInt(int64(code))}
+	case token.STRING:
+		s, err := strconv.Unquote(lit)
+		assert(err == nil)
+		return Const{s}
+	}
+	panic("unreachable")
+}
+
+// MakeZero returns the zero constant for the given type.
+func MakeZero(typ *Type) Const {
+	// TODO(gri) fix this
+	return Const{0}
+}
+
+// Match attempts to match the internal constant representations of x and y.
+// If the attempt is successful, the result is the values of x and y,
+// if necessary converted to have the same internal representation; otherwise
+// the results are invalid.
+func (x Const) Match(y Const) (u, v Const) {
+	switch a := x.val.(type) {
+	case bool:
+		if _, ok := y.val.(bool); ok {
+			u, v = x, y
+		}
+	case *big.Int:
+		switch y.val.(type) {
+		case *big.Int:
+			u, v = x, y
+		case *big.Rat:
+			var z big.Rat
+			z.SetInt(a)
+			u, v = Const{&z}, y
+		case cmplx:
+			var z big.Rat
+			z.SetInt(a)
+			u, v = Const{cmplx{&z, big.NewRat(0, 1)}}, y
+		}
+	case *big.Rat:
+		switch y.val.(type) {
+		case *big.Int:
+			v, u = y.Match(x)
+		case *big.Rat:
+			u, v = x, y
+		case cmplx:
+			u, v = Const{cmplx{a, big.NewRat(0, 0)}}, y
+		}
+	case cmplx:
+		switch y.val.(type) {
+		case *big.Int, *big.Rat:
+			v, u = y.Match(x)
+		case cmplx:
+			u, v = x, y
+		}
+	case string:
+		if _, ok := y.val.(string); ok {
+			u, v = x, y
+		}
+	default:
+		panic("unreachable")
+	}
+	return
+}
+
+// Convert attempts to convert the constant x to a given type.
+// If the attempt is successful, the result is the new constant;
+// otherwise the result is invalid.
+func (x Const) Convert(typ *Type) Const {
+	// TODO(gri) implement this
+	switch x := x.val.(type) {
+	case bool:
+	case *big.Int:
+	case *big.Rat:
+	case cmplx:
+	case string:
+	}
+	return x
+}
+
+func (x Const) String() string {
+	switch x := x.val.(type) {
+	case bool:
+		if x {
+			return "true"
+		}
+		return "false"
+	case *big.Int:
+		return x.String()
+	case *big.Rat:
+		return x.FloatString(10) // 10 digits of precision after decimal point seems fine
+	case cmplx:
+		// TODO(gri) don't print 0 components
+		return x.re.FloatString(10) + " + " + x.im.FloatString(10) + "i"
+	case string:
+		return x
+	}
+	panic("unreachable")
+}
+
+func (x Const) UnaryOp(op token.Token) Const {
+	panic("unimplemented")
+}
+
+func (x Const) BinaryOp(op token.Token, y Const) Const {
+	var z interface{}
+	switch x := x.val.(type) {
+	case bool:
+		z = binaryBoolOp(x, op, y.val.(bool))
+	case *big.Int:
+		z = binaryIntOp(x, op, y.val.(*big.Int))
+	case *big.Rat:
+		z = binaryFloatOp(x, op, y.val.(*big.Rat))
+	case cmplx:
+		z = binaryCmplxOp(x, op, y.val.(cmplx))
+	case string:
+		z = binaryStringOp(x, op, y.val.(string))
+	default:
+		panic("unreachable")
+	}
+	return Const{z}
+}
+
+func binaryBoolOp(x bool, op token.Token, y bool) interface{} {
+	switch op {
+	case token.EQL:
+		return x == y
+	case token.NEQ:
+		return x != y
+	}
+	panic("unreachable")
+}
+
+func binaryIntOp(x *big.Int, op token.Token, y *big.Int) interface{} {
+	var z big.Int
+	switch op {
+	case token.ADD:
+		return z.Add(x, y)
+	case token.SUB:
+		return z.Sub(x, y)
+	case token.MUL:
+		return z.Mul(x, y)
+	case token.QUO:
+		return z.Quo(x, y)
+	case token.REM:
+		return z.Rem(x, y)
+	case token.AND:
+		return z.And(x, y)
+	case token.OR:
+		return z.Or(x, y)
+	case token.XOR:
+		return z.Xor(x, y)
+	case token.AND_NOT:
+		return z.AndNot(x, y)
+	case token.SHL:
+		panic("unimplemented")
+	case token.SHR:
+		panic("unimplemented")
+	case token.EQL:
+		return x.Cmp(y) == 0
+	case token.NEQ:
+		return x.Cmp(y) != 0
+	case token.LSS:
+		return x.Cmp(y) < 0
+	case token.LEQ:
+		return x.Cmp(y) <= 0
+	case token.GTR:
+		return x.Cmp(y) > 0
+	case token.GEQ:
+		return x.Cmp(y) >= 0
+	}
+	panic("unreachable")
+}
+
+func binaryFloatOp(x *big.Rat, op token.Token, y *big.Rat) interface{} {
+	var z big.Rat
+	switch op {
+	case token.ADD:
+		return z.Add(x, y)
+	case token.SUB:
+		return z.Sub(x, y)
+	case token.MUL:
+		return z.Mul(x, y)
+	case token.QUO:
+		return z.Quo(x, y)
+	case token.EQL:
+		return x.Cmp(y) == 0
+	case token.NEQ:
+		return x.Cmp(y) != 0
+	case token.LSS:
+		return x.Cmp(y) < 0
+	case token.LEQ:
+		return x.Cmp(y) <= 0
+	case token.GTR:
+		return x.Cmp(y) > 0
+	case token.GEQ:
+		return x.Cmp(y) >= 0
+	}
+	panic("unreachable")
+}
+
+func binaryCmplxOp(x cmplx, op token.Token, y cmplx) interface{} {
+	a, b := x.re, x.im
+	c, d := y.re, y.im
+	switch op {
+	case token.ADD:
+		// (a+c) + i(b+d)
+		var re, im big.Rat
+		re.Add(a, c)
+		im.Add(b, d)
+		return cmplx{&re, &im}
+	case token.SUB:
+		// (a-c) + i(b-d)
+		var re, im big.Rat
+		re.Sub(a, c)
+		im.Sub(b, d)
+		return cmplx{&re, &im}
+	case token.MUL:
+		// (ac-bd) + i(bc+ad)
+		var ac, bd, bc, ad big.Rat
+		ac.Mul(a, c)
+		bd.Mul(b, d)
+		bc.Mul(b, c)
+		ad.Mul(a, d)
+		var re, im big.Rat
+		re.Sub(&ac, &bd)
+		im.Add(&bc, &ad)
+		return cmplx{&re, &im}
+	case token.QUO:
+		// (ac+bd)/s + i(bc-ad)/s, with s = cc + dd
+		var ac, bd, bc, ad, s big.Rat
+		ac.Mul(a, c)
+		bd.Mul(b, d)
+		bc.Mul(b, c)
+		ad.Mul(a, d)
+		s.Add(c.Mul(c, c), d.Mul(d, d))
+		var re, im big.Rat
+		re.Add(&ac, &bd)
+		re.Quo(&re, &s)
+		im.Sub(&bc, &ad)
+		im.Quo(&im, &s)
+		return cmplx{&re, &im}
+	case token.EQL:
+		return a.Cmp(c) == 0 && b.Cmp(d) == 0
+	case token.NEQ:
+		return a.Cmp(c) != 0 || b.Cmp(d) != 0
+	}
+	panic("unreachable")
+}
+
+func binaryStringOp(x string, op token.Token, y string) interface{} {
+	switch op {
+	case token.ADD:
+		return x + y
+	case token.EQL:
+		return x == y
+	case token.NEQ:
+		return x != y
+	case token.LSS:
+		return x < y
+	case token.LEQ:
+		return x <= y
+	case token.GTR:
+		return x > y
+	case token.GEQ:
+		return x >= y
+	}
+	panic("unreachable")
+}
diff --git a/src/pkg/go/types/exportdata.go b/src/pkg/go/types/exportdata.go
new file mode 100644
index 000000000..383520320
--- /dev/null
+++ b/src/pkg/go/types/exportdata.go
@@ -0,0 +1,132 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements ExportData.
+
+package types
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func readGopackHeader(buf *bufio.Reader) (name string, size int, err os.Error) {
+	// See $GOROOT/include/ar.h.
+	hdr := make([]byte, 64+12+6+6+8+10+2)
+	_, err = io.ReadFull(buf, hdr)
+	if err != nil {
+		return
+	}
+	if trace {
+		fmt.Printf("header: %s", hdr)
+	}
+	s := strings.TrimSpace(string(hdr[64+12+6+6+8:][:10]))
+	size, err = strconv.Atoi(s)
+	if err != nil || hdr[len(hdr)-2] != '`' || hdr[len(hdr)-1] != '\n' {
+		err = os.NewError("invalid archive header")
+		return
+	}
+	name = strings.TrimSpace(string(hdr[:64]))
+	return
+}
+
+type dataReader struct {
+	*bufio.Reader
+	io.Closer
+}
+
+// ExportData returns a readCloser positioned at the beginning of the
+// export data section of the given object/archive file, or an error.
+// It is the caller's responsibility to close the readCloser.
+//
+func ExportData(filename string) (rc io.ReadCloser, err os.Error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		if err != nil {
+			file.Close()
+			// Add file name to error.
+			err = fmt.Errorf("reading export data: %s: %v", filename, err)
+		}
+	}()
+
+	buf := bufio.NewReader(file)
+
+	// Read first line to make sure this is an object file.
+	line, err := buf.ReadSlice('\n')
+	if err != nil {
+		return
+	}
+	if string(line) == "!\n" {
+		// Archive file.  Scan to __.PKGDEF, which should
+		// be second archive entry.
+		var name string
+		var size int
+
+		// First entry should be __.SYMDEF.
+		// Read and discard.
+		if name, size, err = readGopackHeader(buf); err != nil {
+			return
+		}
+		if name != "__.SYMDEF" {
+			err = os.NewError("go archive does not begin with __.SYMDEF")
+			return
+		}
+		const block = 4096
+		tmp := make([]byte, block)
+		for size > 0 {
+			n := size
+			if n > block {
+				n = block
+			}
+			_, err = io.ReadFull(buf, tmp[:n])
+			if err != nil {
+				return
+			}
+			size -= n
+		}
+
+		// Second entry should be __.PKGDEF.
+		if name, size, err = readGopackHeader(buf); err != nil {
+			return
+		}
+		if name != "__.PKGDEF" {
+			err = os.NewError("go archive is missing __.PKGDEF")
+			return
+		}
+
+		// Read first line of __.PKGDEF data, so that line
+		// is once again the first line of the input.
+		line, err = buf.ReadSlice('\n')
+		if err != nil {
+			return
+		}
+	}
+
+	// Now at __.PKGDEF in archive or still at beginning of file.
+	// Either way, line should begin with "go object ".
+	if !strings.HasPrefix(string(line), "go object ") {
+		err = os.NewError("not a go object file")
+		return
+	}
+
+	// Skip over object header to export data.
+	// Begins after first line with $$.
+	for line[0] != '$' {
+		line, err = buf.ReadSlice('\n')
+		if err != nil {
+			return
+		}
+	}
+
+	rc = &dataReader{buf, file}
+	return
+}
diff --git a/src/pkg/go/types/gcimporter.go b/src/pkg/go/types/gcimporter.go
new file mode 100644
index 000000000..6ab1806b6
--- /dev/null
+++ b/src/pkg/go/types/gcimporter.go
@@ -0,0 +1,799 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements an ast.Importer for gc generated object files.
+// TODO(gri) Eventually move this into a separate package outside types.
+
+package types
+
+import (
+	"big"
+	"fmt"
+	"go/ast"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"scanner"
+	"strconv"
+)
+
+const trace = false // set to true for debugging
+
+var (
+	pkgRoot = filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_"+runtime.GOARCH)
+	pkgExts = [...]string{".a", ".5", ".6", ".8"}
+)
+
+// findPkg returns the filename and package id for an import path.
+// If no file was found, an empty filename is returned.
+func findPkg(path string) (filename, id string) {
+	if len(path) == 0 {
+		return
+	}
+
+	id = path
+	var noext string
+	switch path[0] {
+	default:
+		// "x" -> "$GOROOT/pkg/$GOOS_$GOARCH/x.ext", "x"
+		noext = filepath.Join(pkgRoot, path)
+
+	case '.':
+		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
+		cwd, err := os.Getwd()
+		if err != nil {
+			return
+		}
+		noext = filepath.Join(cwd, path)
+		id = noext
+
+	case '/':
+		// "/x" -> "/x.ext", "/x"
+		noext = path
+	}
+
+	// try extensions
+	for _, ext := range pkgExts {
+		filename = noext + ext
+		if f, err := os.Stat(filename); err == nil && f.IsRegular() {
+			return
+		}
+	}
+
+	filename = "" // not found
+	return
+}
+
+// gcParser parses the exports inside a gc compiler-produced
+// object/archive file and populates its scope with the results.
+type gcParser struct {
+	scanner scanner.Scanner
+	tok     int                    // current token
+	lit     string                 // literal string; only valid for Ident, Int, String tokens
+	id      string                 // package id of imported package
+	imports map[string]*ast.Object // package id -> package object
+}
+
+func (p *gcParser) init(filename, id string, src io.Reader, imports map[string]*ast.Object) {
+	p.scanner.Init(src)
+	p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) }
+	p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments
+	p.scanner.Whitespace = 1<<'\t' | 1<<' '
+	p.scanner.Filename = filename // for good error messages
+	p.next()
+	p.id = id
+	p.imports = imports
+}
+
+func (p *gcParser) next() {
+	p.tok = p.scanner.Scan()
+	switch p.tok {
+	case scanner.Ident, scanner.Int, scanner.String:
+		p.lit = p.scanner.TokenText()
+	default:
+		p.lit = ""
+	}
+	if trace {
+		fmt.Printf("%s: %q -> %q\n", scanner.TokenString(p.tok), p.scanner.TokenText(), p.lit)
+	}
+}
+
+// GcImporter implements the ast.Importer signature.
+func GcImporter(imports map[string]*ast.Object, path string) (pkg *ast.Object, err os.Error) {
+	if path == "unsafe" {
+		return Unsafe, nil
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			err = r.(importError) // will re-panic if r is not an importError
+			if trace {
+				panic(err) // force a stack trace
+			}
+		}
+	}()
+
+	filename, id := findPkg(path)
+	if filename == "" {
+		err = os.NewError("can't find import: " + id)
+		return
+	}
+
+	if pkg = imports[id]; pkg != nil {
+		return // package was imported before
+	}
+
+	buf, err := ExportData(filename)
+	if err != nil {
+		return
+	}
+	defer buf.Close()
+
+	if trace {
+		fmt.Printf("importing %s (%s)\n", id, filename)
+	}
+
+	var p gcParser
+	p.init(filename, id, buf, imports)
+	pkg = p.parseExport()
+	return
+}
+
+// ----------------------------------------------------------------------------
+// Error handling
+
+// Internal errors are boxed as importErrors.
+type importError struct {
+	pos scanner.Position
+	err os.Error
+}
+
+func (e importError) String() string {
+	return fmt.Sprintf("import error %s (byte offset = %d): %s", e.pos, e.pos.Offset, e.err)
+}
+
+func (p *gcParser) error(err interface{}) {
+	if s, ok := err.(string); ok {
+		err = os.NewError(s)
+	}
+	// panic with a runtime.Error if err is not an os.Error
+	panic(importError{p.scanner.Pos(), err.(os.Error)})
+}
+
+func (p *gcParser) errorf(format string, args ...interface{}) {
+	p.error(fmt.Sprintf(format, args...))
+}
+
+func (p *gcParser) expect(tok int) string {
+	lit := p.lit
+	if p.tok != tok {
+		p.errorf("expected %q, got %q (%q)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit)
+	}
+	p.next()
+	return lit
+}
+
+func (p *gcParser) expectSpecial(tok string) {
+	sep := 'x' // not white space
+	i := 0
+	for i < len(tok) && p.tok == int(tok[i]) && sep > ' ' {
+		sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token
+		p.next()
+		i++
+	}
+	if i < len(tok) {
+		p.errorf("expected %q, got %q", tok, tok[0:i])
+	}
+}
+
+func (p *gcParser) expectKeyword(keyword string) {
+	lit := p.expect(scanner.Ident)
+	if lit != keyword {
+		p.errorf("expected keyword %s, got %q", keyword, lit)
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Import declarations
+
+// ImportPath = string_lit .
+//
+func (p *gcParser) parsePkgId() *ast.Object {
+	id, err := strconv.Unquote(p.expect(scanner.String))
+	if err != nil {
+		p.error(err)
+	}
+
+	switch id {
+	case "":
+		// id == "" stands for the imported package id
+		// (only known at time of package installation)
+		id = p.id
+	case "unsafe":
+		// package unsafe is not in the imports map - handle explicitly
+		return Unsafe
+	}
+
+	pkg := p.imports[id]
+	if pkg == nil {
+		scope = ast.NewScope(nil)
+		pkg = ast.NewObj(ast.Pkg, "")
+		pkg.Data = scope
+		p.imports[id] = pkg
+	}
+
+	return pkg
+}
+
+// dotIdentifier = ( ident | '·' ) { ident | int | '·' } .
+func (p *gcParser) parseDotIdent() string {
+	ident := ""
+	if p.tok != scanner.Int {
+		sep := 'x' // not white space
+		for (p.tok == scanner.Ident || p.tok == scanner.Int || p.tok == '·') && sep > ' ' {
+			ident += p.lit
+			sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token
+			p.next()
+		}
+	}
+	if ident == "" {
+		p.expect(scanner.Ident) // use expect() for error handling
+	}
+	return ident
+}
+
+// ExportedName = ImportPath "." dotIdentifier .
+//
+func (p *gcParser) parseExportedName(kind ast.ObjKind) *ast.Object {
+	pkg := p.parsePkgId()
+	p.expect('.')
+	name := p.parseDotIdent()
+
+	// a type may have been declared before - if it exists
+	// already in the respective package scope, return that
+	// type
+	scope := pkg.Data.(*ast.Scope)
+	if kind == ast.Typ {
+		if obj := scope.Lookup(name); obj != nil {
+			assert(obj.Kind == ast.Typ)
+			return obj
+		}
+	}
+
+	// any other object must be a newly declared object -
+	// create it and insert it into the package scope
+	obj := ast.NewObj(kind, name)
+	if scope.Insert(obj) != nil {
+		p.errorf("already declared: %s", obj.Name)
+	}
+
+	// a new type object is a named type and may be referred
+	// to before the underlying type is known - set it up
+	if kind == ast.Typ {
+		obj.Type = &Name{Obj: obj}
+	}
+
+	return obj
+}
+
+// ----------------------------------------------------------------------------
+// Types
+
+// BasicType = identifier .
+//
+func (p *gcParser) parseBasicType() Type {
+	obj := Universe.Lookup(p.expect(scanner.Ident))
+	if obj == nil || obj.Kind != ast.Typ {
+		p.errorf("not a basic type: %s", obj.Name)
+	}
+	return obj.Type.(Type)
+}
+
+// ArrayType = "[" int_lit "]" Type .
+//
+func (p *gcParser) parseArrayType() Type {
+	// "[" already consumed and lookahead known not to be "]"
+	lit := p.expect(scanner.Int)
+	p.expect(']')
+	elt := p.parseType()
+	n, err := strconv.Atoui64(lit)
+	if err != nil {
+		p.error(err)
+	}
+	return &Array{Len: n, Elt: elt}
+}
+
+// MapType = "map" "[" Type "]" Type .
+//
+func (p *gcParser) parseMapType() Type {
+	p.expectKeyword("map")
+	p.expect('[')
+	key := p.parseType()
+	p.expect(']')
+	elt := p.parseType()
+	return &Map{Key: key, Elt: elt}
+}
+
+// Name = identifier | "?" .
+//
+func (p *gcParser) parseName() (name string) {
+	switch p.tok {
+	case scanner.Ident:
+		name = p.lit
+		p.next()
+	case '?':
+		// anonymous
+		p.next()
+	default:
+		p.error("name expected")
+	}
+	return
+}
+
+// Field = Name Type [ ":" string_lit ] .
+//
+func (p *gcParser) parseField() (fld *ast.Object, tag string) {
+	name := p.parseName()
+	ftyp := p.parseType()
+	if name == "" {
+		// anonymous field - ftyp must be T or *T and T must be a type name
+		if _, ok := Deref(ftyp).(*Name); !ok {
+			p.errorf("anonymous field expected")
+		}
+	}
+	if p.tok == ':' {
+		p.next()
+		tag = p.expect(scanner.String)
+	}
+	fld = ast.NewObj(ast.Var, name)
+	fld.Type = ftyp
+	return
+}
+
+// StructType = "struct" "{" [ FieldList ] "}" .
+// FieldList  = Field { ";" Field } .
+//
+func (p *gcParser) parseStructType() Type {
+	var fields []*ast.Object
+	var tags []string
+
+	parseField := func() {
+		fld, tag := p.parseField()
+		fields = append(fields, fld)
+		tags = append(tags, tag)
+	}
+
+	p.expectKeyword("struct")
+	p.expect('{')
+	if p.tok != '}' {
+		parseField()
+		for p.tok == ';' {
+			p.next()
+			parseField()
+		}
+	}
+	p.expect('}')
+
+	return &Struct{Fields: fields, Tags: tags}
+}
+
+// Parameter = ( identifier | "?" ) [ "..." ] Type [ ":" string_lit ] .
+//
+func (p *gcParser) parseParameter() (par *ast.Object, isVariadic bool) {
+	name := p.parseName()
+	if name == "" {
+		name = "_" // cannot access unnamed identifiers
+	}
+	if p.tok == '.' {
+		p.expectSpecial("...")
+		isVariadic = true
+	}
+	ptyp := p.parseType()
+	// ignore argument tag
+	if p.tok == ':' {
+		p.next()
+		p.expect(scanner.String)
+	}
+	par = ast.NewObj(ast.Var, name)
+	par.Type = ptyp
+	return
+}
+
+// Parameters    = "(" [ ParameterList ] ")" .
+// ParameterList = { Parameter "," } Parameter .
+//
+func (p *gcParser) parseParameters() (list []*ast.Object, isVariadic bool) {
+	parseParameter := func() {
+		par, variadic := p.parseParameter()
+		list = append(list, par)
+		if variadic {
+			if isVariadic {
+				p.error("... not on final argument")
+			}
+			isVariadic = true
+		}
+	}
+
+	p.expect('(')
+	if p.tok != ')' {
+		parseParameter()
+		for p.tok == ',' {
+			p.next()
+			parseParameter()
+		}
+	}
+	p.expect(')')
+
+	return
+}
+
+// Signature = Parameters [ Result ] .
+// Result    = Type | Parameters .
+//
+func (p *gcParser) parseSignature() *Func {
+	params, isVariadic := p.parseParameters()
+
+	// optional result type
+	var results []*ast.Object
+	switch p.tok {
+	case scanner.Ident, scanner.String, '[', '*', '<':
+		// single, unnamed result
+		result := ast.NewObj(ast.Var, "_")
+		result.Type = p.parseType()
+		results = []*ast.Object{result}
+	case '(':
+		// named or multiple result(s)
+		var variadic bool
+		results, variadic = p.parseParameters()
+		if variadic {
+			p.error("... not permitted on result type")
+		}
+	}
+
+	return &Func{Params: params, Results: results, IsVariadic: isVariadic}
+}
+
+// MethodSpec = identifier Signature .
+//
+func (p *gcParser) parseMethodSpec() *ast.Object {
+	if p.tok == scanner.Ident {
+		p.expect(scanner.Ident)
+	} else {
+		// TODO(gri) should this be parseExportedName here?
+		p.parsePkgId()
+		p.expect('.')
+		p.parseDotIdent()
+	}
+	p.parseSignature()
+
+	// TODO(gri) compute method object
+	return ast.NewObj(ast.Fun, "_")
+}
+
+// InterfaceType = "interface" "{" [ MethodList ] "}" .
+// MethodList    = MethodSpec { ";" MethodSpec } .
+//
+func (p *gcParser) parseInterfaceType() Type {
+	var methods ObjList
+
+	parseMethod := func() {
+		meth := p.parseMethodSpec()
+		methods = append(methods, meth)
+	}
+
+	p.expectKeyword("interface")
+	p.expect('{')
+	if p.tok != '}' {
+		parseMethod()
+		for p.tok == ';' {
+			p.next()
+			parseMethod()
+		}
+	}
+	p.expect('}')
+
+	methods.Sort()
+	return &Interface{Methods: methods}
+}
+
+// ChanType = ( "chan" [ "<-" ] | "<-" "chan" ) Type .
+//
+func (p *gcParser) parseChanType() Type {
+	dir := ast.SEND | ast.RECV
+	if p.tok == scanner.Ident {
+		p.expectKeyword("chan")
+		if p.tok == '<' {
+			p.expectSpecial("<-")
+			dir = ast.SEND
+		}
+	} else {
+		p.expectSpecial("<-")
+		p.expectKeyword("chan")
+		dir = ast.RECV
+	}
+	elt := p.parseType()
+	return &Chan{Dir: dir, Elt: elt}
+}
+
+// Type =
+//	BasicType | TypeName | ArrayType | SliceType | StructType |
+//      PointerType | FuncType | InterfaceType | MapType | ChanType |
+//      "(" Type ")" .
+// BasicType = ident .
+// TypeName = ExportedName .
+// SliceType = "[" "]" Type .
+// PointerType = "*" Type .
+// FuncType = "func" Signature .
+//
+func (p *gcParser) parseType() Type {
+	switch p.tok {
+	case scanner.Ident:
+		switch p.lit {
+		default:
+			return p.parseBasicType()
+		case "struct":
+			return p.parseStructType()
+		case "func":
+			// FuncType
+			p.next()
+			return p.parseSignature()
+		case "interface":
+			return p.parseInterfaceType()
+		case "map":
+			return p.parseMapType()
+		case "chan":
+			return p.parseChanType()
+		}
+	case scanner.String:
+		// TypeName
+		return p.parseExportedName(ast.Typ).Type.(Type)
+	case '[':
+		p.next() // look ahead
+		if p.tok == ']' {
+			// SliceType
+			p.next()
+			return &Slice{Elt: p.parseType()}
+		}
+		return p.parseArrayType()
+	case '*':
+		// PointerType
+		p.next()
+		return &Pointer{Base: p.parseType()}
+	case '<':
+		return p.parseChanType()
+	case '(':
+		// "(" Type ")"
+		p.next()
+		typ := p.parseType()
+		p.expect(')')
+		return typ
+	}
+	p.errorf("expected type, got %s (%q)", scanner.TokenString(p.tok), p.lit)
+	return nil
+}
+
+// ----------------------------------------------------------------------------
+// Declarations
+
+// ImportDecl = "import" identifier string_lit .
+//
+func (p *gcParser) parseImportDecl() {
+	p.expectKeyword("import")
+	// The identifier has no semantic meaning in the import data.
+	// It exists so that error messages can print the real package
+	// name: binary.ByteOrder instead of "encoding/binary".ByteOrder.
+	name := p.expect(scanner.Ident)
+	pkg := p.parsePkgId()
+	assert(pkg.Name == "" || pkg.Name == name)
+	pkg.Name = name
+}
+
+// int_lit = [ "+" | "-" ] { "0" ... "9" } .
+//
+func (p *gcParser) parseInt() (sign, val string) {
+	switch p.tok {
+	case '-':
+		p.next()
+		sign = "-"
+	case '+':
+		p.next()
+	}
+	val = p.expect(scanner.Int)
+	return
+}
+
+// number = int_lit [ "p" int_lit ] .
+//
+func (p *gcParser) parseNumber() Const {
+	// mantissa
+	sign, val := p.parseInt()
+	mant, ok := new(big.Int).SetString(sign+val, 10)
+	assert(ok)
+
+	if p.lit == "p" {
+		// exponent (base 2)
+		p.next()
+		sign, val = p.parseInt()
+		exp, err := strconv.Atoui(val)
+		if err != nil {
+			p.error(err)
+		}
+		if sign == "-" {
+			denom := big.NewInt(1)
+			denom.Lsh(denom, exp)
+			return Const{new(big.Rat).SetFrac(mant, denom)}
+		}
+		if exp > 0 {
+			mant.Lsh(mant, exp)
+		}
+		return Const{new(big.Rat).SetInt(mant)}
+	}
+
+	return Const{mant}
+}
+
+// ConstDecl   = "const" ExportedName [ Type ] "=" Literal .
+// Literal     = bool_lit | int_lit | float_lit | complex_lit | string_lit .
+// bool_lit    = "true" | "false" .
+// complex_lit = "(" float_lit "+" float_lit ")" .
+// string_lit  = `"` { unicode_char } `"` .
+//
+func (p *gcParser) parseConstDecl() {
+	p.expectKeyword("const")
+	obj := p.parseExportedName(ast.Con)
+	var x Const
+	var typ Type
+	if p.tok != '=' {
+		obj.Type = p.parseType()
+	}
+	p.expect('=')
+	switch p.tok {
+	case scanner.Ident:
+		// bool_lit
+		if p.lit != "true" && p.lit != "false" {
+			p.error("expected true or false")
+		}
+		x = Const{p.lit == "true"}
+		typ = Bool.Underlying
+		p.next()
+	case '-', scanner.Int:
+		// int_lit
+		x = p.parseNumber()
+		typ = Int.Underlying
+		if _, ok := x.val.(*big.Rat); ok {
+			typ = Float64.Underlying
+		}
+	case '(':
+		// complex_lit
+		p.next()
+		re := p.parseNumber()
+		p.expect('+')
+		im := p.parseNumber()
+		p.expect(')')
+		x = Const{cmplx{re.val.(*big.Rat), im.val.(*big.Rat)}}
+		typ = Complex128.Underlying
+	case scanner.String:
+		// string_lit
+		x = MakeConst(token.STRING, p.lit)
+		p.next()
+		typ = String.Underlying
+	default:
+		p.error("expected literal")
+	}
+	if obj.Type == nil {
+		obj.Type = typ
+	}
+	obj.Data = x
+}
+
+// TypeDecl = "type" ExportedName Type .
+//
+func (p *gcParser) parseTypeDecl() {
+	p.expectKeyword("type")
+	obj := p.parseExportedName(ast.Typ)
+
+	// The type object may have been imported before and thus already
+	// have a type associated with it. We still need to parse the type
+	// structure, but throw it away if the object already has a type.
+	// This ensures that all imports refer to the same type object for
+	// a given type declaration.
+	typ := p.parseType()
+
+	if name := obj.Type.(*Name); name.Underlying == nil {
+		assert(Underlying(typ) == typ)
+		name.Underlying = typ
+	}
+}
+
+// VarDecl = "var" ExportedName Type .
+//
+func (p *gcParser) parseVarDecl() {
+	p.expectKeyword("var")
+	obj := p.parseExportedName(ast.Var)
+	obj.Type = p.parseType()
+}
+
+// FuncDecl = "func" ExportedName Signature .
+//
+func (p *gcParser) parseFuncDecl() {
+	// "func" already consumed
+	obj := p.parseExportedName(ast.Fun)
+	obj.Type = p.parseSignature()
+}
+
+// MethodDecl = "func" Receiver identifier Signature .
+// Receiver   = "(" ( identifier | "?" ) [ "*" ] ExportedName ")" .
+//
+func (p *gcParser) parseMethodDecl() {
+	// "func" already consumed
+	p.expect('(')
+	p.parseParameter() // receiver
+	p.expect(')')
+	p.expect(scanner.Ident)
+	p.parseSignature()
+}
+
+// Decl = [ ImportDecl | ConstDecl | TypeDecl | VarDecl | FuncDecl | MethodDecl ] "\n" .
+//
+func (p *gcParser) parseDecl() {
+	switch p.lit {
+	case "import":
+		p.parseImportDecl()
+	case "const":
+		p.parseConstDecl()
+	case "type":
+		p.parseTypeDecl()
+	case "var":
+		p.parseVarDecl()
+	case "func":
+		p.next() // look ahead
+		if p.tok == '(' {
+			p.parseMethodDecl()
+		} else {
+			p.parseFuncDecl()
+		}
+	}
+	p.expect('\n')
+}
+
+// ----------------------------------------------------------------------------
+// Export
+
+// Export        = "PackageClause { Decl } "$$" .
+// PackageClause = "package" identifier [ "safe" ] "\n" .
+//
+func (p *gcParser) parseExport() *ast.Object {
+	p.expectKeyword("package")
+	name := p.expect(scanner.Ident)
+	if p.tok != '\n' {
+		// A package is safe if it was compiled with the -u flag,
+		// which disables the unsafe package.
+		// TODO(gri) remember "safe" package
+		p.expectKeyword("safe")
+	}
+	p.expect('\n')
+
+	assert(p.imports[p.id] == nil)
+	pkg := ast.NewObj(ast.Pkg, name)
+	pkg.Data = ast.NewScope(nil)
+	p.imports[p.id] = pkg
+
+	for p.tok != '$' && p.tok != scanner.EOF {
+		p.parseDecl()
+	}
+
+	if ch := p.scanner.Peek(); p.tok != '$' || ch != '$' {
+		// don't call next()/expect() since reading past the
+		// export data may cause scanner errors (e.g. NUL chars)
+		p.errorf("expected '$$', got %s %c", scanner.TokenString(p.tok), ch)
+	}
+
+	if n := p.scanner.ErrorCount; n != 0 {
+		p.errorf("expected no scanner errors, got %d", n)
+	}
+
+	return pkg
+}
diff --git a/src/pkg/go/types/gcimporter_test.go b/src/pkg/go/types/gcimporter_test.go
new file mode 100644
index 000000000..ec87f5d51
--- /dev/null
+++ b/src/pkg/go/types/gcimporter_test.go
@@ -0,0 +1,101 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package types
+
+import (
+	"exec"
+	"go/ast"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+var gcName, gcPath string // compiler name and path
+
+func init() {
+	// determine compiler
+	switch runtime.GOARCH {
+	case "386":
+		gcName = "8g"
+	case "amd64":
+		gcName = "6g"
+	case "arm":
+		gcName = "5g"
+	default:
+		gcName = "unknown-GOARCH-compiler"
+		gcPath = gcName
+		return
+	}
+	gcPath, _ = exec.LookPath(gcName)
+}
+
+func compile(t *testing.T, dirname, filename string) {
+	cmd := exec.Command(gcPath, filename)
+	cmd.Dir = dirname
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("%s %s failed: %s", gcName, filename, err)
+		return
+	}
+	t.Logf("%s", string(out))
+}
+
+// Use the same global imports map for all tests. The effect is
+// as if all tested packages were imported into a single package.
+var imports = make(map[string]*ast.Object)
+
+func testPath(t *testing.T, path string) bool {
+	_, err := GcImporter(imports, path)
+	if err != nil {
+		t.Errorf("testPath(%s): %s", path, err)
+		return false
+	}
+	return true
+}
+
+const maxTime = 3e9 // maximum allotted testing time in ns
+
+func testDir(t *testing.T, dir string, endTime int64) (nimports int) {
+	dirname := filepath.Join(pkgRoot, dir)
+	list, err := ioutil.ReadDir(dirname)
+	if err != nil {
+		t.Errorf("testDir(%s): %s", dirname, err)
+	}
+	for _, f := range list {
+		if time.Nanoseconds() >= endTime {
+			t.Log("testing time used up")
+			return
+		}
+		switch {
+		case f.IsRegular():
+			// try extensions
+			for _, ext := range pkgExts {
+				if strings.HasSuffix(f.Name, ext) {
+					name := f.Name[0 : len(f.Name)-len(ext)] // remove extension
+					if testPath(t, filepath.Join(dir, name)) {
+						nimports++
+					}
+				}
+			}
+		case f.IsDirectory():
+			nimports += testDir(t, filepath.Join(dir, f.Name), endTime)
+		}
+	}
+	return
+}
+
+func TestGcImport(t *testing.T) {
+	compile(t, "testdata", "exports.go")
+
+	nimports := 0
+	if testPath(t, "./testdata/exports") {
+		nimports++
+	}
+	nimports += testDir(t, "", time.Nanoseconds()+maxTime) // installed packages
+	t.Logf("tested %d imports", nimports)
+}
diff --git a/src/pkg/go/types/testdata/exports.go b/src/pkg/go/types/testdata/exports.go
new file mode 100644
index 000000000..ed63bf9ad
--- /dev/null
+++ b/src/pkg/go/types/testdata/exports.go
@@ -0,0 +1,84 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is used to generate an object file which
+// serves as test file for gcimporter_test.go.
+
+package exports
+
+import (
+	"go/ast"
+)
+
+const (
+	C0 int = 0
+	C1     = 3.14159265
+	C2     = 2.718281828i
+	C3     = -123.456e-789
+	C4     = +123.456E+789
+	C5     = 1234i
+	C6     = "foo\n"
+	C7     = `bar\n`
+)
+
+type (
+	T1  int
+	T2  [10]int
+	T3  []int
+	T4  *int
+	T5  chan int
+	T6a chan<- int
+	T6b chan (<-chan int)
+	T6c chan<- (chan int)
+	T7  <-chan *ast.File
+	T8  struct{}
+	T9  struct {
+		a    int
+		b, c float32
+		d    []string `go:"tag"`
+	}
+	T10 struct {
+		T8
+		T9
+		_ *T10
+	}
+	T11 map[int]string
+	T12 interface{}
+	T13 interface {
+		m1()
+		m2(int) float32
+	}
+	T14 interface {
+		T12
+		T13
+		m3(x ...struct{}) []T9
+	}
+	T15 func()
+	T16 func(int)
+	T17 func(x int)
+	T18 func() float32
+	T19 func() (x float32)
+	T20 func(...interface{})
+	T21 struct{ next *T21 }
+	T22 struct{ link *T23 }
+	T23 struct{ link *T22 }
+	T24 *T24
+	T25 *T26
+	T26 *T27
+	T27 *T25
+	T28 func(T28) T28
+)
+
+var (
+	V0 int
+	V1 = -991.0
+)
+
+func F1()         {}
+func F2(x int)    {}
+func F3() int     { return 0 }
+func F4() float32 { return 0 }
+func F5(a, b, c int, u, v, w struct{ x, y T1 }, more ...interface{}) (p, q, r chan<- T10)
+
+func (p *T1) M1()
diff --git a/src/pkg/go/types/testdata/test0.src b/src/pkg/go/types/testdata/test0.src
new file mode 100644
index 000000000..84a1abe27
--- /dev/null
+++ b/src/pkg/go/types/testdata/test0.src
@@ -0,0 +1,154 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// type declarations
+
+package test0
+
+import "unsafe"
+
+const pi = 3.1415
+
+type (
+	N undeclared /* ERROR "undeclared" */
+	B bool
+	I int32
+	A [10]P
+	T struct {
+		x, y P
+	}
+	P *T
+	R (*R)
+	F func(A) I
+	Y interface {
+		f(A) I
+	}
+	S [](((P)))
+	M map[I]F
+	C chan<- I
+)
+
+
+type (
+	p1 pi /* ERROR "not a package" */ .foo
+	p2 unsafe.Pointer
+)
+
+
+type (
+	Pi pi /* ERROR "not a type" */
+
+	a /* DISABLED "illegal cycle" */ a
+	a /* ERROR "redeclared" */ int
+
+	// where the cycle error appears depends on the
+	// order in which declarations are processed
+	// (which depends on the order in which a map
+	// is iterated through)
+	b c
+	c /* DISABLED "illegal cycle" */ d
+	d e
+	e b
+
+	t *t
+
+	U V
+	V *W
+	W U
+
+	P1 *S2
+	P2 P1
+
+	S0 struct {
+	}
+	S1 struct {
+		a, b, c int
+		u, v, a /* ERROR "redeclared" */ float32
+	}
+	S2 struct {
+		U // anonymous field
+		// TODO(gri) recognize double-declaration below
+		// U /* ERROR "redeclared" */ int
+	}
+	S3 struct {
+		x S2
+	}
+	S4/* DISABLED "illegal cycle" */ struct {
+		S4
+	}
+	S5 struct {
+		S6
+	}
+	S6 /* DISABLED "illegal cycle" */ struct {
+		field S7
+	}
+	S7 struct {
+		S5
+	}
+
+	L1 []L1
+	L2 []int
+
+	A1 [10]int
+	A2 /* DISABLED "illegal cycle" */ [10]A2
+	A3 /* DISABLED "illegal cycle" */ [10]struct {
+		x A4
+	}
+	A4 [10]A3
+
+	F1 func()
+	F2 func(x, y, z float32)
+	F3 func(x, y, x /* ERROR "redeclared" */ float32)
+	F4 func() (x, y, x /* ERROR "redeclared" */ float32)
+	F5 func(x int) (x /* ERROR "redeclared" */ float32)
+	F6 func(x ...int)
+
+	I1 interface{}
+	I2 interface {
+		m1()
+	}
+	I3 interface {
+		m1()
+		m1 /* ERROR "redeclared" */ ()
+	}
+	I4 interface {
+		m1(x, y, x /* ERROR "redeclared" */ float32)
+		m2() (x, y, x /* ERROR "redeclared" */ float32)
+		m3(x int) (x /* ERROR "redeclared" */ float32)
+	}
+	I5 interface {
+		m1(I5)
+	}
+	I6 interface {
+		S0 /* ERROR "non-interface" */
+	}
+	I7 interface {
+		I1
+		I1
+	}
+	I8 /* DISABLED "illegal cycle" */ interface {
+		I8
+	}
+	I9 /* DISABLED "illegal cycle" */ interface {
+		I10
+	}
+	I10 interface {
+		I11
+	}
+	I11 interface {
+		I9
+	}
+
+	C1 chan int
+	C2 <-chan int
+	C3 chan<- C3
+	C4 chan C5
+	C5 chan C6
+	C6 chan C4
+
+	M1 map[Last]string
+	M2 map[string]M2
+
+	Last int
+)
diff --git a/src/pkg/go/types/types.go b/src/pkg/go/types/types.go
new file mode 100644
index 000000000..3aa896892
--- /dev/null
+++ b/src/pkg/go/types/types.go
@@ -0,0 +1,255 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// PACKAGE UNDER CONSTRUCTION. ANY AND ALL PARTS MAY CHANGE.
+// Package types declares the types used to represent Go types.
+//
+package types
+
+import (
+	"go/ast"
+	"sort"
+)
+
+// All types implement the Type interface.
+type Type interface {
+	isType()
+}
+
+// All concrete types embed ImplementsType which
+// ensures that all types implement the Type interface.
+type ImplementsType struct{}
+
+func (t *ImplementsType) isType() {}
+
+// A Bad type is a non-nil placeholder type when we don't know a type.
+type Bad struct {
+	ImplementsType
+	Msg string // for better error reporting/debugging
+}
+
+// A Basic represents a (unnamed) basic type.
+type Basic struct {
+	ImplementsType
+	// TODO(gri) need a field specifying the exact basic type
+}
+
+// An Array represents an array type [Len]Elt.
+type Array struct {
+	ImplementsType
+	Len uint64
+	Elt Type
+}
+
+// A Slice represents a slice type []Elt.
+type Slice struct {
+	ImplementsType
+	Elt Type
+}
+
+// A Struct represents a struct type struct{...}.
+// Anonymous fields are represented by objects with empty names.
+type Struct struct {
+	ImplementsType
+	Fields ObjList  // struct fields; or nil
+	Tags   []string // corresponding tags; or nil
+	// TODO(gri) This type needs some rethinking:
+	// - at the moment anonymous fields are marked with "" object names,
+	//   and their names have to be reconstructed
+	// - there is no scope for fast lookup (but the parser creates one)
+}
+
+// A Pointer represents a pointer type *Base.
+type Pointer struct {
+	ImplementsType
+	Base Type
+}
+
+// A Func represents a function type func(...) (...).
+// Unnamed parameters are represented by objects with empty names.
+type Func struct {
+	ImplementsType
+	Recv       *ast.Object // nil if not a method
+	Params     ObjList     // (incoming) parameters from left to right; or nil
+	Results    ObjList     // (outgoing) results from left to right; or nil
+	IsVariadic bool        // true if the last parameter's type is of the form ...T
+}
+
+// An Interface represents an interface type interface{...}.
+type Interface struct {
+	ImplementsType
+	Methods ObjList // interface methods sorted by name; or nil
+}
+
+// A Map represents a map type map[Key]Elt.
+type Map struct {
+	ImplementsType
+	Key, Elt Type
+}
+
+// A Chan represents a channel type chan Elt, <-chan Elt, or chan<-Elt.
+type Chan struct {
+	ImplementsType
+	Dir ast.ChanDir
+	Elt Type
+}
+
+// A Name represents a named type as declared in a type declaration.
+type Name struct {
+	ImplementsType
+	Underlying Type        // nil if not fully declared
+	Obj        *ast.Object // corresponding declared object
+	// TODO(gri) need to remember fields and methods.
+}
+
+// If typ is a pointer type, Deref returns the pointer's base type;
+// otherwise it returns typ.
+func Deref(typ Type) Type {
+	if typ, ok := typ.(*Pointer); ok {
+		return typ.Base
+	}
+	return typ
+}
+
+// Underlying returns the underlying type of a type.
+func Underlying(typ Type) Type {
+	if typ, ok := typ.(*Name); ok {
+		utyp := typ.Underlying
+		if _, ok := utyp.(*Basic); !ok {
+			return utyp
+		}
+		// the underlying type of a type name referring
+		// to an (untyped) basic type is the basic type
+		// name
+	}
+	return typ
+}
+
+// An ObjList represents an ordered (in some fashion) list of objects.
+type ObjList []*ast.Object
+
+// ObjList implements sort.Interface.
+func (list ObjList) Len() int           { return len(list) }
+func (list ObjList) Less(i, j int) bool { return list[i].Name < list[j].Name }
+func (list ObjList) Swap(i, j int)      { list[i], list[j] = list[j], list[i] }
+
+// Sort sorts an object list by object name.
+func (list ObjList) Sort() { sort.Sort(list) }
+
+// identicalTypes returns true if both lists a and b have the
+// same length and corresponding objects have identical types.
+func identicalTypes(a, b ObjList) bool {
+	if len(a) == len(b) {
+		for i, x := range a {
+			y := b[i]
+			if !Identical(x.Type.(Type), y.Type.(Type)) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+// Identical returns true if two types are identical.
+func Identical(x, y Type) bool {
+	if x == y {
+		return true
+	}
+
+	switch x := x.(type) {
+	case *Bad:
+		// A Bad type is always identical to any other type
+		// (to avoid spurious follow-up errors).
+		return true
+
+	case *Basic:
+		if y, ok := y.(*Basic); ok {
+			panic("unimplemented")
+			_ = y
+		}
+
+	case *Array:
+		// Two array types are identical if they have identical element types
+		// and the same array length.
+		if y, ok := y.(*Array); ok {
+			return x.Len == y.Len && Identical(x.Elt, y.Elt)
+		}
+
+	case *Slice:
+		// Two slice types are identical if they have identical element types.
+		if y, ok := y.(*Slice); ok {
+			return Identical(x.Elt, y.Elt)
+		}
+
+	case *Struct:
+		// Two struct types are identical if they have the same sequence of fields,
+		// and if corresponding fields have the same names, and identical types,
+		// and identical tags. Two anonymous fields are considered to have the same
+		// name. Lower-case field names from different packages are always different.
+		if y, ok := y.(*Struct); ok {
+			// TODO(gri) handle structs from different packages
+			if identicalTypes(x.Fields, y.Fields) {
+				for i, f := range x.Fields {
+					g := y.Fields[i]
+					if f.Name != g.Name || x.Tags[i] != y.Tags[i] {
+						return false
+					}
+				}
+				return true
+			}
+		}
+
+	case *Pointer:
+		// Two pointer types are identical if they have identical base types.
+		if y, ok := y.(*Pointer); ok {
+			return Identical(x.Base, y.Base)
+		}
+
+	case *Func:
+		// Two function types are identical if they have the same number of parameters
+		// and result values, corresponding parameter and result types are identical,
+		// and either both functions are variadic or neither is. Parameter and result
+		// names are not required to match.
+		if y, ok := y.(*Func); ok {
+			return identicalTypes(x.Params, y.Params) &&
+				identicalTypes(x.Results, y.Results) &&
+				x.IsVariadic == y.IsVariadic
+		}
+
+	case *Interface:
+		// Two interface types are identical if they have the same set of methods with
+		// the same names and identical function types. Lower-case method names from
+		// different packages are always different. The order of the methods is irrelevant.
+		if y, ok := y.(*Interface); ok {
+			return identicalTypes(x.Methods, y.Methods) // methods are sorted
+		}
+
+	case *Map:
+		// Two map types are identical if they have identical key and value types.
+		if y, ok := y.(*Map); ok {
+			return Identical(x.Key, y.Key) && Identical(x.Elt, y.Elt)
+		}
+
+	case *Chan:
+		// Two channel types are identical if they have identical value types
+		// and the same direction.
+		if y, ok := y.(*Chan); ok {
+			return x.Dir == y.Dir && Identical(x.Elt, y.Elt)
+		}
+
+	case *Name:
+		// Two named types are identical if their type names originate
+		// in the same type declaration.
+		if y, ok := y.(*Name); ok {
+			return x.Obj == y.Obj ||
+				// permit bad objects to be equal to avoid
+				// follow up errors
+				x.Obj != nil && x.Obj.Kind == ast.Bad ||
+				y.Obj != nil && y.Obj.Kind == ast.Bad
+		}
+	}
+
+	return false
+}
diff --git a/src/pkg/go/types/universe.go b/src/pkg/go/types/universe.go
new file mode 100644
index 000000000..6ae88e5f9
--- /dev/null
+++ b/src/pkg/go/types/universe.go
@@ -0,0 +1,108 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// FILE UNDER CONSTRUCTION. ANY AND ALL PARTS MAY CHANGE.
+// This file implements the universe and unsafe package scopes.
+
+package types
+
+import "go/ast"
+
+var (
+	scope    *ast.Scope // current scope to use for initialization
+	Universe *ast.Scope
+	Unsafe   *ast.Object // package unsafe
+)
+
+func define(kind ast.ObjKind, name string) *ast.Object {
+	obj := ast.NewObj(kind, name)
+	if scope.Insert(obj) != nil {
+		panic("types internal error: double declaration")
+	}
+	return obj
+}
+
+func defType(name string) *Name {
+	obj := define(ast.Typ, name)
+	typ := &Name{Underlying: &Basic{}, Obj: obj}
+	obj.Type = typ
+	return typ
+}
+
+func defConst(name string) {
+	obj := define(ast.Con, name)
+	_ = obj // TODO(gri) fill in other properties
+}
+
+func defFun(name string) {
+	obj := define(ast.Fun, name)
+	_ = obj // TODO(gri) fill in other properties
+}
+
+var (
+	Bool,
+	Int,
+	Float64,
+	Complex128,
+	String *Name
+)
+
+func init() {
+	scope = ast.NewScope(nil)
+	Universe = scope
+
+	Bool = defType("bool")
+	defType("byte") // TODO(gri) should be an alias for uint8
+	defType("complex64")
+	Complex128 = defType("complex128")
+	defType("float32")
+	Float64 = defType("float64")
+	defType("int8")
+	defType("int16")
+	defType("int32")
+	defType("int64")
+	String = defType("string")
+	defType("uint8")
+	defType("uint16")
+	defType("uint32")
+	defType("uint64")
+	Int = defType("int")
+	defType("uint")
+	defType("uintptr")
+
+	defConst("true")
+	defConst("false")
+	defConst("iota")
+	defConst("nil")
+
+	defFun("append")
+	defFun("cap")
+	defFun("close")
+	defFun("complex")
+	defFun("copy")
+	defFun("imag")
+	defFun("len")
+	defFun("make")
+	defFun("new")
+	defFun("panic")
+	defFun("print")
+	defFun("println")
+	defFun("real")
+	defFun("recover")
+
+	scope = ast.NewScope(nil)
+	Unsafe = ast.NewObj(ast.Pkg, "unsafe")
+	Unsafe.Data = scope
+
+	defType("Pointer")
+
+	defFun("Alignof")
+	defFun("New")
+	defFun("NewArray")
+	defFun("Offsetof")
+	defFun("Reflect")
+	defFun("Sizeof")
+	defFun("Typeof")
+	defFun("Unreflect")
+}
diff --git a/src/pkg/gob/Makefile b/src/pkg/gob/Makefile
new file mode 100644
index 000000000..68007c189
--- /dev/null
+++ b/src/pkg/gob/Makefile
@@ -0,0 +1,25 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.inc
+
+TARG=gob
+GOFILES=\
+	decode.go\
+	decoder.go\
+	doc.go\
+	encode.go\
+	encoder.go\
+	error.go\
+	type.go\
+
+include ../../Make.pkg
+
+# Help for debugging. Requires adding debug.go to the gob package as well.
+
+dump:	dump.$O
+	$(LD) -o dump $<
+
+dump.$O:	dump.go
+	$(GC) $<
diff --git a/src/pkg/gob/codec_test.go b/src/pkg/gob/codec_test.go
new file mode 100644
index 000000000..a5fb91cda
--- /dev/null
+++ b/src/pkg/gob/codec_test.go
@@ -0,0 +1,1406 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"math"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"unsafe"
+)
+
+// Guarantee encoding format by comparing some encodings to hand-written values
+type EncodeT struct {
+	x uint64
+	b []byte
+}
+
+var encodeT = []EncodeT{
+	{0x00, []byte{0x00}},
+	{0x0F, []byte{0x0F}},
+	{0xFF, []byte{0xFF, 0xFF}},
+	{0xFFFF, []byte{0xFE, 0xFF, 0xFF}},
+	{0xFFFFFF, []byte{0xFD, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFF, []byte{0xFC, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFF, []byte{0xFB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFF, []byte{0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFFFF, []byte{0xF9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0xFFFFFFFFFFFFFFFF, []byte{0xF8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}},
+	{0x1111, []byte{0xFE, 0x11, 0x11}},
+	{0x1111111111111111, []byte{0xF8, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}},
+	{0x8888888888888888, []byte{0xF8, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88}},
+	{1 << 63, []byte{0xF8, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
+}
+
+// testError is meant to be used as a deferred function to turn a panic(gobError) into a
+// plain test.Error call.
+func testError(t *testing.T) {
+	if e := recover(); e != nil {
+		t.Error(e.(gobError).Error) // Will re-panic if not one of our errors, such as a runtime error.
+	}
+	return
+}
+
+// Test basic encode/decode routines for unsigned integers
+func TestUintCodec(t *testing.T) {
+	defer testError(t)
+	b := new(bytes.Buffer)
+	encState := newEncoderState(b)
+	for _, tt := range encodeT {
+		b.Reset()
+		encState.encodeUint(tt.x)
+		if !bytes.Equal(tt.b, b.Bytes()) {
+			t.Errorf("encodeUint: %#x encode: expected % x got % x", tt.x, tt.b, b.Bytes())
+		}
+	}
+	decState := newDecodeState(b)
+	for u := uint64(0); ; u = (u + 1) * 7 {
+		b.Reset()
+		encState.encodeUint(u)
+		v := decState.decodeUint()
+		if u != v {
+			t.Errorf("Encode/Decode: sent %#x received %#x", u, v)
+		}
+		if u&(1<<63) != 0 {
+			break
+		}
+	}
+}
+
+func verifyInt(i int64, t *testing.T) {
+	defer testError(t)
+	var b = new(bytes.Buffer)
+	encState := newEncoderState(b)
+	encState.encodeInt(i)
+	decState := newDecodeState(b)
+	decState.buf = make([]byte, 8)
+	j := decState.decodeInt()
+	if i != j {
+		t.Errorf("Encode/Decode: sent %#x received %#x", uint64(i), uint64(j))
+	}
+}
+
+// Test basic encode/decode routines for signed integers
+func TestIntCodec(t *testing.T) {
+	for u := uint64(0); ; u = (u + 1) * 7 {
+		// Do positive and negative values
+		i := int64(u)
+		verifyInt(i, t)
+		verifyInt(-i, t)
+		verifyInt(^i, t)
+		if u&(1<<63) != 0 {
+			break
+		}
+	}
+	verifyInt(-1<<63, t) // a tricky case
+}
+
+// The result of encoding a true boolean with field number 7
+var boolResult = []byte{0x07, 0x01}
+// The result of encoding a number 17 with field number 7
+var signedResult = []byte{0x07, 2 * 17}
+var unsignedResult = []byte{0x07, 17}
+var floatResult = []byte{0x07, 0xFE, 0x31, 0x40}
+// The result of encoding a number 17+19i with field number 7
+var complexResult = []byte{0x07, 0xFE, 0x31, 0x40, 0xFE, 0x33, 0x40}
+// The result of encoding "hello" with field number 7
+var bytesResult = []byte{0x07, 0x05, 'h', 'e', 'l', 'l', 'o'}
+
+func newDecodeState(buf *bytes.Buffer) *decoderState {
+	d := new(decoderState)
+	d.b = buf
+	d.buf = make([]byte, uint64Size)
+	return d
+}
+
+func newEncoderState(b *bytes.Buffer) *encoderState {
+	b.Reset()
+	state := &encoderState{enc: nil, b: b}
+	state.fieldnum = -1
+	return state
+}
+
+// Test instruction execution for encoding.
+// Do not run the machine yet; instead do individual instructions crafted by hand.
+func TestScalarEncInstructions(t *testing.T) {
+	var b = new(bytes.Buffer)
+
+	// bool
+	{
+		data := struct{ a bool }{true}
+		instr := &encInstr{encBool, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(boolResult, b.Bytes()) {
+			t.Errorf("bool enc instructions: expected % x got % x", boolResult, b.Bytes())
+		}
+	}
+
+	// int
+	{
+		b.Reset()
+		data := struct{ a int }{17}
+		instr := &encInstr{encInt, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint
+	{
+		b.Reset()
+		data := struct{ a uint }{17}
+		instr := &encInstr{encUint, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int8
+	{
+		b.Reset()
+		data := struct{ a int8 }{17}
+		instr := &encInstr{encInt8, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int8 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint8
+	{
+		b.Reset()
+		data := struct{ a uint8 }{17}
+		instr := &encInstr{encUint8, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint8 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int16
+	{
+		b.Reset()
+		data := struct{ a int16 }{17}
+		instr := &encInstr{encInt16, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int16 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint16
+	{
+		b.Reset()
+		data := struct{ a uint16 }{17}
+		instr := &encInstr{encUint16, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint16 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int32
+	{
+		b.Reset()
+		data := struct{ a int32 }{17}
+		instr := &encInstr{encInt32, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int32 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint32
+	{
+		b.Reset()
+		data := struct{ a uint32 }{17}
+		instr := &encInstr{encUint32, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint32 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// int64
+	{
+		b.Reset()
+		data := struct{ a int64 }{17}
+		instr := &encInstr{encInt64, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(signedResult, b.Bytes()) {
+			t.Errorf("int64 enc instructions: expected % x got % x", signedResult, b.Bytes())
+		}
+	}
+
+	// uint64
+	{
+		b.Reset()
+		data := struct{ a uint64 }{17}
+		instr := &encInstr{encUint64, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(unsignedResult, b.Bytes()) {
+			t.Errorf("uint64 enc instructions: expected % x got % x", unsignedResult, b.Bytes())
+		}
+	}
+
+	// float32
+	{
+		b.Reset()
+		data := struct{ a float32 }{17}
+		instr := &encInstr{encFloat32, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(floatResult, b.Bytes()) {
+			t.Errorf("float32 enc instructions: expected % x got % x", floatResult, b.Bytes())
+		}
+	}
+
+	// float64
+	{
+		b.Reset()
+		data := struct{ a float64 }{17}
+		instr := &encInstr{encFloat64, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(floatResult, b.Bytes()) {
+			t.Errorf("float64 enc instructions: expected % x got % x", floatResult, b.Bytes())
+		}
+	}
+
+	// bytes == []uint8
+	{
+		b.Reset()
+		data := struct{ a []byte }{[]byte("hello")}
+		instr := &encInstr{encUint8Array, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(bytesResult, b.Bytes()) {
+			t.Errorf("bytes enc instructions: expected % x got % x", bytesResult, b.Bytes())
+		}
+	}
+
+	// string
+	{
+		b.Reset()
+		data := struct{ a string }{"hello"}
+		instr := &encInstr{encString, 6, 0, 0}
+		state := newEncoderState(b)
+		instr.op(instr, state, unsafe.Pointer(&data))
+		if !bytes.Equal(bytesResult, b.Bytes()) {
+			t.Errorf("string enc instructions: expected % x got % x", bytesResult, b.Bytes())
+		}
+	}
+}
+
+func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, p unsafe.Pointer) {
+	defer testError(t)
+	v := int(state.decodeUint())
+	if v+state.fieldnum != 6 {
+		t.Fatalf("decoding field number %d, got %d", 6, v+state.fieldnum)
+	}
+	instr.op(instr, state, decIndirect(p, instr.indir))
+	state.fieldnum = 6
+}
+
+func newDecodeStateFromData(data []byte) *decoderState {
+	b := bytes.NewBuffer(data)
+	state := newDecodeState(b)
+	state.fieldnum = -1
+	return state
+}
+
+// Test instruction execution for decoding.
+// Do not run the machine yet; instead do individual instructions crafted by hand.
+func TestScalarDecInstructions(t *testing.T) {
+	ovfl := os.NewError("overflow")
+
+	// bool
+	{
+		var data struct {
+			a bool
+		}
+		instr := &decInstr{decBool, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(boolResult)
+		execDec("bool", instr, state, t, unsafe.Pointer(&data))
+		if data.a != true {
+			t.Errorf("bool a = %v not true", data.a)
+		}
+	}
+	// int
+	{
+		var data struct {
+			a int
+		}
+		instr := &decInstr{decOpTable[reflect.Int], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int a = %v not 17", data.a)
+		}
+	}
+
+	// uint
+	{
+		var data struct {
+			a uint
+		}
+		instr := &decInstr{decOpTable[reflect.Uint], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint a = %v not 17", data.a)
+		}
+	}
+
+	// int8
+	{
+		var data struct {
+			a int8
+		}
+		instr := &decInstr{decInt8, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int8", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int8 a = %v not 17", data.a)
+		}
+	}
+
+	// uint8
+	{
+		var data struct {
+			a uint8
+		}
+		instr := &decInstr{decUint8, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint8", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint8 a = %v not 17", data.a)
+		}
+	}
+
+	// int16
+	{
+		var data struct {
+			a int16
+		}
+		instr := &decInstr{decInt16, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int16", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int16 a = %v not 17", data.a)
+		}
+	}
+
+	// uint16
+	{
+		var data struct {
+			a uint16
+		}
+		instr := &decInstr{decUint16, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint16", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint16 a = %v not 17", data.a)
+		}
+	}
+
+	// int32
+	{
+		var data struct {
+			a int32
+		}
+		instr := &decInstr{decInt32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int32 a = %v not 17", data.a)
+		}
+	}
+
+	// uint32
+	{
+		var data struct {
+			a uint32
+		}
+		instr := &decInstr{decUint32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint32 a = %v not 17", data.a)
+		}
+	}
+
+	// uintptr
+	{
+		var data struct {
+			a uintptr
+		}
+		instr := &decInstr{decOpTable[reflect.Uintptr], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uintptr", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uintptr a = %v not 17", data.a)
+		}
+	}
+
+	// int64
+	{
+		var data struct {
+			a int64
+		}
+		instr := &decInstr{decInt64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(signedResult)
+		execDec("int64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("int64 a = %v not 17", data.a)
+		}
+	}
+
+	// uint64
+	{
+		var data struct {
+			a uint64
+		}
+		instr := &decInstr{decUint64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(unsignedResult)
+		execDec("uint64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("uint64 a = %v not 17", data.a)
+		}
+	}
+
+	// float32
+	{
+		var data struct {
+			a float32
+		}
+		instr := &decInstr{decFloat32, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(floatResult)
+		execDec("float32", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("float32 a = %v not 17", data.a)
+		}
+	}
+
+	// float64
+	{
+		var data struct {
+			a float64
+		}
+		instr := &decInstr{decFloat64, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(floatResult)
+		execDec("float64", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17 {
+			t.Errorf("float64 a = %v not 17", data.a)
+		}
+	}
+
+	// complex64
+	{
+		var data struct {
+			a complex64
+		}
+		instr := &decInstr{decOpTable[reflect.Complex64], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(complexResult)
+		execDec("complex", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17+19i {
+			t.Errorf("complex a = %v not 17+19i", data.a)
+		}
+	}
+
+	// complex128
+	{
+		var data struct {
+			a complex128
+		}
+		instr := &decInstr{decOpTable[reflect.Complex128], 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(complexResult)
+		execDec("complex", instr, state, t, unsafe.Pointer(&data))
+		if data.a != 17+19i {
+			t.Errorf("complex a = %v not 17+19i", data.a)
+		}
+	}
+
+	// bytes == []uint8
+	{
+		var data struct {
+			a []byte
+		}
+		instr := &decInstr{decUint8Array, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(bytesResult)
+		execDec("bytes", instr, state, t, unsafe.Pointer(&data))
+		if string(data.a) != "hello" {
+			t.Errorf(`bytes a = %q not "hello"`, string(data.a))
+		}
+	}
+
+	// string
+	{
+		var data struct {
+			a string
+		}
+		instr := &decInstr{decString, 6, 0, 0, ovfl}
+		state := newDecodeStateFromData(bytesResult)
+		execDec("bytes", instr, state, t, unsafe.Pointer(&data))
+		if data.a != "hello" {
+			t.Errorf(`bytes a = %q not "hello"`, data.a)
+		}
+	}
+}
+
+func TestEndToEnd(t *testing.T) {
+	type T2 struct {
+		T string
+	}
+	s1 := "string1"
+	s2 := "string2"
+	type T1 struct {
+		A, B, C  int
+		M        map[string]*float64
+		EmptyMap map[string]int // to check that we receive a non-nil map.
+		N        *[3]float64
+		Strs     *[2]string
+		Int64s   *[]int64
+		RI       complex64
+		S        string
+		Y        []byte
+		T        *T2
+	}
+	pi := 3.14159
+	e := 2.71828
+	t1 := &T1{
+		A:        17,
+		B:        18,
+		C:        -5,
+		M:        map[string]*float64{"pi": &pi, "e": &e},
+		EmptyMap: make(map[string]int),
+		N:        &[3]float64{1.5, 2.5, 3.5},
+		Strs:     &[2]string{s1, s2},
+		Int64s:   &[]int64{77, 89, 123412342134},
+		RI:       17 - 23i,
+		S:        "Now is the time",
+		Y:        []byte("hello, sailor"),
+		T:        &T2{"this is T2"},
+	}
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(t1)
+	if err != nil {
+		t.Error("encode:", err)
+	}
+	var _t1 T1
+	err = NewDecoder(b).Decode(&_t1)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(t1, &_t1) {
+		t.Errorf("encode expected %v got %v", *t1, _t1)
+	}
+	// Be absolutely sure the received map is non-nil.
+	if t1.EmptyMap == nil {
+		t.Errorf("nil map sent")
+	}
+	if _t1.EmptyMap == nil {
+		t.Errorf("nil map received")
+	}
+}
+
+func TestOverflow(t *testing.T) {
+	type inputT struct {
+		Maxi int64
+		Mini int64
+		Maxu uint64
+		Maxf float64
+		Minf float64
+		Maxc complex128
+		Minc complex128
+	}
+	var it inputT
+	var err os.Error
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	dec := NewDecoder(b)
+
+	// int8
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt8 + 1,
+	}
+	type outi8 struct {
+		Maxi int8
+		Mini int8
+	}
+	var o1 outi8
+	enc.Encode(it)
+	err = dec.Decode(&o1)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int8:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt8 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o1)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int8:", err)
+	}
+
+	// int16
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt16 + 1,
+	}
+	type outi16 struct {
+		Maxi int16
+		Mini int16
+	}
+	var o2 outi16
+	enc.Encode(it)
+	err = dec.Decode(&o2)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int16:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt16 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o2)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int16:", err)
+	}
+
+	// int32
+	b.Reset()
+	it = inputT{
+		Maxi: math.MaxInt32 + 1,
+	}
+	type outi32 struct {
+		Maxi int32
+		Mini int32
+	}
+	var o3 outi32
+	enc.Encode(it)
+	err = dec.Decode(&o3)
+	if err == nil || err.String() != `value for "Maxi" out of range` {
+		t.Error("wrong overflow error for int32:", err)
+	}
+	it = inputT{
+		Mini: math.MinInt32 - 1,
+	}
+	b.Reset()
+	enc.Encode(it)
+	err = dec.Decode(&o3)
+	if err == nil || err.String() != `value for "Mini" out of range` {
+		t.Error("wrong underflow error for int32:", err)
+	}
+
+	// uint8
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint8 + 1,
+	}
+	type outu8 struct {
+		Maxu uint8
+	}
+	var o4 outu8
+	enc.Encode(it)
+	err = dec.Decode(&o4)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint8:", err)
+	}
+
+	// uint16
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint16 + 1,
+	}
+	type outu16 struct {
+		Maxu uint16
+	}
+	var o5 outu16
+	enc.Encode(it)
+	err = dec.Decode(&o5)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint16:", err)
+	}
+
+	// uint32
+	b.Reset()
+	it = inputT{
+		Maxu: math.MaxUint32 + 1,
+	}
+	type outu32 struct {
+		Maxu uint32
+	}
+	var o6 outu32
+	enc.Encode(it)
+	err = dec.Decode(&o6)
+	if err == nil || err.String() != `value for "Maxu" out of range` {
+		t.Error("wrong overflow error for uint32:", err)
+	}
+
+	// float32
+	b.Reset()
+	it = inputT{
+		Maxf: math.MaxFloat32 * 2,
+	}
+	type outf32 struct {
+		Maxf float32
+		Minf float32
+	}
+	var o7 outf32
+	enc.Encode(it)
+	err = dec.Decode(&o7)
+	if err == nil || err.String() != `value for "Maxf" out of range` {
+		t.Error("wrong overflow error for float32:", err)
+	}
+
+	// complex64
+	b.Reset()
+	it = inputT{
+		Maxc: complex(math.MaxFloat32*2, math.MaxFloat32*2),
+	}
+	type outc64 struct {
+		Maxc complex64
+		Minc complex64
+	}
+	var o8 outc64
+	enc.Encode(it)
+	err = dec.Decode(&o8)
+	if err == nil || err.String() != `value for "Maxc" out of range` {
+		t.Error("wrong overflow error for complex64:", err)
+	}
+}
+
+func TestNesting(t *testing.T) {
+	type RT struct {
+		A    string
+		Next *RT
+	}
+	rt := new(RT)
+	rt.A = "level1"
+	rt.Next = new(RT)
+	rt.Next.A = "level2"
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(rt)
+	var drt RT
+	dec := NewDecoder(b)
+	err := dec.Decode(&drt)
+	if err != nil {
+		t.Fatal("decoder error:", err)
+	}
+	if drt.A != rt.A {
+		t.Errorf("nesting: encode expected %v got %v", *rt, drt)
+	}
+	if drt.Next == nil {
+		t.Errorf("nesting: recursion failed")
+	}
+	if drt.Next.A != rt.Next.A {
+		t.Errorf("nesting: encode expected %v got %v", *rt.Next, *drt.Next)
+	}
+}
+
+// These three structures have the same data with different indirections
+type T0 struct {
+	A int
+	B int
+	C int
+	D int
+}
+type T1 struct {
+	A int
+	B *int
+	C **int
+	D ***int
+}
+type T2 struct {
+	A ***int
+	B **int
+	C *int
+	D int
+}
+
+func TestAutoIndirection(t *testing.T) {
+	// First transfer t1 into t0
+	var t1 T1
+	t1.A = 17
+	t1.B = new(int)
+	*t1.B = 177
+	t1.C = new(*int)
+	*t1.C = new(int)
+	**t1.C = 1777
+	t1.D = new(**int)
+	*t1.D = new(*int)
+	**t1.D = new(int)
+	***t1.D = 17777
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	enc.Encode(t1)
+	dec := NewDecoder(b)
+	var t0 T0
+	dec.Decode(&t0)
+	if t0.A != 17 || t0.B != 177 || t0.C != 1777 || t0.D != 17777 {
+		t.Errorf("t1->t0: expected {17 177 1777 17777}; got %v", t0)
+	}
+
+	// Now transfer t2 into t0
+	var t2 T2
+	t2.D = 17777
+	t2.C = new(int)
+	*t2.C = 1777
+	t2.B = new(*int)
+	*t2.B = new(int)
+	**t2.B = 177
+	t2.A = new(**int)
+	*t2.A = new(*int)
+	**t2.A = new(int)
+	***t2.A = 17
+	b.Reset()
+	enc.Encode(t2)
+	t0 = T0{}
+	dec.Decode(&t0)
+	if t0.A != 17 || t0.B != 177 || t0.C != 1777 || t0.D != 17777 {
+		t.Errorf("t2->t0 expected {17 177 1777 17777}; got %v", t0)
+	}
+
+	// Now transfer t0 into t1
+	t0 = T0{17, 177, 1777, 17777}
+	b.Reset()
+	enc.Encode(t0)
+	t1 = T1{}
+	dec.Decode(&t1)
+	if t1.A != 17 || *t1.B != 177 || **t1.C != 1777 || ***t1.D != 17777 {
+		t.Errorf("t0->t1 expected {17 177 1777 17777}; got {%d %d %d %d}", t1.A, *t1.B, **t1.C, ***t1.D)
+	}
+
+	// Now transfer t0 into t2
+	b.Reset()
+	enc.Encode(t0)
+	t2 = T2{}
+	dec.Decode(&t2)
+	if ***t2.A != 17 || **t2.B != 177 || *t2.C != 1777 || t2.D != 17777 {
+		t.Errorf("t0->t2 expected {17 177 1777 17777}; got {%d %d %d %d}", ***t2.A, **t2.B, *t2.C, t2.D)
+	}
+
+	// Now do t2 again but without pre-allocated pointers.
+	b.Reset()
+	enc.Encode(t0)
+	***t2.A = 0
+	**t2.B = 0
+	*t2.C = 0
+	t2.D = 0
+	dec.Decode(&t2)
+	if ***t2.A != 17 || **t2.B != 177 || *t2.C != 1777 || t2.D != 17777 {
+		t.Errorf("t0->t2 expected {17 177 1777 17777}; got {%d %d %d %d}", ***t2.A, **t2.B, *t2.C, t2.D)
+	}
+}
+
+type RT0 struct {
+	A int
+	B string
+	C float64
+}
+type RT1 struct {
+	C      float64
+	B      string
+	A      int
+	NotSet string
+}
+
+func TestReorderedFields(t *testing.T) {
+	var rt0 RT0
+	rt0.A = 17
+	rt0.B = "hello"
+	rt0.C = 3.14159
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(rt0)
+	dec := NewDecoder(b)
+	var rt1 RT1
+	// Wire type is RT0, local type is RT1.
+	err := dec.Decode(&rt1)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if rt0.A != rt1.A || rt0.B != rt1.B || rt0.C != rt1.C {
+		t.Errorf("rt1->rt0: expected %v; got %v", rt0, rt1)
+	}
+}
+
+// Like an RT0 but with fields we'll ignore on the decode side.
+type IT0 struct {
+	A        int64
+	B        string
+	Ignore_d []int
+	Ignore_e [3]float64
+	Ignore_f bool
+	Ignore_g string
+	Ignore_h []byte
+	Ignore_i *RT1
+	Ignore_m map[string]int
+	C        float64
+}
+
+func TestIgnoredFields(t *testing.T) {
+	var it0 IT0
+	it0.A = 17
+	it0.B = "hello"
+	it0.C = 3.14159
+	it0.Ignore_d = []int{1, 2, 3}
+	it0.Ignore_e[0] = 1.0
+	it0.Ignore_e[1] = 2.0
+	it0.Ignore_e[2] = 3.0
+	it0.Ignore_f = true
+	it0.Ignore_g = "pay no attention"
+	it0.Ignore_h = []byte("to the curtain")
+	it0.Ignore_i = &RT1{3.1, "hi", 7, "hello"}
+	it0.Ignore_m = map[string]int{"one": 1, "two": 2}
+
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(it0)
+	dec := NewDecoder(b)
+	var rt1 RT1
+	// Wire type is IT0, local type is RT1.
+	err := dec.Decode(&rt1)
+	if err != nil {
+		t.Error("error: ", err)
+	}
+	if int(it0.A) != rt1.A || it0.B != rt1.B || it0.C != rt1.C {
+		t.Errorf("rt0->rt1: expected %v; got %v", it0, rt1)
+	}
+}
+
+func TestBadRecursiveType(t *testing.T) {
+	type Rec ***Rec
+	var rec Rec
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(&rec)
+	if err == nil {
+		t.Error("expected error; got none")
+	} else if strings.Index(err.String(), "recursive") < 0 {
+		t.Error("expected recursive type error; got", err)
+	}
+	// Can't test decode easily because we can't encode one, so we can't pass one to a Decoder.
+}
+
+type Bad0 struct {
+	CH chan int
+	C  float64
+}
+
+func TestInvalidField(t *testing.T) {
+	var bad0 Bad0
+	bad0.CH = make(chan int)
+	b := new(bytes.Buffer)
+	dummyEncoder := new(Encoder) // sufficient for this purpose.
+	dummyEncoder.encode(b, reflect.ValueOf(&bad0), userType(reflect.TypeOf(&bad0)))
+	if err := dummyEncoder.err; err == nil {
+		t.Error("expected error; got none")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("expected type error; got", err)
+	}
+}
+
+type Indirect struct {
+	A ***[3]int
+	S ***[]int
+	M ****map[string]int
+}
+
+type Direct struct {
+	A [3]int
+	S []int
+	M map[string]int
+}
+
+func TestIndirectSliceMapArray(t *testing.T) {
+	// Marshal indirect, unmarshal to direct.
+	i := new(Indirect)
+	i.A = new(**[3]int)
+	*i.A = new(*[3]int)
+	**i.A = new([3]int)
+	***i.A = [3]int{1, 2, 3}
+	i.S = new(**[]int)
+	*i.S = new(*[]int)
+	**i.S = new([]int)
+	***i.S = []int{4, 5, 6}
+	i.M = new(***map[string]int)
+	*i.M = new(**map[string]int)
+	**i.M = new(*map[string]int)
+	***i.M = new(map[string]int)
+	****i.M = map[string]int{"one": 1, "two": 2, "three": 3}
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(i)
+	dec := NewDecoder(b)
+	var d Direct
+	err := dec.Decode(&d)
+	if err != nil {
+		t.Error("error: ", err)
+	}
+	if len(d.A) != 3 || d.A[0] != 1 || d.A[1] != 2 || d.A[2] != 3 {
+		t.Errorf("indirect to direct: d.A is %v not %v", d.A, ***i.A)
+	}
+	if len(d.S) != 3 || d.S[0] != 4 || d.S[1] != 5 || d.S[2] != 6 {
+		t.Errorf("indirect to direct: d.S is %v not %v", d.S, ***i.S)
+	}
+	if len(d.M) != 3 || d.M["one"] != 1 || d.M["two"] != 2 || d.M["three"] != 3 {
+		t.Errorf("indirect to direct: d.M is %v not %v", d.M, ***i.M)
+	}
+	// Marshal direct, unmarshal to indirect.
+	d.A = [3]int{11, 22, 33}
+	d.S = []int{44, 55, 66}
+	d.M = map[string]int{"four": 4, "five": 5, "six": 6}
+	i = new(Indirect)
+	b.Reset()
+	NewEncoder(b).Encode(d)
+	dec = NewDecoder(b)
+	err = dec.Decode(&i)
+	if err != nil {
+		t.Fatal("error: ", err)
+	}
+	if len(***i.A) != 3 || (***i.A)[0] != 11 || (***i.A)[1] != 22 || (***i.A)[2] != 33 {
+		t.Errorf("direct to indirect: ***i.A is %v not %v", ***i.A, d.A)
+	}
+	if len(***i.S) != 3 || (***i.S)[0] != 44 || (***i.S)[1] != 55 || (***i.S)[2] != 66 {
+		t.Errorf("direct to indirect: ***i.S is %v not %v", ***i.S, ***i.S)
+	}
+	if len(****i.M) != 3 || (****i.M)["four"] != 4 || (****i.M)["five"] != 5 || (****i.M)["six"] != 6 {
+		t.Errorf("direct to indirect: ****i.M is %v not %v", ****i.M, d.M)
+	}
+}
+
+// An interface with several implementations
+type Squarer interface {
+	Square() int
+}
+
+type Int int
+
+func (i Int) Square() int {
+	return int(i * i)
+}
+
+type Float float64
+
+func (f Float) Square() int {
+	return int(f * f)
+}
+
+type Vector []int
+
+func (v Vector) Square() int {
+	sum := 0
+	for _, x := range v {
+		sum += x * x
+	}
+	return sum
+}
+
+type Point struct {
+	X, Y int
+}
+
+func (p Point) Square() int {
+	return p.X*p.X + p.Y*p.Y
+}
+
+// A struct with interfaces in it.
+type InterfaceItem struct {
+	I             int
+	Sq1, Sq2, Sq3 Squarer
+	F             float64
+	Sq            []Squarer
+}
+
+// The same struct without interfaces
+type NoInterfaceItem struct {
+	I int
+	F float64
+}
+
+func TestInterface(t *testing.T) {
+	iVal := Int(3)
+	fVal := Float(5)
+	// Sending a Vector will require that the receiver define a type in the middle of
+	// receiving the value for item2.
+	vVal := Vector{1, 2, 3}
+	b := new(bytes.Buffer)
+	item1 := &InterfaceItem{1, iVal, fVal, vVal, 11.5, []Squarer{iVal, fVal, nil, vVal}}
+	// Register the types.
+	Register(Int(0))
+	Register(Float(0))
+	Register(Vector{})
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := InterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if item2.I != item1.I {
+		t.Error("normal int did not decode correctly")
+	}
+	if item2.Sq1 == nil || item2.Sq1.Square() != iVal.Square() {
+		t.Error("Int did not decode correctly")
+	}
+	if item2.Sq2 == nil || item2.Sq2.Square() != fVal.Square() {
+		t.Error("Float did not decode correctly")
+	}
+	if item2.Sq3 == nil || item2.Sq3.Square() != vVal.Square() {
+		t.Error("Vector did not decode correctly")
+	}
+	if item2.F != item1.F {
+		t.Error("normal float did not decode correctly")
+	}
+	// Now check that we received a slice of Squarers correctly, including a nil element
+	if len(item1.Sq) != len(item2.Sq) {
+		t.Fatalf("[]Squarer length wrong: got %d; expected %d", len(item2.Sq), len(item1.Sq))
+	}
+	for i, v1 := range item1.Sq {
+		v2 := item2.Sq[i]
+		if v1 == nil || v2 == nil {
+			if v1 != nil || v2 != nil {
+				t.Errorf("item %d inconsistent nils", i)
+			}
+			continue
+			if v1.Square() != v2.Square() {
+				t.Errorf("item %d inconsistent values: %v %v", i, v1, v2)
+			}
+		}
+	}
+}
+
+// A struct with all basic types, stored in interfaces.
+type BasicInterfaceItem struct {
+	Int, Int8, Int16, Int32, Int64      interface{}
+	Uint, Uint8, Uint16, Uint32, Uint64 interface{}
+	Float32, Float64                    interface{}
+	Complex64, Complex128               interface{}
+	Bool                                interface{}
+	String                              interface{}
+	Bytes                               interface{}
+}
+
+func TestInterfaceBasic(t *testing.T) {
+	b := new(bytes.Buffer)
+	item1 := &BasicInterfaceItem{
+		int(1), int8(1), int16(1), int32(1), int64(1),
+		uint(1), uint8(1), uint16(1), uint32(1), uint64(1),
+		float32(1), 1.0,
+		complex64(1i), complex128(1i),
+		true,
+		"hello",
+		[]byte("sailor"),
+	}
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := &BasicInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(item1, item2) {
+		t.Errorf("encode expected %v got %v", item1, item2)
+	}
+	// Hand check a couple for correct types.
+	if v, ok := item2.Bool.(bool); !ok || !v {
+		t.Error("boolean should be true")
+	}
+	if v, ok := item2.String.(string); !ok || v != item1.String.(string) {
+		t.Errorf("string should be %v is %v", item1.String, v)
+	}
+}
+
+type String string
+
+type PtrInterfaceItem struct {
+	Str1 interface{} // basic
+	Str2 interface{} // derived
+}
+
+// We'll send pointers; should receive values.
+// Also check that we can register T but send *T.
+func TestInterfacePointer(t *testing.T) {
+	b := new(bytes.Buffer)
+	str1 := "howdy"
+	str2 := String("kiddo")
+	item1 := &PtrInterfaceItem{
+		&str1,
+		&str2,
+	}
+	// Register the type.
+	Register(str2)
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := &PtrInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	// Hand test for correct types and values.
+	if v, ok := item2.Str1.(string); !ok || v != str1 {
+		t.Errorf("basic string failed: %q should be %q", v, str1)
+	}
+	if v, ok := item2.Str2.(String); !ok || v != str2 {
+		t.Errorf("derived type String failed: %q should be %q", v, str2)
+	}
+}
+
+func TestIgnoreInterface(t *testing.T) {
+	iVal := Int(3)
+	fVal := Float(5)
+	// Sending a Point will require that the receiver define a type in the middle of
+	// receiving the value for item2.
+	pVal := Point{2, 3}
+	b := new(bytes.Buffer)
+	item1 := &InterfaceItem{1, iVal, fVal, pVal, 11.5, nil}
+	// Register the types.
+	Register(Int(0))
+	Register(Float(0))
+	Register(Point{})
+	err := NewEncoder(b).Encode(item1)
+	if err != nil {
+		t.Error("expected no encode error; got", err)
+	}
+
+	item2 := NoInterfaceItem{}
+	err = NewDecoder(b).Decode(&item2)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if item2.I != item1.I {
+		t.Error("normal int did not decode correctly")
+	}
+	if item2.F != item2.F {
+		t.Error("normal float did not decode correctly")
+	}
+}
+
+type U struct {
+	A int
+	B string
+	c float64
+	D uint
+}
+
+func TestUnexportedFields(t *testing.T) {
+	var u0 U
+	u0.A = 17
+	u0.B = "hello"
+	u0.c = 3.14159
+	u0.D = 23
+	b := new(bytes.Buffer)
+	NewEncoder(b).Encode(u0)
+	dec := NewDecoder(b)
+	var u1 U
+	u1.c = 1234.
+	err := dec.Decode(&u1)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if u0.A != u0.A || u0.B != u1.B || u0.D != u1.D {
+		t.Errorf("u1->u0: expected %v; got %v", u0, u1)
+	}
+	if u1.c != 1234. {
+		t.Error("u1.c modified")
+	}
+}
+
+var singletons = []interface{}{
+	true,
+	7,
+	3.2,
+	"hello",
+	[3]int{11, 22, 33},
+	[]float32{0.5, 0.25, 0.125},
+	map[string]int{"one": 1, "two": 2},
+}
+
+func TestDebugSingleton(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	b := new(bytes.Buffer)
+	// Accumulate a number of values and print them out all at once.
+	for _, x := range singletons {
+		err := NewEncoder(b).Encode(x)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	debugFunc(b)
+}
+
+// A type that won't be defined in the gob until we send it in an interface value.
+type OnTheFly struct {
+	A int
+}
+
+type DT struct {
+	//	X OnTheFly
+	A     int
+	B     string
+	C     float64
+	I     interface{}
+	J     interface{}
+	I_nil interface{}
+	M     map[string]int
+	T     [3]int
+	S     []string
+}
+
+func TestDebugStruct(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	Register(OnTheFly{})
+	var dt DT
+	dt.A = 17
+	dt.B = "hello"
+	dt.C = 3.14159
+	dt.I = 271828
+	dt.J = OnTheFly{3}
+	dt.I_nil = nil
+	dt.M = map[string]int{"one": 1, "two": 2}
+	dt.T = [3]int{11, 22, 33}
+	dt.S = []string{"hi", "joe"}
+	b := new(bytes.Buffer)
+	err := NewEncoder(b).Encode(dt)
+	if err != nil {
+		t.Fatal("encode:", err)
+	}
+	debugBuffer := bytes.NewBuffer(b.Bytes())
+	dt2 := &DT{}
+	err = NewDecoder(b).Decode(&dt2)
+	if err != nil {
+		t.Error("decode:", err)
+	}
+	debugFunc(debugBuffer)
+}
diff --git a/src/pkg/gob/debug.go b/src/pkg/gob/debug.go
new file mode 100644
index 000000000..ce8a6ff5e
--- /dev/null
+++ b/src/pkg/gob/debug.go
@@ -0,0 +1,686 @@
+package gob
+
+// This file is not normally included in the gob package.  Used only for debugging the package itself.
+// Add debug.go to the files listed in the Makefile to add Debug to the gob package.
+// Except for reading uints, it is an implementation of a reader that is independent of
+// the one implemented by Decoder.
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+)
+
+var dumpBytes = false // If true, print the remaining bytes in the input buffer at each item.
+
+// Init installs the debugging facility. If this file is not compiled in the
+// package, the tests in codec_test.go are no-ops.
+func init() {
+	debugFunc = Debug
+}
+
+var (
+	blanks = bytes.Repeat([]byte{' '}, 3*10)
+	empty  = []byte(": \n")
+	tabs   = strings.Repeat("\t", 100)
+)
+
+// tab indents itself when printed.
+type tab int
+
+func (t tab) String() string {
+	n := int(t)
+	if n > len(tabs) {
+		n = len(tabs)
+	}
+	return tabs[0:n]
+}
+
+func (t tab) print() {
+	fmt.Fprint(os.Stderr, t)
+}
+
+// A peekReader wraps an io.Reader, allowing one to peek ahead to see
+// what's coming without stealing the data from the client of the Reader.
+type peekReader struct {
+	r    io.Reader
+	data []byte // read-ahead data
+}
+
+// newPeekReader returns a peekReader that wraps r.
+func newPeekReader(r io.Reader) *peekReader {
+	return &peekReader{r: r}
+}
+
+// Read is the usual method. It will first take data that has been read ahead.
+func (p *peekReader) Read(b []byte) (n int, err os.Error) {
+	if len(p.data) == 0 {
+		return p.r.Read(b)
+	}
+	// Satisfy what's possible from the read-ahead data.
+	n = copy(b, p.data)
+	// Move data down to beginning of slice, to avoid endless growth
+	copy(p.data, p.data[n:])
+	p.data = p.data[:len(p.data)-n]
+	return
+}
+
+// peek returns as many bytes as possible from the unread
+// portion of the stream, up to the length of b.
+func (p *peekReader) peek(b []byte) (n int, err os.Error) {
+	if len(p.data) > 0 {
+		n = copy(b, p.data)
+		if n == len(b) {
+			return
+		}
+		b = b[n:]
+	}
+	if len(b) == 0 {
+		return
+	}
+	m, e := io.ReadFull(p.r, b)
+	if m > 0 {
+		p.data = append(p.data, b[:m]...)
+	}
+	n += m
+	if e == io.ErrUnexpectedEOF {
+		// That means m > 0 but we reached EOF. If we got data
+		// we won't complain about not being able to peek enough.
+		if n > 0 {
+			e = nil
+		} else {
+			e = os.EOF
+		}
+	}
+	return n, e
+}
+
+type debugger struct {
+	mutex          sync.Mutex
+	remain         int  // the number of bytes known to remain in the input
+	remainingKnown bool // the value of 'remain' is valid
+	r              *peekReader
+	wireType       map[typeId]*wireType
+	tmp            []byte // scratch space for decoding uints.
+}
+
+// dump prints the next nBytes of the input.
+// It arranges to print the output aligned from call to
+// call, to make it easy to see what has been consumed.
+func (deb *debugger) dump(format string, args ...interface{}) {
+	if !dumpBytes {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+" ", args...)
+	if !deb.remainingKnown {
+		return
+	}
+	if deb.remain < 0 {
+		fmt.Fprintf(os.Stderr, "remaining byte count is negative! %d\n", deb.remain)
+		return
+	}
+	data := make([]byte, deb.remain)
+	n, _ := deb.r.peek(data)
+	if n == 0 {
+		os.Stderr.Write(empty)
+		return
+	}
+	b := new(bytes.Buffer)
+	fmt.Fprintf(b, "[%d]{\n", deb.remain)
+	// Blanks until first byte
+	lineLength := 0
+	if n := len(data); n%10 != 0 {
+		lineLength = 10 - n%10
+		fmt.Fprintf(b, "\t%s", blanks[:lineLength*3])
+	}
+	// 10 bytes per line
+	for len(data) > 0 {
+		if lineLength == 0 {
+			fmt.Fprint(b, "\t")
+		}
+		m := 10 - lineLength
+		lineLength = 0
+		if m > len(data) {
+			m = len(data)
+		}
+		fmt.Fprintf(b, "% x\n", data[:m])
+		data = data[m:]
+	}
+	fmt.Fprint(b, "}\n")
+	os.Stderr.Write(b.Bytes())
+}
+
+// Debug prints a human-readable representation of the gob data read from r.
+func Debug(r io.Reader) {
+	err := debug(r)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "gob debug: %s\n", err)
+	}
+}
+
+// debug implements Debug, but catches panics and returns
+// them as errors to be printed by Debug.
+func debug(r io.Reader) (err os.Error) {
+	defer catchError(&err)
+	fmt.Fprintln(os.Stderr, "Start of debugging")
+	deb := &debugger{
+		r:        newPeekReader(r),
+		wireType: make(map[typeId]*wireType),
+		tmp:      make([]byte, 16),
+	}
+	if b, ok := r.(*bytes.Buffer); ok {
+		deb.remain = b.Len()
+		deb.remainingKnown = true
+	}
+	deb.gobStream()
+	return
+}
+
+// note that we've consumed some bytes
+func (deb *debugger) consumed(n int) {
+	if deb.remainingKnown {
+		deb.remain -= n
+	}
+}
+
+// int64 decodes and returns the next integer, which must be present.
+// Don't call this if you could be at EOF.
+func (deb *debugger) int64() int64 {
+	return toInt(deb.uint64())
+}
+
+// uint64 returns and decodes the next unsigned integer, which must be present.
+// Don't call this if you could be at EOF.
+// TODO: handle errors better.
+func (deb *debugger) uint64() uint64 {
+	n, w, err := decodeUintReader(deb.r, deb.tmp)
+	if err != nil {
+		errorf("debug: read error: %s", err)
+	}
+	deb.consumed(w)
+	return n
+}
+
+// GobStream:
+//	DelimitedMessage* (until EOF)
+func (deb *debugger) gobStream() {
+	// Make sure we're single-threaded through here.
+	deb.mutex.Lock()
+	defer deb.mutex.Unlock()
+
+	for deb.delimitedMessage(0) {
+	}
+}
+
+// DelimitedMessage:
+//	uint(lengthOfMessage) Message
+func (deb *debugger) delimitedMessage(indent tab) bool {
+	for {
+		n := deb.loadBlock(true)
+		if n < 0 {
+			return false
+		}
+		deb.dump("Delimited message of length %d", n)
+		deb.message(indent)
+	}
+	return true
+}
+
+// loadBlock preps us to read a message
+// of the length specified next in the input. It returns
+// the length of the block. The argument tells whether
+// an EOF is acceptable now.  If it is and one is found,
+// the return value is negative.
+func (deb *debugger) loadBlock(eofOK bool) int {
+	n64, w, err := decodeUintReader(deb.r, deb.tmp) // deb.uint64 will error at EOF
+	if err != nil {
+		if eofOK && err == os.EOF {
+			return -1
+		}
+		errorf("debug: unexpected error: %s", err)
+	}
+	deb.consumed(w)
+	n := int(n64)
+	if n < 0 {
+		errorf("huge value for message length: %d", n64)
+	}
+	return int(n)
+}
+
+// Message:
+//	TypeSequence TypedValue
+// TypeSequence
+//	(TypeDefinition DelimitedTypeDefinition*)?
+// DelimitedTypeDefinition:
+//	uint(lengthOfTypeDefinition) TypeDefinition
+// TypedValue:
+//	int(typeId) Value
+func (deb *debugger) message(indent tab) bool {
+	for {
+		// Convert the uint64 to a signed integer typeId
+		uid := deb.int64()
+		id := typeId(uid)
+		deb.dump("type id=%d", id)
+		if id < 0 {
+			deb.typeDefinition(indent, -id)
+			n := deb.loadBlock(false)
+			deb.dump("Message of length %d", n)
+			continue
+		} else {
+			deb.value(indent, id)
+			break
+		}
+	}
+	return true
+}
+
+// Helper methods to make it easy to scan a type descriptor.
+
+// common returns the CommonType at the input point.
+func (deb *debugger) common() CommonType {
+	fieldNum := -1
+	name := ""
+	id := typeId(0)
+	for {
+		delta := deb.delta(-1)
+		if delta == 0 {
+			break
+		}
+		fieldNum += delta
+		switch fieldNum {
+		case 0:
+			name = deb.string()
+		case 1:
+			// Id typeId
+			id = deb.typeId()
+		default:
+			errorf("corrupted CommonType")
+		}
+	}
+	return CommonType{name, id}
+}
+
+// uint returns the unsigned int at the input point, as a uint (not uint64).
+func (deb *debugger) uint() uint {
+	return uint(deb.uint64())
+}
+
+// int returns the signed int at the input point, as an int (not int64).
+func (deb *debugger) int() int {
+	return int(deb.int64())
+}
+
+// typeId returns the type id at the input point.
+func (deb *debugger) typeId() typeId {
+	return typeId(deb.int64())
+}
+
+// string returns the string at the input point.
+func (deb *debugger) string() string {
+	x := int(deb.uint64())
+	b := make([]byte, x)
+	nb, _ := deb.r.Read(b)
+	if nb != x {
+		errorf("corrupted type")
+	}
+	deb.consumed(nb)
+	return string(b)
+}
+
+// delta returns the field delta at the input point.  The expect argument,
+// if non-negative, identifies what the value should be.
+func (deb *debugger) delta(expect int) int {
+	delta := int(deb.uint64())
+	if delta < 0 || (expect >= 0 && delta != expect) {
+		errorf("decode: corrupted type: delta %d expected %d", delta, expect)
+	}
+	return delta
+}
+
+// TypeDefinition:
+//	[int(-typeId) (already read)] encodingOfWireType
+func (deb *debugger) typeDefinition(indent tab, id typeId) {
+	deb.dump("type definition for id %d", id)
+	// Encoding is of a wireType. Decode the structure as usual
+	fieldNum := -1
+	wire := new(wireType)
+	// A wireType defines a single field.
+	delta := deb.delta(-1)
+	fieldNum += delta
+	switch fieldNum {
+	case 0: // array type, one field of {{Common}, elem, length}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is type Id of elem
+		deb.delta(1)
+		id := deb.typeId()
+		// Field number 3 is length
+		deb.delta(1)
+		length := deb.int()
+		wire.ArrayT = &arrayType{com, id, length}
+
+	case 1: // slice type, one field of {{Common}, elem}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is type Id of elem
+		deb.delta(1)
+		id := deb.typeId()
+		wire.SliceT = &sliceType{com, id}
+
+	case 2: // struct type, one field of {{Common}, []fieldType}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is slice of FieldType
+		deb.delta(1)
+		numField := int(deb.uint())
+		field := make([]*fieldType, numField)
+		for i := 0; i < numField; i++ {
+			field[i] = new(fieldType)
+			deb.delta(1) // field 0 of fieldType: name
+			field[i].Name = deb.string()
+			deb.delta(1) // field 1 of fieldType: id
+			field[i].Id = deb.typeId()
+			deb.delta(0) // end of fieldType
+		}
+		wire.StructT = &structType{com, field}
+
+	case 3: // map type, one field of {{Common}, key, elem}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		// Field number 1 is type Id of key
+		deb.delta(1)
+		keyId := deb.typeId()
+		// Field number 2 is type Id of elem
+		deb.delta(1)
+		elemId := deb.typeId()
+		wire.MapT = &mapType{com, keyId, elemId}
+	case 4: // GobEncoder type, one field of {{Common}}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		wire.GobEncoderT = &gobEncoderType{com}
+	default:
+		errorf("bad field in type %d", fieldNum)
+	}
+	deb.printWireType(indent, wire)
+	deb.delta(0) // end inner type (arrayType, etc.)
+	deb.delta(0) // end wireType
+	// Remember we've seen this type.
+	deb.wireType[id] = wire
+}
+
+// Value:
+//	SingletonValue | StructValue
+func (deb *debugger) value(indent tab, id typeId) {
+	wire, ok := deb.wireType[id]
+	if ok && wire.StructT != nil {
+		deb.structValue(indent, id)
+	} else {
+		deb.singletonValue(indent, id)
+	}
+}
+
+// SingletonValue:
+//	uint(0) FieldValue
+func (deb *debugger) singletonValue(indent tab, id typeId) {
+	deb.dump("Singleton value")
+	// is it a builtin type?
+	wire := deb.wireType[id]
+	_, ok := builtinIdToType[id]
+	if !ok && wire == nil {
+		errorf("type id %d not defined", id)
+	}
+	m := deb.uint64()
+	if m != 0 {
+		errorf("expected zero; got %d", m)
+	}
+	deb.fieldValue(indent, id)
+}
+
+// InterfaceValue:
+//	NilInterfaceValue | NonNilInterfaceValue
+func (deb *debugger) interfaceValue(indent tab) {
+	deb.dump("Start of interface value")
+	if nameLen := deb.uint64(); nameLen == 0 {
+		deb.nilInterfaceValue(indent)
+	} else {
+		deb.nonNilInterfaceValue(indent, int(nameLen))
+	}
+}
+
+// NilInterfaceValue:
+//	uint(0) [already read]
+func (deb *debugger) nilInterfaceValue(indent tab) int {
+	fmt.Fprintf(os.Stderr, "%snil interface\n", indent)
+	return 0
+}
+
+// NonNilInterfaceValue:
+//	ConcreteTypeName TypeSequence InterfaceContents
+// ConcreteTypeName:
+//	uint(lengthOfName) [already read=n] name
+// InterfaceContents:
+//	int(concreteTypeId) DelimitedValue
+// DelimitedValue:
+//	uint(length) Value
+func (deb *debugger) nonNilInterfaceValue(indent tab, nameLen int) {
+	// ConcreteTypeName
+	b := make([]byte, nameLen)
+	deb.r.Read(b) // TODO: CHECK THESE READS!!
+	deb.consumed(nameLen)
+	name := string(b)
+
+	for {
+		id := deb.typeId()
+		if id < 0 {
+			deb.typeDefinition(indent, -id)
+			n := deb.loadBlock(false)
+			deb.dump("Nested message of length %d", n)
+		} else {
+			// DelimitedValue
+			x := deb.uint64() // in case we want to ignore the value; we don't.
+			fmt.Fprintf(os.Stderr, "%sinterface value, type %q id=%d; valueLength %d\n", indent, name, id, x)
+			deb.value(indent, id)
+			break
+		}
+	}
+}
+
+// printCommonType prints a common type; used by printWireType.
+func (deb *debugger) printCommonType(indent tab, kind string, common *CommonType) {
+	indent.print()
+	fmt.Fprintf(os.Stderr, "%s %q id=%d\n", kind, common.Name, common.Id)
+}
+
+// printWireType prints the contents of a wireType.
+func (deb *debugger) printWireType(indent tab, wire *wireType) {
+	fmt.Fprintf(os.Stderr, "%stype definition {\n", indent)
+	indent++
+	switch {
+	case wire.ArrayT != nil:
+		deb.printCommonType(indent, "array", &wire.ArrayT.CommonType)
+		fmt.Fprintf(os.Stderr, "%slen %d\n", indent+1, wire.ArrayT.Len)
+		fmt.Fprintf(os.Stderr, "%selemid %d\n", indent+1, wire.ArrayT.Elem)
+	case wire.MapT != nil:
+		deb.printCommonType(indent, "map", &wire.MapT.CommonType)
+		fmt.Fprintf(os.Stderr, "%skey id=%d\n", indent+1, wire.MapT.Key)
+		fmt.Fprintf(os.Stderr, "%selem id=%d\n", indent+1, wire.MapT.Elem)
+	case wire.SliceT != nil:
+		deb.printCommonType(indent, "slice", &wire.SliceT.CommonType)
+		fmt.Fprintf(os.Stderr, "%selem id=%d\n", indent+1, wire.SliceT.Elem)
+	case wire.StructT != nil:
+		deb.printCommonType(indent, "struct", &wire.StructT.CommonType)
+		for i, field := range wire.StructT.Field {
+			fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\tid=%d\n", indent+1, i, field.Name, field.Id)
+		}
+	case wire.GobEncoderT != nil:
+		deb.printCommonType(indent, "GobEncoder", &wire.GobEncoderT.CommonType)
+	}
+	indent--
+	fmt.Fprintf(os.Stderr, "%s}\n", indent)
+}
+
+// fieldValue prints a value of any type, such as a struct field.
+// FieldValue:
+//	builtinValue | ArrayValue | MapValue | SliceValue | StructValue | InterfaceValue
+func (deb *debugger) fieldValue(indent tab, id typeId) {
+	_, ok := builtinIdToType[id]
+	if ok {
+		if id == tInterface {
+			deb.interfaceValue(indent)
+		} else {
+			deb.printBuiltin(indent, id)
+		}
+		return
+	}
+	wire, ok := deb.wireType[id]
+	if !ok {
+		errorf("type id %d not defined", id)
+	}
+	switch {
+	case wire.ArrayT != nil:
+		deb.arrayValue(indent, wire)
+	case wire.MapT != nil:
+		deb.mapValue(indent, wire)
+	case wire.SliceT != nil:
+		deb.sliceValue(indent, wire)
+	case wire.StructT != nil:
+		deb.structValue(indent, id)
+	case wire.GobEncoderT != nil:
+		deb.gobEncoderValue(indent, id)
+	default:
+		panic("bad wire type for field")
+	}
+}
+
+// printBuiltin prints a value not of a fundamental type, that is,
+// one whose type is known to gobs at bootstrap time.
+func (deb *debugger) printBuiltin(indent tab, id typeId) {
+	switch id {
+	case tBool:
+		x := deb.int64()
+		if x == 0 {
+			fmt.Fprintf(os.Stderr, "%sfalse\n", indent)
+		} else {
+			fmt.Fprintf(os.Stderr, "%strue\n", indent)
+		}
+	case tInt:
+		x := deb.int64()
+		fmt.Fprintf(os.Stderr, "%s%d\n", indent, x)
+	case tUint:
+		x := deb.int64()
+		fmt.Fprintf(os.Stderr, "%s%d\n", indent, x)
+	case tFloat:
+		x := deb.uint64()
+		fmt.Fprintf(os.Stderr, "%s%g\n", indent, floatFromBits(x))
+	case tComplex:
+		r := deb.uint64()
+		i := deb.uint64()
+		fmt.Fprintf(os.Stderr, "%s%g+%gi\n", indent, floatFromBits(r), floatFromBits(i))
+	case tBytes:
+		x := int(deb.uint64())
+		b := make([]byte, x)
+		deb.r.Read(b)
+		deb.consumed(x)
+		fmt.Fprintf(os.Stderr, "%s{% x}=%q\n", indent, b, b)
+	case tString:
+		x := int(deb.uint64())
+		b := make([]byte, x)
+		deb.r.Read(b)
+		deb.consumed(x)
+		fmt.Fprintf(os.Stderr, "%s%q\n", indent, b)
+	default:
+		panic("unknown builtin")
+	}
+}
+
+// ArrayValue:
+//	uint(n) FieldValue*n
+func (deb *debugger) arrayValue(indent tab, wire *wireType) {
+	elemId := wire.ArrayT.Elem
+	u := deb.uint64()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		deb.fieldValue(indent, elemId)
+	}
+	if length != wire.ArrayT.Len {
+		fmt.Fprintf(os.Stderr, "%s(wrong length for array: %d should be %d)\n", indent, length, wire.ArrayT.Len)
+	}
+}
+
+// MapValue:
+//	uint(n) (FieldValue FieldValue)*n  [n (key, value) pairs]
+func (deb *debugger) mapValue(indent tab, wire *wireType) {
+	keyId := wire.MapT.Key
+	elemId := wire.MapT.Elem
+	u := deb.uint64()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		deb.fieldValue(indent+1, keyId)
+		deb.fieldValue(indent+1, elemId)
+	}
+}
+
+// SliceValue:
+//	uint(n) (n FieldValue)
+func (deb *debugger) sliceValue(indent tab, wire *wireType) {
+	elemId := wire.SliceT.Elem
+	u := deb.uint64()
+	length := int(u)
+	deb.dump("Start of slice of length %d", length)
+
+	for i := 0; i < length; i++ {
+		deb.fieldValue(indent, elemId)
+	}
+}
+
+// StructValue:
+//	(uint(fieldDelta) FieldValue)*
+func (deb *debugger) structValue(indent tab, id typeId) {
+	deb.dump("Start of struct value of %q id=%d\n<<\n", id.name(), id)
+	fmt.Fprintf(os.Stderr, "%s%s struct {\n", indent, id.name())
+	wire, ok := deb.wireType[id]
+	if !ok {
+		errorf("type id %d not defined", id)
+	}
+	strct := wire.StructT
+	fieldNum := -1
+	indent++
+	for {
+		delta := deb.uint64()
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldNum += int(delta)
+		if fieldNum < 0 || fieldNum >= len(strct.Field) {
+			deb.dump("field number out of range: prevField=%d delta=%d", fieldNum-int(delta), delta)
+			break
+		}
+		fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\n", indent, fieldNum, wire.StructT.Field[fieldNum].Name)
+		deb.fieldValue(indent+1, strct.Field[fieldNum].Id)
+	}
+	indent--
+	fmt.Fprintf(os.Stderr, "%s} // end %s struct\n", indent, id.name())
+	deb.dump(">> End of struct value of type %d %q", id, id.name())
+}
+
+// GobEncoderValue:
+//	uint(n) byte*n
+func (deb *debugger) gobEncoderValue(indent tab, id typeId) {
+	len := deb.uint64()
+	deb.dump("GobEncoder value of %q id=%d, length %d\n", id.name(), id, len)
+	fmt.Fprintf(os.Stderr, "%s%s (implements GobEncoder)\n", indent, id.name())
+	data := make([]byte, len)
+	_, err := deb.r.Read(data)
+	if err != nil {
+		errorf("gobEncoder data read: %s", err)
+	}
+	fmt.Fprintf(os.Stderr, "%s[% .2x]\n", indent+1, data)
+}
diff --git a/src/pkg/gob/decode.go b/src/pkg/gob/decode.go
new file mode 100644
index 000000000..bf7cb95f2
--- /dev/null
+++ b/src/pkg/gob/decode.go
@@ -0,0 +1,1275 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+// TODO(rsc): When garbage collector changes, revisit
+// the allocations in this file that use unsafe.Pointer.
+
+import (
+	"bytes"
+	"io"
+	"math"
+	"os"
+	"reflect"
+	"unsafe"
+)
+
+var (
+	errBadUint = os.NewError("gob: encoded unsigned integer out of range")
+	errBadType = os.NewError("gob: unknown type id or corrupted data")
+	errRange   = os.NewError("gob: bad data: field numbers out of bounds")
+)
+
+// decoderState is the execution state of an instance of the decoder. A new state
+// is created for nested objects.
+type decoderState struct {
+	dec *Decoder
+	// The buffer is stored with an extra indirection because it may be replaced
+	// if we load a type during decode (when reading an interface value).
+	b        *bytes.Buffer
+	fieldnum int // the last field number read.
+	buf      []byte
+	next     *decoderState // for free list
+}
+
+// We pass the bytes.Buffer separately for easier testing of the infrastructure
+// without requiring a full Decoder.
+func (dec *Decoder) newDecoderState(buf *bytes.Buffer) *decoderState {
+	d := dec.freeList
+	if d == nil {
+		d = new(decoderState)
+		d.dec = dec
+		d.buf = make([]byte, uint64Size)
+	} else {
+		dec.freeList = d.next
+	}
+	d.b = buf
+	return d
+}
+
+func (dec *Decoder) freeDecoderState(d *decoderState) {
+	d.next = dec.freeList
+	dec.freeList = d
+}
+
+func overflow(name string) os.Error {
+	return os.NewError(`value for "` + name + `" out of range`)
+}
+
+// decodeUintReader reads an encoded unsigned integer from an io.Reader.
+// Used only by the Decoder to read the message length.
+func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Error) {
+	width = 1
+	_, err = r.Read(buf[0:width])
+	if err != nil {
+		return
+	}
+	b := buf[0]
+	if b <= 0x7f {
+		return uint64(b), width, nil
+	}
+	nb := -int(int8(b))
+	if nb > uint64Size {
+		err = errBadUint
+		return
+	}
+	var n int
+	n, err = io.ReadFull(r, buf[0:nb])
+	if err != nil {
+		if err == os.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return
+	}
+	// Could check that the high byte is zero but it's not worth it.
+	for i := 0; i < n; i++ {
+		x <<= 8
+		x |= uint64(buf[i])
+		width++
+	}
+	return
+}
+
+// decodeUint reads an encoded unsigned integer from state.r.
+// Does not check for overflow.
+func (state *decoderState) decodeUint() (x uint64) {
+	b, err := state.b.ReadByte()
+	if err != nil {
+		error(err)
+	}
+	if b <= 0x7f {
+		return uint64(b)
+	}
+	nb := -int(int8(b))
+	if nb > uint64Size {
+		error(errBadUint)
+	}
+	n, err := state.b.Read(state.buf[0:nb])
+	if err != nil {
+		error(err)
+	}
+	// Don't need to check error; it's safe to loop regardless.
+	// Could check that the high byte is zero but it's not worth it.
+	for i := 0; i < n; i++ {
+		x <<= 8
+		x |= uint64(state.buf[i])
+	}
+	return x
+}
+
+// decodeInt reads an encoded signed integer from state.r.
+// Does not check for overflow.
+func (state *decoderState) decodeInt() int64 {
+	x := state.decodeUint()
+	if x&1 != 0 {
+		return ^int64(x >> 1)
+	}
+	return int64(x >> 1)
+}
+
+// decOp is the signature of a decoding operator for a given type.
+type decOp func(i *decInstr, state *decoderState, p unsafe.Pointer)
+
+// The 'instructions' of the decoding machine
+type decInstr struct {
+	op     decOp
+	field  int      // field number of the wire type
+	indir  int      // how many pointer indirections to reach the value in the struct
+	offset uintptr  // offset in the structure of the field to encode
+	ovfl   os.Error // error message for overflow/underflow (for arrays, of the elements)
+}
+
+// Since the encoder writes no zeros, if we arrive at a decoder we have
+// a value to extract and store.  The field number has already been read
+// (it's how we knew to call this decoder).
+// Each decoder is responsible for handling any indirections associated
+// with the data structure.  If any pointer so reached is nil, allocation must
+// be done.
+
+// Walk the pointer hierarchy, allocating if we find a nil.  Stop one before the end.
+func decIndirect(p unsafe.Pointer, indir int) unsafe.Pointer {
+	for ; indir > 1; indir-- {
+		if *(*unsafe.Pointer)(p) == nil {
+			// Allocation required
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(unsafe.Pointer))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	return p
+}
+
+// ignoreUint discards a uint value with no destination.
+func ignoreUint(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	state.decodeUint()
+}
+
+// ignoreTwoUints discards a uint value with no destination. It's used to skip
+// complex values.
+func ignoreTwoUints(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	state.decodeUint()
+	state.decodeUint()
+}
+
+// decBool decodes a uint and stores it as a boolean through p.
+func decBool(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(bool))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*bool)(p) = state.decodeUint() != 0
+}
+
+// decInt8 decodes an integer and stores it as an int8 through p.
+func decInt8(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt8 || math.MaxInt8 < v {
+		error(i.ovfl)
+	} else {
+		*(*int8)(p) = int8(v)
+	}
+}
+
+// decUint8 decodes an unsigned integer and stores it as a uint8 through p.
+func decUint8(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint8 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint8)(p) = uint8(v)
+	}
+}
+
+// decInt16 decodes an integer and stores it as an int16 through p.
+func decInt16(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int16))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt16 || math.MaxInt16 < v {
+		error(i.ovfl)
+	} else {
+		*(*int16)(p) = int16(v)
+	}
+}
+
+// decUint16 decodes an unsigned integer and stores it as a uint16 through p.
+func decUint16(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint16))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint16 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint16)(p) = uint16(v)
+	}
+}
+
+// decInt32 decodes an integer and stores it as an int32 through p.
+func decInt32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeInt()
+	if v < math.MinInt32 || math.MaxInt32 < v {
+		error(i.ovfl)
+	} else {
+		*(*int32)(p) = int32(v)
+	}
+}
+
+// decUint32 decodes an unsigned integer and stores it as a uint32 through p.
+func decUint32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	v := state.decodeUint()
+	if math.MaxUint32 < v {
+		error(i.ovfl)
+	} else {
+		*(*uint32)(p) = uint32(v)
+	}
+}
+
+// decInt64 decodes an integer and stores it as an int64 through p.
+func decInt64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(int64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*int64)(p) = int64(state.decodeInt())
+}
+
+// decUint64 decodes an unsigned integer and stores it as a uint64 through p.
+func decUint64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*uint64)(p) = uint64(state.decodeUint())
+}
+
+// Floating-point numbers are transmitted as uint64s holding the bits
+// of the underlying representation.  They are sent byte-reversed, with
+// the exponent end coming out first, so integer floating point numbers
+// (for example) transmit more compactly.  This routine does the
+// unswizzling.
+func floatFromBits(u uint64) float64 {
+	var v uint64
+	for i := 0; i < 8; i++ {
+		v <<= 8
+		v |= u & 0xFF
+		u >>= 8
+	}
+	return math.Float64frombits(v)
+}
+
+// storeFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point
+// number, and stores it through p. It's a helper function for float32 and complex64.
+func storeFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	v := floatFromBits(state.decodeUint())
+	av := v
+	if av < 0 {
+		av = -av
+	}
+	// +Inf is OK in both 32- and 64-bit floats.  Underflow is always OK.
+	if math.MaxFloat32 < av && av <= math.MaxFloat64 {
+		error(i.ovfl)
+	} else {
+		*(*float32)(p) = float32(v)
+	}
+}
+
+// decFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point
+// number, and stores it through p.
+func decFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(float32))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	storeFloat32(i, state, p)
+}
+
+// decFloat64 decodes an unsigned integer, treats it as a 64-bit floating-point
+// number, and stores it through p.
+func decFloat64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(float64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	*(*float64)(p) = floatFromBits(uint64(state.decodeUint()))
+}
+
+// decComplex64 decodes a pair of unsigned integers, treats them as a
+// pair of floating point numbers, and stores them as a complex64 through p.
+// The real part comes first.
+func decComplex64(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex64))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	storeFloat32(i, state, p)
+	storeFloat32(i, state, unsafe.Pointer(uintptr(p)+unsafe.Sizeof(float32(0))))
+}
+
+// decComplex128 decodes a pair of unsigned integers, treats them as a
+// pair of floating point numbers, and stores them as a complex128 through p.
+// The real part comes first.
+func decComplex128(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex128))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	real := floatFromBits(uint64(state.decodeUint()))
+	imag := floatFromBits(uint64(state.decodeUint()))
+	*(*complex128)(p) = complex(real, imag)
+}
+
+// decUint8Array decodes byte array and stores through p a slice header
+// describing the data.
+// uint8 arrays are encoded as an unsigned count followed by the raw bytes.
+func decUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new([]uint8))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	b := make([]uint8, state.decodeUint())
+	state.b.Read(b)
+	*(*[]uint8)(p) = b
+}
+
+// decString decodes byte array and stores through p a string header
+// describing the data.
+// Strings are encoded as an unsigned count followed by the raw bytes.
+func decString(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	if i.indir > 0 {
+		if *(*unsafe.Pointer)(p) == nil {
+			*(*unsafe.Pointer)(p) = unsafe.Pointer(new(string))
+		}
+		p = *(*unsafe.Pointer)(p)
+	}
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+	// It would be a shame to do the obvious thing here,
+	//	*(*string)(p) = string(b)
+	// because we've already allocated the storage and this would
+	// allocate again and copy.  So we do this ugly hack, which is even
+	// even more unsafe than it looks as it depends the memory
+	// representation of a string matching the beginning of the memory
+	// representation of a byte slice (a byte slice is longer).
+	*(*string)(p) = *(*string)(unsafe.Pointer(&b))
+}
+
+// ignoreUint8Array skips over the data for a byte slice value with no destination.
+func ignoreUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) {
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+}
+
+// Execution engine
+
+// The encoder engine is an array of instructions indexed by field number of the incoming
+// decoder.  It is executed with random access according to field number.
+type decEngine struct {
+	instr    []decInstr
+	numInstr int // the number of active instructions
+}
+
+// allocate makes sure storage is available for an object of underlying type rtyp
+// that is indir levels of indirection through p.
+func allocate(rtyp reflect.Type, p uintptr, indir int) uintptr {
+	if indir == 0 {
+		return p
+	}
+	up := unsafe.Pointer(p)
+	if indir > 1 {
+		up = decIndirect(up, indir)
+	}
+	if *(*unsafe.Pointer)(up) == nil {
+		// Allocate object.
+		*(*unsafe.Pointer)(up) = unsafe.New(rtyp)
+	}
+	return *(*uintptr)(up)
+}
+
+// decodeSingle decodes a top-level value that is not a struct and stores it through p.
+// Such values are preceded by a zero, making them have the memory layout of a
+// struct field (although with an illegal field number).
+func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, p uintptr) (err os.Error) {
+	indir := ut.indir
+	if ut.isGobDecoder {
+		indir = int(ut.decIndir)
+	}
+	p = allocate(ut.base, p, indir)
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = singletonField
+	basep := p
+	delta := int(state.decodeUint())
+	if delta != 0 {
+		errorf("decode: corrupted data: non-zero delta for singleton")
+	}
+	instr := &engine.instr[singletonField]
+	ptr := unsafe.Pointer(basep) // offset will be zero
+	if instr.indir > 1 {
+		ptr = decIndirect(ptr, instr.indir)
+	}
+	instr.op(instr, state, ptr)
+	dec.freeDecoderState(state)
+	return nil
+}
+
+// decodeSingle decodes a top-level struct and stores it through p.
+// Indir is for the value, not the type.  At the time of the call it may
+// differ from ut.indir, which was computed when the engine was built.
+// This state cannot arise for decodeSingle, which is called directly
+// from the user's value, not from the innards of an engine.
+func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p uintptr, indir int) {
+	p = allocate(ut.base, p, indir)
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = -1
+	basep := p
+	for state.b.Len() > 0 {
+		delta := int(state.decodeUint())
+		if delta < 0 {
+			errorf("decode: corrupted data: negative delta")
+		}
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldnum := state.fieldnum + delta
+		if fieldnum >= len(engine.instr) {
+			error(errRange)
+			break
+		}
+		instr := &engine.instr[fieldnum]
+		p := unsafe.Pointer(basep + instr.offset)
+		if instr.indir > 1 {
+			p = decIndirect(p, instr.indir)
+		}
+		instr.op(instr, state, p)
+		state.fieldnum = fieldnum
+	}
+	dec.freeDecoderState(state)
+}
+
+// ignoreStruct discards the data for a struct with no destination.
+func (dec *Decoder) ignoreStruct(engine *decEngine) {
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = -1
+	for state.b.Len() > 0 {
+		delta := int(state.decodeUint())
+		if delta < 0 {
+			errorf("ignore decode: corrupted data: negative delta")
+		}
+		if delta == 0 { // struct terminator is zero delta fieldnum
+			break
+		}
+		fieldnum := state.fieldnum + delta
+		if fieldnum >= len(engine.instr) {
+			error(errRange)
+		}
+		instr := &engine.instr[fieldnum]
+		instr.op(instr, state, unsafe.Pointer(nil))
+		state.fieldnum = fieldnum
+	}
+	dec.freeDecoderState(state)
+}
+
+// ignoreSingle discards the data for a top-level non-struct value with no
+// destination. It's used when calling Decode with a nil value.
+func (dec *Decoder) ignoreSingle(engine *decEngine) {
+	state := dec.newDecoderState(&dec.buf)
+	state.fieldnum = singletonField
+	delta := int(state.decodeUint())
+	if delta != 0 {
+		errorf("decode: corrupted data: non-zero delta for singleton")
+	}
+	instr := &engine.instr[singletonField]
+	instr.op(instr, state, unsafe.Pointer(nil))
+	dec.freeDecoderState(state)
+}
+
+// decodeArrayHelper does the work for decoding arrays and slices.
+func (dec *Decoder) decodeArrayHelper(state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.Error) {
+	instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl}
+	for i := 0; i < length; i++ {
+		up := unsafe.Pointer(p)
+		if elemIndir > 1 {
+			up = decIndirect(up, elemIndir)
+		}
+		elemOp(instr, state, up)
+		p += uintptr(elemWid)
+	}
+}
+
+// decodeArray decodes an array and stores it through p, that is, p points to the zeroth element.
+// The length is an unsigned integer preceding the elements.  Even though the length is redundant
+// (it's part of the type), it's a useful check and is included in the encoding.
+func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, length, indir, elemIndir int, ovfl os.Error) {
+	if indir > 0 {
+		p = allocate(atyp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	if n := state.decodeUint(); n != uint64(length) {
+		errorf("length mismatch in decodeArray")
+	}
+	dec.decodeArrayHelper(state, p, elemOp, elemWid, length, elemIndir, ovfl)
+}
+
+// decodeIntoValue is a helper for map decoding.  Since maps are decoded using reflection,
+// unlike the other items we can't use a pointer directly.
+func decodeIntoValue(state *decoderState, op decOp, indir int, v reflect.Value, ovfl os.Error) reflect.Value {
+	instr := &decInstr{op, 0, indir, 0, ovfl}
+	up := unsafe.Pointer(unsafeAddr(v))
+	if indir > 1 {
+		up = decIndirect(up, indir)
+	}
+	op(instr, state, up)
+	return v
+}
+
+// decodeMap decodes a map and stores its header through p.
+// Maps are encoded as a length followed by key:value pairs.
+// Because the internals of maps are not visible to us, we must
+// use reflection rather than pointer magic.
+func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, p uintptr, keyOp, elemOp decOp, indir, keyIndir, elemIndir int, ovfl os.Error) {
+	if indir > 0 {
+		p = allocate(mtyp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	up := unsafe.Pointer(p)
+	if *(*unsafe.Pointer)(up) == nil { // maps are represented as a pointer in the runtime
+		// Allocate map.
+		*(*unsafe.Pointer)(up) = unsafe.Pointer(reflect.MakeMap(mtyp).Pointer())
+	}
+	// Maps cannot be accessed by moving addresses around the way
+	// that slices etc. can.  We must recover a full reflection value for
+	// the iteration.
+	v := reflect.ValueOf(unsafe.Unreflect(mtyp, unsafe.Pointer(p)))
+	n := int(state.decodeUint())
+	for i := 0; i < n; i++ {
+		key := decodeIntoValue(state, keyOp, keyIndir, allocValue(mtyp.Key()), ovfl)
+		elem := decodeIntoValue(state, elemOp, elemIndir, allocValue(mtyp.Elem()), ovfl)
+		v.SetMapIndex(key, elem)
+	}
+}
+
+// ignoreArrayHelper does the work for discarding arrays and slices.
+func (dec *Decoder) ignoreArrayHelper(state *decoderState, elemOp decOp, length int) {
+	instr := &decInstr{elemOp, 0, 0, 0, os.NewError("no error")}
+	for i := 0; i < length; i++ {
+		elemOp(instr, state, nil)
+	}
+}
+
+// ignoreArray discards the data for an array value with no destination.
+func (dec *Decoder) ignoreArray(state *decoderState, elemOp decOp, length int) {
+	if n := state.decodeUint(); n != uint64(length) {
+		errorf("length mismatch in ignoreArray")
+	}
+	dec.ignoreArrayHelper(state, elemOp, length)
+}
+
+// ignoreMap discards the data for a map value with no destination.
+func (dec *Decoder) ignoreMap(state *decoderState, keyOp, elemOp decOp) {
+	n := int(state.decodeUint())
+	keyInstr := &decInstr{keyOp, 0, 0, 0, os.NewError("no error")}
+	elemInstr := &decInstr{elemOp, 0, 0, 0, os.NewError("no error")}
+	for i := 0; i < n; i++ {
+		keyOp(keyInstr, state, nil)
+		elemOp(elemInstr, state, nil)
+	}
+}
+
+// decodeSlice decodes a slice and stores the slice header through p.
+// Slices are encoded as an unsigned length followed by the elements.
+func (dec *Decoder) decodeSlice(atyp reflect.Type, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl os.Error) {
+	n := int(uintptr(state.decodeUint()))
+	if indir > 0 {
+		up := unsafe.Pointer(p)
+		if *(*unsafe.Pointer)(up) == nil {
+			// Allocate the slice header.
+			*(*unsafe.Pointer)(up) = unsafe.Pointer(new([]unsafe.Pointer))
+		}
+		p = *(*uintptr)(up)
+	}
+	// Allocate storage for the slice elements, that is, the underlying array.
+	// Always write a header at p.
+	hdrp := (*reflect.SliceHeader)(unsafe.Pointer(p))
+	hdrp.Data = uintptr(unsafe.NewArray(atyp.Elem(), n))
+	hdrp.Len = n
+	hdrp.Cap = n
+	dec.decodeArrayHelper(state, hdrp.Data, elemOp, elemWid, n, elemIndir, ovfl)
+}
+
+// ignoreSlice skips over the data for a slice value with no destination.
+func (dec *Decoder) ignoreSlice(state *decoderState, elemOp decOp) {
+	dec.ignoreArrayHelper(state, elemOp, int(state.decodeUint()))
+}
+
+// setInterfaceValue sets an interface value to a concrete value,
+// but first it checks that the assignment will succeed.
+func setInterfaceValue(ivalue reflect.Value, value reflect.Value) {
+	if !value.Type().AssignableTo(ivalue.Type()) {
+		errorf("cannot assign value of type %s to %s", value.Type(), ivalue.Type())
+	}
+	ivalue.Set(value)
+}
+
+// decodeInterface decodes an interface value and stores it through p.
+// Interfaces are encoded as the name of a concrete type followed by a value.
+// If the name is empty, the value is nil and no value is sent.
+func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p uintptr, indir int) {
+	// Create a writable interface reflect.Value.  We need one even for the nil case.
+	ivalue := allocValue(ityp)
+	// Read the name of the concrete type.
+	b := make([]byte, state.decodeUint())
+	state.b.Read(b)
+	name := string(b)
+	if name == "" {
+		// Copy the representation of the nil interface value to the target.
+		// This is horribly unsafe and special.
+		*(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.InterfaceData()
+		return
+	}
+	// The concrete type must be registered.
+	typ, ok := nameToConcreteType[name]
+	if !ok {
+		errorf("name not registered for interface: %q", name)
+	}
+	// Read the type id of the concrete value.
+	concreteId := dec.decodeTypeSequence(true)
+	if concreteId < 0 {
+		error(dec.err)
+	}
+	// Byte count of value is next; we don't care what it is (it's there
+	// in case we want to ignore the value by skipping it completely).
+	state.decodeUint()
+	// Read the concrete value.
+	value := allocValue(typ)
+	dec.decodeValue(concreteId, value)
+	if dec.err != nil {
+		error(dec.err)
+	}
+	// Allocate the destination interface value.
+	if indir > 0 {
+		p = allocate(ityp, p, 1) // All but the last level has been allocated by dec.Indirect
+	}
+	// Assign the concrete value to the interface.
+	// Tread carefully; it might not satisfy the interface.
+	setInterfaceValue(ivalue, value)
+	// Copy the representation of the interface value to the target.
+	// This is horribly unsafe and special.
+	*(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.InterfaceData()
+}
+
+// ignoreInterface discards the data for an interface value with no destination.
+func (dec *Decoder) ignoreInterface(state *decoderState) {
+	// Read the name of the concrete type.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+	id := dec.decodeTypeSequence(true)
+	if id < 0 {
+		error(dec.err)
+	}
+	// At this point, the decoder buffer contains a delimited value. Just toss it.
+	state.b.Next(int(state.decodeUint()))
+}
+
+// decodeGobDecoder decodes something implementing the GobDecoder interface.
+// The data is encoded as a byte slice.
+func (dec *Decoder) decodeGobDecoder(state *decoderState, v reflect.Value) {
+	// Read the bytes for the value.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+	// We know it's a GobDecoder, so just call the method directly.
+	err = v.Interface().(GobDecoder).GobDecode(b)
+	if err != nil {
+		error(err)
+	}
+}
+
+// ignoreGobDecoder discards the data for a GobDecoder value with no destination.
+func (dec *Decoder) ignoreGobDecoder(state *decoderState) {
+	// Read the bytes for the value.
+	b := make([]byte, state.decodeUint())
+	_, err := state.b.Read(b)
+	if err != nil {
+		error(err)
+	}
+}
+
+// Index by Go types.
+var decOpTable = [...]decOp{
+	reflect.Bool:       decBool,
+	reflect.Int8:       decInt8,
+	reflect.Int16:      decInt16,
+	reflect.Int32:      decInt32,
+	reflect.Int64:      decInt64,
+	reflect.Uint8:      decUint8,
+	reflect.Uint16:     decUint16,
+	reflect.Uint32:     decUint32,
+	reflect.Uint64:     decUint64,
+	reflect.Float32:    decFloat32,
+	reflect.Float64:    decFloat64,
+	reflect.Complex64:  decComplex64,
+	reflect.Complex128: decComplex128,
+	reflect.String:     decString,
+}
+
+// Indexed by gob types.  tComplex will be added during type.init().
+var decIgnoreOpMap = map[typeId]decOp{
+	tBool:    ignoreUint,
+	tInt:     ignoreUint,
+	tUint:    ignoreUint,
+	tFloat:   ignoreUint,
+	tBytes:   ignoreUint8Array,
+	tString:  ignoreUint8Array,
+	tComplex: ignoreTwoUints,
+}
+
+// decOpFor returns the decoding op for the base type under rt and
+// the indirection count to reach it.
+func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) (*decOp, int) {
+	ut := userType(rt)
+	// If the type implements GobEncoder, we handle it without further processing.
+	if ut.isGobDecoder {
+		return dec.gobDecodeOpFor(ut)
+	}
+	// If this type is already in progress, it's a recursive type (e.g. map[string]*T).
+	// Return the pointer to the op we're already building.
+	if opPtr := inProgress[rt]; opPtr != nil {
+		return opPtr, ut.indir
+	}
+	typ := ut.base
+	indir := ut.indir
+	var op decOp
+	k := typ.Kind()
+	if int(k) < len(decOpTable) {
+		op = decOpTable[k]
+	}
+	if op == nil {
+		inProgress[rt] = &op
+		// Special cases
+		switch t := typ; t.Kind() {
+		case reflect.Array:
+			name = "element of " + name
+			elemId := dec.wireType[wireId].ArrayT.Elem
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.decodeArray(t, state, uintptr(p), *elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl)
+			}
+
+		case reflect.Map:
+			name = "element of " + name
+			keyId := dec.wireType[wireId].MapT.Key
+			elemId := dec.wireType[wireId].MapT.Elem
+			keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name, inProgress)
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				up := unsafe.Pointer(p)
+				state.dec.decodeMap(t, state, uintptr(up), *keyOp, *elemOp, i.indir, keyIndir, elemIndir, ovfl)
+			}
+
+		case reflect.Slice:
+			name = "element of " + name
+			if t.Elem().Kind() == reflect.Uint8 {
+				op = decUint8Array
+				break
+			}
+			var elemId typeId
+			if tt, ok := builtinIdToType[wireId]; ok {
+				elemId = tt.(*sliceType).Elem
+			} else {
+				elemId = dec.wireType[wireId].SliceT.Elem
+			}
+			elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress)
+			ovfl := overflow(name)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.decodeSlice(t, state, uintptr(p), *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl)
+			}
+
+		case reflect.Struct:
+			// Generate a closure that calls out to the engine for the nested type.
+			enginePtr, err := dec.getDecEnginePtr(wireId, userType(typ))
+			if err != nil {
+				error(err)
+			}
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				// indirect through enginePtr to delay evaluation for recursive structs.
+				dec.decodeStruct(*enginePtr, userType(typ), uintptr(p), i.indir)
+			}
+		case reflect.Interface:
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.decodeInterface(t, state, uintptr(p), i.indir)
+			}
+		}
+	}
+	if op == nil {
+		errorf("decode can't handle type %s", rt.String())
+	}
+	return &op, indir
+}
+
+// decIgnoreOpFor returns the decoding op for a field that has no destination.
+func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp {
+	op, ok := decIgnoreOpMap[wireId]
+	if !ok {
+		if wireId == tInterface {
+			// Special case because it's a method: the ignored item might
+			// define types and we need to record their state in the decoder.
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreInterface(state)
+			}
+			return op
+		}
+		// Special cases
+		wire := dec.wireType[wireId]
+		switch {
+		case wire == nil:
+			errorf("bad data: undefined type %s", wireId.string())
+		case wire.ArrayT != nil:
+			elemId := wire.ArrayT.Elem
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreArray(state, elemOp, wire.ArrayT.Len)
+			}
+
+		case wire.MapT != nil:
+			keyId := dec.wireType[wireId].MapT.Key
+			elemId := dec.wireType[wireId].MapT.Elem
+			keyOp := dec.decIgnoreOpFor(keyId)
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreMap(state, keyOp, elemOp)
+			}
+
+		case wire.SliceT != nil:
+			elemId := wire.SliceT.Elem
+			elemOp := dec.decIgnoreOpFor(elemId)
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreSlice(state, elemOp)
+			}
+
+		case wire.StructT != nil:
+			// Generate a closure that calls out to the engine for the nested type.
+			enginePtr, err := dec.getIgnoreEnginePtr(wireId)
+			if err != nil {
+				error(err)
+			}
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				// indirect through enginePtr to delay evaluation for recursive structs
+				state.dec.ignoreStruct(*enginePtr)
+			}
+
+		case wire.GobEncoderT != nil:
+			op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+				state.dec.ignoreGobDecoder(state)
+			}
+		}
+	}
+	if op == nil {
+		errorf("bad data: ignore can't handle type %s", wireId.string())
+	}
+	return op
+}
+
+// gobDecodeOpFor returns the op for a type that is known to implement
+// GobDecoder.
+func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) (*decOp, int) {
+	rcvrType := ut.user
+	if ut.decIndir == -1 {
+		rcvrType = reflect.PtrTo(rcvrType)
+	} else if ut.decIndir > 0 {
+		for i := int8(0); i < ut.decIndir; i++ {
+			rcvrType = rcvrType.Elem()
+		}
+	}
+	var op decOp
+	op = func(i *decInstr, state *decoderState, p unsafe.Pointer) {
+		// Caller has gotten us to within one indirection of our value.
+		if i.indir > 0 {
+			if *(*unsafe.Pointer)(p) == nil {
+				*(*unsafe.Pointer)(p) = unsafe.New(ut.base)
+			}
+		}
+		// Now p is a pointer to the base type.  Do we need to climb out to
+		// get to the receiver type?
+		var v reflect.Value
+		if ut.decIndir == -1 {
+			v = reflect.ValueOf(unsafe.Unreflect(rcvrType, unsafe.Pointer(&p)))
+		} else {
+			v = reflect.ValueOf(unsafe.Unreflect(rcvrType, p))
+		}
+		state.dec.decodeGobDecoder(state, v)
+	}
+	return &op, int(ut.indir)
+
+}
+
+// compatibleType asks: Are these two gob Types compatible?
+// Answers the question for basic types, arrays, maps and slices, plus
+// GobEncoder/Decoder pairs.
+// Structs are considered ok; fields will be checked later.
+func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId, inProgress map[reflect.Type]typeId) bool {
+	if rhs, ok := inProgress[fr]; ok {
+		return rhs == fw
+	}
+	inProgress[fr] = fw
+	ut := userType(fr)
+	wire, ok := dec.wireType[fw]
+	// If fr is a GobDecoder, the wire type must be GobEncoder.
+	// And if fr is not a GobDecoder, the wire type must not be either.
+	if ut.isGobDecoder != (ok && wire.GobEncoderT != nil) { // the parentheses look odd but are correct.
+		return false
+	}
+	if ut.isGobDecoder { // This test trumps all others.
+		return true
+	}
+	switch t := ut.base; t.Kind() {
+	default:
+		// chan, etc: cannot handle.
+		return false
+	case reflect.Bool:
+		return fw == tBool
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return fw == tInt
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return fw == tUint
+	case reflect.Float32, reflect.Float64:
+		return fw == tFloat
+	case reflect.Complex64, reflect.Complex128:
+		return fw == tComplex
+	case reflect.String:
+		return fw == tString
+	case reflect.Interface:
+		return fw == tInterface
+	case reflect.Array:
+		if !ok || wire.ArrayT == nil {
+			return false
+		}
+		array := wire.ArrayT
+		return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem, inProgress)
+	case reflect.Map:
+		if !ok || wire.MapT == nil {
+			return false
+		}
+		MapType := wire.MapT
+		return dec.compatibleType(t.Key(), MapType.Key, inProgress) && dec.compatibleType(t.Elem(), MapType.Elem, inProgress)
+	case reflect.Slice:
+		// Is it an array of bytes?
+		if t.Elem().Kind() == reflect.Uint8 {
+			return fw == tBytes
+		}
+		// Extract and compare element types.
+		var sw *sliceType
+		if tt, ok := builtinIdToType[fw]; ok {
+			sw = tt.(*sliceType)
+		} else {
+			sw = dec.wireType[fw].SliceT
+		}
+		elem := userType(t.Elem()).base
+		return sw != nil && dec.compatibleType(elem, sw.Elem, inProgress)
+	case reflect.Struct:
+		return true
+	}
+	return true
+}
+
+// typeString returns a human-readable description of the type identified by remoteId.
+func (dec *Decoder) typeString(remoteId typeId) string {
+	if t := idToType[remoteId]; t != nil {
+		// globally known type.
+		return t.string()
+	}
+	return dec.wireType[remoteId].string()
+}
+
+// compileSingle compiles the decoder engine for a non-struct top-level value, including
+// GobDecoders.
+func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) {
+	rt := ut.base
+	if ut.isGobDecoder {
+		rt = ut.user
+	}
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, 1) // one item
+	name := rt.String()                // best we can do
+	if !dec.compatibleType(rt, remoteId, make(map[reflect.Type]typeId)) {
+		return nil, os.NewError("gob: wrong type received for local value " + name + ": " + dec.typeString(remoteId))
+	}
+	op, indir := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp))
+	ovfl := os.NewError(`value for "` + name + `" out of range`)
+	engine.instr[singletonField] = decInstr{*op, singletonField, indir, 0, ovfl}
+	engine.numInstr = 1
+	return
+}
+
+// compileIgnoreSingle compiles the decoder engine for a non-struct top-level value that will be discarded.
+func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err os.Error) {
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, 1) // one item
+	op := dec.decIgnoreOpFor(remoteId)
+	ovfl := overflow(dec.typeString(remoteId))
+	engine.instr[0] = decInstr{op, 0, 0, 0, ovfl}
+	engine.numInstr = 1
+	return
+}
+
+// compileDec compiles the decoder engine for a value.  If the value is not a struct,
+// it calls out to compileSingle.
+func (dec *Decoder) compileDec(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) {
+	rt := ut.base
+	srt := rt
+	if srt.Kind() != reflect.Struct ||
+		ut.isGobDecoder {
+		return dec.compileSingle(remoteId, ut)
+	}
+	var wireStruct *structType
+	// Builtin types can come from global pool; the rest must be defined by the decoder.
+	// Also we know we're decoding a struct now, so the client must have sent one.
+	if t, ok := builtinIdToType[remoteId]; ok {
+		wireStruct, _ = t.(*structType)
+	} else {
+		wire := dec.wireType[remoteId]
+		if wire == nil {
+			error(errBadType)
+		}
+		wireStruct = wire.StructT
+	}
+	if wireStruct == nil {
+		errorf("type mismatch in decoder: want struct type %s; got non-struct", rt.String())
+	}
+	engine = new(decEngine)
+	engine.instr = make([]decInstr, len(wireStruct.Field))
+	seen := make(map[reflect.Type]*decOp)
+	// Loop over the fields of the wire type.
+	for fieldnum := 0; fieldnum < len(wireStruct.Field); fieldnum++ {
+		wireField := wireStruct.Field[fieldnum]
+		if wireField.Name == "" {
+			errorf("empty name for remote field of type %s", wireStruct.Name)
+		}
+		ovfl := overflow(wireField.Name)
+		// Find the field of the local type with the same name.
+		localField, present := srt.FieldByName(wireField.Name)
+		// TODO(r): anonymous names
+		if !present || !isExported(wireField.Name) {
+			op := dec.decIgnoreOpFor(wireField.Id)
+			engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl}
+			continue
+		}
+		if !dec.compatibleType(localField.Type, wireField.Id, make(map[reflect.Type]typeId)) {
+			errorf("wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name)
+		}
+		op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen)
+		engine.instr[fieldnum] = decInstr{*op, fieldnum, indir, uintptr(localField.Offset), ovfl}
+		engine.numInstr++
+	}
+	return
+}
+
+// getDecEnginePtr returns the engine for the specified type.
+func (dec *Decoder) getDecEnginePtr(remoteId typeId, ut *userTypeInfo) (enginePtr **decEngine, err os.Error) {
+	rt := ut.base
+	decoderMap, ok := dec.decoderCache[rt]
+	if !ok {
+		decoderMap = make(map[typeId]**decEngine)
+		dec.decoderCache[rt] = decoderMap
+	}
+	if enginePtr, ok = decoderMap[remoteId]; !ok {
+		// To handle recursive types, mark this engine as underway before compiling.
+		enginePtr = new(*decEngine)
+		decoderMap[remoteId] = enginePtr
+		*enginePtr, err = dec.compileDec(remoteId, ut)
+		if err != nil {
+			decoderMap[remoteId] = nil, false
+		}
+	}
+	return
+}
+
+// emptyStruct is the type we compile into when ignoring a struct value.
+type emptyStruct struct{}
+
+var emptyStructType = reflect.TypeOf(emptyStruct{})
+
+// getDecEnginePtr returns the engine for the specified type when the value is to be discarded.
+func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, err os.Error) {
+	var ok bool
+	if enginePtr, ok = dec.ignorerCache[wireId]; !ok {
+		// To handle recursive types, mark this engine as underway before compiling.
+		enginePtr = new(*decEngine)
+		dec.ignorerCache[wireId] = enginePtr
+		wire := dec.wireType[wireId]
+		if wire != nil && wire.StructT != nil {
+			*enginePtr, err = dec.compileDec(wireId, userType(emptyStructType))
+		} else {
+			*enginePtr, err = dec.compileIgnoreSingle(wireId)
+		}
+		if err != nil {
+			dec.ignorerCache[wireId] = nil, false
+		}
+	}
+	return
+}
+
+// decodeValue decodes the data stream representing a value and stores it in val.
+func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) {
+	defer catchError(&dec.err)
+	// If the value is nil, it means we should just ignore this item.
+	if !val.IsValid() {
+		dec.decodeIgnoredValue(wireId)
+		return
+	}
+	// Dereference down to the underlying struct type.
+	ut := userType(val.Type())
+	base := ut.base
+	var enginePtr **decEngine
+	enginePtr, dec.err = dec.getDecEnginePtr(wireId, ut)
+	if dec.err != nil {
+		return
+	}
+	engine := *enginePtr
+	if st := base; st.Kind() == reflect.Struct && !ut.isGobDecoder {
+		if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 {
+			name := base.Name()
+			errorf("type mismatch: no fields matched compiling decoder for %s", name)
+		}
+		dec.decodeStruct(engine, ut, uintptr(unsafeAddr(val)), ut.indir)
+	} else {
+		dec.decodeSingle(engine, ut, uintptr(unsafeAddr(val)))
+	}
+}
+
+// decodeIgnoredValue decodes the data stream representing a value of the specified type and discards it.
+func (dec *Decoder) decodeIgnoredValue(wireId typeId) {
+	var enginePtr **decEngine
+	enginePtr, dec.err = dec.getIgnoreEnginePtr(wireId)
+	if dec.err != nil {
+		return
+	}
+	wire := dec.wireType[wireId]
+	if wire != nil && wire.StructT != nil {
+		dec.ignoreStruct(*enginePtr)
+	} else {
+		dec.ignoreSingle(*enginePtr)
+	}
+}
+
+func init() {
+	var iop, uop decOp
+	switch reflect.TypeOf(int(0)).Bits() {
+	case 32:
+		iop = decInt32
+		uop = decUint32
+	case 64:
+		iop = decInt64
+		uop = decUint64
+	default:
+		panic("gob: unknown size of int/uint")
+	}
+	decOpTable[reflect.Int] = iop
+	decOpTable[reflect.Uint] = uop
+
+	// Finally uintptr
+	switch reflect.TypeOf(uintptr(0)).Bits() {
+	case 32:
+		uop = decUint32
+	case 64:
+		uop = decUint64
+	default:
+		panic("gob: unknown size of uintptr")
+	}
+	decOpTable[reflect.Uintptr] = uop
+}
+
+// Gob assumes it can call UnsafeAddr on any Value
+// in order to get a pointer it can copy data from.
+// Values that have just been created and do not point
+// into existing structs or slices cannot be addressed,
+// so simulate it by returning a pointer to a copy.
+// Each call allocates once.
+func unsafeAddr(v reflect.Value) uintptr {
+	if v.CanAddr() {
+		return v.UnsafeAddr()
+	}
+	x := reflect.New(v.Type()).Elem()
+	x.Set(v)
+	return x.UnsafeAddr()
+}
+
+// Gob depends on being able to take the address
+// of zeroed Values it creates, so use this wrapper instead
+// of the standard reflect.Zero.
+// Each call allocates once.
+func allocValue(t reflect.Type) reflect.Value {
+	return reflect.New(t).Elem()
+}
diff --git a/src/pkg/gob/decoder.go b/src/pkg/gob/decoder.go
new file mode 100644
index 000000000..281947132
--- /dev/null
+++ b/src/pkg/gob/decoder.go
@@ -0,0 +1,202 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// A Decoder manages the receipt of type and data information read from the
+// remote side of a connection.
+type Decoder struct {
+	mutex        sync.Mutex                              // each item must be received atomically
+	r            io.Reader                               // source of the data
+	buf          bytes.Buffer                            // buffer for more efficient i/o from r
+	wireType     map[typeId]*wireType                    // map from remote ID to local description
+	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
+	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
+	freeList     *decoderState                           // list of free decoderStates; avoids reallocation
+	countBuf     []byte                                  // used for decoding integers while parsing messages
+	tmp          []byte                                  // temporary storage for i/o; saves reallocating
+	err          os.Error
+}
+
+// NewDecoder returns a new decoder that reads from the io.Reader.
+func NewDecoder(r io.Reader) *Decoder {
+	dec := new(Decoder)
+	dec.r = bufio.NewReader(r)
+	dec.wireType = make(map[typeId]*wireType)
+	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
+	dec.ignorerCache = make(map[typeId]**decEngine)
+	dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes
+
+	return dec
+}
+
+// recvType loads the definition of a type.
+func (dec *Decoder) recvType(id typeId) {
+	// Have we already seen this type?  That's an error
+	if id < firstUserId || dec.wireType[id] != nil {
+		dec.err = os.NewError("gob: duplicate type received")
+		return
+	}
+
+	// Type:
+	wire := new(wireType)
+	dec.decodeValue(tWireType, reflect.ValueOf(wire))
+	if dec.err != nil {
+		return
+	}
+	// Remember we've seen this type.
+	dec.wireType[id] = wire
+}
+
+// recvMessage reads the next count-delimited item from the input. It is the converse
+// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
+func (dec *Decoder) recvMessage() bool {
+	// Read a count.
+	nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
+	if err != nil {
+		dec.err = err
+		return false
+	}
+	dec.readMessage(int(nbytes))
+	return dec.err == nil
+}
+
+// readMessage reads the next nbytes bytes from the input.
+func (dec *Decoder) readMessage(nbytes int) {
+	// Allocate the buffer.
+	if cap(dec.tmp) < nbytes {
+		dec.tmp = make([]byte, nbytes+100) // room to grow
+	}
+	dec.tmp = dec.tmp[:nbytes]
+
+	// Read the data
+	_, dec.err = io.ReadFull(dec.r, dec.tmp)
+	if dec.err != nil {
+		if dec.err == os.EOF {
+			dec.err = io.ErrUnexpectedEOF
+		}
+		return
+	}
+	dec.buf.Write(dec.tmp)
+}
+
+// toInt turns an encoded uint64 into an int, according to the marshaling rules.
+func toInt(x uint64) int64 {
+	i := int64(x >> 1)
+	if x&1 != 0 {
+		i = ^i
+	}
+	return i
+}
+
+func (dec *Decoder) nextInt() int64 {
+	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
+	if err != nil {
+		dec.err = err
+	}
+	return toInt(n)
+}
+
+func (dec *Decoder) nextUint() uint64 {
+	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
+	if err != nil {
+		dec.err = err
+	}
+	return n
+}
+
+// decodeTypeSequence parses:
+// TypeSequence
+//	(TypeDefinition DelimitedTypeDefinition*)?
+// and returns the type id of the next value.  It returns -1 at
+// EOF.  Upon return, the remainder of dec.buf is the value to be
+// decoded.  If this is an interface value, it can be ignored by
+// simply resetting that buffer.
+func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
+	for dec.err == nil {
+		if dec.buf.Len() == 0 {
+			if !dec.recvMessage() {
+				break
+			}
+		}
+		// Receive a type id.
+		id := typeId(dec.nextInt())
+		if id >= 0 {
+			// Value follows.
+			return id
+		}
+		// Type definition for (-id) follows.
+		dec.recvType(-id)
+		// When decoding an interface, after a type there may be a
+		// DelimitedValue still in the buffer.  Skip its count.
+		// (Alternatively, the buffer is empty and the byte count
+		// will be absorbed by recvMessage.)
+		if dec.buf.Len() > 0 {
+			if !isInterface {
+				dec.err = os.NewError("extra data in buffer")
+				break
+			}
+			dec.nextUint()
+		}
+	}
+	return -1
+}
+
+// Decode reads the next value from the connection and stores
+// it in the data represented by the empty interface value.
+// If e is nil, the value will be discarded. Otherwise,
+// the value underlying e must be a pointer to the
+// correct type for the next data item received.
+func (dec *Decoder) Decode(e interface{}) os.Error {
+	if e == nil {
+		return dec.DecodeValue(reflect.Value{})
+	}
+	value := reflect.ValueOf(e)
+	// If e represents a value as opposed to a pointer, the answer won't
+	// get back to the caller.  Make sure it's a pointer.
+	if value.Type().Kind() != reflect.Ptr {
+		dec.err = os.NewError("gob: attempt to decode into a non-pointer")
+		return dec.err
+	}
+	return dec.DecodeValue(value)
+}
+
+// DecodeValue reads the next value from the connection.
+// If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value.
+// Otherwise, it stores the value into v.  In that case, v must represent
+// a non-nil pointer to data or be an assignable reflect.Value (v.CanSet())
+func (dec *Decoder) DecodeValue(v reflect.Value) os.Error {
+	if v.IsValid() {
+		if v.Kind() == reflect.Ptr && !v.IsNil() {
+			// That's okay, we'll store through the pointer.
+		} else if !v.CanSet() {
+			return os.NewError("gob: DecodeValue of unassignable value")
+		}
+	}
+	// Make sure we're single-threaded through here.
+	dec.mutex.Lock()
+	defer dec.mutex.Unlock()
+
+	dec.buf.Reset() // In case data lingers from previous invocation.
+	dec.err = nil
+	id := dec.decodeTypeSequence(false)
+	if dec.err == nil {
+		dec.decodeValue(id, v)
+	}
+	return dec.err
+}
+
+// If debug.go is compiled into the program , debugFunc prints a human-readable
+// representation of the gob data read from r by calling that file's Debug function.
+// Otherwise it is nil.
+var debugFunc func(io.Reader)
diff --git a/src/pkg/gob/doc.go b/src/pkg/gob/doc.go
new file mode 100644
index 000000000..35d882afb
--- /dev/null
+++ b/src/pkg/gob/doc.go
@@ -0,0 +1,360 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package gob manages streams of gobs - binary values exchanged between an
+Encoder (transmitter) and a Decoder (receiver).  A typical use is transporting
+arguments and results of remote procedure calls (RPCs) such as those provided by
+package "rpc".
+
+A stream of gobs is self-describing.  Each data item in the stream is preceded by
+a specification of its type, expressed in terms of a small set of predefined
+types.  Pointers are not transmitted, but the things they point to are
+transmitted; that is, the values are flattened.  Recursive types work fine, but
+recursive values (data with cycles) are problematic.  This may change.
+
+To use gobs, create an Encoder and present it with a series of data items as
+values or addresses that can be dereferenced to values.  The Encoder makes sure
+all type information is sent before it is needed.  At the receive side, a
+Decoder retrieves values from the encoded stream and unpacks them into local
+variables.
+
+The source and destination values/types need not correspond exactly.  For structs,
+fields (identified by name) that are in the source but absent from the receiving
+variable will be ignored.  Fields that are in the receiving variable but missing
+from the transmitted type or value will be ignored in the destination.  If a field
+with the same name is present in both, their types must be compatible. Both the
+receiver and transmitter will do all necessary indirection and dereferencing to
+convert between gobs and actual Go values.  For instance, a gob type that is
+schematically,
+
+	struct { A, B int }
+
+can be sent from or received into any of these Go types:
+
+	struct { A, B int }	// the same
+	*struct { A, B int }	// extra indirection of the struct
+	struct { *A, **B int }	// extra indirection of the fields
+	struct { A, B int64 }	// different concrete value type; see below
+
+It may also be received into any of these:
+
+	struct { A, B int }	// the same
+	struct { B, A int }	// ordering doesn't matter; matching is by name
+	struct { A, B, C int }	// extra field (C) ignored
+	struct { B int }	// missing field (A) ignored; data will be dropped
+	struct { B, C int }	// missing field (A) ignored; extra field (C) ignored.
+
+Attempting to receive into these types will draw a decode error:
+
+	struct { A int; B uint }	// change of signedness for B
+	struct { A int; B float }	// change of type for B
+	struct { }			// no field names in common
+	struct { C, D int }		// no field names in common
+
+Integers are transmitted two ways: arbitrary precision signed integers or
+arbitrary precision unsigned integers.  There is no int8, int16 etc.
+discrimination in the gob format; there are only signed and unsigned integers.  As
+described below, the transmitter sends the value in a variable-length encoding;
+the receiver accepts the value and stores it in the destination variable.
+Floating-point numbers are always sent using IEEE-754 64-bit precision (see
+below).
+
+Signed integers may be received into any signed integer variable: int, int16, etc.;
+unsigned integers may be received into any unsigned integer variable; and floating
+point values may be received into any floating point variable.  However,
+the destination variable must be able to represent the value or the decode
+operation will fail.
+
+Structs, arrays and slices are also supported.  Strings and arrays of bytes are
+supported with a special, efficient representation (see below).
+
+Functions and channels cannot be sent in a gob.  Attempting
+to encode a value that contains one will fail.
+
+The rest of this comment documents the encoding, details that are not important
+for most users.  Details are presented bottom-up.
+
+An unsigned integer is sent one of two ways.  If it is less than 128, it is sent
+as a byte with that value.  Otherwise it is sent as a minimal-length big-endian
+(high byte first) byte stream holding the value, preceded by one byte holding the
+byte count, negated.  Thus 0 is transmitted as (00), 7 is transmitted as (07) and
+256 is transmitted as (FE 01 00).
+
+A boolean is encoded within an unsigned integer: 0 for false, 1 for true.
+
+A signed integer, i, is encoded within an unsigned integer, u.  Within u, bits 1
+upward contain the value; bit 0 says whether they should be complemented upon
+receipt.  The encode algorithm looks like this:
+
+	uint u;
+	if i < 0 {
+		u = (^i << 1) | 1	// complement i, bit 0 is 1
+	} else {
+		u = (i << 1)	// do not complement i, bit 0 is 0
+	}
+	encodeUnsigned(u)
+
+The low bit is therefore analogous to a sign bit, but making it the complement bit
+instead guarantees that the largest negative integer is not a special case.  For
+example, -129=^128=(^256>>1) encodes as (FE 01 01).
+
+Floating-point numbers are always sent as a representation of a float64 value.
+That value is converted to a uint64 using math.Float64bits.  The uint64 is then
+byte-reversed and sent as a regular unsigned integer.  The byte-reversal means the
+exponent and high-precision part of the mantissa go first.  Since the low bits are
+often zero, this can save encoding bytes.  For instance, 17.0 is encoded in only
+three bytes (FE 31 40).
+
+Strings and slices of bytes are sent as an unsigned count followed by that many
+uninterpreted bytes of the value.
+
+All other slices and arrays are sent as an unsigned count followed by that many
+elements using the standard gob encoding for their type, recursively.
+
+Maps are sent as an unsigned count followed by that man key, element
+pairs. Empty but non-nil maps are sent, so if the sender has allocated
+a map, the receiver will allocate a map even no elements are
+transmitted.
+
+Structs are sent as a sequence of (field number, field value) pairs.  The field
+value is sent using the standard gob encoding for its type, recursively.  If a
+field has the zero value for its type, it is omitted from the transmission.  The
+field number is defined by the type of the encoded struct: the first field of the
+encoded type is field 0, the second is field 1, etc.  When encoding a value, the
+field numbers are delta encoded for efficiency and the fields are always sent in
+order of increasing field number; the deltas are therefore unsigned.  The
+initialization for the delta encoding sets the field number to -1, so an unsigned
+integer field 0 with value 7 is transmitted as unsigned delta = 1, unsigned value
+= 7 or (01 07).  Finally, after all the fields have been sent a terminating mark
+denotes the end of the struct.  That mark is a delta=0 value, which has
+representation (00).
+
+Interface types are not checked for compatibility; all interface types are
+treated, for transmission, as members of a single "interface" type, analogous to
+int or []byte - in effect they're all treated as interface{}.  Interface values
+are transmitted as a string identifying the concrete type being sent (a name
+that must be pre-defined by calling Register), followed by a byte count of the
+length of the following data (so the value can be skipped if it cannot be
+stored), followed by the usual encoding of concrete (dynamic) value stored in
+the interface value.  (A nil interface value is identified by the empty string
+and transmits no value.) Upon receipt, the decoder verifies that the unpacked
+concrete item satisfies the interface of the receiving variable.
+
+The representation of types is described below.  When a type is defined on a given
+connection between an Encoder and Decoder, it is assigned a signed integer type
+id.  When Encoder.Encode(v) is called, it makes sure there is an id assigned for
+the type of v and all its elements and then it sends the pair (typeid, encoded-v)
+where typeid is the type id of the encoded type of v and encoded-v is the gob
+encoding of the value v.
+
+To define a type, the encoder chooses an unused, positive type id and sends the
+pair (-type id, encoded-type) where encoded-type is the gob encoding of a wireType
+description, constructed from these types:
+
+	type wireType struct {
+		ArrayT  *ArrayType
+		SliceT  *SliceType
+		StructT *StructType
+		MapT    *MapType
+	}
+	type ArrayType struct {
+		CommonType
+		Elem typeId
+		Len  int
+	}
+	type CommonType struct {
+		Name string // the name of the struct type
+		Id  int    // the id of the type, repeated so it's inside the type
+	}
+	type SliceType struct {
+		CommonType
+		Elem typeId
+	}
+	type StructType struct {
+		CommonType
+		Field []*fieldType // the fields of the struct.
+	}
+	type FieldType struct {
+		Name string // the name of the field.
+		Id   int    // the type id of the field, which must be already defined
+	}
+	type MapType struct {
+		CommonType
+		Key  typeId
+		Elem typeId
+	}
+
+If there are nested type ids, the types for all inner type ids must be defined
+before the top-level type id is used to describe an encoded-v.
+
+For simplicity in setup, the connection is defined to understand these types a
+priori, as well as the basic gob types int, uint, etc.  Their ids are:
+
+	bool        1
+	int         2
+	uint        3
+	float       4
+	[]byte      5
+	string      6
+	complex     7
+	interface   8
+	// gap for reserved ids.
+	WireType    16
+	ArrayType   17
+	CommonType  18
+	SliceType   19
+	StructType  20
+	FieldType   21
+	// 22 is slice of fieldType.
+	MapType     23
+
+Finally, each message created by a call to Encode is preceded by an encoded
+unsigned integer count of the number of bytes remaining in the message.  After
+the initial type name, interface values are wrapped the same way; in effect, the
+interface value acts like a recursive invocation of Encode.
+
+In summary, a gob stream looks like
+
+	(byteCount (-type id, encoding of a wireType)* (type id, encoding of a value))*
+
+where * signifies zero or more repetitions and the type id of a value must
+be predefined or be defined before the value in the stream.
+*/
+package gob
+
+/*
+Grammar:
+
+Tokens starting with a lower case letter are terminals; int(n)
+and uint(n) represent the signed/unsigned encodings of the value n.
+
+GobStream:
+	DelimitedMessage*
+DelimitedMessage:
+	uint(lengthOfMessage) Message
+Message:
+	TypeSequence TypedValue
+TypeSequence
+	(TypeDefinition DelimitedTypeDefinition*)?
+DelimitedTypeDefinition:
+	uint(lengthOfTypeDefinition) TypeDefinition
+TypedValue:
+	int(typeId) Value
+TypeDefinition:
+	int(-typeId) encodingOfWireType
+Value:
+	SingletonValue | StructValue
+SingletonValue:
+	uint(0) FieldValue
+FieldValue:
+	builtinValue | ArrayValue | MapValue | SliceValue | StructValue | InterfaceValue
+InterfaceValue:
+	NilInterfaceValue | NonNilInterfaceValue
+NilInterfaceValue:
+	uint(0)
+NonNilInterfaceValue:
+	ConcreteTypeName TypeSequence InterfaceContents
+ConcreteTypeName:
+	uint(lengthOfName) [already read=n] name
+InterfaceContents:
+	int(concreteTypeId) DelimitedValue
+DelimitedValue:
+	uint(length) Value
+ArrayValue:
+	uint(n) FieldValue*n [n elements]
+MapValue:
+	uint(n) (FieldValue FieldValue)*n  [n (key, value) pairs]
+SliceValue:
+	uint(n) FieldValue*n [n elements]
+StructValue:
+	(uint(fieldDelta) FieldValue)*
+*/
+
+/*
+For implementers and the curious, here is an encoded example.  Given
+	type Point struct {X, Y int}
+and the value
+	p := Point{22, 33}
+the bytes transmitted that encode p will be:
+	1f ff 81 03 01 01 05 50 6f 69 6e 74 01 ff 82 00
+	01 02 01 01 58 01 04 00 01 01 59 01 04 00 00 00
+	07 ff 82 01 2c 01 42 00
+They are determined as follows.
+
+Since this is the first transmission of type Point, the type descriptor
+for Point itself must be sent before the value.  This is the first type
+we've sent on this Encoder, so it has type id 65 (0 through 64 are
+reserved).
+
+	1f	// This item (a type descriptor) is 31 bytes long.
+	ff 81	// The negative of the id for the type we're defining, -65.
+		// This is one byte (indicated by FF = -1) followed by
+		// ^-65<<1 | 1.  The low 1 bit signals to complement the
+		// rest upon receipt.
+
+	// Now we send a type descriptor, which is itself a struct (wireType).
+	// The type of wireType itself is known (it's built in, as is the type of
+	// all its components), so we just need to send a *value* of type wireType
+	// that represents type "Point".
+	// Here starts the encoding of that value.
+	// Set the field number implicitly to -1; this is done at the beginning
+	// of every struct, including nested structs.
+	03	// Add 3 to field number; now 2 (wireType.structType; this is a struct).
+		// structType starts with an embedded commonType, which appears
+		// as a regular structure here too.
+	01	// add 1 to field number (now 0); start of embedded commonType.
+	01	// add 1 to field number (now 0, the name of the type)
+	05	// string is (unsigned) 5 bytes long
+	50 6f 69 6e 74	// wireType.structType.commonType.name = "Point"
+	01	// add 1 to field number (now 1, the id of the type)
+	ff 82	// wireType.structType.commonType._id = 65
+	00	// end of embedded wiretype.structType.commonType struct
+	01	// add 1 to field number (now 1, the field array in wireType.structType)
+	02	// There are two fields in the type (len(structType.field))
+	01	// Start of first field structure; add 1 to get field number 0: field[0].name
+	01	// 1 byte
+	58	// structType.field[0].name = "X"
+	01	// Add 1 to get field number 1: field[0].id
+	04	// structType.field[0].typeId is 2 (signed int).
+	00	// End of structType.field[0]; start structType.field[1]; set field number to -1.
+	01	// Add 1 to get field number 0: field[1].name
+	01	// 1 byte
+	59	// structType.field[1].name = "Y"
+	01	// Add 1 to get field number 1: field[0].id
+	04	// struct.Type.field[1].typeId is 2 (signed int).
+	00	// End of structType.field[1]; end of structType.field.
+	00	// end of wireType.structType structure
+	00	// end of wireType structure
+
+Now we can send the Point value.  Again the field number resets to -1:
+
+	07	// this value is 7 bytes long
+	ff 82	// the type number, 65 (1 byte (-FF) followed by 65<<1)
+	01	// add one to field number, yielding field 0
+	2c	// encoding of signed "22" (0x22 = 44 = 22<<1); Point.x = 22
+	01	// add one to field number, yielding field 1
+	42	// encoding of signed "33" (0x42 = 66 = 33<<1); Point.y = 33
+	00	// end of structure
+
+The type encoding is long and fairly intricate but we send it only once.
+If p is transmitted a second time, the type is already known so the
+output will be just:
+
+	07 ff 82 01 2c 01 42 00
+
+A single non-struct value at top level is transmitted like a field with
+delta tag 0.  For instance, a signed integer with value 3 presented as
+the argument to Encode will emit:
+
+	03 04 00 06
+
+Which represents:
+
+	03	// this value is 3 bytes long
+	04	// the type number, 2, represents an integer
+	00	// tag delta 0
+	06	// value 3
+
+*/
diff --git a/src/pkg/gob/dump.go b/src/pkg/gob/dump.go
new file mode 100644
index 000000000..1555f0fbb
--- /dev/null
+++ b/src/pkg/gob/dump.go
@@ -0,0 +1,22 @@
+package main
+
+// Need to compile package gob with debug.go to build this program.
+
+import (
+	"fmt"
+	"gob"
+	"os"
+)
+
+func main() {
+	var err os.Error
+	file := os.Stdin
+	if len(os.Args) > 1 {
+		file, err = os.Open(os.Args[1])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "dump: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	gob.Debug(file)
+}
diff --git a/src/pkg/gob/encode.go b/src/pkg/gob/encode.go
new file mode 100644
index 000000000..317014efd
--- /dev/null
+++ b/src/pkg/gob/encode.go
@@ -0,0 +1,717 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"unsafe"
+)
+
+const uint64Size = int(unsafe.Sizeof(uint64(0)))
+
+// encoderState is the global execution state of an instance of the encoder.
+// Field numbers are delta encoded and always increase. The field
+// number is initialized to -1 so 0 comes out as delta(1). A delta of
+// 0 terminates the structure.
+type encoderState struct {
+	enc      *Encoder
+	b        *bytes.Buffer
+	sendZero bool                 // encoding an array element or map key/value pair; send zero values
+	fieldnum int                  // the last field number written.
+	buf      [1 + uint64Size]byte // buffer used by the encoder; here to avoid allocation.
+	next     *encoderState        // for free list
+}
+
+func (enc *Encoder) newEncoderState(b *bytes.Buffer) *encoderState {
+	e := enc.freeList
+	if e == nil {
+		e = new(encoderState)
+		e.enc = enc
+	} else {
+		enc.freeList = e.next
+	}
+	e.sendZero = false
+	e.fieldnum = 0
+	e.b = b
+	return e
+}
+
+func (enc *Encoder) freeEncoderState(e *encoderState) {
+	e.next = enc.freeList
+	enc.freeList = e
+}
+
+// Unsigned integers have a two-state encoding.  If the number is less
+// than 128 (0 through 0x7F), its value is written directly.
+// Otherwise the value is written in big-endian byte order preceded
+// by the byte length, negated.
+
+// encodeUint writes an encoded unsigned integer to state.b.
+func (state *encoderState) encodeUint(x uint64) {
+	if x <= 0x7F {
+		err := state.b.WriteByte(uint8(x))
+		if err != nil {
+			error(err)
+		}
+		return
+	}
+	var n, m int
+	m = uint64Size
+	for n = 1; x > 0; n++ {
+		state.buf[m] = uint8(x)
+		x >>= 8
+		m--
+	}
+	state.buf[m] = uint8(-(n - 1))
+	n, err := state.b.Write(state.buf[m : uint64Size+1])
+	if err != nil {
+		error(err)
+	}
+}
+
+// encodeInt writes an encoded signed integer to state.w.
+// The low bit of the encoding says whether to bit complement the (other bits of the)
+// uint to recover the int.
+func (state *encoderState) encodeInt(i int64) {
+	var x uint64
+	if i < 0 {
+		x = uint64(^i<<1) | 1
+	} else {
+		x = uint64(i << 1)
+	}
+	state.encodeUint(uint64(x))
+}
+
+// encOp is the signature of an encoding operator for a given type.
+type encOp func(i *encInstr, state *encoderState, p unsafe.Pointer)
+
+// The 'instructions' of the encoding machine
+type encInstr struct {
+	op     encOp
+	field  int     // field number
+	indir  int     // how many pointer indirections to reach the value in the struct
+	offset uintptr // offset in the structure of the field to encode
+}
+
+// update emits a field number and updates the state to record its value for delta encoding.
+// If the instruction pointer is nil, it does nothing
+func (state *encoderState) update(instr *encInstr) {
+	if instr != nil {
+		state.encodeUint(uint64(instr.field - state.fieldnum))
+		state.fieldnum = instr.field
+	}
+}
+
+// Each encoder for a composite is responsible for handling any
+// indirections associated with the elements of the data structure.
+// If any pointer so reached is nil, no bytes are written.  If the
+// data item is zero, no bytes are written.  Single values - ints,
+// strings etc. - are indirected before calling their encoders.
+// Otherwise, the output (for a scalar) is the field number, as an
+// encoded integer, followed by the field data in its appropriate
+// format.
+
+// encIndirect dereferences p indir times and returns the result.
+func encIndirect(p unsafe.Pointer, indir int) unsafe.Pointer {
+	for ; indir > 0; indir-- {
+		p = *(*unsafe.Pointer)(p)
+		if p == nil {
+			return unsafe.Pointer(nil)
+		}
+	}
+	return p
+}
+
+// encBool encodes the bool with address p as an unsigned 0 or 1.
+func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	b := *(*bool)(p)
+	if b || state.sendZero {
+		state.update(i)
+		if b {
+			state.encodeUint(1)
+		} else {
+			state.encodeUint(0)
+		}
+	}
+}
+
+// encInt encodes the int with address p.
+func encInt(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint encodes the uint with address p.
+func encUint(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt8 encodes the int8 with address p.
+func encInt8(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int8)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint8 encodes the uint8 with address p.
+func encUint8(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint8)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt16 encodes the int16 with address p.
+func encInt16(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int16)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint16 encodes the uint16 with address p.
+func encUint16(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint16)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt32 encodes the int32 with address p.
+func encInt32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := int64(*(*int32)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encUint encodes the uint32 with address p.
+func encUint32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uint32)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encInt64 encodes the int64 with address p.
+func encInt64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := *(*int64)(p)
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeInt(v)
+	}
+}
+
+// encInt64 encodes the uint64 with address p.
+func encUint64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := *(*uint64)(p)
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encUintptr encodes the uintptr with address p.
+func encUintptr(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	v := uint64(*(*uintptr)(p))
+	if v != 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// floatBits returns a uint64 holding the bits of a floating-point number.
+// Floating-point numbers are transmitted as uint64s holding the bits
+// of the underlying representation.  They are sent byte-reversed, with
+// the exponent end coming out first, so integer floating point numbers
+// (for example) transmit more compactly.  This routine does the
+// swizzling.
+func floatBits(f float64) uint64 {
+	u := math.Float64bits(f)
+	var v uint64
+	for i := 0; i < 8; i++ {
+		v <<= 8
+		v |= u & 0xFF
+		u >>= 8
+	}
+	return v
+}
+
+// encFloat32 encodes the float32 with address p.
+func encFloat32(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	f := *(*float32)(p)
+	if f != 0 || state.sendZero {
+		v := floatBits(float64(f))
+		state.update(i)
+		state.encodeUint(v)
+	}
+}
+
+// encFloat64 encodes the float64 with address p.
+func encFloat64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	f := *(*float64)(p)
+	if f != 0 || state.sendZero {
+		state.update(i)
+		v := floatBits(f)
+		state.encodeUint(v)
+	}
+}
+
+// encComplex64 encodes the complex64 with address p.
+// Complex numbers are just a pair of floating-point numbers, real part first.
+func encComplex64(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	c := *(*complex64)(p)
+	if c != 0+0i || state.sendZero {
+		rpart := floatBits(float64(real(c)))
+		ipart := floatBits(float64(imag(c)))
+		state.update(i)
+		state.encodeUint(rpart)
+		state.encodeUint(ipart)
+	}
+}
+
+// encComplex128 encodes the complex128 with address p.
+func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	c := *(*complex128)(p)
+	if c != 0+0i || state.sendZero {
+		rpart := floatBits(real(c))
+		ipart := floatBits(imag(c))
+		state.update(i)
+		state.encodeUint(rpart)
+		state.encodeUint(ipart)
+	}
+}
+
+// encUint8Array encodes the byte slice whose header has address p.
+// Byte arrays are encoded as an unsigned count followed by the raw bytes.
+func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	b := *(*[]byte)(p)
+	if len(b) > 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(uint64(len(b)))
+		state.b.Write(b)
+	}
+}
+
+// encString encodes the string whose header has address p.
+// Strings are encoded as an unsigned count followed by the raw bytes.
+func encString(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	s := *(*string)(p)
+	if len(s) > 0 || state.sendZero {
+		state.update(i)
+		state.encodeUint(uint64(len(s)))
+		state.b.WriteString(s)
+	}
+}
+
+// encStructTerminator encodes the end of an encoded struct
+// as delta field number of 0.
+func encStructTerminator(i *encInstr, state *encoderState, p unsafe.Pointer) {
+	state.encodeUint(0)
+}
+
+// Execution engine
+
+// encEngine an array of instructions indexed by field number of the encoding
+// data, typically a struct.  It is executed top to bottom, walking the struct.
+type encEngine struct {
+	instr []encInstr
+}
+
+const singletonField = 0
+
+// encodeSingle encodes a single top-level non-struct value.
+func (enc *Encoder) encodeSingle(b *bytes.Buffer, engine *encEngine, basep uintptr) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = singletonField
+	// There is no surrounding struct to frame the transmission, so we must
+	// generate data even if the item is zero.  To do this, set sendZero.
+	state.sendZero = true
+	instr := &engine.instr[singletonField]
+	p := unsafe.Pointer(basep) // offset will be zero
+	if instr.indir > 0 {
+		if p = encIndirect(p, instr.indir); p == nil {
+			return
+		}
+	}
+	instr.op(instr, state, p)
+	enc.freeEncoderState(state)
+}
+
+// encodeStruct encodes a single struct value.
+func (enc *Encoder) encodeStruct(b *bytes.Buffer, engine *encEngine, basep uintptr) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	for i := 0; i < len(engine.instr); i++ {
+		instr := &engine.instr[i]
+		p := unsafe.Pointer(basep + instr.offset)
+		if instr.indir > 0 {
+			if p = encIndirect(p, instr.indir); p == nil {
+				continue
+			}
+		}
+		instr.op(instr, state, p)
+	}
+	enc.freeEncoderState(state)
+}
+
+// encodeArray encodes the array whose 0th element is at p.
+func (enc *Encoder) encodeArray(b *bytes.Buffer, p uintptr, op encOp, elemWid uintptr, elemIndir int, length int) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.sendZero = true
+	state.encodeUint(uint64(length))
+	for i := 0; i < length; i++ {
+		elemp := p
+		up := unsafe.Pointer(elemp)
+		if elemIndir > 0 {
+			if up = encIndirect(up, elemIndir); up == nil {
+				errorf("encodeArray: nil element")
+			}
+			elemp = uintptr(up)
+		}
+		op(nil, state, unsafe.Pointer(elemp))
+		p += uintptr(elemWid)
+	}
+	enc.freeEncoderState(state)
+}
+
+// encodeReflectValue is a helper for maps. It encodes the value v.
+func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir int) {
+	for i := 0; i < indir && v.IsValid(); i++ {
+		v = reflect.Indirect(v)
+	}
+	if !v.IsValid() {
+		errorf("encodeReflectValue: nil element")
+	}
+	op(nil, state, unsafe.Pointer(unsafeAddr(v)))
+}
+
+// encodeMap encodes a map as unsigned count followed by key:value pairs.
+// Because map internals are not exposed, we must use reflection rather than
+// addresses.
+func (enc *Encoder) encodeMap(b *bytes.Buffer, mv reflect.Value, keyOp, elemOp encOp, keyIndir, elemIndir int) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.sendZero = true
+	keys := mv.MapKeys()
+	state.encodeUint(uint64(len(keys)))
+	for _, key := range keys {
+		encodeReflectValue(state, key, keyOp, keyIndir)
+		encodeReflectValue(state, mv.MapIndex(key), elemOp, elemIndir)
+	}
+	enc.freeEncoderState(state)
+}
+
+// encodeInterface encodes the interface value iv.
+// To send an interface, we send a string identifying the concrete type, followed
+// by the type identifier (which might require defining that type right now), followed
+// by the concrete value.  A nil value gets sent as the empty string for the name,
+// followed by no value.
+func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) {
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.sendZero = true
+	if iv.IsNil() {
+		state.encodeUint(0)
+		return
+	}
+
+	ut := userType(iv.Elem().Type())
+	name, ok := concreteTypeToName[ut.base]
+	if !ok {
+		errorf("type not registered for interface: %s", ut.base)
+	}
+	// Send the name.
+	state.encodeUint(uint64(len(name)))
+	_, err := state.b.WriteString(name)
+	if err != nil {
+		error(err)
+	}
+	// Define the type id if necessary.
+	enc.sendTypeDescriptor(enc.writer(), state, ut)
+	// Send the type id.
+	enc.sendTypeId(state, ut)
+	// Encode the value into a new buffer.  Any nested type definitions
+	// should be written to b, before the encoded value.
+	enc.pushWriter(b)
+	data := new(bytes.Buffer)
+	enc.encode(data, iv.Elem(), ut)
+	if enc.err != nil {
+		error(enc.err)
+	}
+	enc.popWriter()
+	enc.writeMessage(b, data)
+	if enc.err != nil {
+		error(err)
+	}
+	enc.freeEncoderState(state)
+}
+
+// isZero returns whether the value is the zero of its type.
+func isZero(val reflect.Value) bool {
+	switch val.Kind() {
+	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+		return val.Len() == 0
+	case reflect.Bool:
+		return !val.Bool()
+	case reflect.Complex64, reflect.Complex128:
+		return val.Complex() == 0
+	case reflect.Chan, reflect.Func, reflect.Ptr:
+		return val.IsNil()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return val.Int() == 0
+	case reflect.Float32, reflect.Float64:
+		return val.Float() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return val.Uint() == 0
+	}
+	panic("unknown type in isZero " + val.Type().String())
+}
+
+// encGobEncoder encodes a value that implements the GobEncoder interface.
+// The data is sent as a byte array.
+func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, v reflect.Value) {
+	// TODO: should we catch panics from the called method?
+	// We know it's a GobEncoder, so just call the method directly.
+	data, err := v.Interface().(GobEncoder).GobEncode()
+	if err != nil {
+		error(err)
+	}
+	state := enc.newEncoderState(b)
+	state.fieldnum = -1
+	state.encodeUint(uint64(len(data)))
+	state.b.Write(data)
+	enc.freeEncoderState(state)
+}
+
+var encOpTable = [...]encOp{
+	reflect.Bool:       encBool,
+	reflect.Int:        encInt,
+	reflect.Int8:       encInt8,
+	reflect.Int16:      encInt16,
+	reflect.Int32:      encInt32,
+	reflect.Int64:      encInt64,
+	reflect.Uint:       encUint,
+	reflect.Uint8:      encUint8,
+	reflect.Uint16:     encUint16,
+	reflect.Uint32:     encUint32,
+	reflect.Uint64:     encUint64,
+	reflect.Uintptr:    encUintptr,
+	reflect.Float32:    encFloat32,
+	reflect.Float64:    encFloat64,
+	reflect.Complex64:  encComplex64,
+	reflect.Complex128: encComplex128,
+	reflect.String:     encString,
+}
+
+// encOpFor returns (a pointer to) the encoding op for the base type under rt and
+// the indirection count to reach it.
+func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp) (*encOp, int) {
+	ut := userType(rt)
+	// If the type implements GobEncoder, we handle it without further processing.
+	if ut.isGobEncoder {
+		return enc.gobEncodeOpFor(ut)
+	}
+	// If this type is already in progress, it's a recursive type (e.g. map[string]*T).
+	// Return the pointer to the op we're already building.
+	if opPtr := inProgress[rt]; opPtr != nil {
+		return opPtr, ut.indir
+	}
+	typ := ut.base
+	indir := ut.indir
+	k := typ.Kind()
+	var op encOp
+	if int(k) < len(encOpTable) {
+		op = encOpTable[k]
+	}
+	if op == nil {
+		inProgress[rt] = &op
+		// Special cases
+		switch t := typ; t.Kind() {
+		case reflect.Slice:
+			if t.Elem().Kind() == reflect.Uint8 {
+				op = encUint8Array
+				break
+			}
+			// Slices have a header; we decode it to find the underlying array.
+			elemOp, indir := enc.encOpFor(t.Elem(), inProgress)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				slice := (*reflect.SliceHeader)(p)
+				if !state.sendZero && slice.Len == 0 {
+					return
+				}
+				state.update(i)
+				state.enc.encodeArray(state.b, slice.Data, *elemOp, t.Elem().Size(), indir, int(slice.Len))
+			}
+		case reflect.Array:
+			// True arrays have size in the type.
+			elemOp, indir := enc.encOpFor(t.Elem(), inProgress)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				state.update(i)
+				state.enc.encodeArray(state.b, uintptr(p), *elemOp, t.Elem().Size(), indir, t.Len())
+			}
+		case reflect.Map:
+			keyOp, keyIndir := enc.encOpFor(t.Key(), inProgress)
+			elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				// Maps cannot be accessed by moving addresses around the way
+				// that slices etc. can.  We must recover a full reflection value for
+				// the iteration.
+				v := reflect.ValueOf(unsafe.Unreflect(t, unsafe.Pointer(p)))
+				mv := reflect.Indirect(v)
+				// We send zero-length (but non-nil) maps because the
+				// receiver might want to use the map.  (Maps don't use append.)
+				if !state.sendZero && mv.IsNil() {
+					return
+				}
+				state.update(i)
+				state.enc.encodeMap(state.b, mv, *keyOp, *elemOp, keyIndir, elemIndir)
+			}
+		case reflect.Struct:
+			// Generate a closure that calls out to the engine for the nested type.
+			enc.getEncEngine(userType(typ))
+			info := mustGetTypeInfo(typ)
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				state.update(i)
+				// indirect through info to delay evaluation for recursive structs
+				state.enc.encodeStruct(state.b, info.encoder, uintptr(p))
+			}
+		case reflect.Interface:
+			op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+				// Interfaces transmit the name and contents of the concrete
+				// value they contain.
+				v := reflect.ValueOf(unsafe.Unreflect(t, unsafe.Pointer(p)))
+				iv := reflect.Indirect(v)
+				if !state.sendZero && (!iv.IsValid() || iv.IsNil()) {
+					return
+				}
+				state.update(i)
+				state.enc.encodeInterface(state.b, iv)
+			}
+		}
+	}
+	if op == nil {
+		errorf("can't happen: encode type %s", rt.String())
+	}
+	return &op, indir
+}
+
+// gobEncodeOpFor returns the op for a type that is known to implement
+// GobEncoder.
+func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) {
+	rt := ut.user
+	if ut.encIndir == -1 {
+		rt = reflect.PtrTo(rt)
+	} else if ut.encIndir > 0 {
+		for i := int8(0); i < ut.encIndir; i++ {
+			rt = rt.Elem()
+		}
+	}
+	var op encOp
+	op = func(i *encInstr, state *encoderState, p unsafe.Pointer) {
+		var v reflect.Value
+		if ut.encIndir == -1 {
+			// Need to climb up one level to turn value into pointer.
+			v = reflect.ValueOf(unsafe.Unreflect(rt, unsafe.Pointer(&p)))
+		} else {
+			v = reflect.ValueOf(unsafe.Unreflect(rt, p))
+		}
+		if !state.sendZero && isZero(v) {
+			return
+		}
+		state.update(i)
+		state.enc.encodeGobEncoder(state.b, v)
+	}
+	return &op, int(ut.encIndir) // encIndir: op will get called with p == address of receiver.
+}
+
+// compileEnc returns the engine to compile the type.
+func (enc *Encoder) compileEnc(ut *userTypeInfo) *encEngine {
+	srt := ut.base
+	engine := new(encEngine)
+	seen := make(map[reflect.Type]*encOp)
+	rt := ut.base
+	if ut.isGobEncoder {
+		rt = ut.user
+	}
+	if !ut.isGobEncoder &&
+		srt.Kind() == reflect.Struct {
+		for fieldNum, wireFieldNum := 0, 0; fieldNum < srt.NumField(); fieldNum++ {
+			f := srt.Field(fieldNum)
+			if !isExported(f.Name) {
+				continue
+			}
+			op, indir := enc.encOpFor(f.Type, seen)
+			engine.instr = append(engine.instr, encInstr{*op, wireFieldNum, indir, uintptr(f.Offset)})
+			wireFieldNum++
+		}
+		if srt.NumField() > 0 && len(engine.instr) == 0 {
+			errorf("type %s has no exported fields", rt)
+		}
+		engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, 0, 0})
+	} else {
+		engine.instr = make([]encInstr, 1)
+		op, indir := enc.encOpFor(rt, seen)
+		engine.instr[0] = encInstr{*op, singletonField, indir, 0} // offset is zero
+	}
+	return engine
+}
+
+// getEncEngine returns the engine to compile the type.
+// typeLock must be held (or we're in initialization and guaranteed single-threaded).
+func (enc *Encoder) getEncEngine(ut *userTypeInfo) *encEngine {
+	info, err1 := getTypeInfo(ut)
+	if err1 != nil {
+		error(err1)
+	}
+	if info.encoder == nil {
+		// mark this engine as underway before compiling to handle recursive types.
+		info.encoder = new(encEngine)
+		info.encoder = enc.compileEnc(ut)
+	}
+	return info.encoder
+}
+
+// lockAndGetEncEngine is a function that locks and compiles.
+// This lets us hold the lock only while compiling, not when encoding.
+func (enc *Encoder) lockAndGetEncEngine(ut *userTypeInfo) *encEngine {
+	typeLock.Lock()
+	defer typeLock.Unlock()
+	return enc.getEncEngine(ut)
+}
+
+func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInfo) {
+	defer catchError(&enc.err)
+	engine := enc.lockAndGetEncEngine(ut)
+	indir := ut.indir
+	if ut.isGobEncoder {
+		indir = int(ut.encIndir)
+	}
+	for i := 0; i < indir; i++ {
+		value = reflect.Indirect(value)
+	}
+	if !ut.isGobEncoder && value.Type().Kind() == reflect.Struct {
+		enc.encodeStruct(b, engine, unsafeAddr(value))
+	} else {
+		enc.encodeSingle(b, engine, unsafeAddr(value))
+	}
+}
diff --git a/src/pkg/gob/encoder.go b/src/pkg/gob/encoder.go
new file mode 100644
index 000000000..96101d92b
--- /dev/null
+++ b/src/pkg/gob/encoder.go
@@ -0,0 +1,243 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"reflect"
+	"sync"
+)
+
+// An Encoder manages the transmission of type and data information to the
+// other side of a connection.
+type Encoder struct {
+	mutex      sync.Mutex              // each item must be sent atomically
+	w          []io.Writer             // where to send the data
+	sent       map[reflect.Type]typeId // which types we've already sent
+	countState *encoderState           // stage for writing counts
+	freeList   *encoderState           // list of free encoderStates; avoids reallocation
+	buf        []byte                  // for collecting the output.
+	byteBuf    bytes.Buffer            // buffer for top-level encoderState
+	err        os.Error
+}
+
+// NewEncoder returns a new encoder that will transmit on the io.Writer.
+func NewEncoder(w io.Writer) *Encoder {
+	enc := new(Encoder)
+	enc.w = []io.Writer{w}
+	enc.sent = make(map[reflect.Type]typeId)
+	enc.countState = enc.newEncoderState(new(bytes.Buffer))
+	return enc
+}
+
+// writer() returns the innermost writer the encoder is using
+func (enc *Encoder) writer() io.Writer {
+	return enc.w[len(enc.w)-1]
+}
+
+// pushWriter adds a writer to the encoder.
+func (enc *Encoder) pushWriter(w io.Writer) {
+	enc.w = append(enc.w, w)
+}
+
+// popWriter pops the innermost writer.
+func (enc *Encoder) popWriter() {
+	enc.w = enc.w[0 : len(enc.w)-1]
+}
+
+func (enc *Encoder) badType(rt reflect.Type) {
+	enc.setError(os.NewError("gob: can't encode type " + rt.String()))
+}
+
+func (enc *Encoder) setError(err os.Error) {
+	if enc.err == nil { // remember the first.
+		enc.err = err
+	}
+}
+
+// writeMessage sends the data item preceded by a unsigned count of its length.
+func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) {
+	enc.countState.encodeUint(uint64(b.Len()))
+	// Build the buffer.
+	countLen := enc.countState.b.Len()
+	total := countLen + b.Len()
+	if total > len(enc.buf) {
+		enc.buf = make([]byte, total+1000) // extra for growth
+	}
+	// Place the length before the data.
+	// TODO(r): avoid the extra copy here.
+	enc.countState.b.Read(enc.buf[0:countLen])
+	// Now the data.
+	b.Read(enc.buf[countLen:total])
+	// Write the data.
+	_, err := w.Write(enc.buf[0:total])
+	if err != nil {
+		enc.setError(err)
+	}
+}
+
+// sendActualType sends the requested type, without further investigation, unless
+// it's been sent before.
+func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTypeInfo, actual reflect.Type) (sent bool) {
+	if _, alreadySent := enc.sent[actual]; alreadySent {
+		return false
+	}
+	typeLock.Lock()
+	info, err := getTypeInfo(ut)
+	typeLock.Unlock()
+	if err != nil {
+		enc.setError(err)
+		return
+	}
+	// Send the pair (-id, type)
+	// Id:
+	state.encodeInt(-int64(info.id))
+	// Type:
+	enc.encode(state.b, reflect.ValueOf(info.wire), wireTypeUserInfo)
+	enc.writeMessage(w, state.b)
+	if enc.err != nil {
+		return
+	}
+
+	// Remember we've sent this type, both what the user gave us and the base type.
+	enc.sent[ut.base] = info.id
+	if ut.user != ut.base {
+		enc.sent[ut.user] = info.id
+	}
+	// Now send the inner types
+	switch st := actual; st.Kind() {
+	case reflect.Struct:
+		for i := 0; i < st.NumField(); i++ {
+			enc.sendType(w, state, st.Field(i).Type)
+		}
+	case reflect.Array, reflect.Slice:
+		enc.sendType(w, state, st.Elem())
+	case reflect.Map:
+		enc.sendType(w, state, st.Key())
+		enc.sendType(w, state, st.Elem())
+	}
+	return true
+}
+
+// sendType sends the type info to the other side, if necessary. 
+func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
+	ut := userType(origt)
+	if ut.isGobEncoder {
+		// The rules are different: regardless of the underlying type's representation,
+		// we need to tell the other side that this exact type is a GobEncoder.
+		return enc.sendActualType(w, state, ut, ut.user)
+	}
+
+	// It's a concrete value, so drill down to the base type.
+	switch rt := ut.base; rt.Kind() {
+	default:
+		// Basic types and interfaces do not need to be described.
+		return
+	case reflect.Slice:
+		// If it's []uint8, don't send; it's considered basic.
+		if rt.Elem().Kind() == reflect.Uint8 {
+			return
+		}
+		// Otherwise we do send.
+		break
+	case reflect.Array:
+		// arrays must be sent so we know their lengths and element types.
+		break
+	case reflect.Map:
+		// maps must be sent so we know their lengths and key/value types.
+		break
+	case reflect.Struct:
+		// structs must be sent so we know their fields.
+		break
+	case reflect.Chan, reflect.Func:
+		// Probably a bad field in a struct.
+		enc.badType(rt)
+		return
+	}
+
+	return enc.sendActualType(w, state, ut, ut.base)
+}
+
+// Encode transmits the data item represented by the empty interface value,
+// guaranteeing that all necessary type information has been transmitted first.
+func (enc *Encoder) Encode(e interface{}) os.Error {
+	return enc.EncodeValue(reflect.ValueOf(e))
+}
+
+// sendTypeDescriptor makes sure the remote side knows about this type.
+// It will send a descriptor if this is the first time the type has been
+// sent.
+func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) {
+	// Make sure the type is known to the other side.
+	// First, have we already sent this type?
+	rt := ut.base
+	if ut.isGobEncoder {
+		rt = ut.user
+	}
+	if _, alreadySent := enc.sent[rt]; !alreadySent {
+		// No, so send it.
+		sent := enc.sendType(w, state, rt)
+		if enc.err != nil {
+			return
+		}
+		// If the type info has still not been transmitted, it means we have
+		// a singleton basic type (int, []byte etc.) at top level.  We don't
+		// need to send the type info but we do need to update enc.sent.
+		if !sent {
+			typeLock.Lock()
+			info, err := getTypeInfo(ut)
+			typeLock.Unlock()
+			if err != nil {
+				enc.setError(err)
+				return
+			}
+			enc.sent[rt] = info.id
+		}
+	}
+}
+
+// sendTypeId sends the id, which must have already been defined.
+func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) {
+	// Identify the type of this top-level value.
+	state.encodeInt(int64(enc.sent[ut.base]))
+}
+
+// EncodeValue transmits the data item represented by the reflection value,
+// guaranteeing that all necessary type information has been transmitted first.
+func (enc *Encoder) EncodeValue(value reflect.Value) os.Error {
+	// Make sure we're single-threaded through here, so multiple
+	// goroutines can share an encoder.
+	enc.mutex.Lock()
+	defer enc.mutex.Unlock()
+
+	// Remove any nested writers remaining due to previous errors.
+	enc.w = enc.w[0:1]
+
+	ut, err := validUserType(value.Type())
+	if err != nil {
+		return err
+	}
+
+	enc.err = nil
+	enc.byteBuf.Reset()
+	state := enc.newEncoderState(&enc.byteBuf)
+
+	enc.sendTypeDescriptor(enc.writer(), state, ut)
+	enc.sendTypeId(state, ut)
+	if enc.err != nil {
+		return enc.err
+	}
+
+	// Encode the object.
+	enc.encode(state.b, value, ut)
+	if enc.err == nil {
+		enc.writeMessage(enc.writer(), state.b)
+	}
+
+	enc.freeEncoderState(state)
+	return enc.err
+}
diff --git a/src/pkg/gob/encoder_test.go b/src/pkg/gob/encoder_test.go
new file mode 100644
index 000000000..f5ee423cb
--- /dev/null
+++ b/src/pkg/gob/encoder_test.go
@@ -0,0 +1,580 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type ET2 struct {
+	X string
+}
+
+type ET1 struct {
+	A    int
+	Et2  *ET2
+	Next *ET1
+}
+
+// Like ET1 but with a different name for a field
+type ET3 struct {
+	A             int
+	Et2           *ET2
+	DifferentNext *ET1
+}
+
+// Like ET1 but with a different type for a field
+type ET4 struct {
+	A    int
+	Et2  float64
+	Next int
+}
+
+func TestEncoderDecoder(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	et1 := new(ET1)
+	et1.A = 7
+	et1.Et2 = new(ET2)
+	err := enc.Encode(et1)
+	if err != nil {
+		t.Error("encoder fail:", err)
+	}
+	dec := NewDecoder(b)
+	newEt1 := new(ET1)
+	err = dec.Decode(newEt1)
+	if err != nil {
+		t.Fatal("error decoding ET1:", err)
+	}
+
+	if !reflect.DeepEqual(et1, newEt1) {
+		t.Fatalf("invalid data for et1: expected %+v; got %+v", *et1, *newEt1)
+	}
+	if b.Len() != 0 {
+		t.Error("not at eof;", b.Len(), "bytes left")
+	}
+
+	enc.Encode(et1)
+	newEt1 = new(ET1)
+	err = dec.Decode(newEt1)
+	if err != nil {
+		t.Fatal("round 2: error decoding ET1:", err)
+	}
+	if !reflect.DeepEqual(et1, newEt1) {
+		t.Fatalf("round 2: invalid data for et1: expected %+v; got %+v", *et1, *newEt1)
+	}
+	if b.Len() != 0 {
+		t.Error("round 2: not at eof;", b.Len(), "bytes left")
+	}
+
+	// Now test with a running encoder/decoder pair that we recognize a type mismatch.
+	err = enc.Encode(et1)
+	if err != nil {
+		t.Error("round 3: encoder fail:", err)
+	}
+	newEt2 := new(ET2)
+	err = dec.Decode(newEt2)
+	if err == nil {
+		t.Fatal("round 3: expected `bad type' error decoding ET2")
+	}
+}
+
+// Run one value through the encoder/decoder, but use the wrong type.
+// Input is always an ET1; we compare it to whatever is under 'e'.
+func badTypeCheck(e interface{}, shouldFail bool, msg string, t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	et1 := new(ET1)
+	et1.A = 7
+	et1.Et2 = new(ET2)
+	err := enc.Encode(et1)
+	if err != nil {
+		t.Error("encoder fail:", err)
+	}
+	dec := NewDecoder(b)
+	err = dec.Decode(e)
+	if shouldFail && err == nil {
+		t.Error("expected error for", msg)
+	}
+	if !shouldFail && err != nil {
+		t.Error("unexpected error for", msg, err)
+	}
+}
+
+// Test that we recognize a bad type the first time.
+func TestWrongTypeDecoder(t *testing.T) {
+	badTypeCheck(new(ET2), true, "no fields in common", t)
+	badTypeCheck(new(ET3), false, "different name of field", t)
+	badTypeCheck(new(ET4), true, "different type of field", t)
+}
+
+func corruptDataCheck(s string, err os.Error, t *testing.T) {
+	b := bytes.NewBufferString(s)
+	dec := NewDecoder(b)
+	err1 := dec.Decode(new(ET2))
+	if err1 != err {
+		t.Errorf("from %q expected error %s; got %s", s, err, err1)
+	}
+}
+
+// Check that we survive bad data.
+func TestBadData(t *testing.T) {
+	corruptDataCheck("", os.EOF, t)
+	corruptDataCheck("\x7Fhi", io.ErrUnexpectedEOF, t)
+	corruptDataCheck("\x03now is the time for all good men", errBadType, t)
+}
+
+// Types not supported by the Encoder.
+var unsupportedValues = []interface{}{
+	make(chan int),
+	func(a int) bool { return true },
+}
+
+func TestUnsupported(t *testing.T) {
+	var b bytes.Buffer
+	enc := NewEncoder(&b)
+	for _, v := range unsupportedValues {
+		err := enc.Encode(v)
+		if err == nil {
+			t.Errorf("expected error for %T; got none", v)
+		}
+	}
+}
+
+func encAndDec(in, out interface{}) os.Error {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(in)
+	if err != nil {
+		return err
+	}
+	dec := NewDecoder(b)
+	err = dec.Decode(out)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestTypeToPtrType(t *testing.T) {
+	// Encode a T, decode a *T
+	type Type0 struct {
+		A int
+	}
+	t0 := Type0{7}
+	t0p := new(Type0)
+	if err := encAndDec(t0, t0p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestPtrTypeToType(t *testing.T) {
+	// Encode a *T, decode a T
+	type Type1 struct {
+		A uint
+	}
+	t1p := &Type1{17}
+	var t1 Type1
+	if err := encAndDec(t1, t1p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTypeToPtrPtrPtrPtrType(t *testing.T) {
+	type Type2 struct {
+		A ****float64
+	}
+	t2 := Type2{}
+	t2.A = new(***float64)
+	*t2.A = new(**float64)
+	**t2.A = new(*float64)
+	***t2.A = new(float64)
+	****t2.A = 27.4
+	t2pppp := new(***Type2)
+	if err := encAndDec(t2, t2pppp); err != nil {
+		t.Fatal(err)
+	}
+	if ****(****t2pppp).A != ****t2.A {
+		t.Errorf("wrong value after decode: %g not %g", ****(****t2pppp).A, ****t2.A)
+	}
+}
+
+func TestSlice(t *testing.T) {
+	type Type3 struct {
+		A []string
+	}
+	t3p := &Type3{[]string{"hello", "world"}}
+	var t3 Type3
+	if err := encAndDec(t3, t3p); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestValueError(t *testing.T) {
+	// Encode a *T, decode a T
+	type Type4 struct {
+		A int
+	}
+	t4p := &Type4{3}
+	var t4 Type4 // note: not a pointer.
+	if err := encAndDec(t4p, t4); err == nil || strings.Index(err.String(), "pointer") < 0 {
+		t.Error("expected error about pointer; got", err)
+	}
+}
+
+func TestArray(t *testing.T) {
+	type Type5 struct {
+		A [3]string
+		B [3]byte
+	}
+	type Type6 struct {
+		A [2]string // can't hold t5.a
+	}
+	t5 := Type5{[3]string{"hello", ",", "world"}, [3]byte{1, 2, 3}}
+	var t5p Type5
+	if err := encAndDec(t5, &t5p); err != nil {
+		t.Error(err)
+	}
+	var t6 Type6
+	if err := encAndDec(t5, &t6); err == nil {
+		t.Error("should fail with mismatched array sizes")
+	}
+}
+
+func TestRecursiveMapType(t *testing.T) {
+	type recursiveMap map[string]recursiveMap
+	r1 := recursiveMap{"A": recursiveMap{"B": nil, "C": nil}, "D": nil}
+	r2 := make(recursiveMap)
+	if err := encAndDec(r1, &r2); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestRecursiveSliceType(t *testing.T) {
+	type recursiveSlice []recursiveSlice
+	r1 := recursiveSlice{0: recursiveSlice{0: nil}, 1: nil}
+	r2 := make(recursiveSlice, 0)
+	if err := encAndDec(r1, &r2); err != nil {
+		t.Error(err)
+	}
+}
+
+// Regression test for bug: must send zero values inside arrays
+func TestDefaultsInArray(t *testing.T) {
+	type Type7 struct {
+		B []bool
+		I []int
+		S []string
+		F []float64
+	}
+	t7 := Type7{
+		[]bool{false, false, true},
+		[]int{0, 0, 1},
+		[]string{"hi", "", "there"},
+		[]float64{0, 0, 1},
+	}
+	var t7p Type7
+	if err := encAndDec(t7, &t7p); err != nil {
+		t.Error(err)
+	}
+}
+
+var testInt int
+var testFloat32 float32
+var testString string
+var testSlice []string
+var testMap map[string]int
+var testArray [7]int
+
+type SingleTest struct {
+	in  interface{}
+	out interface{}
+	err string
+}
+
+var singleTests = []SingleTest{
+	{17, &testInt, ""},
+	{float32(17.5), &testFloat32, ""},
+	{"bike shed", &testString, ""},
+	{[]string{"bike", "shed", "paint", "color"}, &testSlice, ""},
+	{map[string]int{"seven": 7, "twelve": 12}, &testMap, ""},
+	{[7]int{4, 55, 0, 0, 0, 0, 0}, &testArray, ""}, // case that once triggered a bug
+	{[7]int{4, 55, 1, 44, 22, 66, 1234}, &testArray, ""},
+
+	// Decode errors
+	{172, &testFloat32, "wrong type"},
+}
+
+func TestSingletons(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	dec := NewDecoder(b)
+	for _, test := range singleTests {
+		b.Reset()
+		err := enc.Encode(test.in)
+		if err != nil {
+			t.Errorf("error encoding %v: %s", test.in, err)
+			continue
+		}
+		err = dec.Decode(test.out)
+		switch {
+		case err != nil && test.err == "":
+			t.Errorf("error decoding %v: %s", test.in, err)
+			continue
+		case err == nil && test.err != "":
+			t.Errorf("expected error decoding %v: %s", test.in, test.err)
+			continue
+		case err != nil && test.err != "":
+			if strings.Index(err.String(), test.err) < 0 {
+				t.Errorf("wrong error decoding %v: wanted %s, got %v", test.in, test.err, err)
+			}
+			continue
+		}
+		// Get rid of the pointer in the rhs
+		val := reflect.ValueOf(test.out).Elem().Interface()
+		if !reflect.DeepEqual(test.in, val) {
+			t.Errorf("decoding singleton: expected %v got %v", test.in, val)
+		}
+	}
+}
+
+func TestStructNonStruct(t *testing.T) {
+	type Struct struct {
+		A string
+	}
+	type NonStruct string
+	s := Struct{"hello"}
+	var sp Struct
+	if err := encAndDec(s, &sp); err != nil {
+		t.Error(err)
+	}
+	var ns NonStruct
+	if err := encAndDec(s, &ns); err == nil {
+		t.Error("should get error for struct/non-struct")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("for struct/non-struct expected type error; got", err)
+	}
+	// Now try the other way
+	var nsp NonStruct
+	if err := encAndDec(ns, &nsp); err != nil {
+		t.Error(err)
+	}
+	if err := encAndDec(ns, &s); err == nil {
+		t.Error("should get error for non-struct/struct")
+	} else if strings.Index(err.String(), "type") < 0 {
+		t.Error("for non-struct/struct expected type error; got", err)
+	}
+}
+
+type interfaceIndirectTestI interface {
+	F() bool
+}
+
+type interfaceIndirectTestT struct{}
+
+func (this *interfaceIndirectTestT) F() bool {
+	return true
+}
+
+// A version of a bug reported on golang-nuts.  Also tests top-level
+// slice of interfaces.  The issue was registering *T caused T to be
+// stored as the concrete type.
+func TestInterfaceIndirect(t *testing.T) {
+	Register(&interfaceIndirectTestT{})
+	b := new(bytes.Buffer)
+	w := []interfaceIndirectTestI{&interfaceIndirectTestT{}}
+	err := NewEncoder(b).Encode(w)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+
+	var r []interfaceIndirectTestI
+	err = NewDecoder(b).Decode(&r)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+}
+
+// Now follow various tests that decode into things that can't represent the
+// encoded value, all of which should be legal.
+
+// Also, when the ignored object contains an interface value, it may define
+// types. Make sure that skipping the value still defines the types by using
+// the encoder/decoder pair to send a value afterwards.  If an interface
+// is sent, its type in the test is always NewType0, so this checks that the
+// encoder and decoder don't skew with respect to type definitions.
+
+type Struct0 struct {
+	I interface{}
+}
+
+type NewType0 struct {
+	S string
+}
+
+type ignoreTest struct {
+	in, out interface{}
+}
+
+var ignoreTests = []ignoreTest{
+	// Decode normal struct into an empty struct
+	{&struct{ A int }{23}, &struct{}{}},
+	// Decode normal struct into a nil.
+	{&struct{ A int }{23}, nil},
+	// Decode singleton string into a nil.
+	{"hello, world", nil},
+	// Decode singleton slice into a nil.
+	{[]int{1, 2, 3, 4}, nil},
+	// Decode struct containing an interface into a nil.
+	{&Struct0{&NewType0{"value0"}}, nil},
+	// Decode singleton slice of interfaces into a nil.
+	{[]interface{}{"hi", &NewType0{"value1"}, 23}, nil},
+}
+
+func TestDecodeIntoNothing(t *testing.T) {
+	Register(new(NewType0))
+	for i, test := range ignoreTests {
+		b := new(bytes.Buffer)
+		enc := NewEncoder(b)
+		err := enc.Encode(test.in)
+		if err != nil {
+			t.Errorf("%d: encode error %s:", i, err)
+			continue
+		}
+		dec := NewDecoder(b)
+		err = dec.Decode(test.out)
+		if err != nil {
+			t.Errorf("%d: decode error: %s", i, err)
+			continue
+		}
+		// Now see if the encoder and decoder are in a consistent state.
+		str := fmt.Sprintf("Value %d", i)
+		err = enc.Encode(&NewType0{str})
+		if err != nil {
+			t.Fatalf("%d: NewType0 encode error: %s", i, err)
+		}
+		ns := new(NewType0)
+		err = dec.Decode(ns)
+		if err != nil {
+			t.Fatalf("%d: NewType0 decode error: %s", i, err)
+		}
+		if ns.S != str {
+			t.Fatalf("%d: expected %q got %q", i, str, ns.S)
+		}
+	}
+}
+
+// Another bug from golang-nuts, involving nested interfaces.
+type Bug0Outer struct {
+	Bug0Field interface{}
+}
+
+type Bug0Inner struct {
+	A int
+}
+
+func TestNestedInterfaces(t *testing.T) {
+	var buf bytes.Buffer
+	e := NewEncoder(&buf)
+	d := NewDecoder(&buf)
+	Register(new(Bug0Outer))
+	Register(new(Bug0Inner))
+	f := &Bug0Outer{&Bug0Outer{&Bug0Inner{7}}}
+	var v interface{} = f
+	err := e.Encode(&v)
+	if err != nil {
+		t.Fatal("Encode:", err)
+	}
+	err = d.Decode(&v)
+	if err != nil {
+		t.Fatal("Decode:", err)
+	}
+	// Make sure it decoded correctly.
+	outer1, ok := v.(*Bug0Outer)
+	if !ok {
+		t.Fatalf("v not Bug0Outer: %T", v)
+	}
+	outer2, ok := outer1.Bug0Field.(*Bug0Outer)
+	if !ok {
+		t.Fatalf("v.Bug0Field not Bug0Outer: %T", outer1.Bug0Field)
+	}
+	inner, ok := outer2.Bug0Field.(*Bug0Inner)
+	if !ok {
+		t.Fatalf("v.Bug0Field.Bug0Field not Bug0Inner: %T", outer2.Bug0Field)
+	}
+	if inner.A != 7 {
+		t.Fatalf("final value %d; expected %d", inner.A, 7)
+	}
+}
+
+// The bugs keep coming. We forgot to send map subtypes before the map.
+
+type Bug1Elem struct {
+	Name string
+	Id   int
+}
+
+type Bug1StructMap map[string]Bug1Elem
+
+func bug1EncDec(in Bug1StructMap, out *Bug1StructMap) os.Error {
+	return nil
+}
+
+func TestMapBug1(t *testing.T) {
+	in := make(Bug1StructMap)
+	in["val1"] = Bug1Elem{"elem1", 1}
+	in["val2"] = Bug1Elem{"elem2", 2}
+
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(in)
+	if err != nil {
+		t.Fatal("encode:", err)
+	}
+	dec := NewDecoder(b)
+	out := make(Bug1StructMap)
+	err = dec.Decode(&out)
+	if err != nil {
+		t.Fatal("decode:", err)
+	}
+	if !reflect.DeepEqual(in, out) {
+		t.Errorf("mismatch: %v %v", in, out)
+	}
+}
+
+func TestGobMapInterfaceEncode(t *testing.T) {
+	m := map[string]interface{}{
+		"up": uintptr(0),
+		"i0": []int{-1},
+		"i1": []int8{-1},
+		"i2": []int16{-1},
+		"i3": []int32{-1},
+		"i4": []int64{-1},
+		"u0": []uint{1},
+		"u1": []uint8{1},
+		"u2": []uint16{1},
+		"u3": []uint32{1},
+		"u4": []uint64{1},
+		"f0": []float32{1},
+		"f1": []float64{1},
+		"c0": []complex64{complex(2, -2)},
+		"c1": []complex128{complex(2, float64(-2))},
+		"us": []uintptr{0},
+		"bo": []bool{false},
+		"st": []string{"s"},
+	}
+	buf := bytes.NewBuffer(nil)
+	enc := NewEncoder(buf)
+	err := enc.Encode(m)
+	if err != nil {
+		t.Errorf("gob.Encode map: %s", err)
+	}
+}
diff --git a/src/pkg/gob/error.go b/src/pkg/gob/error.go
new file mode 100644
index 000000000..bfd38fc16
--- /dev/null
+++ b/src/pkg/gob/error.go
@@ -0,0 +1,42 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"fmt"
+	"os"
+)
+
+// Errors in decoding and encoding are handled using panic and recover.
+// Panics caused by user error (that is, everything except run-time panics
+// such as "index out of bounds" errors) do not leave the file that caused
+// them, but are instead turned into plain os.Error returns.  Encoding and
+// decoding functions and methods that do not return an os.Error either use
+// panic to report an error or are guaranteed error-free.
+
+// A gobError wraps an os.Error and is used to distinguish errors (panics) generated in this package.
+type gobError struct {
+	os.Error
+}
+
+// errorf is like error but takes Printf-style arguments to construct an os.Error.
+// It always prefixes the message with "gob: ".
+func errorf(format string, args ...interface{}) {
+	error(fmt.Errorf("gob: "+format, args...))
+}
+
+// error wraps the argument error and uses it as the argument to panic.
+func error(err os.Error) {
+	panic(gobError{Error: err})
+}
+
+// catchError is meant to be used as a deferred function to turn a panic(gobError) into a
+// plain os.Error.  It overwrites the error return of the function that deferred its call.
+func catchError(err *os.Error) {
+	if e := recover(); e != nil {
+		*err = e.(gobError).Error // Will re-panic if not one of our errors, such as a runtime error.
+	}
+	return
+}
diff --git a/src/pkg/gob/gobencdec_test.go b/src/pkg/gob/gobencdec_test.go
new file mode 100644
index 000000000..371a43c8f
--- /dev/null
+++ b/src/pkg/gob/gobencdec_test.go
@@ -0,0 +1,490 @@
+// Copyright 20011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains tests of the GobEncoder/GobDecoder support.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+// Types that implement the GobEncoder/Decoder interfaces.
+
+type ByteStruct struct {
+	a byte // not an exported field
+}
+
+type StringStruct struct {
+	s string // not an exported field
+}
+
+type ArrayStruct struct {
+	a [8192]byte // not an exported field
+}
+
+type Gobber int
+
+type ValueGobber string // encodes with a value, decodes with a pointer.
+
+// The relevant methods
+
+func (g *ByteStruct) GobEncode() ([]byte, os.Error) {
+	b := make([]byte, 3)
+	b[0] = g.a
+	b[1] = g.a + 1
+	b[2] = g.a + 2
+	return b, nil
+}
+
+func (g *ByteStruct) GobDecode(data []byte) os.Error {
+	if g == nil {
+		return os.NewError("NIL RECEIVER")
+	}
+	// Expect N sequential-valued bytes.
+	if len(data) == 0 {
+		return os.EOF
+	}
+	g.a = data[0]
+	for i, c := range data {
+		if c != g.a+byte(i) {
+			return os.NewError("invalid data sequence")
+		}
+	}
+	return nil
+}
+
+func (g *StringStruct) GobEncode() ([]byte, os.Error) {
+	return []byte(g.s), nil
+}
+
+func (g *StringStruct) GobDecode(data []byte) os.Error {
+	// Expect N sequential-valued bytes.
+	if len(data) == 0 {
+		return os.EOF
+	}
+	a := data[0]
+	for i, c := range data {
+		if c != a+byte(i) {
+			return os.NewError("invalid data sequence")
+		}
+	}
+	g.s = string(data)
+	return nil
+}
+
+func (a *ArrayStruct) GobEncode() ([]byte, os.Error) {
+	return a.a[:], nil
+}
+
+func (a *ArrayStruct) GobDecode(data []byte) os.Error {
+	if len(data) != len(a.a) {
+		return os.NewError("wrong length in array decode")
+	}
+	copy(a.a[:], data)
+	return nil
+}
+
+func (g *Gobber) GobEncode() ([]byte, os.Error) {
+	return []byte(fmt.Sprintf("VALUE=%d", *g)), nil
+}
+
+func (g *Gobber) GobDecode(data []byte) os.Error {
+	_, err := fmt.Sscanf(string(data), "VALUE=%d", (*int)(g))
+	return err
+}
+
+func (v ValueGobber) GobEncode() ([]byte, os.Error) {
+	return []byte(fmt.Sprintf("VALUE=%s", v)), nil
+}
+
+func (v *ValueGobber) GobDecode(data []byte) os.Error {
+	_, err := fmt.Sscanf(string(data), "VALUE=%s", (*string)(v))
+	return err
+}
+
+// Structs that include GobEncodable fields.
+
+type GobTest0 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *ByteStruct
+}
+
+type GobTest1 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *StringStruct
+}
+
+type GobTest2 struct {
+	X int    // guarantee we have  something in common with GobTest*
+	G string // not a GobEncoder - should give us errors
+}
+
+type GobTest3 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *Gobber
+}
+
+type GobTest4 struct {
+	X int // guarantee we have  something in common with GobTest*
+	V ValueGobber
+}
+
+type GobTest5 struct {
+	X int // guarantee we have  something in common with GobTest*
+	V *ValueGobber
+}
+
+type GobTestIgnoreEncoder struct {
+	X int // guarantee we have  something in common with GobTest*
+}
+
+type GobTestValueEncDec struct {
+	X int          // guarantee we have  something in common with GobTest*
+	G StringStruct // not a pointer.
+}
+
+type GobTestIndirectEncDec struct {
+	X int             // guarantee we have  something in common with GobTest*
+	G ***StringStruct // indirections to the receiver.
+}
+
+type GobTestArrayEncDec struct {
+	X int         // guarantee we have  something in common with GobTest*
+	A ArrayStruct // not a pointer.
+}
+
+type GobTestIndirectArrayEncDec struct {
+	X int            // guarantee we have  something in common with GobTest*
+	A ***ArrayStruct // indirections to a large receiver.
+}
+
+func TestGobEncoderField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{17, &ByteStruct{'A'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.G.a)
+	}
+	// Now a field that's not a structure.
+	b.Reset()
+	gobber := Gobber(23)
+	err = enc.Encode(GobTest3{17, &gobber})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := new(GobTest3)
+	err = dec.Decode(y)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if *y.G != 23 {
+		t.Errorf("expected '23 got %d", *y.G)
+	}
+}
+
+// Even though the field is a value, we can still take its address
+// and should be able to call the methods.
+func TestGobEncoderValueField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTestValueEncDec{17, StringStruct{"HIJKL"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestValueEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.s != "HIJKL" {
+		t.Errorf("expected `HIJKL` got %s", x.G.s)
+	}
+}
+
+// GobEncode/Decode should work even if the value is
+// more indirect than the receiver.
+func TestGobEncoderIndirectField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	s := &StringStruct{"HIJKL"}
+	sp := &s
+	err := enc.Encode(GobTestIndirectEncDec{17, &sp})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIndirectEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if (***x.G).s != "HIJKL" {
+		t.Errorf("expected `HIJKL` got %s", (***x.G).s)
+	}
+}
+
+// Test with a large field with methods.
+func TestGobEncoderArrayField(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	var a GobTestArrayEncDec
+	a.X = 17
+	for i := range a.A.a {
+		a.A.a[i] = byte(i)
+	}
+	err := enc.Encode(a)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestArrayEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	for i, v := range x.A.a {
+		if v != byte(i) {
+			t.Errorf("expected %x got %x", byte(i), v)
+			break
+		}
+	}
+}
+
+// Test an indirection to a large field with methods.
+func TestGobEncoderIndirectArrayField(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	var a GobTestIndirectArrayEncDec
+	a.X = 17
+	var array ArrayStruct
+	ap := &array
+	app := &ap
+	a.A = &app
+	for i := range array.a {
+		array.a[i] = byte(i)
+	}
+	err := enc.Encode(a)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIndirectArrayEncDec)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	for i, v := range (***x.A).a {
+		if v != byte(i) {
+			t.Errorf("expected %x got %x", byte(i), v)
+			break
+		}
+	}
+}
+
+// As long as the fields have the same name and implement the
+// interface, we can cross-connect them.  Not sure it's useful
+// and may even be bad but it works and it's hard to prevent
+// without exposing the contents of the object, which would
+// defeat the purpose.
+func TestGobEncoderFieldsOfDifferentType(t *testing.T) {
+	// first, string in field to byte in field
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.G.a)
+	}
+	// now the other direction, byte in field to string in field
+	b.Reset()
+	err = enc.Encode(GobTest0{17, &ByteStruct{'X'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := new(GobTest1)
+	err = dec.Decode(y)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if y.G.s != "XYZ" {
+		t.Fatalf("expected `XYZ` got %c", y.G.s)
+	}
+}
+
+// Test that we can encode a value and decode into a pointer.
+func TestGobEncoderValueEncoder(t *testing.T) {
+	// first, string in field to byte in field
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest4{17, ValueGobber("hello")})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest5)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if *x.V != "hello" {
+		t.Errorf("expected `hello` got %s", x.V)
+	}
+}
+
+func TestGobEncoderFieldTypeError(t *testing.T) {
+	// GobEncoder to non-decoder: error
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := &GobTest2{}
+	err = dec.Decode(x)
+	if err == nil {
+		t.Fatal("expected decode error for mismatched fields (encoder to non-decoder)")
+	}
+	if strings.Index(err.String(), "type") < 0 {
+		t.Fatal("expected type error; got", err)
+	}
+	// Non-encoder to GobDecoder: error
+	b.Reset()
+	err = enc.Encode(GobTest2{17, "ABC"})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := &GobTest1{}
+	err = dec.Decode(y)
+	if err == nil {
+		t.Fatal("expected decode error for mismatched fields (non-encoder to decoder)")
+	}
+	if strings.Index(err.String(), "type") < 0 {
+		t.Fatal("expected type error; got", err)
+	}
+}
+
+// Even though ByteStruct is a struct, it's treated as a singleton at the top level.
+func TestGobEncoderStructSingleton(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(&ByteStruct{'A'})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(ByteStruct)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.a)
+	}
+}
+
+func TestGobEncoderNonStructSingleton(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(Gobber(1234))
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	var x Gobber
+	err = dec.Decode(&x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x != 1234 {
+		t.Errorf("expected 1234 got %c", x)
+	}
+}
+
+func TestGobEncoderIgnoreStructField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{17, &ByteStruct{'A'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIgnoreEncoder)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 17 {
+		t.Errorf("expected 17 got %c", x.X)
+	}
+}
+
+func TestGobEncoderIgnoreNonStructField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	gobber := Gobber(23)
+	err := enc.Encode(GobTest3{17, &gobber})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIgnoreEncoder)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 17 {
+		t.Errorf("expected 17 got %c", x.X)
+	}
+}
+
+func TestGobEncoderIgnoreNilEncoder(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{X: 18}) // G is nil
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 18 {
+		t.Errorf("expected x.X = 18, got %v", x.X)
+	}
+	if x.G != nil {
+		t.Errorf("expected x.G = nil, got %v", x.G)
+	}
+}
diff --git a/src/pkg/gob/timing_test.go b/src/pkg/gob/timing_test.go
new file mode 100644
index 000000000..2a2be7336
--- /dev/null
+++ b/src/pkg/gob/timing_test.go
@@ -0,0 +1,94 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"testing"
+)
+
+type Bench struct {
+	A int
+	B float64
+	C string
+	D []byte
+}
+
+func benchmarkEndToEnd(r io.Reader, w io.Writer, b *testing.B) {
+	b.StopTimer()
+	enc := NewEncoder(w)
+	dec := NewDecoder(r)
+	bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")}
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		if enc.Encode(bench) != nil {
+			panic("encode error")
+		}
+		if dec.Decode(bench) != nil {
+			panic("decode error")
+		}
+	}
+}
+
+func BenchmarkEndToEndPipe(b *testing.B) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		panic("can't get pipe:" + err.String())
+	}
+	benchmarkEndToEnd(r, w, b)
+}
+
+func BenchmarkEndToEndByteBuffer(b *testing.B) {
+	var buf bytes.Buffer
+	benchmarkEndToEnd(&buf, &buf, b)
+}
+
+func TestCountEncodeMallocs(t *testing.T) {
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")}
+	runtime.UpdateMemStats()
+	mallocs := 0 - runtime.MemStats.Mallocs
+	const count = 1000
+	for i := 0; i < count; i++ {
+		err := enc.Encode(bench)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	runtime.UpdateMemStats()
+	mallocs += runtime.MemStats.Mallocs
+	fmt.Printf("mallocs per encode of type Bench: %d\n", mallocs/count)
+}
+
+func TestCountDecodeMallocs(t *testing.T) {
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")}
+	const count = 1000
+	for i := 0; i < count; i++ {
+		err := enc.Encode(bench)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	dec := NewDecoder(&buf)
+	runtime.UpdateMemStats()
+	mallocs := 0 - runtime.MemStats.Mallocs
+	for i := 0; i < count; i++ {
+		*bench = Bench{}
+		err := dec.Decode(&bench)
+		if err != nil {
+			t.Fatal("decode:", err)
+		}
+	}
+	runtime.UpdateMemStats()
+	mallocs += runtime.MemStats.Mallocs
+	fmt.Printf("mallocs per decode of type Bench: %d\n", mallocs/count)
+}
diff --git a/src/pkg/gob/type.go b/src/pkg/gob/type.go
new file mode 100644
index 000000000..b2f716c4b
--- /dev/null
+++ b/src/pkg/gob/type.go
@@ -0,0 +1,786 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"sync"
+	"unicode"
+	"utf8"
+)
+
+// userTypeInfo stores the information associated with a type the user has handed
+// to the package.  It's computed once and stored in a map keyed by reflection
+// type.
+type userTypeInfo struct {
+	user         reflect.Type // the type the user handed us
+	base         reflect.Type // the base type after all indirections
+	indir        int          // number of indirections to reach the base type
+	isGobEncoder bool         // does the type implement GobEncoder?
+	isGobDecoder bool         // does the type implement GobDecoder?
+	encIndir     int8         // number of indirections to reach the receiver type; may be negative
+	decIndir     int8         // number of indirections to reach the receiver type; may be negative
+}
+
+var (
+	// Protected by an RWMutex because we read it a lot and write
+	// it only when we see a new type, typically when compiling.
+	userTypeLock  sync.RWMutex
+	userTypeCache = make(map[reflect.Type]*userTypeInfo)
+)
+
+// validType returns, and saves, the information associated with user-provided type rt.
+// If the user type is not valid, err will be non-nil.  To be used when the error handler
+// is not set up.
+func validUserType(rt reflect.Type) (ut *userTypeInfo, err os.Error) {
+	userTypeLock.RLock()
+	ut = userTypeCache[rt]
+	userTypeLock.RUnlock()
+	if ut != nil {
+		return
+	}
+	// Now set the value under the write lock.
+	userTypeLock.Lock()
+	defer userTypeLock.Unlock()
+	if ut = userTypeCache[rt]; ut != nil {
+		// Lost the race; not a problem.
+		return
+	}
+	ut = new(userTypeInfo)
+	ut.base = rt
+	ut.user = rt
+	// A type that is just a cycle of pointers (such as type T *T) cannot
+	// be represented in gobs, which need some concrete data.  We use a
+	// cycle detection algorithm from Knuth, Vol 2, Section 3.1, Ex 6,
+	// pp 539-540.  As we step through indirections, run another type at
+	// half speed. If they meet up, there's a cycle.
+	slowpoke := ut.base // walks half as fast as ut.base
+	for {
+		pt := ut.base
+		if pt.Kind() != reflect.Ptr {
+			break
+		}
+		ut.base = pt.Elem()
+		if ut.base == slowpoke { // ut.base lapped slowpoke
+			// recursive pointer type.
+			return nil, os.NewError("can't represent recursive pointer type " + ut.base.String())
+		}
+		if ut.indir%2 == 0 {
+			slowpoke = slowpoke.Elem()
+		}
+		ut.indir++
+	}
+	ut.isGobEncoder, ut.encIndir = implementsInterface(ut.user, gobEncoderInterfaceType)
+	ut.isGobDecoder, ut.decIndir = implementsInterface(ut.user, gobDecoderInterfaceType)
+	userTypeCache[rt] = ut
+	return
+}
+
+var (
+	gobEncoderInterfaceType = reflect.TypeOf((*GobEncoder)(nil)).Elem()
+	gobDecoderInterfaceType = reflect.TypeOf((*GobDecoder)(nil)).Elem()
+)
+
+// implementsInterface reports whether the type implements the
+// gobEncoder/gobDecoder interface.
+// It also returns the number of indirections required to get to the
+// implementation.
+func implementsInterface(typ, gobEncDecType reflect.Type) (success bool, indir int8) {
+	if typ == nil {
+		return
+	}
+	rt := typ
+	// The type might be a pointer and we need to keep
+	// dereferencing to the base type until we find an implementation.
+	for {
+		if rt.Implements(gobEncDecType) {
+			return true, indir
+		}
+		if p := rt; p.Kind() == reflect.Ptr {
+			indir++
+			if indir > 100 { // insane number of indirections
+				return false, 0
+			}
+			rt = p.Elem()
+			continue
+		}
+		break
+	}
+	// No luck yet, but if this is a base type (non-pointer), the pointer might satisfy.
+	if typ.Kind() != reflect.Ptr {
+		// Not a pointer, but does the pointer work?
+		if reflect.PtrTo(typ).Implements(gobEncDecType) {
+			return true, -1
+		}
+	}
+	return false, 0
+}
+
+// userType returns, and saves, the information associated with user-provided type rt.
+// If the user type is not valid, it calls error.
+func userType(rt reflect.Type) *userTypeInfo {
+	ut, err := validUserType(rt)
+	if err != nil {
+		error(err)
+	}
+	return ut
+}
+// A typeId represents a gob Type as an integer that can be passed on the wire.
+// Internally, typeIds are used as keys to a map to recover the underlying type info.
+type typeId int32
+
+var nextId typeId       // incremented for each new type we build
+var typeLock sync.Mutex // set while building a type
+const firstUserId = 64  // lowest id number granted to user
+
+type gobType interface {
+	id() typeId
+	setId(id typeId)
+	name() string
+	string() string // not public; only for debugging
+	safeString(seen map[typeId]bool) string
+}
+
+var types = make(map[reflect.Type]gobType)
+var idToType = make(map[typeId]gobType)
+var builtinIdToType map[typeId]gobType // set in init() after builtins are established
+
+func setTypeId(typ gobType) {
+	nextId++
+	typ.setId(nextId)
+	idToType[nextId] = typ
+}
+
+func (t typeId) gobType() gobType {
+	if t == 0 {
+		return nil
+	}
+	return idToType[t]
+}
+
+// string returns the string representation of the type associated with the typeId.
+func (t typeId) string() string {
+	if t.gobType() == nil {
+		return ""
+	}
+	return t.gobType().string()
+}
+
+// Name returns the name of the type associated with the typeId.
+func (t typeId) name() string {
+	if t.gobType() == nil {
+		return ""
+	}
+	return t.gobType().name()
+}
+
+// Common elements of all types.
+type CommonType struct {
+	Name string
+	Id   typeId
+}
+
+func (t *CommonType) id() typeId { return t.Id }
+
+func (t *CommonType) setId(id typeId) { t.Id = id }
+
+func (t *CommonType) string() string { return t.Name }
+
+func (t *CommonType) safeString(seen map[typeId]bool) string {
+	return t.Name
+}
+
+func (t *CommonType) name() string { return t.Name }
+
+// Create and check predefined types
+// The string for tBytes is "bytes" not "[]byte" to signify its specialness.
+
+var (
+	// Primordial types, needed during initialization.
+	// Always passed as pointers so the interface{} type
+	// goes through without losing its interfaceness.
+	tBool      = bootstrapType("bool", (*bool)(nil), 1)
+	tInt       = bootstrapType("int", (*int)(nil), 2)
+	tUint      = bootstrapType("uint", (*uint)(nil), 3)
+	tFloat     = bootstrapType("float", (*float64)(nil), 4)
+	tBytes     = bootstrapType("bytes", (*[]byte)(nil), 5)
+	tString    = bootstrapType("string", (*string)(nil), 6)
+	tComplex   = bootstrapType("complex", (*complex128)(nil), 7)
+	tInterface = bootstrapType("interface", (*interface{})(nil), 8)
+	// Reserve some Ids for compatible expansion
+	tReserved7 = bootstrapType("_reserved1", (*struct{ r7 int })(nil), 9)
+	tReserved6 = bootstrapType("_reserved1", (*struct{ r6 int })(nil), 10)
+	tReserved5 = bootstrapType("_reserved1", (*struct{ r5 int })(nil), 11)
+	tReserved4 = bootstrapType("_reserved1", (*struct{ r4 int })(nil), 12)
+	tReserved3 = bootstrapType("_reserved1", (*struct{ r3 int })(nil), 13)
+	tReserved2 = bootstrapType("_reserved1", (*struct{ r2 int })(nil), 14)
+	tReserved1 = bootstrapType("_reserved1", (*struct{ r1 int })(nil), 15)
+)
+
+// Predefined because it's needed by the Decoder
+var tWireType = mustGetTypeInfo(reflect.TypeOf(wireType{})).id
+var wireTypeUserInfo *userTypeInfo // userTypeInfo of (*wireType)
+
+func init() {
+	// Some magic numbers to make sure there are no surprises.
+	checkId(16, tWireType)
+	checkId(17, mustGetTypeInfo(reflect.TypeOf(arrayType{})).id)
+	checkId(18, mustGetTypeInfo(reflect.TypeOf(CommonType{})).id)
+	checkId(19, mustGetTypeInfo(reflect.TypeOf(sliceType{})).id)
+	checkId(20, mustGetTypeInfo(reflect.TypeOf(structType{})).id)
+	checkId(21, mustGetTypeInfo(reflect.TypeOf(fieldType{})).id)
+	checkId(23, mustGetTypeInfo(reflect.TypeOf(mapType{})).id)
+
+	builtinIdToType = make(map[typeId]gobType)
+	for k, v := range idToType {
+		builtinIdToType[k] = v
+	}
+
+	// Move the id space upwards to allow for growth in the predefined world
+	// without breaking existing files.
+	if nextId > firstUserId {
+		panic(fmt.Sprintln("nextId too large:", nextId))
+	}
+	nextId = firstUserId
+	registerBasics()
+	wireTypeUserInfo = userType(reflect.TypeOf((*wireType)(nil)))
+}
+
+// Array type
+type arrayType struct {
+	CommonType
+	Elem typeId
+	Len  int
+}
+
+func newArrayType(name string) *arrayType {
+	a := &arrayType{CommonType{Name: name}, 0, 0}
+	return a
+}
+
+func (a *arrayType) init(elem gobType, len int) {
+	// Set our type id before evaluating the element's, in case it's our own.
+	setTypeId(a)
+	a.Elem = elem.id()
+	a.Len = len
+}
+
+func (a *arrayType) safeString(seen map[typeId]bool) string {
+	if seen[a.Id] {
+		return a.Name
+	}
+	seen[a.Id] = true
+	return fmt.Sprintf("[%d]%s", a.Len, a.Elem.gobType().safeString(seen))
+}
+
+func (a *arrayType) string() string { return a.safeString(make(map[typeId]bool)) }
+
+// GobEncoder type (something that implements the GobEncoder interface)
+type gobEncoderType struct {
+	CommonType
+}
+
+func newGobEncoderType(name string) *gobEncoderType {
+	g := &gobEncoderType{CommonType{Name: name}}
+	setTypeId(g)
+	return g
+}
+
+func (g *gobEncoderType) safeString(seen map[typeId]bool) string {
+	return g.Name
+}
+
+func (g *gobEncoderType) string() string { return g.Name }
+
+// Map type
+type mapType struct {
+	CommonType
+	Key  typeId
+	Elem typeId
+}
+
+func newMapType(name string) *mapType {
+	m := &mapType{CommonType{Name: name}, 0, 0}
+	return m
+}
+
+func (m *mapType) init(key, elem gobType) {
+	// Set our type id before evaluating the element's, in case it's our own.
+	setTypeId(m)
+	m.Key = key.id()
+	m.Elem = elem.id()
+}
+
+func (m *mapType) safeString(seen map[typeId]bool) string {
+	if seen[m.Id] {
+		return m.Name
+	}
+	seen[m.Id] = true
+	key := m.Key.gobType().safeString(seen)
+	elem := m.Elem.gobType().safeString(seen)
+	return fmt.Sprintf("map[%s]%s", key, elem)
+}
+
+func (m *mapType) string() string { return m.safeString(make(map[typeId]bool)) }
+
+// Slice type
+type sliceType struct {
+	CommonType
+	Elem typeId
+}
+
+func newSliceType(name string) *sliceType {
+	s := &sliceType{CommonType{Name: name}, 0}
+	return s
+}
+
+func (s *sliceType) init(elem gobType) {
+	// Set our type id before evaluating the element's, in case it's our own.
+	setTypeId(s)
+	s.Elem = elem.id()
+}
+
+func (s *sliceType) safeString(seen map[typeId]bool) string {
+	if seen[s.Id] {
+		return s.Name
+	}
+	seen[s.Id] = true
+	return fmt.Sprintf("[]%s", s.Elem.gobType().safeString(seen))
+}
+
+func (s *sliceType) string() string { return s.safeString(make(map[typeId]bool)) }
+
+// Struct type
+type fieldType struct {
+	Name string
+	Id   typeId
+}
+
+type structType struct {
+	CommonType
+	Field []*fieldType
+}
+
+func (s *structType) safeString(seen map[typeId]bool) string {
+	if s == nil {
+		return ""
+	}
+	if _, ok := seen[s.Id]; ok {
+		return s.Name
+	}
+	seen[s.Id] = true
+	str := s.Name + " = struct { "
+	for _, f := range s.Field {
+		str += fmt.Sprintf("%s %s; ", f.Name, f.Id.gobType().safeString(seen))
+	}
+	str += "}"
+	return str
+}
+
+func (s *structType) string() string { return s.safeString(make(map[typeId]bool)) }
+
+func newStructType(name string) *structType {
+	s := &structType{CommonType{Name: name}, nil}
+	// For historical reasons we set the id here rather than init.
+	// See the comment in newTypeObject for details.
+	setTypeId(s)
+	return s
+}
+
+// newTypeObject allocates a gobType for the reflection type rt.
+// Unless ut represents a GobEncoder, rt should be the base type
+// of ut.
+// This is only called from the encoding side. The decoding side
+// works through typeIds and userTypeInfos alone.
+func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os.Error) {
+	// Does this type implement GobEncoder?
+	if ut.isGobEncoder {
+		return newGobEncoderType(name), nil
+	}
+	var err os.Error
+	var type0, type1 gobType
+	defer func() {
+		if err != nil {
+			types[rt] = nil, false
+		}
+	}()
+	// Install the top-level type before the subtypes (e.g. struct before
+	// fields) so recursive types can be constructed safely.
+	switch t := rt; t.Kind() {
+	// All basic types are easy: they are predefined.
+	case reflect.Bool:
+		return tBool.gobType(), nil
+
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return tInt.gobType(), nil
+
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return tUint.gobType(), nil
+
+	case reflect.Float32, reflect.Float64:
+		return tFloat.gobType(), nil
+
+	case reflect.Complex64, reflect.Complex128:
+		return tComplex.gobType(), nil
+
+	case reflect.String:
+		return tString.gobType(), nil
+
+	case reflect.Interface:
+		return tInterface.gobType(), nil
+
+	case reflect.Array:
+		at := newArrayType(name)
+		types[rt] = at
+		type0, err = getBaseType("", t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		// Historical aside:
+		// For arrays, maps, and slices, we set the type id after the elements
+		// are constructed. This is to retain the order of type id allocation after
+		// a fix made to handle recursive types, which changed the order in
+		// which types are built.  Delaying the setting in this way preserves
+		// type ids while allowing recursive types to be described. Structs,
+		// done below, were already handling recursion correctly so they
+		// assign the top-level id before those of the field.
+		at.init(type0, t.Len())
+		return at, nil
+
+	case reflect.Map:
+		mt := newMapType(name)
+		types[rt] = mt
+		type0, err = getBaseType("", t.Key())
+		if err != nil {
+			return nil, err
+		}
+		type1, err = getBaseType("", t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		mt.init(type0, type1)
+		return mt, nil
+
+	case reflect.Slice:
+		// []byte == []uint8 is a special case
+		if t.Elem().Kind() == reflect.Uint8 {
+			return tBytes.gobType(), nil
+		}
+		st := newSliceType(name)
+		types[rt] = st
+		type0, err = getBaseType(t.Elem().Name(), t.Elem())
+		if err != nil {
+			return nil, err
+		}
+		st.init(type0)
+		return st, nil
+
+	case reflect.Struct:
+		st := newStructType(name)
+		types[rt] = st
+		idToType[st.id()] = st
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			if !isExported(f.Name) {
+				continue
+			}
+			typ := userType(f.Type).base
+			tname := typ.Name()
+			if tname == "" {
+				t := userType(f.Type).base
+				tname = t.String()
+			}
+			gt, err := getBaseType(tname, f.Type)
+			if err != nil {
+				return nil, err
+			}
+			st.Field = append(st.Field, &fieldType{f.Name, gt.id()})
+		}
+		return st, nil
+
+	default:
+		return nil, os.NewError("gob NewTypeObject can't handle type: " + rt.String())
+	}
+	return nil, nil
+}
+
+// isExported reports whether this is an exported - upper case - name.
+func isExported(name string) bool {
+	rune, _ := utf8.DecodeRuneInString(name)
+	return unicode.IsUpper(rune)
+}
+
+// getBaseType returns the Gob type describing the given reflect.Type's base type.
+// typeLock must be held.
+func getBaseType(name string, rt reflect.Type) (gobType, os.Error) {
+	ut := userType(rt)
+	return getType(name, ut, ut.base)
+}
+
+// getType returns the Gob type describing the given reflect.Type.
+// Should be called only when handling GobEncoders/Decoders,
+// which may be pointers.  All other types are handled through the
+// base type, never a pointer.
+// typeLock must be held.
+func getType(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os.Error) {
+	typ, present := types[rt]
+	if present {
+		return typ, nil
+	}
+	typ, err := newTypeObject(name, ut, rt)
+	if err == nil {
+		types[rt] = typ
+	}
+	return typ, err
+}
+
+func checkId(want, got typeId) {
+	if want != got {
+		fmt.Fprintf(os.Stderr, "checkId: %d should be %d\n", int(got), int(want))
+		panic("bootstrap type wrong id: " + got.name() + " " + got.string() + " not " + want.string())
+	}
+}
+
+// used for building the basic types; called only from init().  the incoming
+// interface always refers to a pointer.
+func bootstrapType(name string, e interface{}, expect typeId) typeId {
+	rt := reflect.TypeOf(e).Elem()
+	_, present := types[rt]
+	if present {
+		panic("bootstrap type already present: " + name + ", " + rt.String())
+	}
+	typ := &CommonType{Name: name}
+	types[rt] = typ
+	setTypeId(typ)
+	checkId(expect, nextId)
+	userType(rt) // might as well cache it now
+	return nextId
+}
+
+// Representation of the information we send and receive about this type.
+// Each value we send is preceded by its type definition: an encoded int.
+// However, the very first time we send the value, we first send the pair
+// (-id, wireType).
+// For bootstrapping purposes, we assume that the recipient knows how
+// to decode a wireType; it is exactly the wireType struct here, interpreted
+// using the gob rules for sending a structure, except that we assume the
+// ids for wireType and structType etc. are known.  The relevant pieces
+// are built in encode.go's init() function.
+// To maintain binary compatibility, if you extend this type, always put
+// the new fields last.
+type wireType struct {
+	ArrayT      *arrayType
+	SliceT      *sliceType
+	StructT     *structType
+	MapT        *mapType
+	GobEncoderT *gobEncoderType
+}
+
+func (w *wireType) string() string {
+	const unknown = "unknown type"
+	if w == nil {
+		return unknown
+	}
+	switch {
+	case w.ArrayT != nil:
+		return w.ArrayT.Name
+	case w.SliceT != nil:
+		return w.SliceT.Name
+	case w.StructT != nil:
+		return w.StructT.Name
+	case w.MapT != nil:
+		return w.MapT.Name
+	case w.GobEncoderT != nil:
+		return w.GobEncoderT.Name
+	}
+	return unknown
+}
+
+type typeInfo struct {
+	id      typeId
+	encoder *encEngine
+	wire    *wireType
+}
+
+var typeInfoMap = make(map[reflect.Type]*typeInfo) // protected by typeLock
+
+// typeLock must be held.
+func getTypeInfo(ut *userTypeInfo) (*typeInfo, os.Error) {
+	rt := ut.base
+	if ut.isGobEncoder {
+		// We want the user type, not the base type.
+		rt = ut.user
+	}
+	info, ok := typeInfoMap[rt]
+	if ok {
+		return info, nil
+	}
+	info = new(typeInfo)
+	gt, err := getBaseType(rt.Name(), rt)
+	if err != nil {
+		return nil, err
+	}
+	info.id = gt.id()
+
+	if ut.isGobEncoder {
+		userType, err := getType(rt.Name(), ut, rt)
+		if err != nil {
+			return nil, err
+		}
+		info.wire = &wireType{GobEncoderT: userType.id().gobType().(*gobEncoderType)}
+		typeInfoMap[ut.user] = info
+		return info, nil
+	}
+
+	t := info.id.gobType()
+	switch typ := rt; typ.Kind() {
+	case reflect.Array:
+		info.wire = &wireType{ArrayT: t.(*arrayType)}
+	case reflect.Map:
+		info.wire = &wireType{MapT: t.(*mapType)}
+	case reflect.Slice:
+		// []byte == []uint8 is a special case handled separately
+		if typ.Elem().Kind() != reflect.Uint8 {
+			info.wire = &wireType{SliceT: t.(*sliceType)}
+		}
+	case reflect.Struct:
+		info.wire = &wireType{StructT: t.(*structType)}
+	}
+	typeInfoMap[rt] = info
+	return info, nil
+}
+
+// Called only when a panic is acceptable and unexpected.
+func mustGetTypeInfo(rt reflect.Type) *typeInfo {
+	t, err := getTypeInfo(userType(rt))
+	if err != nil {
+		panic("getTypeInfo: " + err.String())
+	}
+	return t
+}
+
+// GobEncoder is the interface describing data that provides its own
+// representation for encoding values for transmission to a GobDecoder.
+// A type that implements GobEncoder and GobDecoder has complete
+// control over the representation of its data and may therefore
+// contain things such as private fields, channels, and functions,
+// which are not usually transmissible in gob streams.
+//
+// Note: Since gobs can be stored permanently, It is good design
+// to guarantee the encoding used by a GobEncoder is stable as the
+// software evolves.  For instance, it might make sense for GobEncode
+// to include a version number in the encoding.
+type GobEncoder interface {
+	// GobEncode returns a byte slice representing the encoding of the
+	// receiver for transmission to a GobDecoder, usually of the same
+	// concrete type.
+	GobEncode() ([]byte, os.Error)
+}
+
+// GobDecoder is the interface describing data that provides its own
+// routine for decoding transmitted values sent by a GobEncoder.
+type GobDecoder interface {
+	// GobDecode overwrites the receiver, which must be a pointer,
+	// with the value represented by the byte slice, which was written
+	// by GobEncode, usually for the same concrete type.
+	GobDecode([]byte) os.Error
+}
+
+var (
+	nameToConcreteType = make(map[string]reflect.Type)
+	concreteTypeToName = make(map[reflect.Type]string)
+)
+
+// RegisterName is like Register but uses the provided name rather than the
+// type's default.
+func RegisterName(name string, value interface{}) {
+	if name == "" {
+		// reserved for nil
+		panic("attempt to register empty name")
+	}
+	base := userType(reflect.TypeOf(value)).base
+	// Check for incompatible duplicates.
+	if t, ok := nameToConcreteType[name]; ok && t != base {
+		panic("gob: registering duplicate types for " + name)
+	}
+	if n, ok := concreteTypeToName[base]; ok && n != name {
+		panic("gob: registering duplicate names for " + base.String())
+	}
+	// Store the name and type provided by the user....
+	nameToConcreteType[name] = reflect.TypeOf(value)
+	// but the flattened type in the type table, since that's what decode needs.
+	concreteTypeToName[base] = name
+}
+
+// Register records a type, identified by a value for that type, under its
+// internal type name.  That name will identify the concrete type of a value
+// sent or received as an interface variable.  Only types that will be
+// transferred as implementations of interface values need to be registered.
+// Expecting to be used only during initialization, it panics if the mapping
+// between types and names is not a bijection.
+func Register(value interface{}) {
+	// Default to printed representation for unnamed types
+	rt := reflect.TypeOf(value)
+	name := rt.String()
+
+	// But for named types (or pointers to them), qualify with import path.
+	// Dereference one pointer looking for a named type.
+	star := ""
+	if rt.Name() == "" {
+		if pt := rt; pt.Kind() == reflect.Ptr {
+			star = "*"
+			rt = pt
+		}
+	}
+	if rt.Name() != "" {
+		if rt.PkgPath() == "" {
+			name = star + rt.Name()
+		} else {
+			name = star + rt.PkgPath() + "." + rt.Name()
+		}
+	}
+
+	RegisterName(name, value)
+}
+
+func registerBasics() {
+	Register(int(0))
+	Register(int8(0))
+	Register(int16(0))
+	Register(int32(0))
+	Register(int64(0))
+	Register(uint(0))
+	Register(uint8(0))
+	Register(uint16(0))
+	Register(uint32(0))
+	Register(uint64(0))
+	Register(float32(0))
+	Register(float64(0))
+	Register(complex64(0i))
+	Register(complex128(0i))
+	Register(uintptr(0))
+	Register(false)
+	Register("")
+	Register([]byte(nil))
+	Register([]int(nil))
+	Register([]int8(nil))
+	Register([]int16(nil))
+	Register([]int32(nil))
+	Register([]int64(nil))
+	Register([]uint(nil))
+	Register([]uint8(nil))
+	Register([]uint16(nil))
+	Register([]uint32(nil))
+	Register([]uint64(nil))
+	Register([]float32(nil))
+	Register([]float64(nil))
+	Register([]complex64(nil))
+	Register([]complex128(nil))
+	Register([]uintptr(nil))
+	Register([]bool(nil))
+	Register([]string(nil))
+}
diff --git a/src/pkg/gob/type_test.go b/src/pkg/gob/type_test.go
new file mode 100644
index 000000000..411ffb797
--- /dev/null
+++ b/src/pkg/gob/type_test.go
@@ -0,0 +1,153 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gob
+
+import (
+	"reflect"
+	"testing"
+)
+
+type typeT struct {
+	id  typeId
+	str string
+}
+
+var basicTypes = []typeT{
+	{tBool, "bool"},
+	{tInt, "int"},
+	{tUint, "uint"},
+	{tFloat, "float"},
+	{tBytes, "bytes"},
+	{tString, "string"},
+}
+
+func getTypeUnlocked(name string, rt reflect.Type) gobType {
+	typeLock.Lock()
+	defer typeLock.Unlock()
+	t, err := getBaseType(name, rt)
+	if err != nil {
+		panic("getTypeUnlocked: " + err.String())
+	}
+	return t
+}
+
+// Sanity checks
+func TestBasic(t *testing.T) {
+	for _, tt := range basicTypes {
+		if tt.id.string() != tt.str {
+			t.Errorf("checkType: expected %q got %s", tt.str, tt.id.string())
+		}
+		if tt.id == 0 {
+			t.Errorf("id for %q is zero", tt.str)
+		}
+	}
+}
+
+// Reregister some basic types to check registration is idempotent.
+func TestReregistration(t *testing.T) {
+	newtyp := getTypeUnlocked("int", reflect.TypeOf(int(0)))
+	if newtyp != tInt.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+	newtyp = getTypeUnlocked("uint", reflect.TypeOf(uint(0)))
+	if newtyp != tUint.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+	newtyp = getTypeUnlocked("string", reflect.TypeOf("hello"))
+	if newtyp != tString.gobType() {
+		t.Errorf("reregistration of %s got new type", newtyp.string())
+	}
+}
+
+func TestArrayType(t *testing.T) {
+	var a3 [3]int
+	a3int := getTypeUnlocked("foo", reflect.TypeOf(a3))
+	newa3int := getTypeUnlocked("bar", reflect.TypeOf(a3))
+	if a3int != newa3int {
+		t.Errorf("second registration of [3]int creates new type")
+	}
+	var a4 [4]int
+	a4int := getTypeUnlocked("goo", reflect.TypeOf(a4))
+	if a3int == a4int {
+		t.Errorf("registration of [3]int creates same type as [4]int")
+	}
+	var b3 [3]bool
+	a3bool := getTypeUnlocked("", reflect.TypeOf(b3))
+	if a3int == a3bool {
+		t.Errorf("registration of [3]bool creates same type as [3]int")
+	}
+	str := a3bool.string()
+	expected := "[3]bool"
+	if str != expected {
+		t.Errorf("array printed as %q; expected %q", str, expected)
+	}
+}
+
+func TestSliceType(t *testing.T) {
+	var s []int
+	sint := getTypeUnlocked("slice", reflect.TypeOf(s))
+	var news []int
+	newsint := getTypeUnlocked("slice1", reflect.TypeOf(news))
+	if sint != newsint {
+		t.Errorf("second registration of []int creates new type")
+	}
+	var b []bool
+	sbool := getTypeUnlocked("", reflect.TypeOf(b))
+	if sbool == sint {
+		t.Errorf("registration of []bool creates same type as []int")
+	}
+	str := sbool.string()
+	expected := "[]bool"
+	if str != expected {
+		t.Errorf("slice printed as %q; expected %q", str, expected)
+	}
+}
+
+func TestMapType(t *testing.T) {
+	var m map[string]int
+	mapStringInt := getTypeUnlocked("map", reflect.TypeOf(m))
+	var newm map[string]int
+	newMapStringInt := getTypeUnlocked("map1", reflect.TypeOf(newm))
+	if mapStringInt != newMapStringInt {
+		t.Errorf("second registration of map[string]int creates new type")
+	}
+	var b map[string]bool
+	mapStringBool := getTypeUnlocked("", reflect.TypeOf(b))
+	if mapStringBool == mapStringInt {
+		t.Errorf("registration of map[string]bool creates same type as map[string]int")
+	}
+	str := mapStringBool.string()
+	expected := "map[string]bool"
+	if str != expected {
+		t.Errorf("map printed as %q; expected %q", str, expected)
+	}
+}
+
+type Bar struct {
+	X string
+}
+
+// This structure has pointers and refers to itself, making it a good test case.
+type Foo struct {
+	A int
+	B int32 // will become int
+	C string
+	D []byte
+	E *float64    // will become float64
+	F ****float64 // will become float64
+	G *Bar
+	H *Bar // should not interpolate the definition of Bar again
+	I *Foo // will not explode
+}
+
+func TestStructType(t *testing.T) {
+	sstruct := getTypeUnlocked("Foo", reflect.TypeOf(Foo{}))
+	str := sstruct.string()
+	// If we can print it correctly, we built it correctly.
+	expected := "Foo = struct { A int; B int; C string; D bytes; E float; F float; G Bar = struct { X string; }; H Bar; I Foo; }"
+	if str != expected {
+		t.Errorf("struct printed as %q; expected %q", str, expected)
+	}
+}
diff --git a/src/pkg/hash/Makefile b/src/pkg/hash/Makefile
new file mode 100644
index 000000000..56071cb33
--- /dev/null
+++ b/src/pkg/hash/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.inc
+
+TARG=hash
+GOFILES=\
+	hash.go\
+
+include ../../Make.pkg
diff --git a/src/pkg/hash/adler32/Makefile b/src/pkg/hash/adler32/Makefile
new file mode 100644
index 000000000..38ce537ba
--- /dev/null
+++ b/src/pkg/hash/adler32/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/adler32
+GOFILES=\
+	adler32.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/adler32/adler32.go b/src/pkg/hash/adler32/adler32.go
new file mode 100644
index 000000000..84943d9ae
--- /dev/null
+++ b/src/pkg/hash/adler32/adler32.go
@@ -0,0 +1,88 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package adler32 implements the Adler-32 checksum.
+// Defined in RFC 1950:
+//	Adler-32 is composed of two sums accumulated per byte: s1 is
+//	the sum of all bytes, s2 is the sum of all s1 values. Both sums
+//	are done modulo 65521. s1 is initialized to 1, s2 to zero.  The
+//	Adler-32 checksum is stored as s2*65536 + s1 in most-
+//	significant-byte first (network) order.
+package adler32
+
+import (
+	"hash"
+	"os"
+)
+
+const (
+	mod = 65521
+)
+
+// The size of an Adler-32 checksum in bytes.
+const Size = 4
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	// invariant: (a < mod && b < mod) || a <= b
+	// invariant: a + b + 255 <= 0xffffffff
+	a, b uint32
+}
+
+func (d *digest) Reset() { d.a, d.b = 1, 0 }
+
+// New returns a new hash.Hash32 computing the Adler-32 checksum.
+func New() hash.Hash32 {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+func (d *digest) Size() int { return Size }
+
+// Add p to the running checksum a, b.
+func update(a, b uint32, p []byte) (aa, bb uint32) {
+	for _, pi := range p {
+		a += uint32(pi)
+		b += a
+		// invariant: a <= b
+		if b > (0xffffffff-255)/2 {
+			a %= mod
+			b %= mod
+			// invariant: a < mod && b < mod
+		} else {
+			// invariant: a + b + 255 <= 2 * b + 255 <= 0xffffffff
+		}
+	}
+	return a, b
+}
+
+// Return the 32-bit checksum corresponding to a, b.
+func finish(a, b uint32) uint32 {
+	if b >= mod {
+		a %= mod
+		b %= mod
+	}
+	return b<<16 | a
+}
+
+func (d *digest) Write(p []byte) (nn int, err os.Error) {
+	d.a, d.b = update(d.a, d.b, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum32() uint32 { return finish(d.a, d.b) }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 4)
+	s := d.Sum32()
+	p[0] = byte(s >> 24)
+	p[1] = byte(s >> 16)
+	p[2] = byte(s >> 8)
+	p[3] = byte(s)
+	return p
+}
+
+// Checksum returns the Adler-32 checksum of data.
+func Checksum(data []byte) uint32 { return finish(update(1, 0, data)) }
diff --git a/src/pkg/hash/adler32/adler32_test.go b/src/pkg/hash/adler32/adler32_test.go
new file mode 100644
index 000000000..01f931c68
--- /dev/null
+++ b/src/pkg/hash/adler32/adler32_test.go
@@ -0,0 +1,77 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package adler32
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+type _Adler32Test struct {
+	out uint32
+	in  string
+}
+
+var golden = []_Adler32Test{
+	{0x1, ""},
+	{0x620062, "a"},
+	{0x12600c4, "ab"},
+	{0x24d0127, "abc"},
+	{0x3d8018b, "abcd"},
+	{0x5c801f0, "abcde"},
+	{0x81e0256, "abcdef"},
+	{0xadb02bd, "abcdefg"},
+	{0xe000325, "abcdefgh"},
+	{0x118e038e, "abcdefghi"},
+	{0x158603f8, "abcdefghij"},
+	{0x3f090f02, "Discard medicine more than two years old."},
+	{0x46d81477, "He who has a shady past knows that nice guys finish last."},
+	{0x40ee0ee1, "I wouldn't marry him with a ten foot pole."},
+	{0x16661315, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x5b2e1480, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x8c3c09ea, "Nepal premier won't resign."},
+	{0x45ac18fd, "For every action there is an equal and opposite government program."},
+	{0x53c61462, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x7e511e63, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xe4801a6a, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0x61b507df, "size:  a.out:  bad magic"},
+	{0xb8631171, "The major problem is with sendmail.  -Mark Horton"},
+	{0x8b5e1904, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x7cc6102b, "If the enemy is within range, then so are you."},
+	{0x700318e7, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x1e601747, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0xb55b0b09, "C is as portable as Stonehedge!!"},
+	{0x39111dd0, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0x91dd304f, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0x2e5d1316, "How can you write a big system without C++?  -Paul Glick"},
+	{0xd0201df6, "'Invariant assertions' is the most elegant programming technique!  -Tom Szymanski"},
+}
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := New()
+		io.WriteString(c, g.in)
+		s := c.Sum32()
+		if s != g.out {
+			t.Errorf("adler32(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
+
+func BenchmarkGolden(b *testing.B) {
+	b.StopTimer()
+	c := New()
+	var buf bytes.Buffer
+	for _, g := range golden {
+		buf.Write([]byte(g.in))
+	}
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		c.Write(buf.Bytes())
+	}
+}
diff --git a/src/pkg/hash/crc32/Makefile b/src/pkg/hash/crc32/Makefile
new file mode 100644
index 000000000..af8a64cf2
--- /dev/null
+++ b/src/pkg/hash/crc32/Makefile
@@ -0,0 +1,20 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/crc32
+
+ifeq ($(GOARCH), amd64)
+	ARCH_GOFILES=crc32_amd64.go
+	OFILES=crc32_amd64.6
+else
+	ARCH_GOFILES=crc32_generic.go
+endif
+
+GOFILES=\
+	crc32.go\
+	$(ARCH_GOFILES)
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/crc32/crc32.go b/src/pkg/hash/crc32/crc32.go
new file mode 100644
index 000000000..0245b1ee8
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32.go
@@ -0,0 +1,139 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32,
+// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for
+// information.
+package crc32
+
+import (
+	"hash"
+	"os"
+	"sync"
+)
+
+// The size of a CRC-32 checksum in bytes.
+const Size = 4
+
+// Predefined polynomials.
+const (
+	// Far and away the most common CRC-32 polynomial.
+	// Used by ethernet (IEEE 802.3), v.42, fddi, gzip, zip, png, mpeg-2, ...
+	IEEE = 0xedb88320
+
+	// Castagnoli's polynomial, used in iSCSI.
+	// Has better error detection characteristics than IEEE.
+	// http://dx.doi.org/10.1109/26.231911
+	Castagnoli = 0x82f63b78
+
+	// Koopman's polynomial.
+	// Also has better error detection characteristics than IEEE.
+	// http://dx.doi.org/10.1109/DSN.2002.1028931
+	Koopman = 0xeb31d82e
+)
+
+// Table is a 256-word table representing the polynomial for efficient processing.
+type Table [256]uint32
+
+// castagnoliTable points to a lazily initialized Table for the Castagnoli
+// polynomial. MakeTable will always return this value when asked to make a
+// Castagnoli table so we can compare against it to find when the caller is
+// using this polynomial.
+var castagnoliTable *Table
+var castagnoliOnce sync.Once
+
+func castagnoliInit() {
+	castagnoliTable = makeTable(Castagnoli)
+}
+
+// IEEETable is the table for the IEEE polynomial.
+var IEEETable = makeTable(IEEE)
+
+// MakeTable returns the Table constructed from the specified polynomial.
+func MakeTable(poly uint32) *Table {
+	switch poly {
+	case IEEE:
+		return IEEETable
+	case Castagnoli:
+		castagnoliOnce.Do(castagnoliInit)
+		return castagnoliTable
+	}
+	return makeTable(poly)
+}
+
+// makeTable returns the Table constructed from the specified polynomial.
+func makeTable(poly uint32) *Table {
+	t := new(Table)
+	for i := 0; i < 256; i++ {
+		crc := uint32(i)
+		for j := 0; j < 8; j++ {
+			if crc&1 == 1 {
+				crc = (crc >> 1) ^ poly
+			} else {
+				crc >>= 1
+			}
+		}
+		t[i] = crc
+	}
+	return t
+}
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint32
+	tab *Table
+}
+
+// New creates a new hash.Hash32 computing the CRC-32 checksum
+// using the polynomial represented by the Table.
+func New(tab *Table) hash.Hash32 { return &digest{0, tab} }
+
+// NewIEEE creates a new hash.Hash32 computing the CRC-32 checksum
+// using the IEEE polynomial.
+func NewIEEE() hash.Hash32 { return New(IEEETable) }
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) Reset() { d.crc = 0 }
+
+func update(crc uint32, tab *Table, p []byte) uint32 {
+	crc = ^crc
+	for _, v := range p {
+		crc = tab[byte(crc)^v] ^ (crc >> 8)
+	}
+	return ^crc
+}
+
+// Update returns the result of adding the bytes in p to the crc.
+func Update(crc uint32, tab *Table, p []byte) uint32 {
+	if tab == castagnoliTable {
+		return updateCastagnoli(crc, p)
+	}
+	return update(crc, tab, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err os.Error) {
+	d.crc = Update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum32() uint32 { return d.crc }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 4)
+	s := d.Sum32()
+	p[0] = byte(s >> 24)
+	p[1] = byte(s >> 16)
+	p[2] = byte(s >> 8)
+	p[3] = byte(s)
+	return p
+}
+
+// Checksum returns the CRC-32 checksum of data
+// using the polynomial represented by the Table.
+func Checksum(data []byte, tab *Table) uint32 { return Update(0, tab, data) }
+
+// ChecksumIEEE returns the CRC-32 checksum of data
+// using the IEEE polynomial.
+func ChecksumIEEE(data []byte) uint32 { return update(0, IEEETable, data) }
diff --git a/src/pkg/hash/crc32/crc32_amd64.go b/src/pkg/hash/crc32/crc32_amd64.go
new file mode 100644
index 000000000..83349bc6c
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_amd64.go
@@ -0,0 +1,25 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+// This file contains the code to call the SSE 4.2 version of the Castagnoli
+// CRC.
+
+// haveSSE42 is defined in crc_amd64.s and uses CPUID to test for SSE 4.2
+// support.
+func haveSSE42() bool
+
+// castagnoliSSE42 is defined in crc_amd64.s and uses the SSE4.2 CRC32
+// instruction.
+func castagnoliSSE42(uint32, []byte) uint32
+
+var sse42 = haveSSE42()
+
+func updateCastagnoli(crc uint32, p []byte) uint32 {
+	if sse42 {
+		return castagnoliSSE42(crc, p)
+	}
+	return update(crc, castagnoliTable, p)
+}
diff --git a/src/pkg/hash/crc32/crc32_amd64.s b/src/pkg/hash/crc32/crc32_amd64.s
new file mode 100644
index 000000000..a9e5317e1
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_amd64.s
@@ -0,0 +1,62 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// func castagnoliSSE42(crc uint32, p []byte) uint32
+TEXT ·castagnoliSSE42(SB),7,$0
+	MOVL crc+0(FP), AX  // CRC value
+	MOVQ p+8(FP), SI  // data pointer
+	MOVL p+16(FP), CX  // len(p)
+
+	NOTL AX
+
+	/* If there's less than 8 bytes to process, we do it byte-by-byte. */
+	CMPL CX, $8
+	JL cleanup
+
+	/* Process individual bytes until the input is 8-byte aligned. */
+startup:
+	MOVQ SI, BX
+	ANDQ $7, BX
+	JZ aligned
+
+	CRC32B (SI), AX
+	DECL CX
+	INCQ SI
+	JMP startup
+
+aligned:
+	/* The input is now 8-byte aligned and we can process 8-byte chunks. */
+	CMPL CX, $8
+	JL cleanup
+
+	CRC32Q (SI), AX
+	ADDQ $8, SI
+	SUBQ $8, CX
+	JMP aligned
+
+cleanup:
+	/* We may have some bytes left over that we process one at a time. */
+	CMPL CX, $0
+	JE done
+
+	CRC32B (SI), AX
+	INCQ SI
+	DECQ CX
+	JMP cleanup
+
+done:
+	NOTL AX
+	MOVL AX, ret+24(FP)
+	RET
+
+// func haveSSE42() bool
+TEXT ·haveSSE42(SB),7,$0
+	XORQ AX, AX
+	INCL AX
+	CPUID
+	SHRQ $20, CX
+	ANDQ $1, CX
+	MOVB CX, ret+0(FP)
+	RET
+
diff --git a/src/pkg/hash/crc32/crc32_generic.go b/src/pkg/hash/crc32/crc32_generic.go
new file mode 100644
index 000000000..27aabd903
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_generic.go
@@ -0,0 +1,12 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+// The file contains the generic version of updateCastagnoli which just calls
+// the software implementation.
+
+func updateCastagnoli(crc uint32, p []byte) uint32 {
+	return update(crc, castagnoliTable, p)
+}
diff --git a/src/pkg/hash/crc32/crc32_test.go b/src/pkg/hash/crc32/crc32_test.go
new file mode 100644
index 000000000..7e82dd755
--- /dev/null
+++ b/src/pkg/hash/crc32/crc32_test.go
@@ -0,0 +1,97 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc32
+
+import (
+	"io"
+	"testing"
+)
+
+type test struct {
+	ieee, castagnoli uint32
+	in               string
+}
+
+var golden = []test{
+	{0x0, 0x0, ""},
+	{0xe8b7be43, 0xc1d04330, "a"},
+	{0x9e83486d, 0xe2a22936, "ab"},
+	{0x352441c2, 0x364b3fb7, "abc"},
+	{0xed82cd11, 0x92c80a31, "abcd"},
+	{0x8587d865, 0xc450d697, "abcde"},
+	{0x4b8e39ef, 0x53bceff1, "abcdef"},
+	{0x312a6aa6, 0xe627f441, "abcdefg"},
+	{0xaeef2a50, 0xa9421b7, "abcdefgh"},
+	{0x8da988af, 0x2ddc99fc, "abcdefghi"},
+	{0x3981703a, 0xe6599437, "abcdefghij"},
+	{0x6b9cdfe7, 0xb2cc01fe, "Discard medicine more than two years old."},
+	{0xc90ef73f, 0xe28207f, "He who has a shady past knows that nice guys finish last."},
+	{0xb902341f, 0xbe93f964, "I wouldn't marry him with a ten foot pole."},
+	{0x42080e8, 0x9e3be0c3, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x154c6d11, 0xf505ef04, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x4c418325, 0x85d3dc82, "Nepal premier won't resign."},
+	{0x33955150, 0xc5142380, "For every action there is an equal and opposite government program."},
+	{0x26216a4b, 0x75eb77dd, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x1abbe45e, 0x91ebe9f7, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xc89a94f7, 0xf0b1168e, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xab3abe14, 0x572b74e2, "size:  a.out:  bad magic"},
+	{0xbab102b6, 0x8a58a6d5, "The major problem is with sendmail.  -Mark Horton"},
+	{0x999149d7, 0x9c426c50, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x6d52a33c, 0x735400a4, "If the enemy is within range, then so are you."},
+	{0x90631e8d, 0xbec49c95, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x78309130, 0xa95a2079, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0x7d0a377f, 0xde2e65c5, "C is as portable as Stonehedge!!"},
+	{0x8c79fd79, 0x297a88ed, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0xa20b7167, 0x66ed1d8b, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0x8e0bb443, 0xdcded527, "How can you write a big system without C++?  -Paul Glick"},
+}
+
+func TestGolden(t *testing.T) {
+	castagnoliTab := MakeTable(Castagnoli)
+
+	for _, g := range golden {
+		ieee := NewIEEE()
+		io.WriteString(ieee, g.in)
+		s := ieee.Sum32()
+		if s != g.ieee {
+			t.Errorf("IEEE(%s) = 0x%x want 0x%x", g.in, s, g.ieee)
+		}
+
+		castagnoli := New(castagnoliTab)
+		io.WriteString(castagnoli, g.in)
+		s = castagnoli.Sum32()
+		if s != g.castagnoli {
+			t.Errorf("Castagnoli(%s) = 0x%x want 0x%x", g.in, s, g.castagnoli)
+		}
+
+		if len(g.in) > 0 {
+			// The SSE4.2 implementation of this has code to deal
+			// with misaligned data so we ensure that we test that
+			// too.
+			castagnoli = New(castagnoliTab)
+			io.WriteString(castagnoli, g.in[:1])
+			io.WriteString(castagnoli, g.in[1:])
+			s = castagnoli.Sum32()
+			if s != g.castagnoli {
+				t.Errorf("Castagnoli[misaligned](%s) = 0x%x want 0x%x", g.in, s, g.castagnoli)
+			}
+		}
+	}
+}
+
+func BenchmarkCrc32KB(b *testing.B) {
+	b.StopTimer()
+	data := make([]uint8, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = uint8(i)
+	}
+	c := NewIEEE()
+	b.StartTimer()
+	b.SetBytes(int64(len(data)))
+
+	for i := 0; i < b.N; i++ {
+		c.Write(data)
+	}
+}
diff --git a/src/pkg/hash/crc64/Makefile b/src/pkg/hash/crc64/Makefile
new file mode 100644
index 000000000..5f6c3de69
--- /dev/null
+++ b/src/pkg/hash/crc64/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/crc64
+GOFILES=\
+	crc64.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/crc64/crc64.go b/src/pkg/hash/crc64/crc64.go
new file mode 100644
index 000000000..ae37e781c
--- /dev/null
+++ b/src/pkg/hash/crc64/crc64.go
@@ -0,0 +1,97 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package crc64 implements the 64-bit cyclic redundancy check, or CRC-64,
+// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for
+// information.
+package crc64
+
+import (
+	"hash"
+	"os"
+)
+
+// The size of a CRC-64 checksum in bytes.
+const Size = 8
+
+// Predefined polynomials.
+const (
+	// The ISO polynomial, defined in ISO 3309 and used in HDLC.
+	ISO = 0xD800000000000000
+
+	// The ECMA polynomial, defined in ECMA 182.
+	ECMA = 0xC96C5795D7870F42
+)
+
+// Table is a 256-word table representing the polynomial for efficient processing.
+type Table [256]uint64
+
+// MakeTable returns the Table constructed from the specified polynomial.
+func MakeTable(poly uint64) *Table {
+	t := new(Table)
+	for i := 0; i < 256; i++ {
+		crc := uint64(i)
+		for j := 0; j < 8; j++ {
+			if crc&1 == 1 {
+				crc = (crc >> 1) ^ poly
+			} else {
+				crc >>= 1
+			}
+		}
+		t[i] = crc
+	}
+	return t
+}
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint64
+	tab *Table
+}
+
+// New creates a new hash.Hash64 computing the CRC-64 checksum
+// using the polynomial represented by the Table.
+func New(tab *Table) hash.Hash64 { return &digest{0, tab} }
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) Reset() { d.crc = 0 }
+
+func update(crc uint64, tab *Table, p []byte) uint64 {
+	crc = ^crc
+	for _, v := range p {
+		crc = tab[byte(crc)^v] ^ (crc >> 8)
+	}
+	return ^crc
+}
+
+// Update returns the result of adding the bytes in p to the crc.
+func Update(crc uint64, tab *Table, p []byte) uint64 {
+	return update(crc, tab, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err os.Error) {
+	d.crc = update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+func (d *digest) Sum64() uint64 { return d.crc }
+
+func (d *digest) Sum() []byte {
+	p := make([]byte, 8)
+	s := d.Sum64()
+	p[0] = byte(s >> 56)
+	p[1] = byte(s >> 48)
+	p[2] = byte(s >> 40)
+	p[3] = byte(s >> 32)
+	p[4] = byte(s >> 24)
+	p[5] = byte(s >> 16)
+	p[6] = byte(s >> 8)
+	p[7] = byte(s)
+	return p
+}
+
+// Checksum returns the CRC-64 checksum of data
+// using the polynomial represented by the Table.
+func Checksum(data []byte, tab *Table) uint64 { return update(0, tab, data) }
diff --git a/src/pkg/hash/crc64/crc64_test.go b/src/pkg/hash/crc64/crc64_test.go
new file mode 100644
index 000000000..e932524e0
--- /dev/null
+++ b/src/pkg/hash/crc64/crc64_test.go
@@ -0,0 +1,78 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package crc64
+
+import (
+	"io"
+	"testing"
+)
+
+type test struct {
+	out uint64
+	in  string
+}
+
+var golden = []test{
+	{0x0, ""},
+	{0x3420000000000000, "a"},
+	{0x36c4200000000000, "ab"},
+	{0x3776c42000000000, "abc"},
+	{0x336776c420000000, "abcd"},
+	{0x32d36776c4200000, "abcde"},
+	{0x3002d36776c42000, "abcdef"},
+	{0x31b002d36776c420, "abcdefg"},
+	{0xe21b002d36776c4, "abcdefgh"},
+	{0x8b6e21b002d36776, "abcdefghi"},
+	{0x7f5b6e21b002d367, "abcdefghij"},
+	{0x8ec0e7c835bf9cdf, "Discard medicine more than two years old."},
+	{0xc7db1759e2be5ab4, "He who has a shady past knows that nice guys finish last."},
+	{0xfbf9d9603a6fa020, "I wouldn't marry him with a ten foot pole."},
+	{0xeafc4211a6daa0ef, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x3e05b21c7a4dc4da, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x5255866ad6ef28a6, "Nepal premier won't resign."},
+	{0x8a79895be1e9c361, "For every action there is an equal and opposite government program."},
+	{0x8878963a649d4916, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0xa7b9d53ea87eb82f, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0xdb6805c0966a2f9c, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xf3553c65dacdadd2, "size:  a.out:  bad magic"},
+	{0x9d5e034087a676b9, "The major problem is with sendmail.  -Mark Horton"},
+	{0xa6db2d7f8da96417, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x325e00cd2fe819f9, "If the enemy is within range, then so are you."},
+	{0x88c6600ce58ae4c6, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x28c4a3f3b769e078, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0xa698a34c9d9f1dca, "C is as portable as Stonehedge!!"},
+	{0xf6c1e2a8c26c5cfc, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0xd402559dfe9b70c, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0xdb6efff26aa94946, "How can you write a big system without C++?  -Paul Glick"},
+}
+
+var tab = MakeTable(ISO)
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		c := New(tab)
+		io.WriteString(c, g.in)
+		s := c.Sum64()
+		if s != g.out {
+			t.Errorf("crc64(%s) = 0x%x want 0x%x", g.in, s, g.out)
+			t.FailNow()
+		}
+	}
+}
+
+func BenchmarkCrc64KB(b *testing.B) {
+	b.StopTimer()
+	data := make([]uint8, 1024)
+	for i := 0; i < 1024; i++ {
+		data[i] = uint8(i)
+	}
+	c := New(tab)
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		c.Write(data)
+	}
+}
diff --git a/src/pkg/hash/fnv/Makefile b/src/pkg/hash/fnv/Makefile
new file mode 100644
index 000000000..4c8a4ecf0
--- /dev/null
+++ b/src/pkg/hash/fnv/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=hash/fnv
+GOFILES=\
+	fnv.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/hash/fnv/fnv.go b/src/pkg/hash/fnv/fnv.go
new file mode 100644
index 000000000..3ff7d7c75
--- /dev/null
+++ b/src/pkg/hash/fnv/fnv.go
@@ -0,0 +1,131 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package fnv implements FNV-1 and FNV-1a, non-cryptographic hash functions
+// created by Glenn Fowler, Landon Curt Noll, and Phong Vo.
+// See http://isthe.com/chongo/tech/comp/fnv/.
+package fnv
+
+import (
+	"encoding/binary"
+	"hash"
+	"os"
+)
+
+type (
+	sum32  uint32
+	sum32a uint32
+	sum64  uint64
+	sum64a uint64
+)
+
+const (
+	offset32 = 2166136261
+	offset64 = 14695981039346656037
+	prime32  = 16777619
+	prime64  = 1099511628211
+)
+
+// New32 returns a new 32-bit FNV-1 hash.Hash.
+func New32() hash.Hash32 {
+	var s sum32 = offset32
+	return &s
+}
+
+// New32a returns a new 32-bit FNV-1a hash.Hash.
+func New32a() hash.Hash32 {
+	var s sum32a = offset32
+	return &s
+}
+
+// New64 returns a new 64-bit FNV-1 hash.Hash.
+func New64() hash.Hash64 {
+	var s sum64 = offset64
+	return &s
+}
+
+// New64a returns a new 64-bit FNV-1a hash.Hash.
+func New64a() hash.Hash64 {
+	var s sum64a = offset64
+	return &s
+}
+
+func (s *sum32) Reset()  { *s = offset32 }
+func (s *sum32a) Reset() { *s = offset32 }
+func (s *sum64) Reset()  { *s = offset64 }
+func (s *sum64a) Reset() { *s = offset64 }
+
+func (s *sum32) Sum32() uint32  { return uint32(*s) }
+func (s *sum32a) Sum32() uint32 { return uint32(*s) }
+func (s *sum64) Sum64() uint64  { return uint64(*s) }
+func (s *sum64a) Sum64() uint64 { return uint64(*s) }
+
+func (s *sum32) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash *= prime32
+		hash ^= sum32(c)
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum32a) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash ^= sum32a(c)
+		hash *= prime32
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum64) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash *= prime64
+		hash ^= sum64(c)
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum64a) Write(data []byte) (int, os.Error) {
+	hash := *s
+	for _, c := range data {
+		hash ^= sum64a(c)
+		hash *= prime64
+	}
+	*s = hash
+	return len(data), nil
+}
+
+func (s *sum32) Size() int  { return 4 }
+func (s *sum32a) Size() int { return 4 }
+func (s *sum64) Size() int  { return 8 }
+func (s *sum64a) Size() int { return 8 }
+
+func (s *sum32) Sum() []byte {
+	a := make([]byte, 4)
+	binary.BigEndian.PutUint32(a, uint32(*s))
+	return a
+}
+
+func (s *sum32a) Sum() []byte {
+	a := make([]byte, 4)
+	binary.BigEndian.PutUint32(a, uint32(*s))
+	return a
+}
+
+func (s *sum64) Sum() []byte {
+	a := make([]byte, 8)
+	binary.BigEndian.PutUint64(a, uint64(*s))
+	return a
+}
+
+func (s *sum64a) Sum() []byte {
+	a := make([]byte, 8)
+	binary.BigEndian.PutUint64(a, uint64(*s))
+	return a
+}
diff --git a/src/pkg/hash/fnv/fnv_test.go b/src/pkg/hash/fnv/fnv_test.go
new file mode 100644
index 000000000..429230c80
--- /dev/null
+++ b/src/pkg/hash/fnv/fnv_test.go
@@ -0,0 +1,167 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fnv
+
+import (
+	"bytes"
+	"encoding/binary"
+	"hash"
+	"testing"
+)
+
+const testDataSize = 40
+
+type golden struct {
+	sum  []byte
+	text string
+}
+
+var golden32 = []golden{
+	{[]byte{0x81, 0x1c, 0x9d, 0xc5}, ""},
+	{[]byte{0x05, 0x0c, 0x5d, 0x7e}, "a"},
+	{[]byte{0x70, 0x77, 0x2d, 0x38}, "ab"},
+	{[]byte{0x43, 0x9c, 0x2f, 0x4b}, "abc"},
+}
+
+var golden32a = []golden{
+	{[]byte{0x81, 0x1c, 0x9d, 0xc5}, ""},
+	{[]byte{0xe4, 0x0c, 0x29, 0x2c}, "a"},
+	{[]byte{0x4d, 0x25, 0x05, 0xca}, "ab"},
+	{[]byte{0x1a, 0x47, 0xe9, 0x0b}, "abc"},
+}
+
+var golden64 = []golden{
+	{[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, ""},
+	{[]byte{0xaf, 0x63, 0xbd, 0x4c, 0x86, 0x01, 0xb7, 0xbe}, "a"},
+	{[]byte{0x08, 0x32, 0x67, 0x07, 0xb4, 0xeb, 0x37, 0xb8}, "ab"},
+	{[]byte{0xd8, 0xdc, 0xca, 0x18, 0x6b, 0xaf, 0xad, 0xcb}, "abc"},
+}
+
+var golden64a = []golden{
+	{[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, ""},
+	{[]byte{0xaf, 0x63, 0xdc, 0x4c, 0x86, 0x01, 0xec, 0x8c}, "a"},
+	{[]byte{0x08, 0x9c, 0x44, 0x07, 0xb5, 0x45, 0x98, 0x6a}, "ab"},
+	{[]byte{0xe7, 0x1f, 0xa2, 0x19, 0x05, 0x41, 0x57, 0x4b}, "abc"},
+}
+
+func TestGolden32(t *testing.T) {
+	testGolden(t, New32(), golden32)
+}
+
+func TestGolden32a(t *testing.T) {
+	testGolden(t, New32a(), golden32a)
+}
+
+func TestGolden64(t *testing.T) {
+	testGolden(t, New64(), golden64)
+}
+
+func TestGolden64a(t *testing.T) {
+	testGolden(t, New64a(), golden64a)
+}
+
+func testGolden(t *testing.T, hash hash.Hash, gold []golden) {
+	for _, g := range gold {
+		hash.Reset()
+		done, error := hash.Write([]byte(g.text))
+		if error != nil {
+			t.Fatalf("write error: %s", error)
+		}
+		if done != len(g.text) {
+			t.Fatalf("wrote only %d out of %d bytes", done, len(g.text))
+		}
+		if actual := hash.Sum(); !bytes.Equal(g.sum, actual) {
+			t.Errorf("hash(%q) = 0x%x want 0x%x", g.text, actual, g.sum)
+		}
+	}
+}
+
+func TestIntegrity32(t *testing.T) {
+	testIntegrity(t, New32())
+}
+
+func TestIntegrity32a(t *testing.T) {
+	testIntegrity(t, New32a())
+}
+
+func TestIntegrity64(t *testing.T) {
+	testIntegrity(t, New64())
+}
+
+func TestIntegrity64a(t *testing.T) {
+	testIntegrity(t, New64a())
+}
+
+func testIntegrity(t *testing.T, h hash.Hash) {
+	data := []byte{'1', '2', 3, 4, 5}
+	h.Write(data)
+	sum := h.Sum()
+
+	if size := h.Size(); size != len(sum) {
+		t.Fatalf("Size()=%d but len(Sum())=%d", size, len(sum))
+	}
+
+	if a := h.Sum(); !bytes.Equal(sum, a) {
+		t.Fatalf("first Sum()=0x%x, second Sum()=0x%x", sum, a)
+	}
+
+	h.Reset()
+	h.Write(data)
+	if a := h.Sum(); !bytes.Equal(sum, a) {
+		t.Fatalf("Sum()=0x%x, but after Reset() Sum()=0x%x", sum, a)
+	}
+
+	h.Reset()
+	h.Write(data[:2])
+	h.Write(data[2:])
+	if a := h.Sum(); !bytes.Equal(sum, a) {
+		t.Fatalf("Sum()=0x%x, but with partial writes, Sum()=0x%x", sum, a)
+	}
+
+	switch h.Size() {
+	case 4:
+		sum32 := h.(hash.Hash32).Sum32()
+		if sum32 != binary.BigEndian.Uint32(sum) {
+			t.Fatalf("Sum()=0x%x, but Sum32()=0x%x", sum, sum32)
+		}
+	case 8:
+		sum64 := h.(hash.Hash64).Sum64()
+		if sum64 != binary.BigEndian.Uint64(sum) {
+			t.Fatalf("Sum()=0x%x, but Sum64()=0x%x", sum, sum64)
+		}
+	}
+}
+
+func Benchmark32(b *testing.B) {
+	benchmark(b, New32())
+}
+
+func Benchmark32a(b *testing.B) {
+	benchmark(b, New32a())
+}
+
+func Benchmark64(b *testing.B) {
+	benchmark(b, New64())
+}
+
+func Benchmark64a(b *testing.B) {
+	benchmark(b, New64a())
+}
+
+func benchmark(b *testing.B, h hash.Hash) {
+	b.ResetTimer()
+	b.SetBytes(testDataSize)
+	data := make([]byte, testDataSize)
+	for i := range data {
+		data[i] = byte(i + 'a')
+	}
+
+	b.StartTimer()
+	for todo := b.N; todo != 0; todo-- {
+		h.Reset()
+		h.Write(data)
+		h.Sum()
+	}
+}
diff --git a/src/pkg/hash/hash.go b/src/pkg/hash/hash.go
new file mode 100644
index 000000000..3536c0b6a
--- /dev/null
+++ b/src/pkg/hash/hash.go
@@ -0,0 +1,37 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package hash provides interfaces for hash functions.
+package hash
+
+import "io"
+
+// Hash is the common interface implemented by all hash functions.
+type Hash interface {
+	// Write adds more data to the running hash.
+	// It never returns an error.
+	io.Writer
+
+	// Sum returns the current hash, without changing the
+	// underlying hash state.
+	Sum() []byte
+
+	// Reset resets the hash to one with zero bytes written.
+	Reset()
+
+	// Size returns the number of bytes Sum will return.
+	Size() int
+}
+
+// Hash32 is the common interface implemented by all 32-bit hash functions.
+type Hash32 interface {
+	Hash
+	Sum32() uint32
+}
+
+// Hash64 is the common interface implemented by all 64-bit hash functions.
+type Hash64 interface {
+	Hash
+	Sum64() uint64
+}
diff --git a/src/pkg/hash/test_cases.txt b/src/pkg/hash/test_cases.txt
new file mode 100644
index 000000000..26d3ccc05
--- /dev/null
+++ b/src/pkg/hash/test_cases.txt
@@ -0,0 +1,31 @@
+
+a
+ab
+abc
+abcd
+abcde
+abcdef
+abcdefg
+abcdefgh
+abcdefghi
+abcdefghij
+Discard medicine more than two years old.
+He who has a shady past knows that nice guys finish last.
+I wouldn't marry him with a ten foot pole.
+Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave
+The days of the digital watch are numbered.  -Tom Stoppard
+Nepal premier won't resign.
+For every action there is an equal and opposite government program.
+His money is twice tainted: 'taint yours and 'taint mine.
+There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977
+It's a tiny change to the code and not completely disgusting. - Bob Manchek
+size:  a.out:  bad magic
+The major problem is with sendmail.  -Mark Horton
+Give me a rock, paper and scissors and I will move the world.  CCFestoon
+If the enemy is within range, then so are you.
+It's well we cannot hear the screams/That we create in others' dreams.
+You remind me of a TV show, but that's all right: I watch it anyway.
+C is as portable as Stonehedge!!
+Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley
+The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule
+How can you write a big system without C++?  -Paul Glick
diff --git a/src/pkg/hash/test_gen.awk b/src/pkg/hash/test_gen.awk
new file mode 100644
index 000000000..804f78679
--- /dev/null
+++ b/src/pkg/hash/test_gen.awk
@@ -0,0 +1,14 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# awk -f test_gen.awk test_cases.txt
+# generates test case table.
+# edit next line to set particular reference implementation and name.
+BEGIN { cmd = "echo -n `9 sha1sum`"; name = "Sha1Test" }
+{
+	printf("\t%s{ \"", name);
+	printf("%s", $0) |cmd;
+	close(cmd);
+	printf("\", \"%s\" },\n", $0);
+}
diff --git a/src/pkg/html/Makefile b/src/pkg/html/Makefile
new file mode 100644
index 000000000..28dc1a3f5
--- /dev/null
+++ b/src/pkg/html/Makefile
@@ -0,0 +1,17 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.inc
+
+TARG=html
+GOFILES=\
+	const.go\
+	doc.go\
+	entity.go\
+	escape.go\
+	node.go\
+	parse.go\
+	token.go\
+
+include ../../Make.pkg
diff --git a/src/pkg/html/const.go b/src/pkg/html/const.go
new file mode 100644
index 000000000..9078d2601
--- /dev/null
+++ b/src/pkg/html/const.go
@@ -0,0 +1,90 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// Section 11.2.3.2 of the HTML5 specification says "The following elements
+// have varying levels of special parsing rules".
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#the-stack-of-open-elements
+var isSpecialElement = map[string]bool{
+	"address":    true,
+	"applet":     true,
+	"area":       true,
+	"article":    true,
+	"aside":      true,
+	"base":       true,
+	"basefont":   true,
+	"bgsound":    true,
+	"blockquote": true,
+	"body":       true,
+	"br":         true,
+	"button":     true,
+	"caption":    true,
+	"center":     true,
+	"col":        true,
+	"colgroup":   true,
+	"command":    true,
+	"dd":         true,
+	"details":    true,
+	"dir":        true,
+	"div":        true,
+	"dl":         true,
+	"dt":         true,
+	"embed":      true,
+	"fieldset":   true,
+	"figcaption": true,
+	"figure":     true,
+	"footer":     true,
+	"form":       true,
+	"frame":      true,
+	"frameset":   true,
+	"h1":         true,
+	"h2":         true,
+	"h3":         true,
+	"h4":         true,
+	"h5":         true,
+	"h6":         true,
+	"head":       true,
+	"header":     true,
+	"hgroup":     true,
+	"hr":         true,
+	"html":       true,
+	"iframe":     true,
+	"img":        true,
+	"input":      true,
+	"isindex":    true,
+	"li":         true,
+	"link":       true,
+	"listing":    true,
+	"marquee":    true,
+	"menu":       true,
+	"meta":       true,
+	"nav":        true,
+	"noembed":    true,
+	"noframes":   true,
+	"noscript":   true,
+	"object":     true,
+	"ol":         true,
+	"p":          true,
+	"param":      true,
+	"plaintext":  true,
+	"pre":        true,
+	"script":     true,
+	"section":    true,
+	"select":     true,
+	"style":      true,
+	"summary":    true,
+	"table":      true,
+	"tbody":      true,
+	"td":         true,
+	"textarea":   true,
+	"tfoot":      true,
+	"th":         true,
+	"thead":      true,
+	"title":      true,
+	"tr":         true,
+	"ul":         true,
+	"wbr":        true,
+	"xmp":        true,
+}
diff --git a/src/pkg/html/doc.go b/src/pkg/html/doc.go
new file mode 100644
index 000000000..5bc063086
--- /dev/null
+++ b/src/pkg/html/doc.go
@@ -0,0 +1,110 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package html implements an HTML5-compliant tokenizer and parser.
+INCOMPLETE.
+
+Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
+caller's responsibility to ensure that r provides UTF-8 encoded HTML.
+
+	z := html.NewTokenizer(r)
+
+Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
+which parses the next token and returns its type, or an error:
+
+	for {
+		tt := z.Next()
+		if tt == html.ErrorToken {
+			// ...
+			return ...
+		}
+		// Process the current token.
+	}
+
+There are two APIs for retrieving the current token. The high-level API is to
+call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
+allow optionally calling Raw after Next but before Token, Text, TagName, or
+TagAttr. In EBNF notation, the valid call sequence per token is:
+
+	Next {Raw} [ Token | Text | TagName {TagAttr} ]
+
+Token returns an independent data structure that completely describes a token.
+Entities (such as "<") are unescaped, tag names and attribute keys are
+lower-cased, and attributes are collected into a []Attribute. For example:
+
+	for {
+		if z.Next() == html.ErrorToken {
+			// Returning os.EOF indicates success.
+			return z.Error()
+		}
+		emitToken(z.Token())
+	}
+
+The low-level API performs fewer allocations and copies, but the contents of
+the []byte values returned by Text, TagName and TagAttr may change on the next
+call to Next. For example, to extract an HTML page's anchor text:
+
+	depth := 0
+	for {
+		tt := z.Next()
+		switch tt {
+		case ErrorToken:
+			return z.Error()
+		case TextToken:
+			if depth > 0 {
+				// emitBytes should copy the []byte it receives,
+				// if it doesn't process it immediately.
+				emitBytes(z.Text())
+			}
+		case StartTagToken, EndTagToken:
+			tn, _ := z.TagName()
+			if len(tn) == 1 && tn[0] == 'a' {
+				if tt == StartTag {
+					depth++
+				} else {
+					depth--
+				}
+			}
+		}
+	}
+
+A Tokenizer typically skips over HTML comments. To return comment tokens, set
+Tokenizer.ReturnComments to true before looping over calls to Next.
+
+Parsing is done by calling Parse with an io.Reader, which returns the root of
+the parse tree (the document element) as a *Node. It is the caller's
+responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
+example, to process each anchor node in depth-first order:
+
+	doc, err := html.Parse(r)
+	if err != nil {
+		// ...
+	}
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			// Do something with n...
+		}
+		for _, c := range n.Child {
+			f(c)
+		}
+	}
+	f(doc)
+
+The relevant specifications include:
+http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html and
+http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html
+*/
+package html
+
+// The tokenization algorithm implemented by this package is not a line-by-line
+// transliteration of the relatively verbose state-machine in the WHATWG
+// specification. A more direct approach is used instead, where the program
+// counter implies the state, such as whether it is tokenizing a tag or a text
+// node. Specification compliance is verified by checking expected and actual
+// outputs over a test suite rather than aiming for algorithmic fidelity.
+
+// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
+// TODO(nigeltao): How does parsing interact with a JavaScript engine?
diff --git a/src/pkg/html/entity.go b/src/pkg/html/entity.go
new file mode 100644
index 000000000..21263e22d
--- /dev/null
+++ b/src/pkg/html/entity.go
@@ -0,0 +1,2253 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// All entities that do not end with ';' are 6 or fewer bytes long.
+const longestEntityWithoutSemicolon = 6
+
+// entity is a map from HTML entity names to their values. The semicolon matters:
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/named-character-references.html
+// lists both "amp" and "amp;" as two separate entries.
+//
+// Note that the HTML5 list is larger than the HTML4 list at
+// http://www.w3.org/TR/html4/sgml/entities.html
+var entity = map[string]int{
+	"AElig;":                           '\U000000C6',
+	"AMP;":                             '\U00000026',
+	"Aacute;":                          '\U000000C1',
+	"Abreve;":                          '\U00000102',
+	"Acirc;":                           '\U000000C2',
+	"Acy;":                             '\U00000410',
+	"Afr;":                             '\U0001D504',
+	"Agrave;":                          '\U000000C0',
+	"Alpha;":                           '\U00000391',
+	"Amacr;":                           '\U00000100',
+	"And;":                             '\U00002A53',
+	"Aogon;":                           '\U00000104',
+	"Aopf;":                            '\U0001D538',
+	"ApplyFunction;":                   '\U00002061',
+	"Aring;":                           '\U000000C5',
+	"Ascr;":                            '\U0001D49C',
+	"Assign;":                          '\U00002254',
+	"Atilde;":                          '\U000000C3',
+	"Auml;":                            '\U000000C4',
+	"Backslash;":                       '\U00002216',
+	"Barv;":                            '\U00002AE7',
+	"Barwed;":                          '\U00002306',
+	"Bcy;":                             '\U00000411',
+	"Because;":                         '\U00002235',
+	"Bernoullis;":                      '\U0000212C',
+	"Beta;":                            '\U00000392',
+	"Bfr;":                             '\U0001D505',
+	"Bopf;":                            '\U0001D539',
+	"Breve;":                           '\U000002D8',
+	"Bscr;":                            '\U0000212C',
+	"Bumpeq;":                          '\U0000224E',
+	"CHcy;":                            '\U00000427',
+	"COPY;":                            '\U000000A9',
+	"Cacute;":                          '\U00000106',
+	"Cap;":                             '\U000022D2',
+	"CapitalDifferentialD;":            '\U00002145',
+	"Cayleys;":                         '\U0000212D',
+	"Ccaron;":                          '\U0000010C',
+	"Ccedil;":                          '\U000000C7',
+	"Ccirc;":                           '\U00000108',
+	"Cconint;":                         '\U00002230',
+	"Cdot;":                            '\U0000010A',
+	"Cedilla;":                         '\U000000B8',
+	"CenterDot;":                       '\U000000B7',
+	"Cfr;":                             '\U0000212D',
+	"Chi;":                             '\U000003A7',
+	"CircleDot;":                       '\U00002299',
+	"CircleMinus;":                     '\U00002296',
+	"CirclePlus;":                      '\U00002295',
+	"CircleTimes;":                     '\U00002297',
+	"ClockwiseContourIntegral;":        '\U00002232',
+	"CloseCurlyDoubleQuote;":           '\U0000201D',
+	"CloseCurlyQuote;":                 '\U00002019',
+	"Colon;":                           '\U00002237',
+	"Colone;":                          '\U00002A74',
+	"Congruent;":                       '\U00002261',
+	"Conint;":                          '\U0000222F',
+	"ContourIntegral;":                 '\U0000222E',
+	"Copf;":                            '\U00002102',
+	"Coproduct;":                       '\U00002210',
+	"CounterClockwiseContourIntegral;": '\U00002233',
+	"Cross;":                           '\U00002A2F',
+	"Cscr;":                            '\U0001D49E',
+	"Cup;":                             '\U000022D3',
+	"CupCap;":                          '\U0000224D',
+	"DD;":                              '\U00002145',
+	"DDotrahd;":                        '\U00002911',
+	"DJcy;":                            '\U00000402',
+	"DScy;":                            '\U00000405',
+	"DZcy;":                            '\U0000040F',
+	"Dagger;":                          '\U00002021',
+	"Darr;":                            '\U000021A1',
+	"Dashv;":                           '\U00002AE4',
+	"Dcaron;":                          '\U0000010E',
+	"Dcy;":                             '\U00000414',
+	"Del;":                             '\U00002207',
+	"Delta;":                           '\U00000394',
+	"Dfr;":                             '\U0001D507',
+	"DiacriticalAcute;":                '\U000000B4',
+	"DiacriticalDot;":                  '\U000002D9',
+	"DiacriticalDoubleAcute;":          '\U000002DD',
+	"DiacriticalGrave;":                '\U00000060',
+	"DiacriticalTilde;":                '\U000002DC',
+	"Diamond;":                         '\U000022C4',
+	"DifferentialD;":                   '\U00002146',
+	"Dopf;":                            '\U0001D53B',
+	"Dot;":                             '\U000000A8',
+	"DotDot;":                          '\U000020DC',
+	"DotEqual;":                        '\U00002250',
+	"DoubleContourIntegral;":           '\U0000222F',
+	"DoubleDot;":                       '\U000000A8',
+	"DoubleDownArrow;":                 '\U000021D3',
+	"DoubleLeftArrow;":                 '\U000021D0',
+	"DoubleLeftRightArrow;":            '\U000021D4',
+	"DoubleLeftTee;":                   '\U00002AE4',
+	"DoubleLongLeftArrow;":             '\U000027F8',
+	"DoubleLongLeftRightArrow;":        '\U000027FA',
+	"DoubleLongRightArrow;":            '\U000027F9',
+	"DoubleRightArrow;":                '\U000021D2',
+	"DoubleRightTee;":                  '\U000022A8',
+	"DoubleUpArrow;":                   '\U000021D1',
+	"DoubleUpDownArrow;":               '\U000021D5',
+	"DoubleVerticalBar;":               '\U00002225',
+	"DownArrow;":                       '\U00002193',
+	"DownArrowBar;":                    '\U00002913',
+	"DownArrowUpArrow;":                '\U000021F5',
+	"DownBreve;":                       '\U00000311',
+	"DownLeftRightVector;":             '\U00002950',
+	"DownLeftTeeVector;":               '\U0000295E',
+	"DownLeftVector;":                  '\U000021BD',
+	"DownLeftVectorBar;":               '\U00002956',
+	"DownRightTeeVector;":              '\U0000295F',
+	"DownRightVector;":                 '\U000021C1',
+	"DownRightVectorBar;":              '\U00002957',
+	"DownTee;":                         '\U000022A4',
+	"DownTeeArrow;":                    '\U000021A7',
+	"Downarrow;":                       '\U000021D3',
+	"Dscr;":                            '\U0001D49F',
+	"Dstrok;":                          '\U00000110',
+	"ENG;":                             '\U0000014A',
+	"ETH;":                             '\U000000D0',
+	"Eacute;":                          '\U000000C9',
+	"Ecaron;":                          '\U0000011A',
+	"Ecirc;":                           '\U000000CA',
+	"Ecy;":                             '\U0000042D',
+	"Edot;":                            '\U00000116',
+	"Efr;":                             '\U0001D508',
+	"Egrave;":                          '\U000000C8',
+	"Element;":                         '\U00002208',
+	"Emacr;":                           '\U00000112',
+	"EmptySmallSquare;":                '\U000025FB',
+	"EmptyVerySmallSquare;":            '\U000025AB',
+	"Eogon;":                           '\U00000118',
+	"Eopf;":                            '\U0001D53C',
+	"Epsilon;":                         '\U00000395',
+	"Equal;":                           '\U00002A75',
+	"EqualTilde;":                      '\U00002242',
+	"Equilibrium;":                     '\U000021CC',
+	"Escr;":                            '\U00002130',
+	"Esim;":                            '\U00002A73',
+	"Eta;":                             '\U00000397',
+	"Euml;":                            '\U000000CB',
+	"Exists;":                          '\U00002203',
+	"ExponentialE;":                    '\U00002147',
+	"Fcy;":                             '\U00000424',
+	"Ffr;":                             '\U0001D509',
+	"FilledSmallSquare;":               '\U000025FC',
+	"FilledVerySmallSquare;":           '\U000025AA',
+	"Fopf;":                            '\U0001D53D',
+	"ForAll;":                          '\U00002200',
+	"Fouriertrf;":                      '\U00002131',
+	"Fscr;":                            '\U00002131',
+	"GJcy;":                            '\U00000403',
+	"GT;":                              '\U0000003E',
+	"Gamma;":                           '\U00000393',
+	"Gammad;":                          '\U000003DC',
+	"Gbreve;":                          '\U0000011E',
+	"Gcedil;":                          '\U00000122',
+	"Gcirc;":                           '\U0000011C',
+	"Gcy;":                             '\U00000413',
+	"Gdot;":                            '\U00000120',
+	"Gfr;":                             '\U0001D50A',
+	"Gg;":                              '\U000022D9',
+	"Gopf;":                            '\U0001D53E',
+	"GreaterEqual;":                    '\U00002265',
+	"GreaterEqualLess;":                '\U000022DB',
+	"GreaterFullEqual;":                '\U00002267',
+	"GreaterGreater;":                  '\U00002AA2',
+	"GreaterLess;":                     '\U00002277',
+	"GreaterSlantEqual;":               '\U00002A7E',
+	"GreaterTilde;":                    '\U00002273',
+	"Gscr;":                            '\U0001D4A2',
+	"Gt;":                              '\U0000226B',
+	"HARDcy;":                          '\U0000042A',
+	"Hacek;":                           '\U000002C7',
+	"Hat;":                             '\U0000005E',
+	"Hcirc;":                           '\U00000124',
+	"Hfr;":                             '\U0000210C',
+	"HilbertSpace;":                    '\U0000210B',
+	"Hopf;":                            '\U0000210D',
+	"HorizontalLine;":                  '\U00002500',
+	"Hscr;":                            '\U0000210B',
+	"Hstrok;":                          '\U00000126',
+	"HumpDownHump;":                    '\U0000224E',
+	"HumpEqual;":                       '\U0000224F',
+	"IEcy;":                            '\U00000415',
+	"IJlig;":                           '\U00000132',
+	"IOcy;":                            '\U00000401',
+	"Iacute;":                          '\U000000CD',
+	"Icirc;":                           '\U000000CE',
+	"Icy;":                             '\U00000418',
+	"Idot;":                            '\U00000130',
+	"Ifr;":                             '\U00002111',
+	"Igrave;":                          '\U000000CC',
+	"Im;":                              '\U00002111',
+	"Imacr;":                           '\U0000012A',
+	"ImaginaryI;":                      '\U00002148',
+	"Implies;":                         '\U000021D2',
+	"Int;":                             '\U0000222C',
+	"Integral;":                        '\U0000222B',
+	"Intersection;":                    '\U000022C2',
+	"InvisibleComma;":                  '\U00002063',
+	"InvisibleTimes;":                  '\U00002062',
+	"Iogon;":                           '\U0000012E',
+	"Iopf;":                            '\U0001D540',
+	"Iota;":                            '\U00000399',
+	"Iscr;":                            '\U00002110',
+	"Itilde;":                          '\U00000128',
+	"Iukcy;":                           '\U00000406',
+	"Iuml;":                            '\U000000CF',
+	"Jcirc;":                           '\U00000134',
+	"Jcy;":                             '\U00000419',
+	"Jfr;":                             '\U0001D50D',
+	"Jopf;":                            '\U0001D541',
+	"Jscr;":                            '\U0001D4A5',
+	"Jsercy;":                          '\U00000408',
+	"Jukcy;":                           '\U00000404',
+	"KHcy;":                            '\U00000425',
+	"KJcy;":                            '\U0000040C',
+	"Kappa;":                           '\U0000039A',
+	"Kcedil;":                          '\U00000136',
+	"Kcy;":                             '\U0000041A',
+	"Kfr;":                             '\U0001D50E',
+	"Kopf;":                            '\U0001D542',
+	"Kscr;":                            '\U0001D4A6',
+	"LJcy;":                            '\U00000409',
+	"LT;":                              '\U0000003C',
+	"Lacute;":                          '\U00000139',
+	"Lambda;":                          '\U0000039B',
+	"Lang;":                            '\U000027EA',
+	"Laplacetrf;":                      '\U00002112',
+	"Larr;":                            '\U0000219E',
+	"Lcaron;":                          '\U0000013D',
+	"Lcedil;":                          '\U0000013B',
+	"Lcy;":                             '\U0000041B',
+	"LeftAngleBracket;":                '\U000027E8',
+	"LeftArrow;":                       '\U00002190',
+	"LeftArrowBar;":                    '\U000021E4',
+	"LeftArrowRightArrow;":             '\U000021C6',
+	"LeftCeiling;":                     '\U00002308',
+	"LeftDoubleBracket;":               '\U000027E6',
+	"LeftDownTeeVector;":               '\U00002961',
+	"LeftDownVector;":                  '\U000021C3',
+	"LeftDownVectorBar;":               '\U00002959',
+	"LeftFloor;":                       '\U0000230A',
+	"LeftRightArrow;":                  '\U00002194',
+	"LeftRightVector;":                 '\U0000294E',
+	"LeftTee;":                         '\U000022A3',
+	"LeftTeeArrow;":                    '\U000021A4',
+	"LeftTeeVector;":                   '\U0000295A',
+	"LeftTriangle;":                    '\U000022B2',
+	"LeftTriangleBar;":                 '\U000029CF',
+	"LeftTriangleEqual;":               '\U000022B4',
+	"LeftUpDownVector;":                '\U00002951',
+	"LeftUpTeeVector;":                 '\U00002960',
+	"LeftUpVector;":                    '\U000021BF',
+	"LeftUpVectorBar;":                 '\U00002958',
+	"LeftVector;":                      '\U000021BC',
+	"LeftVectorBar;":                   '\U00002952',
+	"Leftarrow;":                       '\U000021D0',
+	"Leftrightarrow;":                  '\U000021D4',
+	"LessEqualGreater;":                '\U000022DA',
+	"LessFullEqual;":                   '\U00002266',
+	"LessGreater;":                     '\U00002276',
+	"LessLess;":                        '\U00002AA1',
+	"LessSlantEqual;":                  '\U00002A7D',
+	"LessTilde;":                       '\U00002272',
+	"Lfr;":                             '\U0001D50F',
+	"Ll;":                              '\U000022D8',
+	"Lleftarrow;":                      '\U000021DA',
+	"Lmidot;":                          '\U0000013F',
+	"LongLeftArrow;":                   '\U000027F5',
+	"LongLeftRightArrow;":              '\U000027F7',
+	"LongRightArrow;":                  '\U000027F6',
+	"Longleftarrow;":                   '\U000027F8',
+	"Longleftrightarrow;":              '\U000027FA',
+	"Longrightarrow;":                  '\U000027F9',
+	"Lopf;":                            '\U0001D543',
+	"LowerLeftArrow;":                  '\U00002199',
+	"LowerRightArrow;":                 '\U00002198',
+	"Lscr;":                            '\U00002112',
+	"Lsh;":                             '\U000021B0',
+	"Lstrok;":                          '\U00000141',
+	"Lt;":                              '\U0000226A',
+	"Map;":                             '\U00002905',
+	"Mcy;":                             '\U0000041C',
+	"MediumSpace;":                     '\U0000205F',
+	"Mellintrf;":                       '\U00002133',
+	"Mfr;":                             '\U0001D510',
+	"MinusPlus;":                       '\U00002213',
+	"Mopf;":                            '\U0001D544',
+	"Mscr;":                            '\U00002133',
+	"Mu;":                              '\U0000039C',
+	"NJcy;":                            '\U0000040A',
+	"Nacute;":                          '\U00000143',
+	"Ncaron;":                          '\U00000147',
+	"Ncedil;":                          '\U00000145',
+	"Ncy;":                             '\U0000041D',
+	"NegativeMediumSpace;":             '\U0000200B',
+	"NegativeThickSpace;":              '\U0000200B',
+	"NegativeThinSpace;":               '\U0000200B',
+	"NegativeVeryThinSpace;":           '\U0000200B',
+	"NestedGreaterGreater;":            '\U0000226B',
+	"NestedLessLess;":                  '\U0000226A',
+	"NewLine;":                         '\U0000000A',
+	"Nfr;":                             '\U0001D511',
+	"NoBreak;":                         '\U00002060',
+	"NonBreakingSpace;":                '\U000000A0',
+	"Nopf;":                            '\U00002115',
+	"Not;":                             '\U00002AEC',
+	"NotCongruent;":                    '\U00002262',
+	"NotCupCap;":                       '\U0000226D',
+	"NotDoubleVerticalBar;":            '\U00002226',
+	"NotElement;":                      '\U00002209',
+	"NotEqual;":                        '\U00002260',
+	"NotExists;":                       '\U00002204',
+	"NotGreater;":                      '\U0000226F',
+	"NotGreaterEqual;":                 '\U00002271',
+	"NotGreaterLess;":                  '\U00002279',
+	"NotGreaterTilde;":                 '\U00002275',
+	"NotLeftTriangle;":                 '\U000022EA',
+	"NotLeftTriangleEqual;":            '\U000022EC',
+	"NotLess;":                         '\U0000226E',
+	"NotLessEqual;":                    '\U00002270',
+	"NotLessGreater;":                  '\U00002278',
+	"NotLessTilde;":                    '\U00002274',
+	"NotPrecedes;":                     '\U00002280',
+	"NotPrecedesSlantEqual;":           '\U000022E0',
+	"NotReverseElement;":               '\U0000220C',
+	"NotRightTriangle;":                '\U000022EB',
+	"NotRightTriangleEqual;":           '\U000022ED',
+	"NotSquareSubsetEqual;":            '\U000022E2',
+	"NotSquareSupersetEqual;":          '\U000022E3',
+	"NotSubsetEqual;":                  '\U00002288',
+	"NotSucceeds;":                     '\U00002281',
+	"NotSucceedsSlantEqual;":           '\U000022E1',
+	"NotSupersetEqual;":                '\U00002289',
+	"NotTilde;":                        '\U00002241',
+	"NotTildeEqual;":                   '\U00002244',
+	"NotTildeFullEqual;":               '\U00002247',
+	"NotTildeTilde;":                   '\U00002249',
+	"NotVerticalBar;":                  '\U00002224',
+	"Nscr;":                            '\U0001D4A9',
+	"Ntilde;":                          '\U000000D1',
+	"Nu;":                              '\U0000039D',
+	"OElig;":                           '\U00000152',
+	"Oacute;":                          '\U000000D3',
+	"Ocirc;":                           '\U000000D4',
+	"Ocy;":                             '\U0000041E',
+	"Odblac;":                          '\U00000150',
+	"Ofr;":                             '\U0001D512',
+	"Ograve;":                          '\U000000D2',
+	"Omacr;":                           '\U0000014C',
+	"Omega;":                           '\U000003A9',
+	"Omicron;":                         '\U0000039F',
+	"Oopf;":                            '\U0001D546',
+	"OpenCurlyDoubleQuote;":            '\U0000201C',
+	"OpenCurlyQuote;":                  '\U00002018',
+	"Or;":                              '\U00002A54',
+	"Oscr;":                            '\U0001D4AA',
+	"Oslash;":                          '\U000000D8',
+	"Otilde;":                          '\U000000D5',
+	"Otimes;":                          '\U00002A37',
+	"Ouml;":                            '\U000000D6',
+	"OverBar;":                         '\U0000203E',
+	"OverBrace;":                       '\U000023DE',
+	"OverBracket;":                     '\U000023B4',
+	"OverParenthesis;":                 '\U000023DC',
+	"PartialD;":                        '\U00002202',
+	"Pcy;":                             '\U0000041F',
+	"Pfr;":                             '\U0001D513',
+	"Phi;":                             '\U000003A6',
+	"Pi;":                              '\U000003A0',
+	"PlusMinus;":                       '\U000000B1',
+	"Poincareplane;":                   '\U0000210C',
+	"Popf;":                            '\U00002119',
+	"Pr;":                              '\U00002ABB',
+	"Precedes;":                        '\U0000227A',
+	"PrecedesEqual;":                   '\U00002AAF',
+	"PrecedesSlantEqual;":              '\U0000227C',
+	"PrecedesTilde;":                   '\U0000227E',
+	"Prime;":                           '\U00002033',
+	"Product;":                         '\U0000220F',
+	"Proportion;":                      '\U00002237',
+	"Proportional;":                    '\U0000221D',
+	"Pscr;":                            '\U0001D4AB',
+	"Psi;":                             '\U000003A8',
+	"QUOT;":                            '\U00000022',
+	"Qfr;":                             '\U0001D514',
+	"Qopf;":                            '\U0000211A',
+	"Qscr;":                            '\U0001D4AC',
+	"RBarr;":                           '\U00002910',
+	"REG;":                             '\U000000AE',
+	"Racute;":                          '\U00000154',
+	"Rang;":                            '\U000027EB',
+	"Rarr;":                            '\U000021A0',
+	"Rarrtl;":                          '\U00002916',
+	"Rcaron;":                          '\U00000158',
+	"Rcedil;":                          '\U00000156',
+	"Rcy;":                             '\U00000420',
+	"Re;":                              '\U0000211C',
+	"ReverseElement;":                  '\U0000220B',
+	"ReverseEquilibrium;":              '\U000021CB',
+	"ReverseUpEquilibrium;":            '\U0000296F',
+	"Rfr;":                             '\U0000211C',
+	"Rho;":                             '\U000003A1',
+	"RightAngleBracket;":               '\U000027E9',
+	"RightArrow;":                      '\U00002192',
+	"RightArrowBar;":                   '\U000021E5',
+	"RightArrowLeftArrow;":             '\U000021C4',
+	"RightCeiling;":                    '\U00002309',
+	"RightDoubleBracket;":              '\U000027E7',
+	"RightDownTeeVector;":              '\U0000295D',
+	"RightDownVector;":                 '\U000021C2',
+	"RightDownVectorBar;":              '\U00002955',
+	"RightFloor;":                      '\U0000230B',
+	"RightTee;":                        '\U000022A2',
+	"RightTeeArrow;":                   '\U000021A6',
+	"RightTeeVector;":                  '\U0000295B',
+	"RightTriangle;":                   '\U000022B3',
+	"RightTriangleBar;":                '\U000029D0',
+	"RightTriangleEqual;":              '\U000022B5',
+	"RightUpDownVector;":               '\U0000294F',
+	"RightUpTeeVector;":                '\U0000295C',
+	"RightUpVector;":                   '\U000021BE',
+	"RightUpVectorBar;":                '\U00002954',
+	"RightVector;":                     '\U000021C0',
+	"RightVectorBar;":                  '\U00002953',
+	"Rightarrow;":                      '\U000021D2',
+	"Ropf;":                            '\U0000211D',
+	"RoundImplies;":                    '\U00002970',
+	"Rrightarrow;":                     '\U000021DB',
+	"Rscr;":                            '\U0000211B',
+	"Rsh;":                             '\U000021B1',
+	"RuleDelayed;":                     '\U000029F4',
+	"SHCHcy;":                          '\U00000429',
+	"SHcy;":                            '\U00000428',
+	"SOFTcy;":                          '\U0000042C',
+	"Sacute;":                          '\U0000015A',
+	"Sc;":                              '\U00002ABC',
+	"Scaron;":                          '\U00000160',
+	"Scedil;":                          '\U0000015E',
+	"Scirc;":                           '\U0000015C',
+	"Scy;":                             '\U00000421',
+	"Sfr;":                             '\U0001D516',
+	"ShortDownArrow;":                  '\U00002193',
+	"ShortLeftArrow;":                  '\U00002190',
+	"ShortRightArrow;":                 '\U00002192',
+	"ShortUpArrow;":                    '\U00002191',
+	"Sigma;":                           '\U000003A3',
+	"SmallCircle;":                     '\U00002218',
+	"Sopf;":                            '\U0001D54A',
+	"Sqrt;":                            '\U0000221A',
+	"Square;":                          '\U000025A1',
+	"SquareIntersection;":              '\U00002293',
+	"SquareSubset;":                    '\U0000228F',
+	"SquareSubsetEqual;":               '\U00002291',
+	"SquareSuperset;":                  '\U00002290',
+	"SquareSupersetEqual;":             '\U00002292',
+	"SquareUnion;":                     '\U00002294',
+	"Sscr;":                            '\U0001D4AE',
+	"Star;":                            '\U000022C6',
+	"Sub;":                             '\U000022D0',
+	"Subset;":                          '\U000022D0',
+	"SubsetEqual;":                     '\U00002286',
+	"Succeeds;":                        '\U0000227B',
+	"SucceedsEqual;":                   '\U00002AB0',
+	"SucceedsSlantEqual;":              '\U0000227D',
+	"SucceedsTilde;":                   '\U0000227F',
+	"SuchThat;":                        '\U0000220B',
+	"Sum;":                             '\U00002211',
+	"Sup;":                             '\U000022D1',
+	"Superset;":                        '\U00002283',
+	"SupersetEqual;":                   '\U00002287',
+	"Supset;":                          '\U000022D1',
+	"THORN;":                           '\U000000DE',
+	"TRADE;":                           '\U00002122',
+	"TSHcy;":                           '\U0000040B',
+	"TScy;":                            '\U00000426',
+	"Tab;":                             '\U00000009',
+	"Tau;":                             '\U000003A4',
+	"Tcaron;":                          '\U00000164',
+	"Tcedil;":                          '\U00000162',
+	"Tcy;":                             '\U00000422',
+	"Tfr;":                             '\U0001D517',
+	"Therefore;":                       '\U00002234',
+	"Theta;":                           '\U00000398',
+	"ThinSpace;":                       '\U00002009',
+	"Tilde;":                           '\U0000223C',
+	"TildeEqual;":                      '\U00002243',
+	"TildeFullEqual;":                  '\U00002245',
+	"TildeTilde;":                      '\U00002248',
+	"Topf;":                            '\U0001D54B',
+	"TripleDot;":                       '\U000020DB',
+	"Tscr;":                            '\U0001D4AF',
+	"Tstrok;":                          '\U00000166',
+	"Uacute;":                          '\U000000DA',
+	"Uarr;":                            '\U0000219F',
+	"Uarrocir;":                        '\U00002949',
+	"Ubrcy;":                           '\U0000040E',
+	"Ubreve;":                          '\U0000016C',
+	"Ucirc;":                           '\U000000DB',
+	"Ucy;":                             '\U00000423',
+	"Udblac;":                          '\U00000170',
+	"Ufr;":                             '\U0001D518',
+	"Ugrave;":                          '\U000000D9',
+	"Umacr;":                           '\U0000016A',
+	"UnderBar;":                        '\U0000005F',
+	"UnderBrace;":                      '\U000023DF',
+	"UnderBracket;":                    '\U000023B5',
+	"UnderParenthesis;":                '\U000023DD',
+	"Union;":                           '\U000022C3',
+	"UnionPlus;":                       '\U0000228E',
+	"Uogon;":                           '\U00000172',
+	"Uopf;":                            '\U0001D54C',
+	"UpArrow;":                         '\U00002191',
+	"UpArrowBar;":                      '\U00002912',
+	"UpArrowDownArrow;":                '\U000021C5',
+	"UpDownArrow;":                     '\U00002195',
+	"UpEquilibrium;":                   '\U0000296E',
+	"UpTee;":                           '\U000022A5',
+	"UpTeeArrow;":                      '\U000021A5',
+	"Uparrow;":                         '\U000021D1',
+	"Updownarrow;":                     '\U000021D5',
+	"UpperLeftArrow;":                  '\U00002196',
+	"UpperRightArrow;":                 '\U00002197',
+	"Upsi;":                            '\U000003D2',
+	"Upsilon;":                         '\U000003A5',
+	"Uring;":                           '\U0000016E',
+	"Uscr;":                            '\U0001D4B0',
+	"Utilde;":                          '\U00000168',
+	"Uuml;":                            '\U000000DC',
+	"VDash;":                           '\U000022AB',
+	"Vbar;":                            '\U00002AEB',
+	"Vcy;":                             '\U00000412',
+	"Vdash;":                           '\U000022A9',
+	"Vdashl;":                          '\U00002AE6',
+	"Vee;":                             '\U000022C1',
+	"Verbar;":                          '\U00002016',
+	"Vert;":                            '\U00002016',
+	"VerticalBar;":                     '\U00002223',
+	"VerticalLine;":                    '\U0000007C',
+	"VerticalSeparator;":               '\U00002758',
+	"VerticalTilde;":                   '\U00002240',
+	"VeryThinSpace;":                   '\U0000200A',
+	"Vfr;":                             '\U0001D519',
+	"Vopf;":                            '\U0001D54D',
+	"Vscr;":                            '\U0001D4B1',
+	"Vvdash;":                          '\U000022AA',
+	"Wcirc;":                           '\U00000174',
+	"Wedge;":                           '\U000022C0',
+	"Wfr;":                             '\U0001D51A',
+	"Wopf;":                            '\U0001D54E',
+	"Wscr;":                            '\U0001D4B2',
+	"Xfr;":                             '\U0001D51B',
+	"Xi;":                              '\U0000039E',
+	"Xopf;":                            '\U0001D54F',
+	"Xscr;":                            '\U0001D4B3',
+	"YAcy;":                            '\U0000042F',
+	"YIcy;":                            '\U00000407',
+	"YUcy;":                            '\U0000042E',
+	"Yacute;":                          '\U000000DD',
+	"Ycirc;":                           '\U00000176',
+	"Ycy;":                             '\U0000042B',
+	"Yfr;":                             '\U0001D51C',
+	"Yopf;":                            '\U0001D550',
+	"Yscr;":                            '\U0001D4B4',
+	"Yuml;":                            '\U00000178',
+	"ZHcy;":                            '\U00000416',
+	"Zacute;":                          '\U00000179',
+	"Zcaron;":                          '\U0000017D',
+	"Zcy;":                             '\U00000417',
+	"Zdot;":                            '\U0000017B',
+	"ZeroWidthSpace;":                  '\U0000200B',
+	"Zeta;":                            '\U00000396',
+	"Zfr;":                             '\U00002128',
+	"Zopf;":                            '\U00002124',
+	"Zscr;":                            '\U0001D4B5',
+	"aacute;":                          '\U000000E1',
+	"abreve;":                          '\U00000103',
+	"ac;":                              '\U0000223E',
+	"acd;":                             '\U0000223F',
+	"acirc;":                           '\U000000E2',
+	"acute;":                           '\U000000B4',
+	"acy;":                             '\U00000430',
+	"aelig;":                           '\U000000E6',
+	"af;":                              '\U00002061',
+	"afr;":                             '\U0001D51E',
+	"agrave;":                          '\U000000E0',
+	"alefsym;":                         '\U00002135',
+	"aleph;":                           '\U00002135',
+	"alpha;":                           '\U000003B1',
+	"amacr;":                           '\U00000101',
+	"amalg;":                           '\U00002A3F',
+	"amp;":                             '\U00000026',
+	"and;":                             '\U00002227',
+	"andand;":                          '\U00002A55',
+	"andd;":                            '\U00002A5C',
+	"andslope;":                        '\U00002A58',
+	"andv;":                            '\U00002A5A',
+	"ang;":                             '\U00002220',
+	"ange;":                            '\U000029A4',
+	"angle;":                           '\U00002220',
+	"angmsd;":                          '\U00002221',
+	"angmsdaa;":                        '\U000029A8',
+	"angmsdab;":                        '\U000029A9',
+	"angmsdac;":                        '\U000029AA',
+	"angmsdad;":                        '\U000029AB',
+	"angmsdae;":                        '\U000029AC',
+	"angmsdaf;":                        '\U000029AD',
+	"angmsdag;":                        '\U000029AE',
+	"angmsdah;":                        '\U000029AF',
+	"angrt;":                           '\U0000221F',
+	"angrtvb;":                         '\U000022BE',
+	"angrtvbd;":                        '\U0000299D',
+	"angsph;":                          '\U00002222',
+	"angst;":                           '\U000000C5',
+	"angzarr;":                         '\U0000237C',
+	"aogon;":                           '\U00000105',
+	"aopf;":                            '\U0001D552',
+	"ap;":                              '\U00002248',
+	"apE;":                             '\U00002A70',
+	"apacir;":                          '\U00002A6F',
+	"ape;":                             '\U0000224A',
+	"apid;":                            '\U0000224B',
+	"apos;":                            '\U00000027',
+	"approx;":                          '\U00002248',
+	"approxeq;":                        '\U0000224A',
+	"aring;":                           '\U000000E5',
+	"ascr;":                            '\U0001D4B6',
+	"ast;":                             '\U0000002A',
+	"asymp;":                           '\U00002248',
+	"asympeq;":                         '\U0000224D',
+	"atilde;":                          '\U000000E3',
+	"auml;":                            '\U000000E4',
+	"awconint;":                        '\U00002233',
+	"awint;":                           '\U00002A11',
+	"bNot;":                            '\U00002AED',
+	"backcong;":                        '\U0000224C',
+	"backepsilon;":                     '\U000003F6',
+	"backprime;":                       '\U00002035',
+	"backsim;":                         '\U0000223D',
+	"backsimeq;":                       '\U000022CD',
+	"barvee;":                          '\U000022BD',
+	"barwed;":                          '\U00002305',
+	"barwedge;":                        '\U00002305',
+	"bbrk;":                            '\U000023B5',
+	"bbrktbrk;":                        '\U000023B6',
+	"bcong;":                           '\U0000224C',
+	"bcy;":                             '\U00000431',
+	"bdquo;":                           '\U0000201E',
+	"becaus;":                          '\U00002235',
+	"because;":                         '\U00002235',
+	"bemptyv;":                         '\U000029B0',
+	"bepsi;":                           '\U000003F6',
+	"bernou;":                          '\U0000212C',
+	"beta;":                            '\U000003B2',
+	"beth;":                            '\U00002136',
+	"between;":                         '\U0000226C',
+	"bfr;":                             '\U0001D51F',
+	"bigcap;":                          '\U000022C2',
+	"bigcirc;":                         '\U000025EF',
+	"bigcup;":                          '\U000022C3',
+	"bigodot;":                         '\U00002A00',
+	"bigoplus;":                        '\U00002A01',
+	"bigotimes;":                       '\U00002A02',
+	"bigsqcup;":                        '\U00002A06',
+	"bigstar;":                         '\U00002605',
+	"bigtriangledown;":                 '\U000025BD',
+	"bigtriangleup;":                   '\U000025B3',
+	"biguplus;":                        '\U00002A04',
+	"bigvee;":                          '\U000022C1',
+	"bigwedge;":                        '\U000022C0',
+	"bkarow;":                          '\U0000290D',
+	"blacklozenge;":                    '\U000029EB',
+	"blacksquare;":                     '\U000025AA',
+	"blacktriangle;":                   '\U000025B4',
+	"blacktriangledown;":               '\U000025BE',
+	"blacktriangleleft;":               '\U000025C2',
+	"blacktriangleright;":              '\U000025B8',
+	"blank;":                           '\U00002423',
+	"blk12;":                           '\U00002592',
+	"blk14;":                           '\U00002591',
+	"blk34;":                           '\U00002593',
+	"block;":                           '\U00002588',
+	"bnot;":                            '\U00002310',
+	"bopf;":                            '\U0001D553',
+	"bot;":                             '\U000022A5',
+	"bottom;":                          '\U000022A5',
+	"bowtie;":                          '\U000022C8',
+	"boxDL;":                           '\U00002557',
+	"boxDR;":                           '\U00002554',
+	"boxDl;":                           '\U00002556',
+	"boxDr;":                           '\U00002553',
+	"boxH;":                            '\U00002550',
+	"boxHD;":                           '\U00002566',
+	"boxHU;":                           '\U00002569',
+	"boxHd;":                           '\U00002564',
+	"boxHu;":                           '\U00002567',
+	"boxUL;":                           '\U0000255D',
+	"boxUR;":                           '\U0000255A',
+	"boxUl;":                           '\U0000255C',
+	"boxUr;":                           '\U00002559',
+	"boxV;":                            '\U00002551',
+	"boxVH;":                           '\U0000256C',
+	"boxVL;":                           '\U00002563',
+	"boxVR;":                           '\U00002560',
+	"boxVh;":                           '\U0000256B',
+	"boxVl;":                           '\U00002562',
+	"boxVr;":                           '\U0000255F',
+	"boxbox;":                          '\U000029C9',
+	"boxdL;":                           '\U00002555',
+	"boxdR;":                           '\U00002552',
+	"boxdl;":                           '\U00002510',
+	"boxdr;":                           '\U0000250C',
+	"boxh;":                            '\U00002500',
+	"boxhD;":                           '\U00002565',
+	"boxhU;":                           '\U00002568',
+	"boxhd;":                           '\U0000252C',
+	"boxhu;":                           '\U00002534',
+	"boxminus;":                        '\U0000229F',
+	"boxplus;":                         '\U0000229E',
+	"boxtimes;":                        '\U000022A0',
+	"boxuL;":                           '\U0000255B',
+	"boxuR;":                           '\U00002558',
+	"boxul;":                           '\U00002518',
+	"boxur;":                           '\U00002514',
+	"boxv;":                            '\U00002502',
+	"boxvH;":                           '\U0000256A',
+	"boxvL;":                           '\U00002561',
+	"boxvR;":                           '\U0000255E',
+	"boxvh;":                           '\U0000253C',
+	"boxvl;":                           '\U00002524',
+	"boxvr;":                           '\U0000251C',
+	"bprime;":                          '\U00002035',
+	"breve;":                           '\U000002D8',
+	"brvbar;":                          '\U000000A6',
+	"bscr;":                            '\U0001D4B7',
+	"bsemi;":                           '\U0000204F',
+	"bsim;":                            '\U0000223D',
+	"bsime;":                           '\U000022CD',
+	"bsol;":                            '\U0000005C',
+	"bsolb;":                           '\U000029C5',
+	"bsolhsub;":                        '\U000027C8',
+	"bull;":                            '\U00002022',
+	"bullet;":                          '\U00002022',
+	"bump;":                            '\U0000224E',
+	"bumpE;":                           '\U00002AAE',
+	"bumpe;":                           '\U0000224F',
+	"bumpeq;":                          '\U0000224F',
+	"cacute;":                          '\U00000107',
+	"cap;":                             '\U00002229',
+	"capand;":                          '\U00002A44',
+	"capbrcup;":                        '\U00002A49',
+	"capcap;":                          '\U00002A4B',
+	"capcup;":                          '\U00002A47',
+	"capdot;":                          '\U00002A40',
+	"caret;":                           '\U00002041',
+	"caron;":                           '\U000002C7',
+	"ccaps;":                           '\U00002A4D',
+	"ccaron;":                          '\U0000010D',
+	"ccedil;":                          '\U000000E7',
+	"ccirc;":                           '\U00000109',
+	"ccups;":                           '\U00002A4C',
+	"ccupssm;":                         '\U00002A50',
+	"cdot;":                            '\U0000010B',
+	"cedil;":                           '\U000000B8',
+	"cemptyv;":                         '\U000029B2',
+	"cent;":                            '\U000000A2',
+	"centerdot;":                       '\U000000B7',
+	"cfr;":                             '\U0001D520',
+	"chcy;":                            '\U00000447',
+	"check;":                           '\U00002713',
+	"checkmark;":                       '\U00002713',
+	"chi;":                             '\U000003C7',
+	"cir;":                             '\U000025CB',
+	"cirE;":                            '\U000029C3',
+	"circ;":                            '\U000002C6',
+	"circeq;":                          '\U00002257',
+	"circlearrowleft;":                 '\U000021BA',
+	"circlearrowright;":                '\U000021BB',
+	"circledR;":                        '\U000000AE',
+	"circledS;":                        '\U000024C8',
+	"circledast;":                      '\U0000229B',
+	"circledcirc;":                     '\U0000229A',
+	"circleddash;":                     '\U0000229D',
+	"cire;":                            '\U00002257',
+	"cirfnint;":                        '\U00002A10',
+	"cirmid;":                          '\U00002AEF',
+	"cirscir;":                         '\U000029C2',
+	"clubs;":                           '\U00002663',
+	"clubsuit;":                        '\U00002663',
+	"colon;":                           '\U0000003A',
+	"colone;":                          '\U00002254',
+	"coloneq;":                         '\U00002254',
+	"comma;":                           '\U0000002C',
+	"commat;":                          '\U00000040',
+	"comp;":                            '\U00002201',
+	"compfn;":                          '\U00002218',
+	"complement;":                      '\U00002201',
+	"complexes;":                       '\U00002102',
+	"cong;":                            '\U00002245',
+	"congdot;":                         '\U00002A6D',
+	"conint;":                          '\U0000222E',
+	"copf;":                            '\U0001D554',
+	"coprod;":                          '\U00002210',
+	"copy;":                            '\U000000A9',
+	"copysr;":                          '\U00002117',
+	"crarr;":                           '\U000021B5',
+	"cross;":                           '\U00002717',
+	"cscr;":                            '\U0001D4B8',
+	"csub;":                            '\U00002ACF',
+	"csube;":                           '\U00002AD1',
+	"csup;":                            '\U00002AD0',
+	"csupe;":                           '\U00002AD2',
+	"ctdot;":                           '\U000022EF',
+	"cudarrl;":                         '\U00002938',
+	"cudarrr;":                         '\U00002935',
+	"cuepr;":                           '\U000022DE',
+	"cuesc;":                           '\U000022DF',
+	"cularr;":                          '\U000021B6',
+	"cularrp;":                         '\U0000293D',
+	"cup;":                             '\U0000222A',
+	"cupbrcap;":                        '\U00002A48',
+	"cupcap;":                          '\U00002A46',
+	"cupcup;":                          '\U00002A4A',
+	"cupdot;":                          '\U0000228D',
+	"cupor;":                           '\U00002A45',
+	"curarr;":                          '\U000021B7',
+	"curarrm;":                         '\U0000293C',
+	"curlyeqprec;":                     '\U000022DE',
+	"curlyeqsucc;":                     '\U000022DF',
+	"curlyvee;":                        '\U000022CE',
+	"curlywedge;":                      '\U000022CF',
+	"curren;":                          '\U000000A4',
+	"curvearrowleft;":                  '\U000021B6',
+	"curvearrowright;":                 '\U000021B7',
+	"cuvee;":                           '\U000022CE',
+	"cuwed;":                           '\U000022CF',
+	"cwconint;":                        '\U00002232',
+	"cwint;":                           '\U00002231',
+	"cylcty;":                          '\U0000232D',
+	"dArr;":                            '\U000021D3',
+	"dHar;":                            '\U00002965',
+	"dagger;":                          '\U00002020',
+	"daleth;":                          '\U00002138',
+	"darr;":                            '\U00002193',
+	"dash;":                            '\U00002010',
+	"dashv;":                           '\U000022A3',
+	"dbkarow;":                         '\U0000290F',
+	"dblac;":                           '\U000002DD',
+	"dcaron;":                          '\U0000010F',
+	"dcy;":                             '\U00000434',
+	"dd;":                              '\U00002146',
+	"ddagger;":                         '\U00002021',
+	"ddarr;":                           '\U000021CA',
+	"ddotseq;":                         '\U00002A77',
+	"deg;":                             '\U000000B0',
+	"delta;":                           '\U000003B4',
+	"demptyv;":                         '\U000029B1',
+	"dfisht;":                          '\U0000297F',
+	"dfr;":                             '\U0001D521',
+	"dharl;":                           '\U000021C3',
+	"dharr;":                           '\U000021C2',
+	"diam;":                            '\U000022C4',
+	"diamond;":                         '\U000022C4',
+	"diamondsuit;":                     '\U00002666',
+	"diams;":                           '\U00002666',
+	"die;":                             '\U000000A8',
+	"digamma;":                         '\U000003DD',
+	"disin;":                           '\U000022F2',
+	"div;":                             '\U000000F7',
+	"divide;":                          '\U000000F7',
+	"divideontimes;":                   '\U000022C7',
+	"divonx;":                          '\U000022C7',
+	"djcy;":                            '\U00000452',
+	"dlcorn;":                          '\U0000231E',
+	"dlcrop;":                          '\U0000230D',
+	"dollar;":                          '\U00000024',
+	"dopf;":                            '\U0001D555',
+	"dot;":                             '\U000002D9',
+	"doteq;":                           '\U00002250',
+	"doteqdot;":                        '\U00002251',
+	"dotminus;":                        '\U00002238',
+	"dotplus;":                         '\U00002214',
+	"dotsquare;":                       '\U000022A1',
+	"doublebarwedge;":                  '\U00002306',
+	"downarrow;":                       '\U00002193',
+	"downdownarrows;":                  '\U000021CA',
+	"downharpoonleft;":                 '\U000021C3',
+	"downharpoonright;":                '\U000021C2',
+	"drbkarow;":                        '\U00002910',
+	"drcorn;":                          '\U0000231F',
+	"drcrop;":                          '\U0000230C',
+	"dscr;":                            '\U0001D4B9',
+	"dscy;":                            '\U00000455',
+	"dsol;":                            '\U000029F6',
+	"dstrok;":                          '\U00000111',
+	"dtdot;":                           '\U000022F1',
+	"dtri;":                            '\U000025BF',
+	"dtrif;":                           '\U000025BE',
+	"duarr;":                           '\U000021F5',
+	"duhar;":                           '\U0000296F',
+	"dwangle;":                         '\U000029A6',
+	"dzcy;":                            '\U0000045F',
+	"dzigrarr;":                        '\U000027FF',
+	"eDDot;":                           '\U00002A77',
+	"eDot;":                            '\U00002251',
+	"eacute;":                          '\U000000E9',
+	"easter;":                          '\U00002A6E',
+	"ecaron;":                          '\U0000011B',
+	"ecir;":                            '\U00002256',
+	"ecirc;":                           '\U000000EA',
+	"ecolon;":                          '\U00002255',
+	"ecy;":                             '\U0000044D',
+	"edot;":                            '\U00000117',
+	"ee;":                              '\U00002147',
+	"efDot;":                           '\U00002252',
+	"efr;":                             '\U0001D522',
+	"eg;":                              '\U00002A9A',
+	"egrave;":                          '\U000000E8',
+	"egs;":                             '\U00002A96',
+	"egsdot;":                          '\U00002A98',
+	"el;":                              '\U00002A99',
+	"elinters;":                        '\U000023E7',
+	"ell;":                             '\U00002113',
+	"els;":                             '\U00002A95',
+	"elsdot;":                          '\U00002A97',
+	"emacr;":                           '\U00000113',
+	"empty;":                           '\U00002205',
+	"emptyset;":                        '\U00002205',
+	"emptyv;":                          '\U00002205',
+	"emsp;":                            '\U00002003',
+	"emsp13;":                          '\U00002004',
+	"emsp14;":                          '\U00002005',
+	"eng;":                             '\U0000014B',
+	"ensp;":                            '\U00002002',
+	"eogon;":                           '\U00000119',
+	"eopf;":                            '\U0001D556',
+	"epar;":                            '\U000022D5',
+	"eparsl;":                          '\U000029E3',
+	"eplus;":                           '\U00002A71',
+	"epsi;":                            '\U000003B5',
+	"epsilon;":                         '\U000003B5',
+	"epsiv;":                           '\U000003F5',
+	"eqcirc;":                          '\U00002256',
+	"eqcolon;":                         '\U00002255',
+	"eqsim;":                           '\U00002242',
+	"eqslantgtr;":                      '\U00002A96',
+	"eqslantless;":                     '\U00002A95',
+	"equals;":                          '\U0000003D',
+	"equest;":                          '\U0000225F',
+	"equiv;":                           '\U00002261',
+	"equivDD;":                         '\U00002A78',
+	"eqvparsl;":                        '\U000029E5',
+	"erDot;":                           '\U00002253',
+	"erarr;":                           '\U00002971',
+	"escr;":                            '\U0000212F',
+	"esdot;":                           '\U00002250',
+	"esim;":                            '\U00002242',
+	"eta;":                             '\U000003B7',
+	"eth;":                             '\U000000F0',
+	"euml;":                            '\U000000EB',
+	"euro;":                            '\U000020AC',
+	"excl;":                            '\U00000021',
+	"exist;":                           '\U00002203',
+	"expectation;":                     '\U00002130',
+	"exponentiale;":                    '\U00002147',
+	"fallingdotseq;":                   '\U00002252',
+	"fcy;":                             '\U00000444',
+	"female;":                          '\U00002640',
+	"ffilig;":                          '\U0000FB03',
+	"fflig;":                           '\U0000FB00',
+	"ffllig;":                          '\U0000FB04',
+	"ffr;":                             '\U0001D523',
+	"filig;":                           '\U0000FB01',
+	"flat;":                            '\U0000266D',
+	"fllig;":                           '\U0000FB02',
+	"fltns;":                           '\U000025B1',
+	"fnof;":                            '\U00000192',
+	"fopf;":                            '\U0001D557',
+	"forall;":                          '\U00002200',
+	"fork;":                            '\U000022D4',
+	"forkv;":                           '\U00002AD9',
+	"fpartint;":                        '\U00002A0D',
+	"frac12;":                          '\U000000BD',
+	"frac13;":                          '\U00002153',
+	"frac14;":                          '\U000000BC',
+	"frac15;":                          '\U00002155',
+	"frac16;":                          '\U00002159',
+	"frac18;":                          '\U0000215B',
+	"frac23;":                          '\U00002154',
+	"frac25;":                          '\U00002156',
+	"frac34;":                          '\U000000BE',
+	"frac35;":                          '\U00002157',
+	"frac38;":                          '\U0000215C',
+	"frac45;":                          '\U00002158',
+	"frac56;":                          '\U0000215A',
+	"frac58;":                          '\U0000215D',
+	"frac78;":                          '\U0000215E',
+	"frasl;":                           '\U00002044',
+	"frown;":                           '\U00002322',
+	"fscr;":                            '\U0001D4BB',
+	"gE;":                              '\U00002267',
+	"gEl;":                             '\U00002A8C',
+	"gacute;":                          '\U000001F5',
+	"gamma;":                           '\U000003B3',
+	"gammad;":                          '\U000003DD',
+	"gap;":                             '\U00002A86',
+	"gbreve;":                          '\U0000011F',
+	"gcirc;":                           '\U0000011D',
+	"gcy;":                             '\U00000433',
+	"gdot;":                            '\U00000121',
+	"ge;":                              '\U00002265',
+	"gel;":                             '\U000022DB',
+	"geq;":                             '\U00002265',
+	"geqq;":                            '\U00002267',
+	"geqslant;":                        '\U00002A7E',
+	"ges;":                             '\U00002A7E',
+	"gescc;":                           '\U00002AA9',
+	"gesdot;":                          '\U00002A80',
+	"gesdoto;":                         '\U00002A82',
+	"gesdotol;":                        '\U00002A84',
+	"gesles;":                          '\U00002A94',
+	"gfr;":                             '\U0001D524',
+	"gg;":                              '\U0000226B',
+	"ggg;":                             '\U000022D9',
+	"gimel;":                           '\U00002137',
+	"gjcy;":                            '\U00000453',
+	"gl;":                              '\U00002277',
+	"glE;":                             '\U00002A92',
+	"gla;":                             '\U00002AA5',
+	"glj;":                             '\U00002AA4',
+	"gnE;":                             '\U00002269',
+	"gnap;":                            '\U00002A8A',
+	"gnapprox;":                        '\U00002A8A',
+	"gne;":                             '\U00002A88',
+	"gneq;":                            '\U00002A88',
+	"gneqq;":                           '\U00002269',
+	"gnsim;":                           '\U000022E7',
+	"gopf;":                            '\U0001D558',
+	"grave;":                           '\U00000060',
+	"gscr;":                            '\U0000210A',
+	"gsim;":                            '\U00002273',
+	"gsime;":                           '\U00002A8E',
+	"gsiml;":                           '\U00002A90',
+	"gt;":                              '\U0000003E',
+	"gtcc;":                            '\U00002AA7',
+	"gtcir;":                           '\U00002A7A',
+	"gtdot;":                           '\U000022D7',
+	"gtlPar;":                          '\U00002995',
+	"gtquest;":                         '\U00002A7C',
+	"gtrapprox;":                       '\U00002A86',
+	"gtrarr;":                          '\U00002978',
+	"gtrdot;":                          '\U000022D7',
+	"gtreqless;":                       '\U000022DB',
+	"gtreqqless;":                      '\U00002A8C',
+	"gtrless;":                         '\U00002277',
+	"gtrsim;":                          '\U00002273',
+	"hArr;":                            '\U000021D4',
+	"hairsp;":                          '\U0000200A',
+	"half;":                            '\U000000BD',
+	"hamilt;":                          '\U0000210B',
+	"hardcy;":                          '\U0000044A',
+	"harr;":                            '\U00002194',
+	"harrcir;":                         '\U00002948',
+	"harrw;":                           '\U000021AD',
+	"hbar;":                            '\U0000210F',
+	"hcirc;":                           '\U00000125',
+	"hearts;":                          '\U00002665',
+	"heartsuit;":                       '\U00002665',
+	"hellip;":                          '\U00002026',
+	"hercon;":                          '\U000022B9',
+	"hfr;":                             '\U0001D525',
+	"hksearow;":                        '\U00002925',
+	"hkswarow;":                        '\U00002926',
+	"hoarr;":                           '\U000021FF',
+	"homtht;":                          '\U0000223B',
+	"hookleftarrow;":                   '\U000021A9',
+	"hookrightarrow;":                  '\U000021AA',
+	"hopf;":                            '\U0001D559',
+	"horbar;":                          '\U00002015',
+	"hscr;":                            '\U0001D4BD',
+	"hslash;":                          '\U0000210F',
+	"hstrok;":                          '\U00000127',
+	"hybull;":                          '\U00002043',
+	"hyphen;":                          '\U00002010',
+	"iacute;":                          '\U000000ED',
+	"ic;":                              '\U00002063',
+	"icirc;":                           '\U000000EE',
+	"icy;":                             '\U00000438',
+	"iecy;":                            '\U00000435',
+	"iexcl;":                           '\U000000A1',
+	"iff;":                             '\U000021D4',
+	"ifr;":                             '\U0001D526',
+	"igrave;":                          '\U000000EC',
+	"ii;":                              '\U00002148',
+	"iiiint;":                          '\U00002A0C',
+	"iiint;":                           '\U0000222D',
+	"iinfin;":                          '\U000029DC',
+	"iiota;":                           '\U00002129',
+	"ijlig;":                           '\U00000133',
+	"imacr;":                           '\U0000012B',
+	"image;":                           '\U00002111',
+	"imagline;":                        '\U00002110',
+	"imagpart;":                        '\U00002111',
+	"imath;":                           '\U00000131',
+	"imof;":                            '\U000022B7',
+	"imped;":                           '\U000001B5',
+	"in;":                              '\U00002208',
+	"incare;":                          '\U00002105',
+	"infin;":                           '\U0000221E',
+	"infintie;":                        '\U000029DD',
+	"inodot;":                          '\U00000131',
+	"int;":                             '\U0000222B',
+	"intcal;":                          '\U000022BA',
+	"integers;":                        '\U00002124',
+	"intercal;":                        '\U000022BA',
+	"intlarhk;":                        '\U00002A17',
+	"intprod;":                         '\U00002A3C',
+	"iocy;":                            '\U00000451',
+	"iogon;":                           '\U0000012F',
+	"iopf;":                            '\U0001D55A',
+	"iota;":                            '\U000003B9',
+	"iprod;":                           '\U00002A3C',
+	"iquest;":                          '\U000000BF',
+	"iscr;":                            '\U0001D4BE',
+	"isin;":                            '\U00002208',
+	"isinE;":                           '\U000022F9',
+	"isindot;":                         '\U000022F5',
+	"isins;":                           '\U000022F4',
+	"isinsv;":                          '\U000022F3',
+	"isinv;":                           '\U00002208',
+	"it;":                              '\U00002062',
+	"itilde;":                          '\U00000129',
+	"iukcy;":                           '\U00000456',
+	"iuml;":                            '\U000000EF',
+	"jcirc;":                           '\U00000135',
+	"jcy;":                             '\U00000439',
+	"jfr;":                             '\U0001D527',
+	"jmath;":                           '\U00000237',
+	"jopf;":                            '\U0001D55B',
+	"jscr;":                            '\U0001D4BF',
+	"jsercy;":                          '\U00000458',
+	"jukcy;":                           '\U00000454',
+	"kappa;":                           '\U000003BA',
+	"kappav;":                          '\U000003F0',
+	"kcedil;":                          '\U00000137',
+	"kcy;":                             '\U0000043A',
+	"kfr;":                             '\U0001D528',
+	"kgreen;":                          '\U00000138',
+	"khcy;":                            '\U00000445',
+	"kjcy;":                            '\U0000045C',
+	"kopf;":                            '\U0001D55C',
+	"kscr;":                            '\U0001D4C0',
+	"lAarr;":                           '\U000021DA',
+	"lArr;":                            '\U000021D0',
+	"lAtail;":                          '\U0000291B',
+	"lBarr;":                           '\U0000290E',
+	"lE;":                              '\U00002266',
+	"lEg;":                             '\U00002A8B',
+	"lHar;":                            '\U00002962',
+	"lacute;":                          '\U0000013A',
+	"laemptyv;":                        '\U000029B4',
+	"lagran;":                          '\U00002112',
+	"lambda;":                          '\U000003BB',
+	"lang;":                            '\U000027E8',
+	"langd;":                           '\U00002991',
+	"langle;":                          '\U000027E8',
+	"lap;":                             '\U00002A85',
+	"laquo;":                           '\U000000AB',
+	"larr;":                            '\U00002190',
+	"larrb;":                           '\U000021E4',
+	"larrbfs;":                         '\U0000291F',
+	"larrfs;":                          '\U0000291D',
+	"larrhk;":                          '\U000021A9',
+	"larrlp;":                          '\U000021AB',
+	"larrpl;":                          '\U00002939',
+	"larrsim;":                         '\U00002973',
+	"larrtl;":                          '\U000021A2',
+	"lat;":                             '\U00002AAB',
+	"latail;":                          '\U00002919',
+	"late;":                            '\U00002AAD',
+	"lbarr;":                           '\U0000290C',
+	"lbbrk;":                           '\U00002772',
+	"lbrace;":                          '\U0000007B',
+	"lbrack;":                          '\U0000005B',
+	"lbrke;":                           '\U0000298B',
+	"lbrksld;":                         '\U0000298F',
+	"lbrkslu;":                         '\U0000298D',
+	"lcaron;":                          '\U0000013E',
+	"lcedil;":                          '\U0000013C',
+	"lceil;":                           '\U00002308',
+	"lcub;":                            '\U0000007B',
+	"lcy;":                             '\U0000043B',
+	"ldca;":                            '\U00002936',
+	"ldquo;":                           '\U0000201C',
+	"ldquor;":                          '\U0000201E',
+	"ldrdhar;":                         '\U00002967',
+	"ldrushar;":                        '\U0000294B',
+	"ldsh;":                            '\U000021B2',
+	"le;":                              '\U00002264',
+	"leftarrow;":                       '\U00002190',
+	"leftarrowtail;":                   '\U000021A2',
+	"leftharpoondown;":                 '\U000021BD',
+	"leftharpoonup;":                   '\U000021BC',
+	"leftleftarrows;":                  '\U000021C7',
+	"leftrightarrow;":                  '\U00002194',
+	"leftrightarrows;":                 '\U000021C6',
+	"leftrightharpoons;":               '\U000021CB',
+	"leftrightsquigarrow;":             '\U000021AD',
+	"leftthreetimes;":                  '\U000022CB',
+	"leg;":                             '\U000022DA',
+	"leq;":                             '\U00002264',
+	"leqq;":                            '\U00002266',
+	"leqslant;":                        '\U00002A7D',
+	"les;":                             '\U00002A7D',
+	"lescc;":                           '\U00002AA8',
+	"lesdot;":                          '\U00002A7F',
+	"lesdoto;":                         '\U00002A81',
+	"lesdotor;":                        '\U00002A83',
+	"lesges;":                          '\U00002A93',
+	"lessapprox;":                      '\U00002A85',
+	"lessdot;":                         '\U000022D6',
+	"lesseqgtr;":                       '\U000022DA',
+	"lesseqqgtr;":                      '\U00002A8B',
+	"lessgtr;":                         '\U00002276',
+	"lesssim;":                         '\U00002272',
+	"lfisht;":                          '\U0000297C',
+	"lfloor;":                          '\U0000230A',
+	"lfr;":                             '\U0001D529',
+	"lg;":                              '\U00002276',
+	"lgE;":                             '\U00002A91',
+	"lhard;":                           '\U000021BD',
+	"lharu;":                           '\U000021BC',
+	"lharul;":                          '\U0000296A',
+	"lhblk;":                           '\U00002584',
+	"ljcy;":                            '\U00000459',
+	"ll;":                              '\U0000226A',
+	"llarr;":                           '\U000021C7',
+	"llcorner;":                        '\U0000231E',
+	"llhard;":                          '\U0000296B',
+	"lltri;":                           '\U000025FA',
+	"lmidot;":                          '\U00000140',
+	"lmoust;":                          '\U000023B0',
+	"lmoustache;":                      '\U000023B0',
+	"lnE;":                             '\U00002268',
+	"lnap;":                            '\U00002A89',
+	"lnapprox;":                        '\U00002A89',
+	"lne;":                             '\U00002A87',
+	"lneq;":                            '\U00002A87',
+	"lneqq;":                           '\U00002268',
+	"lnsim;":                           '\U000022E6',
+	"loang;":                           '\U000027EC',
+	"loarr;":                           '\U000021FD',
+	"lobrk;":                           '\U000027E6',
+	"longleftarrow;":                   '\U000027F5',
+	"longleftrightarrow;":              '\U000027F7',
+	"longmapsto;":                      '\U000027FC',
+	"longrightarrow;":                  '\U000027F6',
+	"looparrowleft;":                   '\U000021AB',
+	"looparrowright;":                  '\U000021AC',
+	"lopar;":                           '\U00002985',
+	"lopf;":                            '\U0001D55D',
+	"loplus;":                          '\U00002A2D',
+	"lotimes;":                         '\U00002A34',
+	"lowast;":                          '\U00002217',
+	"lowbar;":                          '\U0000005F',
+	"loz;":                             '\U000025CA',
+	"lozenge;":                         '\U000025CA',
+	"lozf;":                            '\U000029EB',
+	"lpar;":                            '\U00000028',
+	"lparlt;":                          '\U00002993',
+	"lrarr;":                           '\U000021C6',
+	"lrcorner;":                        '\U0000231F',
+	"lrhar;":                           '\U000021CB',
+	"lrhard;":                          '\U0000296D',
+	"lrm;":                             '\U0000200E',
+	"lrtri;":                           '\U000022BF',
+	"lsaquo;":                          '\U00002039',
+	"lscr;":                            '\U0001D4C1',
+	"lsh;":                             '\U000021B0',
+	"lsim;":                            '\U00002272',
+	"lsime;":                           '\U00002A8D',
+	"lsimg;":                           '\U00002A8F',
+	"lsqb;":                            '\U0000005B',
+	"lsquo;":                           '\U00002018',
+	"lsquor;":                          '\U0000201A',
+	"lstrok;":                          '\U00000142',
+	"lt;":                              '\U0000003C',
+	"ltcc;":                            '\U00002AA6',
+	"ltcir;":                           '\U00002A79',
+	"ltdot;":                           '\U000022D6',
+	"lthree;":                          '\U000022CB',
+	"ltimes;":                          '\U000022C9',
+	"ltlarr;":                          '\U00002976',
+	"ltquest;":                         '\U00002A7B',
+	"ltrPar;":                          '\U00002996',
+	"ltri;":                            '\U000025C3',
+	"ltrie;":                           '\U000022B4',
+	"ltrif;":                           '\U000025C2',
+	"lurdshar;":                        '\U0000294A',
+	"luruhar;":                         '\U00002966',
+	"mDDot;":                           '\U0000223A',
+	"macr;":                            '\U000000AF',
+	"male;":                            '\U00002642',
+	"malt;":                            '\U00002720',
+	"maltese;":                         '\U00002720',
+	"map;":                             '\U000021A6',
+	"mapsto;":                          '\U000021A6',
+	"mapstodown;":                      '\U000021A7',
+	"mapstoleft;":                      '\U000021A4',
+	"mapstoup;":                        '\U000021A5',
+	"marker;":                          '\U000025AE',
+	"mcomma;":                          '\U00002A29',
+	"mcy;":                             '\U0000043C',
+	"mdash;":                           '\U00002014',
+	"measuredangle;":                   '\U00002221',
+	"mfr;":                             '\U0001D52A',
+	"mho;":                             '\U00002127',
+	"micro;":                           '\U000000B5',
+	"mid;":                             '\U00002223',
+	"midast;":                          '\U0000002A',
+	"midcir;":                          '\U00002AF0',
+	"middot;":                          '\U000000B7',
+	"minus;":                           '\U00002212',
+	"minusb;":                          '\U0000229F',
+	"minusd;":                          '\U00002238',
+	"minusdu;":                         '\U00002A2A',
+	"mlcp;":                            '\U00002ADB',
+	"mldr;":                            '\U00002026',
+	"mnplus;":                          '\U00002213',
+	"models;":                          '\U000022A7',
+	"mopf;":                            '\U0001D55E',
+	"mp;":                              '\U00002213',
+	"mscr;":                            '\U0001D4C2',
+	"mstpos;":                          '\U0000223E',
+	"mu;":                              '\U000003BC',
+	"multimap;":                        '\U000022B8',
+	"mumap;":                           '\U000022B8',
+	"nLeftarrow;":                      '\U000021CD',
+	"nLeftrightarrow;":                 '\U000021CE',
+	"nRightarrow;":                     '\U000021CF',
+	"nVDash;":                          '\U000022AF',
+	"nVdash;":                          '\U000022AE',
+	"nabla;":                           '\U00002207',
+	"nacute;":                          '\U00000144',
+	"nap;":                             '\U00002249',
+	"napos;":                           '\U00000149',
+	"napprox;":                         '\U00002249',
+	"natur;":                           '\U0000266E',
+	"natural;":                         '\U0000266E',
+	"naturals;":                        '\U00002115',
+	"nbsp;":                            '\U000000A0',
+	"ncap;":                            '\U00002A43',
+	"ncaron;":                          '\U00000148',
+	"ncedil;":                          '\U00000146',
+	"ncong;":                           '\U00002247',
+	"ncup;":                            '\U00002A42',
+	"ncy;":                             '\U0000043D',
+	"ndash;":                           '\U00002013',
+	"ne;":                              '\U00002260',
+	"neArr;":                           '\U000021D7',
+	"nearhk;":                          '\U00002924',
+	"nearr;":                           '\U00002197',
+	"nearrow;":                         '\U00002197',
+	"nequiv;":                          '\U00002262',
+	"nesear;":                          '\U00002928',
+	"nexist;":                          '\U00002204',
+	"nexists;":                         '\U00002204',
+	"nfr;":                             '\U0001D52B',
+	"nge;":                             '\U00002271',
+	"ngeq;":                            '\U00002271',
+	"ngsim;":                           '\U00002275',
+	"ngt;":                             '\U0000226F',
+	"ngtr;":                            '\U0000226F',
+	"nhArr;":                           '\U000021CE',
+	"nharr;":                           '\U000021AE',
+	"nhpar;":                           '\U00002AF2',
+	"ni;":                              '\U0000220B',
+	"nis;":                             '\U000022FC',
+	"nisd;":                            '\U000022FA',
+	"niv;":                             '\U0000220B',
+	"njcy;":                            '\U0000045A',
+	"nlArr;":                           '\U000021CD',
+	"nlarr;":                           '\U0000219A',
+	"nldr;":                            '\U00002025',
+	"nle;":                             '\U00002270',
+	"nleftarrow;":                      '\U0000219A',
+	"nleftrightarrow;":                 '\U000021AE',
+	"nleq;":                            '\U00002270',
+	"nless;":                           '\U0000226E',
+	"nlsim;":                           '\U00002274',
+	"nlt;":                             '\U0000226E',
+	"nltri;":                           '\U000022EA',
+	"nltrie;":                          '\U000022EC',
+	"nmid;":                            '\U00002224',
+	"nopf;":                            '\U0001D55F',
+	"not;":                             '\U000000AC',
+	"notin;":                           '\U00002209',
+	"notinva;":                         '\U00002209',
+	"notinvb;":                         '\U000022F7',
+	"notinvc;":                         '\U000022F6',
+	"notni;":                           '\U0000220C',
+	"notniva;":                         '\U0000220C',
+	"notnivb;":                         '\U000022FE',
+	"notnivc;":                         '\U000022FD',
+	"npar;":                            '\U00002226',
+	"nparallel;":                       '\U00002226',
+	"npolint;":                         '\U00002A14',
+	"npr;":                             '\U00002280',
+	"nprcue;":                          '\U000022E0',
+	"nprec;":                           '\U00002280',
+	"nrArr;":                           '\U000021CF',
+	"nrarr;":                           '\U0000219B',
+	"nrightarrow;":                     '\U0000219B',
+	"nrtri;":                           '\U000022EB',
+	"nrtrie;":                          '\U000022ED',
+	"nsc;":                             '\U00002281',
+	"nsccue;":                          '\U000022E1',
+	"nscr;":                            '\U0001D4C3',
+	"nshortmid;":                       '\U00002224',
+	"nshortparallel;":                  '\U00002226',
+	"nsim;":                            '\U00002241',
+	"nsime;":                           '\U00002244',
+	"nsimeq;":                          '\U00002244',
+	"nsmid;":                           '\U00002224',
+	"nspar;":                           '\U00002226',
+	"nsqsube;":                         '\U000022E2',
+	"nsqsupe;":                         '\U000022E3',
+	"nsub;":                            '\U00002284',
+	"nsube;":                           '\U00002288',
+	"nsubseteq;":                       '\U00002288',
+	"nsucc;":                           '\U00002281',
+	"nsup;":                            '\U00002285',
+	"nsupe;":                           '\U00002289',
+	"nsupseteq;":                       '\U00002289',
+	"ntgl;":                            '\U00002279',
+	"ntilde;":                          '\U000000F1',
+	"ntlg;":                            '\U00002278',
+	"ntriangleleft;":                   '\U000022EA',
+	"ntrianglelefteq;":                 '\U000022EC',
+	"ntriangleright;":                  '\U000022EB',
+	"ntrianglerighteq;":                '\U000022ED',
+	"nu;":                              '\U000003BD',
+	"num;":                             '\U00000023',
+	"numero;":                          '\U00002116',
+	"numsp;":                           '\U00002007',
+	"nvDash;":                          '\U000022AD',
+	"nvHarr;":                          '\U00002904',
+	"nvdash;":                          '\U000022AC',
+	"nvinfin;":                         '\U000029DE',
+	"nvlArr;":                          '\U00002902',
+	"nvrArr;":                          '\U00002903',
+	"nwArr;":                           '\U000021D6',
+	"nwarhk;":                          '\U00002923',
+	"nwarr;":                           '\U00002196',
+	"nwarrow;":                         '\U00002196',
+	"nwnear;":                          '\U00002927',
+	"oS;":                              '\U000024C8',
+	"oacute;":                          '\U000000F3',
+	"oast;":                            '\U0000229B',
+	"ocir;":                            '\U0000229A',
+	"ocirc;":                           '\U000000F4',
+	"ocy;":                             '\U0000043E',
+	"odash;":                           '\U0000229D',
+	"odblac;":                          '\U00000151',
+	"odiv;":                            '\U00002A38',
+	"odot;":                            '\U00002299',
+	"odsold;":                          '\U000029BC',
+	"oelig;":                           '\U00000153',
+	"ofcir;":                           '\U000029BF',
+	"ofr;":                             '\U0001D52C',
+	"ogon;":                            '\U000002DB',
+	"ograve;":                          '\U000000F2',
+	"ogt;":                             '\U000029C1',
+	"ohbar;":                           '\U000029B5',
+	"ohm;":                             '\U000003A9',
+	"oint;":                            '\U0000222E',
+	"olarr;":                           '\U000021BA',
+	"olcir;":                           '\U000029BE',
+	"olcross;":                         '\U000029BB',
+	"oline;":                           '\U0000203E',
+	"olt;":                             '\U000029C0',
+	"omacr;":                           '\U0000014D',
+	"omega;":                           '\U000003C9',
+	"omicron;":                         '\U000003BF',
+	"omid;":                            '\U000029B6',
+	"ominus;":                          '\U00002296',
+	"oopf;":                            '\U0001D560',
+	"opar;":                            '\U000029B7',
+	"operp;":                           '\U000029B9',
+	"oplus;":                           '\U00002295',
+	"or;":                              '\U00002228',
+	"orarr;":                           '\U000021BB',
+	"ord;":                             '\U00002A5D',
+	"order;":                           '\U00002134',
+	"orderof;":                         '\U00002134',
+	"ordf;":                            '\U000000AA',
+	"ordm;":                            '\U000000BA',
+	"origof;":                          '\U000022B6',
+	"oror;":                            '\U00002A56',
+	"orslope;":                         '\U00002A57',
+	"orv;":                             '\U00002A5B',
+	"oscr;":                            '\U00002134',
+	"oslash;":                          '\U000000F8',
+	"osol;":                            '\U00002298',
+	"otilde;":                          '\U000000F5',
+	"otimes;":                          '\U00002297',
+	"otimesas;":                        '\U00002A36',
+	"ouml;":                            '\U000000F6',
+	"ovbar;":                           '\U0000233D',
+	"par;":                             '\U00002225',
+	"para;":                            '\U000000B6',
+	"parallel;":                        '\U00002225',
+	"parsim;":                          '\U00002AF3',
+	"parsl;":                           '\U00002AFD',
+	"part;":                            '\U00002202',
+	"pcy;":                             '\U0000043F',
+	"percnt;":                          '\U00000025',
+	"period;":                          '\U0000002E',
+	"permil;":                          '\U00002030',
+	"perp;":                            '\U000022A5',
+	"pertenk;":                         '\U00002031',
+	"pfr;":                             '\U0001D52D',
+	"phi;":                             '\U000003C6',
+	"phiv;":                            '\U000003D5',
+	"phmmat;":                          '\U00002133',
+	"phone;":                           '\U0000260E',
+	"pi;":                              '\U000003C0',
+	"pitchfork;":                       '\U000022D4',
+	"piv;":                             '\U000003D6',
+	"planck;":                          '\U0000210F',
+	"planckh;":                         '\U0000210E',
+	"plankv;":                          '\U0000210F',
+	"plus;":                            '\U0000002B',
+	"plusacir;":                        '\U00002A23',
+	"plusb;":                           '\U0000229E',
+	"pluscir;":                         '\U00002A22',
+	"plusdo;":                          '\U00002214',
+	"plusdu;":                          '\U00002A25',
+	"pluse;":                           '\U00002A72',
+	"plusmn;":                          '\U000000B1',
+	"plussim;":                         '\U00002A26',
+	"plustwo;":                         '\U00002A27',
+	"pm;":                              '\U000000B1',
+	"pointint;":                        '\U00002A15',
+	"popf;":                            '\U0001D561',
+	"pound;":                           '\U000000A3',
+	"pr;":                              '\U0000227A',
+	"prE;":                             '\U00002AB3',
+	"prap;":                            '\U00002AB7',
+	"prcue;":                           '\U0000227C',
+	"pre;":                             '\U00002AAF',
+	"prec;":                            '\U0000227A',
+	"precapprox;":                      '\U00002AB7',
+	"preccurlyeq;":                     '\U0000227C',
+	"preceq;":                          '\U00002AAF',
+	"precnapprox;":                     '\U00002AB9',
+	"precneqq;":                        '\U00002AB5',
+	"precnsim;":                        '\U000022E8',
+	"precsim;":                         '\U0000227E',
+	"prime;":                           '\U00002032',
+	"primes;":                          '\U00002119',
+	"prnE;":                            '\U00002AB5',
+	"prnap;":                           '\U00002AB9',
+	"prnsim;":                          '\U000022E8',
+	"prod;":                            '\U0000220F',
+	"profalar;":                        '\U0000232E',
+	"profline;":                        '\U00002312',
+	"profsurf;":                        '\U00002313',
+	"prop;":                            '\U0000221D',
+	"propto;":                          '\U0000221D',
+	"prsim;":                           '\U0000227E',
+	"prurel;":                          '\U000022B0',
+	"pscr;":                            '\U0001D4C5',
+	"psi;":                             '\U000003C8',
+	"puncsp;":                          '\U00002008',
+	"qfr;":                             '\U0001D52E',
+	"qint;":                            '\U00002A0C',
+	"qopf;":                            '\U0001D562',
+	"qprime;":                          '\U00002057',
+	"qscr;":                            '\U0001D4C6',
+	"quaternions;":                     '\U0000210D',
+	"quatint;":                         '\U00002A16',
+	"quest;":                           '\U0000003F',
+	"questeq;":                         '\U0000225F',
+	"quot;":                            '\U00000022',
+	"rAarr;":                           '\U000021DB',
+	"rArr;":                            '\U000021D2',
+	"rAtail;":                          '\U0000291C',
+	"rBarr;":                           '\U0000290F',
+	"rHar;":                            '\U00002964',
+	"racute;":                          '\U00000155',
+	"radic;":                           '\U0000221A',
+	"raemptyv;":                        '\U000029B3',
+	"rang;":                            '\U000027E9',
+	"rangd;":                           '\U00002992',
+	"range;":                           '\U000029A5',
+	"rangle;":                          '\U000027E9',
+	"raquo;":                           '\U000000BB',
+	"rarr;":                            '\U00002192',
+	"rarrap;":                          '\U00002975',
+	"rarrb;":                           '\U000021E5',
+	"rarrbfs;":                         '\U00002920',
+	"rarrc;":                           '\U00002933',
+	"rarrfs;":                          '\U0000291E',
+	"rarrhk;":                          '\U000021AA',
+	"rarrlp;":                          '\U000021AC',
+	"rarrpl;":                          '\U00002945',
+	"rarrsim;":                         '\U00002974',
+	"rarrtl;":                          '\U000021A3',
+	"rarrw;":                           '\U0000219D',
+	"ratail;":                          '\U0000291A',
+	"ratio;":                           '\U00002236',
+	"rationals;":                       '\U0000211A',
+	"rbarr;":                           '\U0000290D',
+	"rbbrk;":                           '\U00002773',
+	"rbrace;":                          '\U0000007D',
+	"rbrack;":                          '\U0000005D',
+	"rbrke;":                           '\U0000298C',
+	"rbrksld;":                         '\U0000298E',
+	"rbrkslu;":                         '\U00002990',
+	"rcaron;":                          '\U00000159',
+	"rcedil;":                          '\U00000157',
+	"rceil;":                           '\U00002309',
+	"rcub;":                            '\U0000007D',
+	"rcy;":                             '\U00000440',
+	"rdca;":                            '\U00002937',
+	"rdldhar;":                         '\U00002969',
+	"rdquo;":                           '\U0000201D',
+	"rdquor;":                          '\U0000201D',
+	"rdsh;":                            '\U000021B3',
+	"real;":                            '\U0000211C',
+	"realine;":                         '\U0000211B',
+	"realpart;":                        '\U0000211C',
+	"reals;":                           '\U0000211D',
+	"rect;":                            '\U000025AD',
+	"reg;":                             '\U000000AE',
+	"rfisht;":                          '\U0000297D',
+	"rfloor;":                          '\U0000230B',
+	"rfr;":                             '\U0001D52F',
+	"rhard;":                           '\U000021C1',
+	"rharu;":                           '\U000021C0',
+	"rharul;":                          '\U0000296C',
+	"rho;":                             '\U000003C1',
+	"rhov;":                            '\U000003F1',
+	"rightarrow;":                      '\U00002192',
+	"rightarrowtail;":                  '\U000021A3',
+	"rightharpoondown;":                '\U000021C1',
+	"rightharpoonup;":                  '\U000021C0',
+	"rightleftarrows;":                 '\U000021C4',
+	"rightleftharpoons;":               '\U000021CC',
+	"rightrightarrows;":                '\U000021C9',
+	"rightsquigarrow;":                 '\U0000219D',
+	"rightthreetimes;":                 '\U000022CC',
+	"ring;":                            '\U000002DA',
+	"risingdotseq;":                    '\U00002253',
+	"rlarr;":                           '\U000021C4',
+	"rlhar;":                           '\U000021CC',
+	"rlm;":                             '\U0000200F',
+	"rmoust;":                          '\U000023B1',
+	"rmoustache;":                      '\U000023B1',
+	"rnmid;":                           '\U00002AEE',
+	"roang;":                           '\U000027ED',
+	"roarr;":                           '\U000021FE',
+	"robrk;":                           '\U000027E7',
+	"ropar;":                           '\U00002986',
+	"ropf;":                            '\U0001D563',
+	"roplus;":                          '\U00002A2E',
+	"rotimes;":                         '\U00002A35',
+	"rpar;":                            '\U00000029',
+	"rpargt;":                          '\U00002994',
+	"rppolint;":                        '\U00002A12',
+	"rrarr;":                           '\U000021C9',
+	"rsaquo;":                          '\U0000203A',
+	"rscr;":                            '\U0001D4C7',
+	"rsh;":                             '\U000021B1',
+	"rsqb;":                            '\U0000005D',
+	"rsquo;":                           '\U00002019',
+	"rsquor;":                          '\U00002019',
+	"rthree;":                          '\U000022CC',
+	"rtimes;":                          '\U000022CA',
+	"rtri;":                            '\U000025B9',
+	"rtrie;":                           '\U000022B5',
+	"rtrif;":                           '\U000025B8',
+	"rtriltri;":                        '\U000029CE',
+	"ruluhar;":                         '\U00002968',
+	"rx;":                              '\U0000211E',
+	"sacute;":                          '\U0000015B',
+	"sbquo;":                           '\U0000201A',
+	"sc;":                              '\U0000227B',
+	"scE;":                             '\U00002AB4',
+	"scap;":                            '\U00002AB8',
+	"scaron;":                          '\U00000161',
+	"sccue;":                           '\U0000227D',
+	"sce;":                             '\U00002AB0',
+	"scedil;":                          '\U0000015F',
+	"scirc;":                           '\U0000015D',
+	"scnE;":                            '\U00002AB6',
+	"scnap;":                           '\U00002ABA',
+	"scnsim;":                          '\U000022E9',
+	"scpolint;":                        '\U00002A13',
+	"scsim;":                           '\U0000227F',
+	"scy;":                             '\U00000441',
+	"sdot;":                            '\U000022C5',
+	"sdotb;":                           '\U000022A1',
+	"sdote;":                           '\U00002A66',
+	"seArr;":                           '\U000021D8',
+	"searhk;":                          '\U00002925',
+	"searr;":                           '\U00002198',
+	"searrow;":                         '\U00002198',
+	"sect;":                            '\U000000A7',
+	"semi;":                            '\U0000003B',
+	"seswar;":                          '\U00002929',
+	"setminus;":                        '\U00002216',
+	"setmn;":                           '\U00002216',
+	"sext;":                            '\U00002736',
+	"sfr;":                             '\U0001D530',
+	"sfrown;":                          '\U00002322',
+	"sharp;":                           '\U0000266F',
+	"shchcy;":                          '\U00000449',
+	"shcy;":                            '\U00000448',
+	"shortmid;":                        '\U00002223',
+	"shortparallel;":                   '\U00002225',
+	"shy;":                             '\U000000AD',
+	"sigma;":                           '\U000003C3',
+	"sigmaf;":                          '\U000003C2',
+	"sigmav;":                          '\U000003C2',
+	"sim;":                             '\U0000223C',
+	"simdot;":                          '\U00002A6A',
+	"sime;":                            '\U00002243',
+	"simeq;":                           '\U00002243',
+	"simg;":                            '\U00002A9E',
+	"simgE;":                           '\U00002AA0',
+	"siml;":                            '\U00002A9D',
+	"simlE;":                           '\U00002A9F',
+	"simne;":                           '\U00002246',
+	"simplus;":                         '\U00002A24',
+	"simrarr;":                         '\U00002972',
+	"slarr;":                           '\U00002190',
+	"smallsetminus;":                   '\U00002216',
+	"smashp;":                          '\U00002A33',
+	"smeparsl;":                        '\U000029E4',
+	"smid;":                            '\U00002223',
+	"smile;":                           '\U00002323',
+	"smt;":                             '\U00002AAA',
+	"smte;":                            '\U00002AAC',
+	"softcy;":                          '\U0000044C',
+	"sol;":                             '\U0000002F',
+	"solb;":                            '\U000029C4',
+	"solbar;":                          '\U0000233F',
+	"sopf;":                            '\U0001D564',
+	"spades;":                          '\U00002660',
+	"spadesuit;":                       '\U00002660',
+	"spar;":                            '\U00002225',
+	"sqcap;":                           '\U00002293',
+	"sqcup;":                           '\U00002294',
+	"sqsub;":                           '\U0000228F',
+	"sqsube;":                          '\U00002291',
+	"sqsubset;":                        '\U0000228F',
+	"sqsubseteq;":                      '\U00002291',
+	"sqsup;":                           '\U00002290',
+	"sqsupe;":                          '\U00002292',
+	"sqsupset;":                        '\U00002290',
+	"sqsupseteq;":                      '\U00002292',
+	"squ;":                             '\U000025A1',
+	"square;":                          '\U000025A1',
+	"squarf;":                          '\U000025AA',
+	"squf;":                            '\U000025AA',
+	"srarr;":                           '\U00002192',
+	"sscr;":                            '\U0001D4C8',
+	"ssetmn;":                          '\U00002216',
+	"ssmile;":                          '\U00002323',
+	"sstarf;":                          '\U000022C6',
+	"star;":                            '\U00002606',
+	"starf;":                           '\U00002605',
+	"straightepsilon;":                 '\U000003F5',
+	"straightphi;":                     '\U000003D5',
+	"strns;":                           '\U000000AF',
+	"sub;":                             '\U00002282',
+	"subE;":                            '\U00002AC5',
+	"subdot;":                          '\U00002ABD',
+	"sube;":                            '\U00002286',
+	"subedot;":                         '\U00002AC3',
+	"submult;":                         '\U00002AC1',
+	"subnE;":                           '\U00002ACB',
+	"subne;":                           '\U0000228A',
+	"subplus;":                         '\U00002ABF',
+	"subrarr;":                         '\U00002979',
+	"subset;":                          '\U00002282',
+	"subseteq;":                        '\U00002286',
+	"subseteqq;":                       '\U00002AC5',
+	"subsetneq;":                       '\U0000228A',
+	"subsetneqq;":                      '\U00002ACB',
+	"subsim;":                          '\U00002AC7',
+	"subsub;":                          '\U00002AD5',
+	"subsup;":                          '\U00002AD3',
+	"succ;":                            '\U0000227B',
+	"succapprox;":                      '\U00002AB8',
+	"succcurlyeq;":                     '\U0000227D',
+	"succeq;":                          '\U00002AB0',
+	"succnapprox;":                     '\U00002ABA',
+	"succneqq;":                        '\U00002AB6',
+	"succnsim;":                        '\U000022E9',
+	"succsim;":                         '\U0000227F',
+	"sum;":                             '\U00002211',
+	"sung;":                            '\U0000266A',
+	"sup;":                             '\U00002283',
+	"sup1;":                            '\U000000B9',
+	"sup2;":                            '\U000000B2',
+	"sup3;":                            '\U000000B3',
+	"supE;":                            '\U00002AC6',
+	"supdot;":                          '\U00002ABE',
+	"supdsub;":                         '\U00002AD8',
+	"supe;":                            '\U00002287',
+	"supedot;":                         '\U00002AC4',
+	"suphsol;":                         '\U000027C9',
+	"suphsub;":                         '\U00002AD7',
+	"suplarr;":                         '\U0000297B',
+	"supmult;":                         '\U00002AC2',
+	"supnE;":                           '\U00002ACC',
+	"supne;":                           '\U0000228B',
+	"supplus;":                         '\U00002AC0',
+	"supset;":                          '\U00002283',
+	"supseteq;":                        '\U00002287',
+	"supseteqq;":                       '\U00002AC6',
+	"supsetneq;":                       '\U0000228B',
+	"supsetneqq;":                      '\U00002ACC',
+	"supsim;":                          '\U00002AC8',
+	"supsub;":                          '\U00002AD4',
+	"supsup;":                          '\U00002AD6',
+	"swArr;":                           '\U000021D9',
+	"swarhk;":                          '\U00002926',
+	"swarr;":                           '\U00002199',
+	"swarrow;":                         '\U00002199',
+	"swnwar;":                          '\U0000292A',
+	"szlig;":                           '\U000000DF',
+	"target;":                          '\U00002316',
+	"tau;":                             '\U000003C4',
+	"tbrk;":                            '\U000023B4',
+	"tcaron;":                          '\U00000165',
+	"tcedil;":                          '\U00000163',
+	"tcy;":                             '\U00000442',
+	"tdot;":                            '\U000020DB',
+	"telrec;":                          '\U00002315',
+	"tfr;":                             '\U0001D531',
+	"there4;":                          '\U00002234',
+	"therefore;":                       '\U00002234',
+	"theta;":                           '\U000003B8',
+	"thetasym;":                        '\U000003D1',
+	"thetav;":                          '\U000003D1',
+	"thickapprox;":                     '\U00002248',
+	"thicksim;":                        '\U0000223C',
+	"thinsp;":                          '\U00002009',
+	"thkap;":                           '\U00002248',
+	"thksim;":                          '\U0000223C',
+	"thorn;":                           '\U000000FE',
+	"tilde;":                           '\U000002DC',
+	"times;":                           '\U000000D7',
+	"timesb;":                          '\U000022A0',
+	"timesbar;":                        '\U00002A31',
+	"timesd;":                          '\U00002A30',
+	"tint;":                            '\U0000222D',
+	"toea;":                            '\U00002928',
+	"top;":                             '\U000022A4',
+	"topbot;":                          '\U00002336',
+	"topcir;":                          '\U00002AF1',
+	"topf;":                            '\U0001D565',
+	"topfork;":                         '\U00002ADA',
+	"tosa;":                            '\U00002929',
+	"tprime;":                          '\U00002034',
+	"trade;":                           '\U00002122',
+	"triangle;":                        '\U000025B5',
+	"triangledown;":                    '\U000025BF',
+	"triangleleft;":                    '\U000025C3',
+	"trianglelefteq;":                  '\U000022B4',
+	"triangleq;":                       '\U0000225C',
+	"triangleright;":                   '\U000025B9',
+	"trianglerighteq;":                 '\U000022B5',
+	"tridot;":                          '\U000025EC',
+	"trie;":                            '\U0000225C',
+	"triminus;":                        '\U00002A3A',
+	"triplus;":                         '\U00002A39',
+	"trisb;":                           '\U000029CD',
+	"tritime;":                         '\U00002A3B',
+	"trpezium;":                        '\U000023E2',
+	"tscr;":                            '\U0001D4C9',
+	"tscy;":                            '\U00000446',
+	"tshcy;":                           '\U0000045B',
+	"tstrok;":                          '\U00000167',
+	"twixt;":                           '\U0000226C',
+	"twoheadleftarrow;":                '\U0000219E',
+	"twoheadrightarrow;":               '\U000021A0',
+	"uArr;":                            '\U000021D1',
+	"uHar;":                            '\U00002963',
+	"uacute;":                          '\U000000FA',
+	"uarr;":                            '\U00002191',
+	"ubrcy;":                           '\U0000045E',
+	"ubreve;":                          '\U0000016D',
+	"ucirc;":                           '\U000000FB',
+	"ucy;":                             '\U00000443',
+	"udarr;":                           '\U000021C5',
+	"udblac;":                          '\U00000171',
+	"udhar;":                           '\U0000296E',
+	"ufisht;":                          '\U0000297E',
+	"ufr;":                             '\U0001D532',
+	"ugrave;":                          '\U000000F9',
+	"uharl;":                           '\U000021BF',
+	"uharr;":                           '\U000021BE',
+	"uhblk;":                           '\U00002580',
+	"ulcorn;":                          '\U0000231C',
+	"ulcorner;":                        '\U0000231C',
+	"ulcrop;":                          '\U0000230F',
+	"ultri;":                           '\U000025F8',
+	"umacr;":                           '\U0000016B',
+	"uml;":                             '\U000000A8',
+	"uogon;":                           '\U00000173',
+	"uopf;":                            '\U0001D566',
+	"uparrow;":                         '\U00002191',
+	"updownarrow;":                     '\U00002195',
+	"upharpoonleft;":                   '\U000021BF',
+	"upharpoonright;":                  '\U000021BE',
+	"uplus;":                           '\U0000228E',
+	"upsi;":                            '\U000003C5',
+	"upsih;":                           '\U000003D2',
+	"upsilon;":                         '\U000003C5',
+	"upuparrows;":                      '\U000021C8',
+	"urcorn;":                          '\U0000231D',
+	"urcorner;":                        '\U0000231D',
+	"urcrop;":                          '\U0000230E',
+	"uring;":                           '\U0000016F',
+	"urtri;":                           '\U000025F9',
+	"uscr;":                            '\U0001D4CA',
+	"utdot;":                           '\U000022F0',
+	"utilde;":                          '\U00000169',
+	"utri;":                            '\U000025B5',
+	"utrif;":                           '\U000025B4',
+	"uuarr;":                           '\U000021C8',
+	"uuml;":                            '\U000000FC',
+	"uwangle;":                         '\U000029A7',
+	"vArr;":                            '\U000021D5',
+	"vBar;":                            '\U00002AE8',
+	"vBarv;":                           '\U00002AE9',
+	"vDash;":                           '\U000022A8',
+	"vangrt;":                          '\U0000299C',
+	"varepsilon;":                      '\U000003F5',
+	"varkappa;":                        '\U000003F0',
+	"varnothing;":                      '\U00002205',
+	"varphi;":                          '\U000003D5',
+	"varpi;":                           '\U000003D6',
+	"varpropto;":                       '\U0000221D',
+	"varr;":                            '\U00002195',
+	"varrho;":                          '\U000003F1',
+	"varsigma;":                        '\U000003C2',
+	"vartheta;":                        '\U000003D1',
+	"vartriangleleft;":                 '\U000022B2',
+	"vartriangleright;":                '\U000022B3',
+	"vcy;":                             '\U00000432',
+	"vdash;":                           '\U000022A2',
+	"vee;":                             '\U00002228',
+	"veebar;":                          '\U000022BB',
+	"veeeq;":                           '\U0000225A',
+	"vellip;":                          '\U000022EE',
+	"verbar;":                          '\U0000007C',
+	"vert;":                            '\U0000007C',
+	"vfr;":                             '\U0001D533',
+	"vltri;":                           '\U000022B2',
+	"vopf;":                            '\U0001D567',
+	"vprop;":                           '\U0000221D',
+	"vrtri;":                           '\U000022B3',
+	"vscr;":                            '\U0001D4CB',
+	"vzigzag;":                         '\U0000299A',
+	"wcirc;":                           '\U00000175',
+	"wedbar;":                          '\U00002A5F',
+	"wedge;":                           '\U00002227',
+	"wedgeq;":                          '\U00002259',
+	"weierp;":                          '\U00002118',
+	"wfr;":                             '\U0001D534',
+	"wopf;":                            '\U0001D568',
+	"wp;":                              '\U00002118',
+	"wr;":                              '\U00002240',
+	"wreath;":                          '\U00002240',
+	"wscr;":                            '\U0001D4CC',
+	"xcap;":                            '\U000022C2',
+	"xcirc;":                           '\U000025EF',
+	"xcup;":                            '\U000022C3',
+	"xdtri;":                           '\U000025BD',
+	"xfr;":                             '\U0001D535',
+	"xhArr;":                           '\U000027FA',
+	"xharr;":                           '\U000027F7',
+	"xi;":                              '\U000003BE',
+	"xlArr;":                           '\U000027F8',
+	"xlarr;":                           '\U000027F5',
+	"xmap;":                            '\U000027FC',
+	"xnis;":                            '\U000022FB',
+	"xodot;":                           '\U00002A00',
+	"xopf;":                            '\U0001D569',
+	"xoplus;":                          '\U00002A01',
+	"xotime;":                          '\U00002A02',
+	"xrArr;":                           '\U000027F9',
+	"xrarr;":                           '\U000027F6',
+	"xscr;":                            '\U0001D4CD',
+	"xsqcup;":                          '\U00002A06',
+	"xuplus;":                          '\U00002A04',
+	"xutri;":                           '\U000025B3',
+	"xvee;":                            '\U000022C1',
+	"xwedge;":                          '\U000022C0',
+	"yacute;":                          '\U000000FD',
+	"yacy;":                            '\U0000044F',
+	"ycirc;":                           '\U00000177',
+	"ycy;":                             '\U0000044B',
+	"yen;":                             '\U000000A5',
+	"yfr;":                             '\U0001D536',
+	"yicy;":                            '\U00000457',
+	"yopf;":                            '\U0001D56A',
+	"yscr;":                            '\U0001D4CE',
+	"yucy;":                            '\U0000044E',
+	"yuml;":                            '\U000000FF',
+	"zacute;":                          '\U0000017A',
+	"zcaron;":                          '\U0000017E',
+	"zcy;":                             '\U00000437',
+	"zdot;":                            '\U0000017C',
+	"zeetrf;":                          '\U00002128',
+	"zeta;":                            '\U000003B6',
+	"zfr;":                             '\U0001D537',
+	"zhcy;":                            '\U00000436',
+	"zigrarr;":                         '\U000021DD',
+	"zopf;":                            '\U0001D56B',
+	"zscr;":                            '\U0001D4CF',
+	"zwj;":                             '\U0000200D',
+	"zwnj;":                            '\U0000200C',
+	"AElig":                            '\U000000C6',
+	"AMP":                              '\U00000026',
+	"Aacute":                           '\U000000C1',
+	"Acirc":                            '\U000000C2',
+	"Agrave":                           '\U000000C0',
+	"Aring":                            '\U000000C5',
+	"Atilde":                           '\U000000C3',
+	"Auml":                             '\U000000C4',
+	"COPY":                             '\U000000A9',
+	"Ccedil":                           '\U000000C7',
+	"ETH":                              '\U000000D0',
+	"Eacute":                           '\U000000C9',
+	"Ecirc":                            '\U000000CA',
+	"Egrave":                           '\U000000C8',
+	"Euml":                             '\U000000CB',
+	"GT":                               '\U0000003E',
+	"Iacute":                           '\U000000CD',
+	"Icirc":                            '\U000000CE',
+	"Igrave":                           '\U000000CC',
+	"Iuml":                             '\U000000CF',
+	"LT":                               '\U0000003C',
+	"Ntilde":                           '\U000000D1',
+	"Oacute":                           '\U000000D3',
+	"Ocirc":                            '\U000000D4',
+	"Ograve":                           '\U000000D2',
+	"Oslash":                           '\U000000D8',
+	"Otilde":                           '\U000000D5',
+	"Ouml":                             '\U000000D6',
+	"QUOT":                             '\U00000022',
+	"REG":                              '\U000000AE',
+	"THORN":                            '\U000000DE',
+	"Uacute":                           '\U000000DA',
+	"Ucirc":                            '\U000000DB',
+	"Ugrave":                           '\U000000D9',
+	"Uuml":                             '\U000000DC',
+	"Yacute":                           '\U000000DD',
+	"aacute":                           '\U000000E1',
+	"acirc":                            '\U000000E2',
+	"acute":                            '\U000000B4',
+	"aelig":                            '\U000000E6',
+	"agrave":                           '\U000000E0',
+	"amp":                              '\U00000026',
+	"aring":                            '\U000000E5',
+	"atilde":                           '\U000000E3',
+	"auml":                             '\U000000E4',
+	"brvbar":                           '\U000000A6',
+	"ccedil":                           '\U000000E7',
+	"cedil":                            '\U000000B8',
+	"cent":                             '\U000000A2',
+	"copy":                             '\U000000A9',
+	"curren":                           '\U000000A4',
+	"deg":                              '\U000000B0',
+	"divide":                           '\U000000F7',
+	"eacute":                           '\U000000E9',
+	"ecirc":                            '\U000000EA',
+	"egrave":                           '\U000000E8',
+	"eth":                              '\U000000F0',
+	"euml":                             '\U000000EB',
+	"frac12":                           '\U000000BD',
+	"frac14":                           '\U000000BC',
+	"frac34":                           '\U000000BE',
+	"gt":                               '\U0000003E',
+	"iacute":                           '\U000000ED',
+	"icirc":                            '\U000000EE',
+	"iexcl":                            '\U000000A1',
+	"igrave":                           '\U000000EC',
+	"iquest":                           '\U000000BF',
+	"iuml":                             '\U000000EF',
+	"laquo":                            '\U000000AB',
+	"lt":                               '\U0000003C',
+	"macr":                             '\U000000AF',
+	"micro":                            '\U000000B5',
+	"middot":                           '\U000000B7',
+	"nbsp":                             '\U000000A0',
+	"not":                              '\U000000AC',
+	"ntilde":                           '\U000000F1',
+	"oacute":                           '\U000000F3',
+	"ocirc":                            '\U000000F4',
+	"ograve":                           '\U000000F2',
+	"ordf":                             '\U000000AA',
+	"ordm":                             '\U000000BA',
+	"oslash":                           '\U000000F8',
+	"otilde":                           '\U000000F5',
+	"ouml":                             '\U000000F6',
+	"para":                             '\U000000B6',
+	"plusmn":                           '\U000000B1',
+	"pound":                            '\U000000A3',
+	"quot":                             '\U00000022',
+	"raquo":                            '\U000000BB',
+	"reg":                              '\U000000AE',
+	"sect":                             '\U000000A7',
+	"shy":                              '\U000000AD',
+	"sup1":                             '\U000000B9',
+	"sup2":                             '\U000000B2',
+	"sup3":                             '\U000000B3',
+	"szlig":                            '\U000000DF',
+	"thorn":                            '\U000000FE',
+	"times":                            '\U000000D7',
+	"uacute":                           '\U000000FA',
+	"ucirc":                            '\U000000FB',
+	"ugrave":                           '\U000000F9',
+	"uml":                              '\U000000A8',
+	"uuml":                             '\U000000FC',
+	"yacute":                           '\U000000FD',
+	"yen":                              '\U000000A5',
+	"yuml":                             '\U000000FF',
+}
+
+// HTML entities that are two unicode codepoints.
+var entity2 = map[string][2]int{
+	// TODO(nigeltao): Handle replacements that are wider than their names.
+	// "nLt;":                     {'\u226A', '\u20D2'},
+	// "nGt;":                     {'\u226B', '\u20D2'},
+	"NotEqualTilde;":           {'\u2242', '\u0338'},
+	"NotGreaterFullEqual;":     {'\u2267', '\u0338'},
+	"NotGreaterGreater;":       {'\u226B', '\u0338'},
+	"NotGreaterSlantEqual;":    {'\u2A7E', '\u0338'},
+	"NotHumpDownHump;":         {'\u224E', '\u0338'},
+	"NotHumpEqual;":            {'\u224F', '\u0338'},
+	"NotLeftTriangleBar;":      {'\u29CF', '\u0338'},
+	"NotLessLess;":             {'\u226A', '\u0338'},
+	"NotLessSlantEqual;":       {'\u2A7D', '\u0338'},
+	"NotNestedGreaterGreater;": {'\u2AA2', '\u0338'},
+	"NotNestedLessLess;":       {'\u2AA1', '\u0338'},
+	"NotPrecedesEqual;":        {'\u2AAF', '\u0338'},
+	"NotRightTriangleBar;":     {'\u29D0', '\u0338'},
+	"NotSquareSubset;":         {'\u228F', '\u0338'},
+	"NotSquareSuperset;":       {'\u2290', '\u0338'},
+	"NotSubset;":               {'\u2282', '\u20D2'},
+	"NotSucceedsEqual;":        {'\u2AB0', '\u0338'},
+	"NotSucceedsTilde;":        {'\u227F', '\u0338'},
+	"NotSuperset;":             {'\u2283', '\u20D2'},
+	"ThickSpace;":              {'\u205F', '\u200A'},
+	"acE;":                     {'\u223E', '\u0333'},
+	"bne;":                     {'\u003D', '\u20E5'},
+	"bnequiv;":                 {'\u2261', '\u20E5'},
+	"caps;":                    {'\u2229', '\uFE00'},
+	"cups;":                    {'\u222A', '\uFE00'},
+	"fjlig;":                   {'\u0066', '\u006A'},
+	"gesl;":                    {'\u22DB', '\uFE00'},
+	"gvertneqq;":               {'\u2269', '\uFE00'},
+	"gvnE;":                    {'\u2269', '\uFE00'},
+	"lates;":                   {'\u2AAD', '\uFE00'},
+	"lesg;":                    {'\u22DA', '\uFE00'},
+	"lvertneqq;":               {'\u2268', '\uFE00'},
+	"lvnE;":                    {'\u2268', '\uFE00'},
+	"nGg;":                     {'\u22D9', '\u0338'},
+	"nGtv;":                    {'\u226B', '\u0338'},
+	"nLl;":                     {'\u22D8', '\u0338'},
+	"nLtv;":                    {'\u226A', '\u0338'},
+	"nang;":                    {'\u2220', '\u20D2'},
+	"napE;":                    {'\u2A70', '\u0338'},
+	"napid;":                   {'\u224B', '\u0338'},
+	"nbump;":                   {'\u224E', '\u0338'},
+	"nbumpe;":                  {'\u224F', '\u0338'},
+	"ncongdot;":                {'\u2A6D', '\u0338'},
+	"nedot;":                   {'\u2250', '\u0338'},
+	"nesim;":                   {'\u2242', '\u0338'},
+	"ngE;":                     {'\u2267', '\u0338'},
+	"ngeqq;":                   {'\u2267', '\u0338'},
+	"ngeqslant;":               {'\u2A7E', '\u0338'},
+	"nges;":                    {'\u2A7E', '\u0338'},
+	"nlE;":                     {'\u2266', '\u0338'},
+	"nleqq;":                   {'\u2266', '\u0338'},
+	"nleqslant;":               {'\u2A7D', '\u0338'},
+	"nles;":                    {'\u2A7D', '\u0338'},
+	"notinE;":                  {'\u22F9', '\u0338'},
+	"notindot;":                {'\u22F5', '\u0338'},
+	"nparsl;":                  {'\u2AFD', '\u20E5'},
+	"npart;":                   {'\u2202', '\u0338'},
+	"npre;":                    {'\u2AAF', '\u0338'},
+	"npreceq;":                 {'\u2AAF', '\u0338'},
+	"nrarrc;":                  {'\u2933', '\u0338'},
+	"nrarrw;":                  {'\u219D', '\u0338'},
+	"nsce;":                    {'\u2AB0', '\u0338'},
+	"nsubE;":                   {'\u2AC5', '\u0338'},
+	"nsubset;":                 {'\u2282', '\u20D2'},
+	"nsubseteqq;":              {'\u2AC5', '\u0338'},
+	"nsucceq;":                 {'\u2AB0', '\u0338'},
+	"nsupE;":                   {'\u2AC6', '\u0338'},
+	"nsupset;":                 {'\u2283', '\u20D2'},
+	"nsupseteqq;":              {'\u2AC6', '\u0338'},
+	"nvap;":                    {'\u224D', '\u20D2'},
+	"nvge;":                    {'\u2265', '\u20D2'},
+	"nvgt;":                    {'\u003E', '\u20D2'},
+	"nvle;":                    {'\u2264', '\u20D2'},
+	"nvlt;":                    {'\u003C', '\u20D2'},
+	"nvltrie;":                 {'\u22B4', '\u20D2'},
+	"nvrtrie;":                 {'\u22B5', '\u20D2'},
+	"nvsim;":                   {'\u223C', '\u20D2'},
+	"race;":                    {'\u223D', '\u0331'},
+	"smtes;":                   {'\u2AAC', '\uFE00'},
+	"sqcaps;":                  {'\u2293', '\uFE00'},
+	"sqcups;":                  {'\u2294', '\uFE00'},
+	"varsubsetneq;":            {'\u228A', '\uFE00'},
+	"varsubsetneqq;":           {'\u2ACB', '\uFE00'},
+	"varsupsetneq;":            {'\u228B', '\uFE00'},
+	"varsupsetneqq;":           {'\u2ACC', '\uFE00'},
+	"vnsub;":                   {'\u2282', '\u20D2'},
+	"vnsup;":                   {'\u2283', '\u20D2'},
+	"vsubnE;":                  {'\u2ACB', '\uFE00'},
+	"vsubne;":                  {'\u228A', '\uFE00'},
+	"vsupnE;":                  {'\u2ACC', '\uFE00'},
+	"vsupne;":                  {'\u228B', '\uFE00'},
+}
diff --git a/src/pkg/html/entity_test.go b/src/pkg/html/entity_test.go
new file mode 100644
index 000000000..2cf49d61d
--- /dev/null
+++ b/src/pkg/html/entity_test.go
@@ -0,0 +1,29 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"testing"
+	"utf8"
+)
+
+func TestEntityLength(t *testing.T) {
+	// We verify that the length of UTF-8 encoding of each value is <= 1 + len(key).
+	// The +1 comes from the leading "&". This property implies that the length of
+	// unescaped text is <= the length of escaped text.
+	for k, v := range entity {
+		if 1+len(k) < utf8.RuneLen(v) {
+			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v))
+		}
+		if len(k) > longestEntityWithoutSemicolon && k[len(k)-1] != ';' {
+			t.Errorf("entity name %s is %d characters, but longestEntityWithoutSemicolon=%d", k, len(k), longestEntityWithoutSemicolon)
+		}
+	}
+	for k, v := range entity2 {
+		if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) {
+			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1]))
+		}
+	}
+}
diff --git a/src/pkg/html/escape.go b/src/pkg/html/escape.go
new file mode 100644
index 000000000..0de97c5ac
--- /dev/null
+++ b/src/pkg/html/escape.go
@@ -0,0 +1,238 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"bytes"
+	"strings"
+	"utf8"
+)
+
+// These replacements permit compatibility with old numeric entities that 
+// assumed Windows-1252 encoding.
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
+var replacementTable = [...]int{
+	'\u20AC', // First entry is what 0x80 should be replaced with.
+	'\u0081',
+	'\u201A',
+	'\u0192',
+	'\u201E',
+	'\u2026',
+	'\u2020',
+	'\u2021',
+	'\u02C6',
+	'\u2030',
+	'\u0160',
+	'\u2039',
+	'\u0152',
+	'\u008D',
+	'\u017D',
+	'\u008F',
+	'\u0090',
+	'\u2018',
+	'\u2019',
+	'\u201C',
+	'\u201D',
+	'\u2022',
+	'\u2013',
+	'\u2014',
+	'\u02DC',
+	'\u2122',
+	'\u0161',
+	'\u203A',
+	'\u0153',
+	'\u009D',
+	'\u017E',
+	'\u0178', // Last entry is 0x9F.
+	// 0x00->'\uFFFD' is handled programmatically. 
+	// 0x0D->'\u000D' is a no-op.
+}
+
+// unescapeEntity reads an entity like "<" from b[src:] and writes the
+// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
+// Precondition: b[src] == '&' && dst <= src.
+// attribute should be true if parsing an attribute value.
+func unescapeEntity(b []byte, dst, src int, attribute bool) (dst1, src1 int) {
+	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
+
+	// i starts at 1 because we already know that s[0] == '&'.
+	i, s := 1, b[src:]
+
+	if len(s) <= 1 {
+		b[dst] = b[src]
+		return dst + 1, src + 1
+	}
+
+	if s[i] == '#' {
+		if len(s) <= 3 { // We need to have at least "&#.".
+			b[dst] = b[src]
+			return dst + 1, src + 1
+		}
+		i++
+		c := s[i]
+		hex := false
+		if c == 'x' || c == 'X' {
+			hex = true
+			i++
+		}
+
+		x := 0
+		for i < len(s) {
+			c = s[i]
+			i++
+			if hex {
+				if '0' <= c && c <= '9' {
+					x = 16*x + int(c) - '0'
+					continue
+				} else if 'a' <= c && c <= 'f' {
+					x = 16*x + int(c) - 'a' + 10
+					continue
+				} else if 'A' <= c && c <= 'F' {
+					x = 16*x + int(c) - 'A' + 10
+					continue
+				}
+			} else if '0' <= c && c <= '9' {
+				x = 10*x + int(c) - '0'
+				continue
+			}
+			if c != ';' {
+				i--
+			}
+			break
+		}
+
+		if i <= 3 { // No characters matched.
+			b[dst] = b[src]
+			return dst + 1, src + 1
+		}
+
+		if 0x80 <= x && x <= 0x9F {
+			// Replace characters from Windows-1252 with UTF-8 equivalents.
+			x = replacementTable[x-0x80]
+		} else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF {
+			// Replace invalid characters with the replacement character.
+			x = '\uFFFD'
+		}
+
+		return dst + utf8.EncodeRune(b[dst:], x), src + i
+	}
+
+	// Consume the maximum number of characters possible, with the
+	// consumed characters matching one of the named references.
+
+	for i < len(s) {
+		c := s[i]
+		i++
+		// Lower-cased characters are more common in entities, so we check for them first.
+		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
+			continue
+		}
+		if c != ';' {
+			i--
+		}
+		break
+	}
+
+	entityName := string(s[1:i])
+	if entityName == "" {
+		// No-op.
+	} else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' {
+		// No-op.
+	} else if x := entity[entityName]; x != 0 {
+		return dst + utf8.EncodeRune(b[dst:], x), src + i
+	} else if x := entity2[entityName]; x[0] != 0 {
+		dst1 := dst + utf8.EncodeRune(b[dst:], x[0])
+		return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i
+	} else if !attribute {
+		maxLen := len(entityName) - 1
+		if maxLen > longestEntityWithoutSemicolon {
+			maxLen = longestEntityWithoutSemicolon
+		}
+		for j := maxLen; j > 1; j-- {
+			if x := entity[entityName[:j]]; x != 0 {
+				return dst + utf8.EncodeRune(b[dst:], x), src + j + 1
+			}
+		}
+	}
+
+	dst1, src1 = dst+i, src+i
+	copy(b[dst:dst1], b[src:src1])
+	return dst1, src1
+}
+
+// unescape unescapes b's entities in-place, so that "a<b" becomes "a"`
+
+func escape(buf *bytes.Buffer, s string) {
+	i := strings.IndexAny(s, escapedChars)
+	for i != -1 {
+		buf.WriteString(s[0:i])
+		var esc string
+		switch s[i] {
+		case '&':
+			esc = "&"
+		case '\'':
+			esc = "'"
+		case '<':
+			esc = "<"
+		case '>':
+			esc = ">"
+		case '"':
+			esc = """
+		default:
+			panic("unrecognized escape character")
+		}
+		s = s[i+1:]
+		buf.WriteString(esc)
+		i = strings.IndexAny(s, escapedChars)
+	}
+	buf.WriteString(s)
+}
+
+// EscapeString escapes special characters like "<" to become "<". It
+// escapes only five such characters: amp, apos, lt, gt and quot.
+// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
+// always true.
+func EscapeString(s string) string {
+	if strings.IndexAny(s, escapedChars) == -1 {
+		return s
+	}
+	buf := bytes.NewBuffer(nil)
+	escape(buf, s)
+	return buf.String()
+}
+
+// UnescapeString unescapes entities like "<" to become "<". It unescapes a
+// larger range of entities than EscapeString escapes. For example, "á"
+// unescapes to "á", as does "á" and "&xE1;".
+// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
+// always true.
+func UnescapeString(s string) string {
+	for _, c := range s {
+		if c == '&' {
+			return string(unescape([]byte(s)))
+		}
+	}
+	return s
+}
diff --git a/src/pkg/html/node.go b/src/pkg/html/node.go
new file mode 100644
index 000000000..4ecfd6ca2
--- /dev/null
+++ b/src/pkg/html/node.go
@@ -0,0 +1,147 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+// A NodeType is the type of a Node.
+type NodeType int
+
+const (
+	ErrorNode NodeType = iota
+	TextNode
+	DocumentNode
+	ElementNode
+	CommentNode
+	DoctypeNode
+	scopeMarkerNode
+)
+
+// Section 11.2.3.3 says "scope markers are inserted when entering applet
+// elements, buttons, object elements, marquees, table cells, and table
+// captions, and are used to prevent formatting from 'leaking'".
+var scopeMarker = Node{Type: scopeMarkerNode}
+
+// A Node consists of a NodeType and some Data (tag name for element nodes,
+// content for text) and are part of a tree of Nodes. Element nodes may also
+// contain a slice of Attributes. Data is unescaped, so that it looks like
+// "a 0 {
+		return (*s)[i-1]
+	}
+	return nil
+}
+
+// index returns the index of the top-most occurence of n in the stack, or -1
+// if n is not present.
+func (s *nodeStack) index(n *Node) int {
+	for i := len(*s) - 1; i >= 0; i-- {
+		if (*s)[i] == n {
+			return i
+		}
+	}
+	return -1
+}
+
+// insert inserts a node at the given index.
+func (s *nodeStack) insert(i int, n *Node) {
+	(*s) = append(*s, nil)
+	copy((*s)[i+1:], (*s)[i:])
+	(*s)[i] = n
+}
+
+// remove removes a node from the stack. It is a no-op if n is not present.
+func (s *nodeStack) remove(n *Node) {
+	i := s.index(n)
+	if i == -1 {
+		return
+	}
+	copy((*s)[i:], (*s)[i+1:])
+	j := len(*s) - 1
+	(*s)[j] = nil
+	*s = (*s)[:j]
+}
+
+// forTag returns the top-most element node with the given tag.
+func (s *nodeStack) forTag(tag string) *Node {
+	for i := len(*s) - 1; i >= 0; i-- {
+		n := (*s)[i]
+		if n.Type == ElementNode && n.Data == tag {
+			return n
+		}
+	}
+	return nil
+}
diff --git a/src/pkg/html/parse.go b/src/pkg/html/parse.go
new file mode 100644
index 000000000..519ebe587
--- /dev/null
+++ b/src/pkg/html/parse.go
@@ -0,0 +1,800 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"io"
+	"os"
+)
+
+// A parser implements the HTML5 parsing algorithm:
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#tree-construction
+type parser struct {
+	// tokenizer provides the tokens for the parser.
+	tokenizer *Tokenizer
+	// tok is the most recently read token.
+	tok Token
+	// Self-closing tags like 
 are re-interpreted as a two-token sequence:
+	// 
 followed by . hasSelfClosingToken is true if we have just read
+	// the synthetic start tag and the next one due is the matching end tag.
+	hasSelfClosingToken bool
+	// doc is the document root element.
+	doc *Node
+	// The stack of open elements (section 11.2.3.2) and active formatting
+	// elements (section 11.2.3.3).
+	oe, afe nodeStack
+	// Element pointers (section 11.2.3.4).
+	head, form *Node
+	// Other parsing state flags (section 11.2.3.5).
+	scripting, framesetOK bool
+}
+
+func (p *parser) top() *Node {
+	if n := p.oe.top(); n != nil {
+		return n
+	}
+	return p.doc
+}
+
+// stopTags for use in popUntil. These come from section 11.2.3.2.
+var (
+	defaultScopeStopTags  = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object"}
+	listItemScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "ol", "ul"}
+	buttonScopeStopTags   = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "button"}
+	tableScopeStopTags    = []string{"html", "table"}
+)
+
+// popUntil pops the stack of open elements at the highest element whose tag
+// is in matchTags, provided there is no higher element in stopTags. It returns
+// whether or not there was such an element. If there was not, popUntil leaves
+// the stack unchanged.
+//
+// For example, if the stack was:
+// ["html", "body", "font", "table", "b", "i", "u"]
+// then popUntil([]string{"html, "table"}, "font") would return false, but
+// popUntil([]string{"html, "table"}, "i") would return true and the resultant
+// stack would be:
+// ["html", "body", "font", "table", "b"]
+//
+// If an element's tag is in both stopTags and matchTags, then the stack will
+// be popped and the function returns true (provided, of course, there was no
+// higher element in the stack that was also in stopTags). For example,
+// popUntil([]string{"html, "table"}, "table") would return true and leave:
+// ["html", "body", "font"]
+func (p *parser) popUntil(stopTags []string, matchTags ...string) bool {
+	for i := len(p.oe) - 1; i >= 0; i-- {
+		tag := p.oe[i].Data
+		for _, t := range matchTags {
+			if t == tag {
+				p.oe = p.oe[:i]
+				return true
+			}
+		}
+		for _, t := range stopTags {
+			if t == tag {
+				return false
+			}
+		}
+	}
+	return false
+}
+
+// addChild adds a child node n to the top element, and pushes n onto the stack
+// of open elements if it is an element node.
+func (p *parser) addChild(n *Node) {
+	p.top().Add(n)
+	if n.Type == ElementNode {
+		p.oe = append(p.oe, n)
+	}
+}
+
+// addText adds text to the preceding node if it is a text node, or else it
+// calls addChild with a new text node.
+func (p *parser) addText(text string) {
+	// TODO: distinguish whitespace text from others.
+	t := p.top()
+	if i := len(t.Child); i > 0 && t.Child[i-1].Type == TextNode {
+		t.Child[i-1].Data += text
+		return
+	}
+	p.addChild(&Node{
+		Type: TextNode,
+		Data: text,
+	})
+}
+
+// addElement calls addChild with an element node.
+func (p *parser) addElement(tag string, attr []Attribute) {
+	p.addChild(&Node{
+		Type: ElementNode,
+		Data: tag,
+		Attr: attr,
+	})
+}
+
+// Section 11.2.3.3.
+func (p *parser) addFormattingElement(tag string, attr []Attribute) {
+	p.addElement(tag, attr)
+	p.afe = append(p.afe, p.top())
+	// TODO.
+}
+
+// Section 11.2.3.3.
+func (p *parser) clearActiveFormattingElements() {
+	for {
+		n := p.afe.pop()
+		if len(p.afe) == 0 || n.Type == scopeMarkerNode {
+			return
+		}
+	}
+}
+
+// Section 11.2.3.3.
+func (p *parser) reconstructActiveFormattingElements() {
+	n := p.afe.top()
+	if n == nil {
+		return
+	}
+	if n.Type == scopeMarkerNode || p.oe.index(n) != -1 {
+		return
+	}
+	i := len(p.afe) - 1
+	for n.Type != scopeMarkerNode && p.oe.index(n) == -1 {
+		if i == 0 {
+			i = -1
+			break
+		}
+		i--
+		n = p.afe[i]
+	}
+	for {
+		i++
+		n = p.afe[i]
+		p.addChild(n.clone())
+		p.afe[i] = n
+		if i == len(p.afe)-1 {
+			break
+		}
+	}
+}
+
+// read reads the next token. This is usually from the tokenizer, but it may
+// be the synthesized end tag implied by a self-closing tag.
+func (p *parser) read() os.Error {
+	if p.hasSelfClosingToken {
+		p.hasSelfClosingToken = false
+		p.tok.Type = EndTagToken
+		p.tok.Attr = nil
+		return nil
+	}
+	p.tokenizer.Next()
+	p.tok = p.tokenizer.Token()
+	switch p.tok.Type {
+	case ErrorToken:
+		return p.tokenizer.Error()
+	case SelfClosingTagToken:
+		p.hasSelfClosingToken = true
+		p.tok.Type = StartTagToken
+	}
+	return nil
+}
+
+// Section 11.2.4.
+func (p *parser) acknowledgeSelfClosingTag() {
+	p.hasSelfClosingToken = false
+}
+
+// An insertion mode (section 11.2.3.1) is the state transition function from
+// a particular state in the HTML5 parser's state machine. It updates the
+// parser's fields depending on parser.token (where ErrorToken means EOF). In
+// addition to returning the next insertionMode state, it also returns whether
+// the token was consumed.
+type insertionMode func(*parser) (insertionMode, bool)
+
+// useTheRulesFor runs the delegate insertionMode over p, returning the actual
+// insertionMode unless the delegate caused a state transition.
+// Section 11.2.3.1, "using the rules for".
+func useTheRulesFor(p *parser, actual, delegate insertionMode) (insertionMode, bool) {
+	im, consumed := delegate(p)
+	if im != delegate {
+		return im, consumed
+	}
+	return actual, consumed
+}
+
+// Section 11.2.5.4.1.
+func initialIM(p *parser) (insertionMode, bool) {
+	if p.tok.Type == DoctypeToken {
+		p.addChild(&Node{
+			Type: DoctypeNode,
+			Data: p.tok.Data,
+		})
+		return beforeHTMLIM, true
+	}
+	// TODO: set "quirks mode"? It's defined in the DOM spec instead of HTML5 proper,
+	// and so switching on "quirks mode" might belong in a different package.
+	return beforeHTMLIM, false
+}
+
+// Section 11.2.5.4.2.
+func beforeHTMLIM(p *parser) (insertionMode, bool) {
+	var (
+		add     bool
+		attr    []Attribute
+		implied bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		implied = true
+	case TextToken:
+		// TODO: distinguish whitespace text from others.
+		implied = true
+	case StartTagToken:
+		if p.tok.Data == "html" {
+			add = true
+			attr = p.tok.Attr
+		} else {
+			implied = true
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "head", "body", "html", "br":
+			implied = true
+		default:
+			// Ignore the token.
+		}
+	}
+	if add || implied {
+		p.addElement("html", attr)
+	}
+	return beforeHeadIM, !implied
+}
+
+// Section 11.2.5.4.3.
+func beforeHeadIM(p *parser) (insertionMode, bool) {
+	var (
+		add     bool
+		attr    []Attribute
+		implied bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		implied = true
+	case TextToken:
+		// TODO: distinguish whitespace text from others.
+		implied = true
+	case StartTagToken:
+		switch p.tok.Data {
+		case "head":
+			add = true
+			attr = p.tok.Attr
+		case "html":
+			return useTheRulesFor(p, beforeHeadIM, inBodyIM)
+		default:
+			implied = true
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "head", "body", "html", "br":
+			implied = true
+		default:
+			// Ignore the token.
+		}
+	}
+	if add || implied {
+		p.addElement("head", attr)
+	}
+	return inHeadIM, !implied
+}
+
+// Section 11.2.5.4.4.
+func inHeadIM(p *parser) (insertionMode, bool) {
+	var (
+		pop     bool
+		implied bool
+	)
+	switch p.tok.Type {
+	case ErrorToken, TextToken:
+		implied = true
+	case StartTagToken:
+		switch p.tok.Data {
+		case "meta":
+			// TODO.
+		case "script":
+			// TODO.
+		default:
+			implied = true
+		}
+	case EndTagToken:
+		if p.tok.Data == "head" {
+			pop = true
+		}
+		// TODO.
+	}
+	if pop || implied {
+		n := p.oe.pop()
+		if n.Data != "head" {
+			panic("html: bad parser state")
+		}
+		return afterHeadIM, !implied
+	}
+	return inHeadIM, !implied
+}
+
+// Section 11.2.5.4.6.
+func afterHeadIM(p *parser) (insertionMode, bool) {
+	var (
+		add        bool
+		attr       []Attribute
+		framesetOK bool
+		implied    bool
+	)
+	switch p.tok.Type {
+	case ErrorToken, TextToken:
+		implied = true
+		framesetOK = true
+	case StartTagToken:
+		switch p.tok.Data {
+		case "html":
+			// TODO.
+		case "body":
+			add = true
+			attr = p.tok.Attr
+			framesetOK = false
+		case "frameset":
+			// TODO.
+		case "base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "title":
+			// TODO.
+		case "head":
+			// TODO.
+		default:
+			implied = true
+			framesetOK = true
+		}
+	case EndTagToken:
+		// TODO.
+	}
+	if add || implied {
+		p.addElement("body", attr)
+		p.framesetOK = framesetOK
+	}
+	return inBodyIM, !implied
+}
+
+// Section 11.2.5.4.7.
+func inBodyIM(p *parser) (insertionMode, bool) {
+	var endP bool
+	switch p.tok.Type {
+	case TextToken:
+		p.reconstructActiveFormattingElements()
+		p.addText(p.tok.Data)
+		p.framesetOK = false
+	case StartTagToken:
+		switch p.tok.Data {
+		case "address", "article", "aside", "blockquote", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "menu", "nav", "ol", "p", "section", "summary", "ul":
+			// TODO: Do the proper "does the stack of open elements has a p element in button scope" algorithm in section 11.2.3.2.
+			n := p.top()
+			if n.Type == ElementNode && n.Data == "p" {
+				endP = true
+			} else {
+				p.addElement(p.tok.Data, p.tok.Attr)
+			}
+		case "h1", "h2", "h3", "h4", "h5", "h6":
+			// TODO: auto-insert 
 if necessary.
+			switch n := p.top(); n.Data {
+			case "h1", "h2", "h3", "h4", "h5", "h6":
+				p.oe.pop()
+			}
+			p.addElement(p.tok.Data, p.tok.Attr)
+		case "a":
+			if n := p.afe.forTag("a"); n != nil {
+				p.inBodyEndTagFormatting("a")
+				p.oe.remove(n)
+				p.afe.remove(n)
+			}
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement(p.tok.Data, p.tok.Attr)
+		case "b", "big", "code", "em", "font", "i", "s", "small", "strike", "strong", "tt", "u":
+			p.reconstructActiveFormattingElements()
+			p.addFormattingElement(p.tok.Data, p.tok.Attr)
+		case "area", "br", "embed", "img", "input", "keygen", "wbr":
+			p.reconstructActiveFormattingElements()
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+			p.framesetOK = false
+		case "table":
+			// TODO: auto-insert 
 if necessary, depending on quirks mode.
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.framesetOK = false
+			return inTableIM, true
+		case "hr":
+			// TODO: auto-insert 
 if necessary.
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.oe.pop()
+			p.acknowledgeSelfClosingTag()
+			p.framesetOK = false
+		default:
+			// TODO.
+			p.addElement(p.tok.Data, p.tok.Attr)
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "body":
+			// TODO: autoclose the stack of open elements.
+			return afterBodyIM, true
+		case "a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u":
+			p.inBodyEndTagFormatting(p.tok.Data)
+		default:
+			// TODO: any other end tag
+			if p.tok.Data == p.top().Data {
+				p.oe.pop()
+			}
+		}
+	}
+	if endP {
+		// TODO: do the proper algorithm.
+		n := p.oe.pop()
+		if n.Type != ElementNode || n.Data != "p" {
+			panic("unreachable")
+		}
+	}
+	return inBodyIM, !endP
+}
+
+func (p *parser) inBodyEndTagFormatting(tag string) {
+	// This is the "adoption agency" algorithm, described at
+	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#adoptionAgency
+
+	// TODO: this is a fairly literal line-by-line translation of that algorithm.
+	// Once the code successfully parses the comprehensive test suite, we should
+	// refactor this code to be more idiomatic.
+
+	// Steps 1-3. The outer loop.
+	for i := 0; i < 8; i++ {
+		// Step 4. Find the formatting element.
+		var formattingElement *Node
+		for j := len(p.afe) - 1; j >= 0; j-- {
+			if p.afe[j].Type == scopeMarkerNode {
+				break
+			}
+			if p.afe[j].Data == tag {
+				formattingElement = p.afe[j]
+				break
+			}
+		}
+		if formattingElement == nil {
+			return
+		}
+		feIndex := p.oe.index(formattingElement)
+		if feIndex == -1 {
+			p.afe.remove(formattingElement)
+			return
+		}
+
+		// Steps 5-6. Find the furthest block.
+		var furthestBlock *Node
+		for _, e := range p.oe[feIndex:] {
+			if isSpecialElement[e.Data] {
+				furthestBlock = e
+				break
+			}
+		}
+		if furthestBlock == nil {
+			e := p.oe.pop()
+			for e != formattingElement {
+				e = p.oe.pop()
+			}
+			p.afe.remove(e)
+			return
+		}
+
+		// Steps 7-8. Find the common ancestor and bookmark node.
+		commonAncestor := p.oe[feIndex-1]
+		bookmark := p.afe.index(formattingElement)
+
+		// Step 9. The inner loop. Find the lastNode to reparent.
+		lastNode := furthestBlock
+		node := furthestBlock
+		x := p.oe.index(node)
+		// Steps 9.1-9.3.
+		for j := 0; j < 3; j++ {
+			// Step 9.4.
+			x--
+			node = p.oe[x]
+			// Step 9.5.
+			if p.afe.index(node) == -1 {
+				p.oe.remove(node)
+				continue
+			}
+			// Step 9.6.
+			if node == formattingElement {
+				break
+			}
+			// Step 9.7.
+			clone := node.clone()
+			p.afe[p.afe.index(node)] = clone
+			p.oe[p.oe.index(node)] = clone
+			node = clone
+			// Step 9.8.
+			if lastNode == furthestBlock {
+				bookmark = p.afe.index(node) + 1
+			}
+			// Step 9.9.
+			if lastNode.Parent != nil {
+				lastNode.Parent.Remove(lastNode)
+			}
+			node.Add(lastNode)
+			// Step 9.10.
+			lastNode = node
+		}
+
+		// Step 10. Reparent lastNode to the common ancestor,
+		// or for misnested table nodes, to the foster parent.
+		if lastNode.Parent != nil {
+			lastNode.Parent.Remove(lastNode)
+		}
+		switch commonAncestor.Data {
+		case "table", "tbody", "tfoot", "thead", "tr":
+			// TODO: fix up misnested table nodes; find the foster parent.
+			fallthrough
+		default:
+			commonAncestor.Add(lastNode)
+		}
+
+		// Steps 11-13. Reparent nodes from the furthest block's children
+		// to a clone of the formatting element.
+		clone := formattingElement.clone()
+		reparentChildren(clone, furthestBlock)
+		furthestBlock.Add(clone)
+
+		// Step 14. Fix up the list of active formatting elements.
+		p.afe.remove(formattingElement)
+		p.afe.insert(bookmark, clone)
+
+		// Step 15. Fix up the stack of open elements.
+		p.oe.remove(formattingElement)
+		p.oe.insert(p.oe.index(furthestBlock)+1, clone)
+	}
+}
+
+// Section 11.2.5.4.9.
+func inTableIM(p *parser) (insertionMode, bool) {
+	var (
+		add      bool
+		data     string
+		attr     []Attribute
+		consumed bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		// Stop parsing.
+		return nil, true
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		switch p.tok.Data {
+		case "tbody", "tfoot", "thead":
+			add = true
+			data = p.tok.Data
+			attr = p.tok.Attr
+			consumed = true
+		case "td", "th", "tr":
+			add = true
+			data = "tbody"
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "table":
+			if p.popUntil(tableScopeStopTags, "table") {
+				// TODO: "reset the insertion mode appropriately" as per 11.2.3.1.
+				return inBodyIM, false
+			}
+			// Ignore the token.
+			return inTableIM, true
+		case "body", "caption", "col", "colgroup", "html", "tbody", "td", "tfoot", "th", "thead", "tr":
+			// Ignore the token.
+			return inTableIM, true
+		}
+	}
+	if add {
+		// TODO: clear the stack back to a table context.
+		p.addElement(data, attr)
+		return inTableBodyIM, consumed
+	}
+	// TODO: return useTheRulesFor(inTableIM, inBodyIM, p) unless etc. etc. foster parenting.
+	return inTableIM, true
+}
+
+// Section 11.2.5.4.13.
+func inTableBodyIM(p *parser) (insertionMode, bool) {
+	var (
+		add      bool
+		data     string
+		attr     []Attribute
+		consumed bool
+	)
+	switch p.tok.Type {
+	case ErrorToken:
+		// TODO.
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		switch p.tok.Data {
+		case "tr":
+			add = true
+			data = p.tok.Data
+			attr = p.tok.Attr
+			consumed = true
+		case "td", "th":
+			add = true
+			data = "tr"
+			consumed = false
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "table":
+			if p.popUntil(tableScopeStopTags, "tbody", "thead", "tfoot") {
+				return inTableIM, false
+			}
+			// Ignore the token.
+			return inTableBodyIM, true
+		case "body", "caption", "col", "colgroup", "html", "td", "th", "tr":
+			// Ignore the token.
+			return inTableBodyIM, true
+		}
+	}
+	if add {
+		// TODO: clear the stack back to a table body context.
+		p.addElement(data, attr)
+		return inRowIM, consumed
+	}
+	return useTheRulesFor(p, inTableBodyIM, inTableIM)
+}
+
+// Section 11.2.5.4.14.
+func inRowIM(p *parser) (insertionMode, bool) {
+	switch p.tok.Type {
+	case ErrorToken:
+		// TODO.
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		switch p.tok.Data {
+		case "td", "th":
+			// TODO: clear the stack back to a table row context.
+			p.addElement(p.tok.Data, p.tok.Attr)
+			p.afe = append(p.afe, &scopeMarker)
+			return inCellIM, true
+		default:
+			// TODO.
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "tr":
+			// TODO.
+		case "table":
+			if p.popUntil(tableScopeStopTags, "tr") {
+				return inTableBodyIM, false
+			}
+			// Ignore the token.
+			return inRowIM, true
+		case "tbody", "tfoot", "thead":
+			// TODO.
+		case "body", "caption", "col", "colgroup", "html", "td", "th":
+			// Ignore the token.
+			return inRowIM, true
+		default:
+			// TODO.
+		}
+	}
+	return useTheRulesFor(p, inRowIM, inTableIM)
+}
+
+// Section 11.2.5.4.15.
+func inCellIM(p *parser) (insertionMode, bool) {
+	var (
+		closeTheCellAndReprocess bool
+	)
+	switch p.tok.Type {
+	case StartTagToken:
+		switch p.tok.Data {
+		case "caption", "col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr":
+			// TODO: check for "td" or "th" in table scope.
+			closeTheCellAndReprocess = true
+		}
+	case EndTagToken:
+		switch p.tok.Data {
+		case "td", "th":
+			// TODO.
+		case "body", "caption", "col", "colgroup", "html":
+			// TODO.
+		case "table", "tbody", "tfoot", "thead", "tr":
+			// TODO: check for matching element in table scope.
+			closeTheCellAndReprocess = true
+		}
+	}
+	if closeTheCellAndReprocess {
+		if p.popUntil(tableScopeStopTags, "td") || p.popUntil(tableScopeStopTags, "th") {
+			p.clearActiveFormattingElements()
+			return inRowIM, false
+		}
+	}
+	return useTheRulesFor(p, inCellIM, inBodyIM)
+}
+
+// Section 11.2.5.4.18.
+func afterBodyIM(p *parser) (insertionMode, bool) {
+	switch p.tok.Type {
+	case ErrorToken:
+		// TODO.
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		// TODO.
+	case EndTagToken:
+		switch p.tok.Data {
+		case "html":
+			// TODO: autoclose the stack of open elements.
+			return afterAfterBodyIM, true
+		default:
+			// TODO.
+		}
+	}
+	return afterBodyIM, true
+}
+
+// Section 11.2.5.4.21.
+func afterAfterBodyIM(p *parser) (insertionMode, bool) {
+	switch p.tok.Type {
+	case ErrorToken:
+		// Stop parsing.
+		return nil, true
+	case TextToken:
+		// TODO.
+	case StartTagToken:
+		if p.tok.Data == "html" {
+			return useTheRulesFor(p, afterAfterBodyIM, inBodyIM)
+		}
+	}
+	return inBodyIM, false
+}
+
+// Parse returns the parse tree for the HTML from the given Reader.
+// The input is assumed to be UTF-8 encoded.
+func Parse(r io.Reader) (*Node, os.Error) {
+	p := &parser{
+		tokenizer: NewTokenizer(r),
+		doc: &Node{
+			Type: DocumentNode,
+		},
+		scripting:  true,
+		framesetOK: true,
+	}
+	// Iterate until EOF. Any other error will cause an early return.
+	im, consumed := initialIM, true
+	for {
+		if consumed {
+			if err := p.read(); err != nil {
+				if err == os.EOF {
+					break
+				}
+				return nil, err
+			}
+		}
+		im, consumed = im(p)
+	}
+	// Loop until the final token (the ErrorToken signifying EOF) is consumed.
+	for {
+		if im, consumed = im(p); consumed {
+			break
+		}
+	}
+	return p.doc, nil
+}
diff --git a/src/pkg/html/parse_test.go b/src/pkg/html/parse_test.go
new file mode 100644
index 000000000..7d918d250
--- /dev/null
+++ b/src/pkg/html/parse_test.go
@@ -0,0 +1,156 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package html
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+func pipeErr(err os.Error) io.Reader {
+	pr, pw := io.Pipe()
+	pw.CloseWithError(err)
+	return pr
+}
+
+func readDat(filename string, c chan io.Reader) {
+	f, err := os.Open("testdata/webkit/" + filename)
+	if err != nil {
+		c <- pipeErr(err)
+		return
+	}
+	defer f.Close()
+
+	// Loop through the lines of the file. Each line beginning with "#" denotes
+	// a new section, which is returned as a separate io.Reader.
+	r := bufio.NewReader(f)
+	var pw *io.PipeWriter
+	for {
+		line, err := r.ReadSlice('\n')
+		if err != nil {
+			if pw != nil {
+				pw.CloseWithError(err)
+				pw = nil
+			} else {
+				c <- pipeErr(err)
+			}
+			return
+		}
+		if len(line) == 0 {
+			continue
+		}
+		if line[0] == '#' {
+			if pw != nil {
+				pw.Close()
+			}
+			var pr *io.PipeReader
+			pr, pw = io.Pipe()
+			c <- pr
+			continue
+		}
+		if line[0] != '|' {
+			// Strip the trailing '\n'.
+			line = line[:len(line)-1]
+		}
+		if pw != nil {
+			if _, err := pw.Write(line); err != nil {
+				pw.CloseWithError(err)
+				pw = nil
+			}
+		}
+	}
+}
+
+func dumpLevel(w io.Writer, n *Node, level int) os.Error {
+	io.WriteString(w, "| ")
+	for i := 0; i < level; i++ {
+		io.WriteString(w, "  ")
+	}
+	switch n.Type {
+	case ErrorNode:
+		return os.NewError("unexpected ErrorNode")
+	case DocumentNode:
+		return os.NewError("unexpected DocumentNode")
+	case ElementNode:
+		fmt.Fprintf(w, "<%s>", EscapeString(n.Data))
+	case TextNode:
+		fmt.Fprintf(w, "%q", EscapeString(n.Data))
+	case CommentNode:
+		return os.NewError("COMMENT")
+	case DoctypeNode:
+		fmt.Fprintf(w, "", EscapeString(n.Data))
+	case scopeMarkerNode:
+		return os.NewError("unexpected scopeMarkerNode")
+	default:
+		return os.NewError("unknown node type")
+	}
+	io.WriteString(w, "\n")
+	for _, c := range n.Child {
+		if err := dumpLevel(w, c, level+1); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func dump(n *Node) (string, os.Error) {
+	if n == nil || len(n.Child) == 0 {
+		return "", nil
+	}
+	b := bytes.NewBuffer(nil)
+	for _, child := range n.Child {
+		if err := dumpLevel(b, child, 0); err != nil {
+			return "", err
+		}
+	}
+	return b.String(), nil
+}
+
+func TestParser(t *testing.T) {
+	// TODO(nigeltao): Process all the .dat files, not just the first one.
+	filenames := []string{
+		"tests1.dat",
+	}
+	for _, filename := range filenames {
+		rc := make(chan io.Reader)
+		go readDat(filename, rc)
+		// TODO(nigeltao): Process all test cases, not just a subset.
+		for i := 0; i < 25; i++ {
+			// Parse the #data section.
+			b, err := ioutil.ReadAll(<-rc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			text := string(b)
+			doc, err := Parse(strings.NewReader(text))
+			if err != nil {
+				t.Fatal(err)
+			}
+			actual, err := dump(doc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// Skip the #error section.
+			if _, err := io.Copy(ioutil.Discard, <-rc); err != nil {
+				t.Fatal(err)
+			}
+			// Compare the parsed tree to the #document section.
+			b, err = ioutil.ReadAll(<-rc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			expected := string(b)
+			if actual != expected {
+				t.Errorf("%s test #%d %q, actual vs expected:\n----\n%s----\n%s----", filename, i, text, actual, expected)
+			}
+		}
+	}
+}
diff --git a/src/pkg/html/testdata/webkit/README b/src/pkg/html/testdata/webkit/README
new file mode 100644
index 000000000..9b4c2d8be
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/README
@@ -0,0 +1,28 @@
+The *.dat files in this directory are copied from The WebKit Open Source
+Project, specifically $WEBKITROOT/LayoutTests/html5lib/resources.
+WebKit is licensed under a BSD style license.
+http://webkit.org/coding/bsd-license.html says:
+
+Copyright (C) 2009 Apple Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
diff --git a/src/pkg/html/testdata/webkit/adoption01.dat b/src/pkg/html/testdata/webkit/adoption01.dat
new file mode 100644
index 000000000..787e1b01e
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/adoption01.dat
@@ -0,0 +1,194 @@
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|     

+|       
+
+#data
+1
23

+#errors
+#document
+| 
+|   
+|   
+|     
+|       "1"
+|     

+|       
+|         "2"
+|       "3"
+
+#data
+1
+#errors
+#document
+| 
+|   
+|   
+|     
+|       "1"
+|     
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end tag (strong) in table context caused voodoo mode.
+Line: 1 Col: 20 End tag (strong) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 24 Unexpected end tag (b) in table context caused voodoo mode.
+Line: 1 Col: 24 End tag (b) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 29 Unexpected end tag (em) in table context caused voodoo mode.
+Line: 1 Col: 29 End tag (em) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 33 Unexpected end tag (i) in table context caused voodoo mode.
+Line: 1 Col: 33 End tag (i) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 37 Unexpected end tag (u) in table context caused voodoo mode.
+Line: 1 Col: 37 End tag (u) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 46 Unexpected end tag (strike) in table context caused voodoo mode.
+Line: 1 Col: 46 End tag (strike) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 50 Unexpected end tag (s) in table context caused voodoo mode.
+Line: 1 Col: 50 End tag (s) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 58 Unexpected end tag (blink) in table context caused voodoo mode.
+Line: 1 Col: 58 Unexpected end tag (blink). Ignored.
+Line: 1 Col: 63 Unexpected end tag (tt) in table context caused voodoo mode.
+Line: 1 Col: 63 End tag (tt) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 69 Unexpected end tag (pre) in table context caused voodoo mode.
+Line: 1 Col: 69 End tag (pre) seen too early. Expected other end tag.
+Line: 1 Col: 75 Unexpected end tag (big) in table context caused voodoo mode.
+Line: 1 Col: 75 End tag (big) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 83 Unexpected end tag (small) in table context caused voodoo mode.
+Line: 1 Col: 83 End tag (small) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 90 Unexpected end tag (font) in table context caused voodoo mode.
+Line: 1 Col: 90 End tag (font) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 99 Unexpected end tag (select) in table context caused voodoo mode.
+Line: 1 Col: 99 Unexpected end tag (select). Ignored.
+Line: 1 Col: 104 Unexpected end tag (h1) in table context caused voodoo mode.
+Line: 1 Col: 104 End tag (h1) seen too early. Expected other end tag.
+Line: 1 Col: 109 Unexpected end tag (h2) in table context caused voodoo mode.
+Line: 1 Col: 109 End tag (h2) seen too early. Expected other end tag.
+Line: 1 Col: 114 Unexpected end tag (h3) in table context caused voodoo mode.
+Line: 1 Col: 114 End tag (h3) seen too early. Expected other end tag.
+Line: 1 Col: 119 Unexpected end tag (h4) in table context caused voodoo mode.
+Line: 1 Col: 119 End tag (h4) seen too early. Expected other end tag.
+Line: 1 Col: 124 Unexpected end tag (h5) in table context caused voodoo mode.
+Line: 1 Col: 124 End tag (h5) seen too early. Expected other end tag.
+Line: 1 Col: 129 Unexpected end tag (h6) in table context caused voodoo mode.
+Line: 1 Col: 129 End tag (h6) seen too early. Expected other end tag.
+Line: 1 Col: 136 Unexpected end tag (body) in the table row phase. Ignored.
+Line: 1 Col: 141 Unexpected end tag (br) in table context caused voodoo mode.
+Line: 1 Col: 141 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 145 Unexpected end tag (a) in table context caused voodoo mode.
+Line: 1 Col: 145 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 151 Unexpected end tag (img) in table context caused voodoo mode.
+Line: 1 Col: 151 This element (img) has no end tag.
+Line: 1 Col: 159 Unexpected end tag (title) in table context caused voodoo mode.
+Line: 1 Col: 159 Unexpected end tag (title). Ignored.
+Line: 1 Col: 166 Unexpected end tag (span) in table context caused voodoo mode.
+Line: 1 Col: 166 Unexpected end tag (span). Ignored.
+Line: 1 Col: 174 Unexpected end tag (style) in table context caused voodoo mode.
+Line: 1 Col: 174 Unexpected end tag (style). Ignored.
+Line: 1 Col: 183 Unexpected end tag (script) in table context caused voodoo mode.
+Line: 1 Col: 183 Unexpected end tag (script). Ignored.
+Line: 1 Col: 196 Unexpected end tag (th). Ignored.
+Line: 1 Col: 201 Unexpected end tag (td). Ignored.
+Line: 1 Col: 206 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 214 This element (frame) has no end tag.
+Line: 1 Col: 221 This element (area) has no end tag.
+Line: 1 Col: 228 Unexpected end tag (link). Ignored.
+Line: 1 Col: 236 This element (param) has no end tag.
+Line: 1 Col: 241 This element (hr) has no end tag.
+Line: 1 Col: 249 This element (input) has no end tag.
+Line: 1 Col: 255 Unexpected end tag (col). Ignored.
+Line: 1 Col: 262 Unexpected end tag (base). Ignored.
+Line: 1 Col: 269 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 280 This element (basefont) has no end tag.
+Line: 1 Col: 290 This element (bgsound) has no end tag.
+Line: 1 Col: 298 This element (embed) has no end tag.
+Line: 1 Col: 307 This element (spacer) has no end tag.
+Line: 1 Col: 311 Unexpected end tag (p). Ignored.
+Line: 1 Col: 316 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 321 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 331 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 342 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 350 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 358 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 366 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 376 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 389 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 398 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 404 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 410 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 426 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 436 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 443 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 448 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 453 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 458 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 465 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 471 This element (wbr) has no end tag.
+Line: 1 Col: 487 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 497 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 506 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 524 Unexpected end tag (html). Ignored.
+Line: 1 Col: 524 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 531 Unexpected end tag (head). Ignored.
+Line: 1 Col: 540 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 548 This element (image) has no end tag.
+Line: 1 Col: 558 This element (isindex) has no end tag.
+Line: 1 Col: 568 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 590 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 601 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 610 Unexpected end tag (option). Ignored.
+Line: 1 Col: 622 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 633 Unexpected end tag (textarea). Ignored.
+#document
+| 
+|   
+|   
+|     

+|     
+|       
+|         
+|     

+
+#data
+
+#errors
+Line: 1 Col: 10 Unexpected start tag (frameset). Expected DOCTYPE.
+Line: 1 Col: 10 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
diff --git a/src/pkg/html/testdata/webkit/tests10.dat b/src/pkg/html/testdata/webkit/tests10.dat
new file mode 100644
index 000000000..4f8df86f2
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests10.dat
@@ -0,0 +1,799 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+a
+#errors
+29: Bogus comment
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+35: Stray “svg” start tag.
+42: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     
+#errors
+43: Stray “svg” start tag.
+50: Stray end tag “svg”
+#document
+| 
+| 
+|   
+|   
+|     

+#errors
+34: Start tag “svg” seen in “table”.
+41: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
foo

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+53: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|     
+
+#data
+
foobar

+#errors
+34: Start tag “svg” seen in “table”.
+46: Stray end tag “g”.
+58: Stray end tag “g”.
+65: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+
+#data
+
foobar

+#errors
+41: Start tag “svg” seen in “table”.
+53: Stray end tag “g”.
+65: Stray end tag “g”.
+72: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+
+#data
+
foobar

+#errors
+45: Start tag “svg” seen in “table”.
+57: Stray end tag “g”.
+69: Stray end tag “g”.
+76: Stray end tag “svg”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     
+|       
+|         
+
+#data
+
foobar

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
+|               
+|                 "foo"
+|               
+|                 "bar"
+|             

+|               "baz"
+
+#data
+
foobar
baz

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+
+#data
+
foobar
baz
quux
+#errors
+70: HTML start tag “p” in a foreign namespace context.
+81: “table” closed but “caption” was still open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|         

+|           "baz"
+|     

+|       "quux"
+
+#data
+
foobarbaz
quux
+#errors
+78: “table” closed but “caption” was still open.
+78: Unclosed elements on stack.
+#document
+| 
+| 
+|   
+|   
+|     
+|       

+|         
+|           
+|             "foo"
+|           
+|             "bar"
+|           "baz"
+|     

+|       "quux"
+
+#data
+foobar
baz
quux
+#errors
+44: Start tag “svg” seen in “table”.
+56: Stray end tag “g”.
+68: Stray end tag “g”.
+71: HTML start tag “p” in a foreign namespace context.
+71: Start tag “p” seen in “table”.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+|     
+|       +|     

+|       "quux"
+
+#data
+


quux
+#errors
+50: Stray “svg” start tag.
+54: Stray “g” start tag.
+62: Stray end tag “g”
+66: Stray “g” start tag.
+74: Stray end tag “g”
+77: Stray “p” start tag.
+88: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|           

+|             
quux
+#errors
+36: Start tag “select” seen in “table”.
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+54: Stray end tag “g”
+58: Stray “g” start tag.
+66: Stray end tag “g”
+69: Stray “p” start tag.
+80: “table” end tag with “select” open.
+#document
+| 
+| 
+|   
+|   
+|     
+|     

+|       "quux"
+
+#data
+foobar
baz
+#errors
+41: Stray “svg” start tag.
+68: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+foobar
baz
+#errors
+34: Stray “svg” start tag.
+61: HTML start tag “p” in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         "foo"
+|       
+|         "bar"
+|     

+|       "baz"
+
+#data
+

+#errors
+31: Stray “svg” start tag.
+35: Stray “g” start tag.
+40: Stray end tag “g”
+44: Stray “g” start tag.
+49: Stray end tag “g”
+52: Stray “p” start tag.
+58: Stray “span” start tag.
+58: End of file seen and there were open elements.
+#document
+| 
+| 
+|   
+|   
+
+#data
+

+#errors
+42: Stray “svg” start tag.
+46: Stray “g” start tag.
+51: Stray end tag “g”
+55: Stray “g” start tag.
+60: Stray end tag “g”
+63: Stray “p” start tag.
+69: Stray “span” start tag.
+#document
+| 
+| 
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     
+|       xlink href="foo"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+
+#data
+bar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     xlink:href="foo"
+|     xml:lang="en"
+|     
+|       
+|         xlink href="foo"
+|         xml lang="en"
+|       "bar"
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|     "a"
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|     "a"
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|       
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|             
+|               "a"
+
+#data
+
a
+#errors
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|             

+|               "a"
+
+#data
+
    a
+#errors
+40: HTML start tag “ul” in a foreign namespace context.
+41: End of file in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
    
+|           
+|           
      
+|             "a"
+
+#data
+
        a
+#errors
+35: HTML start tag “ul” in a foreign namespace context.
+36: End of file in a foreign namespace context.
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|         
+|         
          
+|           "a"
+
+#data
+
          
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
          
+
+#data
+
          
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
          
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
          
+|       
+|         
+|           
+|             
          
+|             
          
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
          
+|           
+|             
          
+|               
+|       
+
+#data
+
          
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             
          
+|               
          
+|       
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|       
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|     

+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|       
+
+#data
+

+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             

+|               
+|                 
+|               
+|           
+|       
+
+#data
+
+#errors
+#document
+| 
+|   
+|   
+|     
+|       
+|         
+|           
+|             
+|               
+|                 
+|               
+|             
+|           
+|       
diff --git a/src/pkg/html/testdata/webkit/tests11.dat b/src/pkg/html/testdata/webkit/tests11.dat
new file mode 100644
index 000000000..638cde479
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests11.dat
@@ -0,0 +1,482 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributeName=""
+|       attributeType=""
+|       baseFrequency=""
+|       baseProfile=""
+|       calcMode=""
+|       clipPathUnits=""
+|       contentScriptType=""
+|       contentStyleType=""
+|       diffuseConstant=""
+|       edgeMode=""
+|       externalResourcesRequired=""
+|       filterRes=""
+|       filterUnits=""
+|       glyphRef=""
+|       gradientTransform=""
+|       gradientUnits=""
+|       kernelMatrix=""
+|       kernelUnitLength=""
+|       keyPoints=""
+|       keySplines=""
+|       keyTimes=""
+|       lengthAdjust=""
+|       limitingConeAngle=""
+|       markerHeight=""
+|       markerUnits=""
+|       markerWidth=""
+|       maskContentUnits=""
+|       maskUnits=""
+|       numOctaves=""
+|       pathLength=""
+|       patternContentUnits=""
+|       patternTransform=""
+|       patternUnits=""
+|       pointsAtX=""
+|       pointsAtY=""
+|       pointsAtZ=""
+|       preserveAlpha=""
+|       preserveAspectRatio=""
+|       primitiveUnits=""
+|       refX=""
+|       refY=""
+|       repeatCount=""
+|       repeatDur=""
+|       requiredExtensions=""
+|       requiredFeatures=""
+|       specularConstant=""
+|       specularExponent=""
+|       spreadMethod=""
+|       startOffset=""
+|       stdDeviation=""
+|       stitchTiles=""
+|       surfaceScale=""
+|       systemLanguage=""
+|       tableValues=""
+|       targetX=""
+|       targetY=""
+|       textLength=""
+|       viewBox=""
+|       viewTarget=""
+|       xChannelSelector=""
+|       yChannelSelector=""
+|       zoomAndPan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       attributename=""
+|       attributetype=""
+|       basefrequency=""
+|       baseprofile=""
+|       calcmode=""
+|       clippathunits=""
+|       contentscripttype=""
+|       contentstyletype=""
+|       diffuseconstant=""
+|       edgemode=""
+|       externalresourcesrequired=""
+|       filterres=""
+|       filterunits=""
+|       glyphref=""
+|       gradienttransform=""
+|       gradientunits=""
+|       kernelmatrix=""
+|       kernelunitlength=""
+|       keypoints=""
+|       keysplines=""
+|       keytimes=""
+|       lengthadjust=""
+|       limitingconeangle=""
+|       markerheight=""
+|       markerunits=""
+|       markerwidth=""
+|       maskcontentunits=""
+|       maskunits=""
+|       numoctaves=""
+|       pathlength=""
+|       patterncontentunits=""
+|       patterntransform=""
+|       patternunits=""
+|       pointsatx=""
+|       pointsaty=""
+|       pointsatz=""
+|       preservealpha=""
+|       preserveaspectratio=""
+|       primitiveunits=""
+|       refx=""
+|       refy=""
+|       repeatcount=""
+|       repeatdur=""
+|       requiredextensions=""
+|       requiredfeatures=""
+|       specularconstant=""
+|       specularexponent=""
+|       spreadmethod=""
+|       startoffset=""
+|       stddeviation=""
+|       stitchtiles=""
+|       surfacescale=""
+|       systemlanguage=""
+|       tablevalues=""
+|       targetx=""
+|       targety=""
+|       textlength=""
+|       viewbox=""
+|       viewtarget=""
+|       xchannelselector=""
+|       ychannelselector=""
+|       zoomandpan=""
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+|       
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       
diff --git a/src/pkg/html/testdata/webkit/tests12.dat b/src/pkg/html/testdata/webkit/tests12.dat
new file mode 100644
index 000000000..63107d277
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests12.dat
@@ -0,0 +1,62 @@
+#data
+
foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     

+|       "foo"
+|       
+|         
+|           
+|             "baz"
+|         
+|           
+|             
+|               
+|                 "eggs"
+|             
+|               
+|                 

+|                   "spam"
+|                 
+|                   
+|                     
+|                       

+|                         
+|             
+|               "quux"
+|       "bar"
+
+#data
+foobazeggs
spam
quuxbar
+#errors
+#document
+| 
+| 
+|   
+|   
+|     "foo"
+|     
+|       
+|         
+|           "baz"
+|       
+|         
+|           
+|             
+|               "eggs"
+|           
+|             
+|               

+|                 "spam"
+|               
+|                 
+|                   
+|                     

+|                       
+|           
+|             "quux"
+|     "bar"
diff --git a/src/pkg/html/testdata/webkit/tests14.dat b/src/pkg/html/testdata/webkit/tests14.dat
new file mode 100644
index 000000000..b8713f885
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests14.dat
@@ -0,0 +1,74 @@
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   abc:def="gh"
+|   
+|   
+|     
+
+#data
+
+#errors
+15: Unexpected start tag html
+#document
+| 
+| 
+|   xml:lang="bar"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   123="456"
+|   789="012"
+|   
+|   
+
+#data
+
+#errors
+#document
+| 
+| 
+|   
+|   
+|     789="012"
diff --git a/src/pkg/html/testdata/webkit/tests15.dat b/src/pkg/html/testdata/webkit/tests15.dat
new file mode 100644
index 000000000..6ce1c0d16
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests15.dat
@@ -0,0 +1,208 @@
+#data
+
 
X
+#errors
+Line: 1 Col: 31 Unexpected end tag (p). Ignored.
+Line: 1 Col: 36 Expected closing tag. Unexpected end of file.
+#document
+| 
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           " "
+|           

+|             "X"
+
+#data
+

+
X
+#errors
+Line: 1 Col: 3 Unexpected start tag (p). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end tag (p). Ignored.
+Line: 2 Col: 4 Expected closing tag. Unexpected end of file.
+#document
+| 
+|   
+|   
+|     

+|       
+|         
+|           
+|     
+|       
+|         
+|           "
+"
+|           

+|             "X"
+
+#data
+ 
+#errors
+Line: 1 Col: 22 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     " "
+
+#data
+
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+
+#data
+
+#errors
+Line: 1 Col: 6 Unexpected start tag (html). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end tag (html) after the (implied) root element.
+#document
+| 
+|   
+|   
+| 
+
+#data
+X
+#errors
+Line: 1 Col: 22 Unexpected end tag (body) after the (implied) root element.
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+
+#data
+<!doctype html><table> X<meta></table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected start tag (meta) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " X"
+|     <meta>
+|     <table>
+
+#data
+<!doctype html><table> x</table>
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+
+#data
+<!doctype html><table> x </table>
+#errors
+Line: 1 Col: 25 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x "
+|     <table>
+
+#data
+<!doctype html><table><tr> x</table>
+#errors
+Line: 1 Col: 28 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " x"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table>X<style> <tr>x </style> </table>
+#errors
+Line: 1 Col: 23 Unexpected non-space characters in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+|     <table>
+|       <style>
+|         " <tr>x "
+|       " "
+
+#data
+<!doctype html><div><table><a>foo</a> <tr><td>bar</td> </tr></table></div>
+#errors
+Line: 1 Col: 30 Unexpected start tag (a) in table context caused voodoo mode.
+Line: 1 Col: 37 Unexpected end tag (a) in table context caused voodoo mode.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|       <a>
+|         "foo"
+|       <table>
+|         " "
+|         <tbody>
+|           <tr>
+|             <td>
+|               "bar"
+|             " "
+
+#data
+<frame></frame></frame><frameset><frame><frameset><frame></frameset><noframes></frameset><noframes>
+#errors
+6: Start tag seen without seeing a doctype first. Expected “<!DOCTYPE html>”.
+13: Stray start tag “frame”.
+21: Stray end tag “frame”.
+29: Stray end tag “frame”.
+39: “frameset” start tag after “body” already open.
+105: End of file seen inside an [R]CDATA element.
+105: End of file seen and there were open elements.
+XXX: These errors are wrong, please fix me!
+#document
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+|     <frameset>
+|       <frame>
+|     <noframes>
+|       "</frameset><noframes>"
+
+#data
+<!DOCTYPE html><object></html>
+#errors
+1: Expected closing tag. Unexpected end of file
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
diff --git a/src/pkg/html/testdata/webkit/tests16.dat b/src/pkg/html/testdata/webkit/tests16.dat
new file mode 100644
index 000000000..937dba9f4
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests16.dat
@@ -0,0 +1,2277 @@
+#data
+<!doctype html><script>
+#errors
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script>a
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<!doctype html><script><
+#errors
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<!doctype html><script></
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<!doctype html><script></S
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<!doctype html><script></SC
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<!doctype html><script></SCR
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<!doctype html><script></SCRI
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<!doctype html><script></SCRIP
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<!doctype html><script></SCRIPT 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script></s
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<!doctype html><script></sc
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<!doctype html><script></scr
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<!doctype html><script></scri
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<!doctype html><script></scrip
+#errors
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<!doctype html><script></script
+#errors
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<!doctype html><script></script 
+#errors
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<!doctype html><script><!
+#errors
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<!doctype html><script><!a
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<!doctype html><script><!-
+#errors
+Line: 1 Col: 26 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<!doctype html><script><!-a
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<!doctype html><script><!--
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--a
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<!doctype html><script><!--<
+#errors
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<!doctype html><script><!--<a
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<!doctype html><script><!--</
+#errors
+Line: 1 Col: 27 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<!doctype html><script><!--</script
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<!doctype html><script><!--</script 
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<s
+#errors
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script
+#errors
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script 
+#errors
+Line: 1 Col: 35 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script <
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script <a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </s
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script
+#errors
+Line: 1 Col: 43 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </scripta
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script 
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script>
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script/
+#errors
+Line: 1 Col: 44 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <
+#errors
+Line: 1 Col: 45 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script <a
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script 
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script/
+#errors
+Line: 1 Col: 53 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script </script </script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<!doctype html><script><!--<script -
+#errors
+Line: 1 Col: 36 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -a
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -<
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --
+#errors
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --a
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --<
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --<"
+|   <body>
+
+#data
+<!doctype html><script><!--<script -->
+#errors
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --><
+#errors
+Line: 1 Col: 39 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></
+#errors
+Line: 1 Col: 40 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script
+#errors
+Line: 1 Col: 46 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script 
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script/
+#errors
+Line: 1 Col: 47 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script --></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script><\/script>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt>--></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>--><!--</script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-- ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- -></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>- - ></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<!doctype html><script><!--<script></script><script></script>-></script>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<!doctype html><script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 49 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<!doctype html><script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 59 Unexpected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 57 Unexpected end of file. Expected end tag (script).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<!doctype html><style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 52 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<!doctype html><style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 66 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><style><!--...</style><!-- --><style>@import ...</style>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<!doctype html><style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 63 Unexpected end tag (style).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<!doctype html><style>...<!--[if IE]><style>...</style>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<!doctype html><title><!--<title>-->
+#errors
+Line: 1 Col: 52 Unexpected end tag (title).
+#document
+| 
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><title></title>
+#errors
+#document
+| 
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 52 Unexpected end of file. Expected end tag (title).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<!doctype html><noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noscript).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<!doctype html><noscript><iframe></noscript>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<!doctype html><noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 64 Unexpected end tag (noframes).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<!doctype html><noframes><body><script><!--...</script></body></noframes></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<!doctype html><textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 64 Unexpected end tag (textarea).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<!doctype html><textarea></textarea></textarea>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<!doctype html><iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 56 Unexpected end tag (iframe).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<!doctype html><iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<!doctype html><xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 44 Unexpected end tag (xmp).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<!doctype html><noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 60 Unexpected end tag (noembed).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script>a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "a"
+|   <body>
+
+#data
+<script><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<"
+|   <body>
+
+#data
+<script></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</"
+|   <body>
+
+#data
+<script></S
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</S"
+|   <body>
+
+#data
+<script></SC
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SC"
+|   <body>
+
+#data
+<script></SCR
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCR"
+|   <body>
+
+#data
+<script></SCRI
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRI"
+|   <body>
+
+#data
+<script></SCRIP
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIP"
+|   <body>
+
+#data
+<script></SCRIPT
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</SCRIPT"
+|   <body>
+
+#data
+<script></SCRIPT 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script></s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</s"
+|   <body>
+
+#data
+<script></sc
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</sc"
+|   <body>
+
+#data
+<script></scr
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scr"
+|   <body>
+
+#data
+<script></scri
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scri"
+|   <body>
+
+#data
+<script></scrip
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 15 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</scrip"
+|   <body>
+
+#data
+<script></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 16 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</script"
+|   <body>
+
+#data
+<script></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 17 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|   <body>
+
+#data
+<script><!
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 10 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!"
+|   <body>
+
+#data
+<script><!a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!a"
+|   <body>
+
+#data
+<script><!-
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-"
+|   <body>
+
+#data
+<script><!-a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!-a"
+|   <body>
+
+#data
+<script><!--
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 12 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--a"
+|   <body>
+
+#data
+<script><!--<
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 13 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<"
+|   <body>
+
+#data
+<script><!--<a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<a"
+|   <body>
+
+#data
+<script><!--</
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</"
+|   <body>
+
+#data
+<script><!--</script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--</script"
+|   <body>
+
+#data
+<script><!--</script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--"
+|   <body>
+
+#data
+<script><!--<s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 14 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<s"
+|   <body>
+
+#data
+<script><!--<script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 19 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script"
+|   <body>
+
+#data
+<script><!--<script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 20 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script "
+|   <body>
+
+#data
+<script><!--<script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <"
+|   <body>
+
+#data
+<script><!--<script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script <a"
+|   <body>
+
+#data
+<script><!--<script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </"
+|   <body>
+
+#data
+<script><!--<script </s
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </s"
+|   <body>
+
+#data
+<script><!--<script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script"
+|   <body>
+
+#data
+<script><!--<script </scripta
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </scripta"
+|   <body>
+
+#data
+<script><!--<script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script>"
+|   <body>
+
+#data
+<script><!--<script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script/"
+|   <body>
+
+#data
+<script><!--<script </script <
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 30 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <"
+|   <body>
+
+#data
+<script><!--<script </script <a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script <a"
+|   <body>
+
+#data
+<script><!--<script </script </
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </"
+|   <body>
+
+#data
+<script><!--<script </script </script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script </script"
+|   <body>
+
+#data
+<script><!--<script </script </script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 38 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script </script </script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script </script "
+|   <body>
+
+#data
+<script><!--<script -
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -"
+|   <body>
+
+#data
+<script><!--<script -a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -a"
+|   <body>
+
+#data
+<script><!--<script --
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --"
+|   <body>
+
+#data
+<script><!--<script --a
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --a"
+|   <body>
+
+#data
+<script><!--<script -->
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --><
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 24 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --><"
+|   <body>
+
+#data
+<script><!--<script --></
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 25 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></"
+|   <body>
+
+#data
+<script><!--<script --></script
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 31 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script --></script"
+|   <body>
+
+#data
+<script><!--<script --></script 
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script/
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 32 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script --></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script -->"
+|   <body>
+
+#data
+<script><!--<script><\/script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script><\/script>-->"
+|   <body>
+
+#data
+<script><!--<script></scr'+'ipt>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt>-->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>--><!--</script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>--><!--"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-- ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>-- >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- -></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- ->"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>- - ></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>- - >"
+|   <body>
+
+#data
+<script><!--<script></script><script></script>-></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></script><script></script>->"
+|   <body>
+
+#data
+<script><!--<script>--!></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 34 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script>--!></script>X"
+|   <body>
+
+#data
+<script><!--<scr'+'ipt></script>--></script>
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 44 Unexpected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<scr'+'ipt>"
+|   <body>
+|     "-->"
+
+#data
+<script><!--<script></scr'+'ipt></script>X
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 42 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "<!--<script></scr'+'ipt></script>X"
+|   <body>
+
+#data
+<style><!--<style></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<style>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--"
+|   <body>
+|     "X"
+
+#data
+<style><!--...</style>...--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 36 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|   <body>
+|     "...-->"
+
+#data
+<style><!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style></style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--<br><html xmlns:v="urn:schemas-microsoft-com:vml"><!--[if !mso]><style>"
+|   <body>
+|     "X"
+
+#data
+<style><!--...<style><!--...--!></style>--></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 51 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--...<style><!--...--!>"
+|   <body>
+|     "-->"
+
+#data
+<style><!--...</style><!-- --><style>@import ...</style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "<!--..."
+|     <!--   -->
+|     <style>
+|       "@import ..."
+|   <body>
+
+#data
+<style>...<style><!--...</style><!-- --></style>
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 48 Unexpected end tag (style).
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<style><!--..."
+|     <!--   -->
+|   <body>
+
+#data
+<style>...<!--[if IE]><style>...</style>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <style>
+|       "...<!--[if IE]><style>..."
+|   <body>
+|     "X"
+
+#data
+<title><!--<title>-->
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end tag (title).
+#document
+| 
+|   
+|     
+|       "<!--<title>"
+|   <body>
+|     "-->"
+
+#data
+<title></title>
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+#document
+| 
+|   
+|     
+|       ""
+|   
+
+#data
+foo/title><link></head><body>X
+#errors
+Line: 1 Col: 7 Unexpected start tag (title). Expected DOCTYPE.
+Line: 1 Col: 37 Unexpected end of file. Expected end tag (title).
+#document
+| <html>
+|   <head>
+|     <title>
+|       "foo/title><link></head><body>X"
+|   <body>
+
+#data
+<noscript><!--<noscript></noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noscript).
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--<noscript>"
+|   <body>
+|     "-->"
+
+#data
+<noscript><!--</noscript>X<noscript>--></noscript>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<!--"
+|   <body>
+|     "X"
+|     <noscript>
+|       "-->"
+
+#data
+<noscript><iframe></noscript>X
+#errors
+Line: 1 Col: 10 Unexpected start tag (noscript). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noscript>
+|       "<iframe>"
+|   <body>
+|     "X"
+
+#data
+<noframes><!--<noframes></noframes>--></noframes>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (noframes).
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<!--<noframes>"
+|   <body>
+|     "-->"
+
+#data
+<noframes><body><script><!--...</script></body></noframes></html>
+#errors
+Line: 1 Col: 10 Unexpected start tag (noframes). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|     <noframes>
+|       "<body><script><!--...</script></body>"
+|   <body>
+
+#data
+<textarea><!--<textarea></textarea>--></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 49 Unexpected end tag (textarea).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "<!--<textarea>"
+|     "-->"
+
+#data
+<textarea></textarea></textarea>
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "</textarea>"
+
+#data
+<iframe><!--<iframe></iframe>--></iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+Line: 1 Col: 41 Unexpected end tag (iframe).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "<!--<iframe>"
+|     "-->"
+
+#data
+<iframe>...<!--X->...<!--/X->...</iframe>
+#errors
+Line: 1 Col: 8 Unexpected start tag (iframe). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+|       "...<!--X->...<!--/X->..."
+
+#data
+<xmp><!--<xmp></xmp>--></xmp>
+#errors
+Line: 1 Col: 5 Unexpected start tag (xmp). Expected DOCTYPE.
+Line: 1 Col: 29 Unexpected end tag (xmp).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+|       "<!--<xmp>"
+|     "-->"
+
+#data
+<noembed><!--<noembed></noembed>--></noembed>
+#errors
+Line: 1 Col: 9 Unexpected start tag (noembed). Expected DOCTYPE.
+Line: 1 Col: 45 Unexpected end tag (noembed).
+#document
+| <html>
+|   <head>
+|   <body>
+|     <noembed>
+|       "<!--<noembed>"
+|     "-->"
+
+#data
+<!doctype html><table>
+
+#errors
+Line 2 Col 0 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "
+"
+
+#data
+<!doctype html><table><td><span><font></span><span>
+#errors
+Line 1 Col 26 Unexpected table cell start tag (td) in the table body phase.
+Line 1 Col 45 Unexpected end tag (span).
+Line 1 Col 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <span>
+|               <font>
+|             <font>
+|               <span>
+
+#data
+<!doctype html><form><table></form><form></table></form>
+#errors
+35: Stray end tag “form”.
+41: Start tag “form” seen in “table”.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <table>
+|         <form>
diff --git a/src/pkg/html/testdata/webkit/tests17.dat b/src/pkg/html/testdata/webkit/tests17.dat
new file mode 100644
index 000000000..7b555f888
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests17.dat
@@ -0,0 +1,153 @@
+#data
+<!doctype html><table><tbody><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><tr><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<!doctype html><table><tr><td><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <select>
+|           <td>
+
+#data
+<!doctype html><table><tr><th><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <th>
+|             <select>
+|           <td>
+
+#data
+<!doctype html><table><caption><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <select>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><select><tr>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><th>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><tbody>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><thead>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><tfoot>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><select><caption>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><table><tr></table>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|     "a"
diff --git a/src/pkg/html/testdata/webkit/tests18.dat b/src/pkg/html/testdata/webkit/tests18.dat
new file mode 100644
index 000000000..680e1f068
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests18.dat
@@ -0,0 +1,269 @@
+#data
+<!doctype html><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!doctype html><table><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+
+#data
+<!doctype html><table><tbody><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+
+#data
+<!doctype html><table><tbody><tr><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><tbody><tr><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><td><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <plaintext>
+|               "</plaintext>"
+
+#data
+<!doctype html><table><caption><plaintext></plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <plaintext>
+|           "</plaintext>"
+
+#data
+<!doctype html><table><tr><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+|           <style>
+|             "</script>"
+
+#data
+<!doctype html><table><tr><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+|           <script>
+|             "</style>"
+
+#data
+<!doctype html><table><caption><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         <style>
+|           "</script>"
+|         "abc"
+
+#data
+<!doctype html><table><td><style></script></style>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <style>
+|               "</script>"
+|             "abc"
+
+#data
+<!doctype html><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+
+#data
+<!doctype html><table><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+|     <table>
+
+#data
+<!doctype html><table><tr><select><script></style></script>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <script>
+|         "</style>"
+|       "abc"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><frameset></frameset><noframes>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+
+#data
+<!doctype html><frameset></frameset><noframes>abc</noframes><!--abc-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+|   <!-- abc -->
+
+#data
+<!doctype html><frameset></frameset></html><noframes>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+
+#data
+<!doctype html><frameset></frameset></html><noframes>abc</noframes><!--abc-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   <noframes>
+|     "abc"
+| <!-- abc -->
+
+#data
+<!doctype html><table><tr></tbody><tfoot>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|       <tfoot>
+
+#data
+<!doctype html><table><td><svg></svg>abc<td>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <svg svg>
+|             "abc"
+|           <td>
diff --git a/src/pkg/html/testdata/webkit/tests19.dat b/src/pkg/html/testdata/webkit/tests19.dat
new file mode 100644
index 000000000..06222f5b9
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests19.dat
@@ -0,0 +1,1220 @@
+#data
+<!doctype html><math><mn DefinitionUrl="foo">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+|       <math mn>
+|         definitionURL="foo"
+
+#data
+<!doctype html><html></p><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <!-- foo -->
+|   <head>
+|   <body>
+
+#data
+<!doctype html><head></head></p><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <!-- foo -->
+|   <body>
+
+#data
+<!doctype html><body><p><pre>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <pre>
+
+#data
+<!doctype html><body><p><listing>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <listing>
+
+#data
+<!doctype html><p><plaintext>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <plaintext>
+
+#data
+<!doctype html><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <h1>
+
+#data
+<!doctype html><form><isindex>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+
+#data
+<!doctype html><isindex action="POST">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       action="POST"
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><isindex prompt="this is isindex">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "this is isindex"
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><isindex type="hidden">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|           type="hidden"
+|       <hr>
+
+#data
+<!doctype html><isindex name="foo">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|       <hr>
+
+#data
+<!doctype html><ruby><p><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <p>
+|       <rp>
+
+#data
+<!doctype html><ruby><div><span><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <span>
+|       <rp>
+
+#data
+<!doctype html><ruby><div><p><rp>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <p>
+|       <rp>
+
+#data
+<!doctype html><ruby><p><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <p>
+|       <rt>
+
+#data
+<!doctype html><ruby><div><span><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <span>
+|       <rt>
+
+#data
+<!doctype html><ruby><div><p><rt>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <ruby>
+|       <div>
+|         <p>
+|       <rt>
+
+#data
+<!doctype html><math/><foo>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+|     <foo>
+
+#data
+<!doctype html><svg/><foo>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <svg svg>
+|     <foo>
+
+#data
+<!doctype html><div></body><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+|   <!-- foo -->
+
+#data
+<!doctype html><h1><div><h3><span></h1>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <h1>
+|       <div>
+|         <h3>
+|           <span>
+|         "foo"
+
+#data
+<!doctype html><p></h3>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       "foo"
+
+#data
+<!doctype html><h3><li>abc</h2>foo
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <h3>
+|       <li>
+|         "abc"
+|     "foo"
+
+#data
+<!doctype html><table>abc<!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "abc"
+|     <table>
+|       <!-- foo -->
+
+#data
+<!doctype html><table>  <!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       "  "
+|       <!-- foo -->
+
+#data
+<!doctype html><table> b <!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     " b "
+|     <table>
+|       <!-- foo -->
+
+#data
+<!doctype html><select><option><option>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <option>
+
+#data
+<!doctype html><select><option></optgroup>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+
+#data
+<!doctype html><select><option></optgroup>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+
+#data
+<!doctype html><p><math><mi><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mi>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mo><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mo>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mn><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mn>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><ms><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math ms>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><p><math><mtext><p><h1>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mtext>
+|           <p>
+|           <h1>
+
+#data
+<!doctype html><frameset></noframes>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html c=d><body></html><html a=b>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <body>
+
+#data
+<!doctype html><html c=d><frameset></frameset></html><html a=b>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html><!--foo-->
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+| <!-- foo -->
+
+#data
+<!doctype html><html><frameset></frameset></html>  
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|   "  "
+
+#data
+<!doctype html><html><frameset></frameset></html>abc
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html><p>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><html><frameset></frameset></html></p>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<html><frameset></frameset></html><!doctype html>
+#errors
+#document
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><body><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!doctype html><p><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><p>a<frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       "a"
+
+#data
+<!doctype html><p> <frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><pre><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <pre>
+
+#data
+<!doctype html><listing><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <listing>
+
+#data
+<!doctype html><li><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <li>
+
+#data
+<!doctype html><dd><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dd>
+
+#data
+<!doctype html><dt><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+
+#data
+<!doctype html><button><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <button>
+
+#data
+<!doctype html><applet><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <applet>
+
+#data
+<!doctype html><marquee><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <marquee>
+
+#data
+<!doctype html><object><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <object>
+
+#data
+<!doctype html><table><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+
+#data
+<!doctype html><area><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <area>
+
+#data
+<!doctype html><basefont><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <basefont>
+|   <frameset>
+
+#data
+<!doctype html><bgsound><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <bgsound>
+|   <frameset>
+
+#data
+<!doctype html><br><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <br>
+
+#data
+<!doctype html><embed><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <embed>
+
+#data
+<!doctype html><img><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <img>
+
+#data
+<!doctype html><input><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <input>
+
+#data
+<!doctype html><keygen><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <keygen>
+
+#data
+<!doctype html><wbr><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <wbr>
+
+#data
+<!doctype html><hr><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <hr>
+
+#data
+<!doctype html><textarea></textarea><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+
+#data
+<!doctype html><xmp></xmp><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <xmp>
+
+#data
+<!doctype html><iframe></iframe><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <iframe>
+
+#data
+<!doctype html><select></select><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+
+#data
+<!doctype html><svg></svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><math></math><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><svg><foreignObject><div> <frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<!doctype html><svg>a</svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <svg svg>
+|       "a"
+
+#data
+<!doctype html><svg> </svg><frameset><frame>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+|     <frame>
+
+#data
+<html>aaa<frameset></frameset>
+#errors
+#document
+| <html>
+|   <head>
+|   <body>
+|     "aaa"
+
+#data
+<html> a <frameset></frameset>
+#errors
+#document
+| <html>
+|   <head>
+|   <body>
+|     "a "
+
+#data
+<!doctype html><div><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!doctype html><div><body><frameset>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <div>
+
+#data
+<!doctype html><p><math></p>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|     "a"
+
+#data
+<!doctype html><p><math><mn><span></p>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <math math>
+|         <math mn>
+|           <span>
+|             <p>
+|             "a"
+
+#data
+<!doctype html><math></html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <math math>
+
+#data
+<!doctype html><meta charset="ascii">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <meta>
+|       charset="ascii"
+|   <body>
+
+#data
+<!doctype html><meta http-equiv="content-type" content="text/html;charset=ascii">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <meta>
+|       content="text/html;charset=ascii"
+|       http-equiv="content-type"
+|   <body>
+
+#data
+<!doctype html><head><!--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--><meta charset="utf8">
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <!-- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -->
+|     <meta>
+|       charset="utf8"
+|   <body>
+
+#data
+<!doctype html><html a=b><head></head><html c=d>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   a="b"
+|   c="d"
+|   <head>
+|   <body>
+
+#data
+<!doctype html><image/>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <img>
+
+#data
+<!doctype html>a<i>b<table>c<b>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "a"
+|     <i>
+|       "bc"
+|       <b>
+|         "de"
+|       "f"
+|       <table>
+
+#data
+<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+|     <table>
+
+#data
+<!doctype html><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+
+#data
+<!doctype html><table><i>a<b>b<div>c</i>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|       <div>
+|         <i>
+|           "c"
+|     <table>
+
+#data
+<!doctype html><table><i>a<b>b<div>c<a>d</i>e</b>f
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <b>
+|         "b"
+|     <b>
+|     <div>
+|       <b>
+|         <i>
+|           "c"
+|           <a>
+|             "d"
+|         <a>
+|           "e"
+|       <a>
+|         "f"
+|     <table>
+
+#data
+<!doctype html><table><i>a<div>b<tr>c<b>d</i>e
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <i>
+|       "a"
+|       <div>
+|         "b"
+|     <i>
+|       "c"
+|       <b>
+|         "d"
+|     <b>
+|       "e"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!doctype html><table><td><table><i>a<div>b<b>c</i>d
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             <i>
+|               "a"
+|             <div>
+|               <i>
+|                 "b"
+|                 <b>
+|                   "c"
+|               <b>
+|                 "d"
+|             <table>
+
+#data
+<!doctype html><body><bgsound>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <bgsound>
+
+#data
+<!doctype html><body><basefont>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <basefont>
+
+#data
+<!doctype html><a><b></a><basefont>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <a>
+|       <b>
+|     <basefont>
+
+#data
+<!doctype html><a><b></a><bgsound>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <a>
+|       <b>
+|     <bgsound>
+
+#data
+<!doctype html><figcaption><article></figcaption>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <figcaption>
+|       <article>
+|     "a"
+
+#data
+<!doctype html><summary><article></summary>a
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <summary>
+|       <article>
+|     "a"
+
+#data
+<!doctype html><p><a><plaintext>b
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|       <a>
+|     <plaintext>
+|       <a>
+|         "b"
diff --git a/src/pkg/html/testdata/webkit/tests2.dat b/src/pkg/html/testdata/webkit/tests2.dat
new file mode 100644
index 000000000..60d859221
--- /dev/null
+++ b/src/pkg/html/testdata/webkit/tests2.dat
@@ -0,0 +1,763 @@
+#data
+<!DOCTYPE html>Test
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "Test"
+
+#data
+<textarea>test</div>test
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+Line: 1 Col: 24 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <textarea>
+|       "test</div>test"
+
+#data
+<table><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 11 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+
+#data
+<table><td>test</tbody></table>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected table cell start tag (td) in the table body phase.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<frame>test
+#errors
+Line: 1 Col: 7 Unexpected start tag (frame). Expected DOCTYPE.
+Line: 1 Col: 7 Unexpected start tag frame. Ignored.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test"
+
+#data
+<!DOCTYPE html><frameset>test
+#errors
+Line: 1 Col: 29 Unepxected characters in the frameset phase. Characters ignored.
+Line: 1 Col: 29 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><frameset><!DOCTYPE html>
+#errors
+Line: 1 Col: 40 Unexpected DOCTYPE. Ignored.
+Line: 1 Col: 40 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <frameset>
+
+#data
+<!DOCTYPE html><font><p><b>test</font>
+#errors
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+Line: 1 Col: 38 End tag (font) violates step 1, paragraph 3 of the adoption agency algorithm.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|     <p>
+|       <font>
+|         <b>
+|           "test"
+
+#data
+<!DOCTYPE html><dt><div><dd>
+#errors
+Line: 1 Col: 28 Missing end tag (div, dt).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <dt>
+|       <div>
+|     <dd>
+
+#data
+<script></x
+#errors
+Line: 1 Col: 8 Unexpected start tag (script). Expected DOCTYPE.
+Line: 1 Col: 11 Unexpected end of file. Expected end tag (script).
+#document
+| <html>
+|   <head>
+|     <script>
+|       "</x"
+|   <body>
+
+#data
+<table><plaintext><td>
+#errors
+Line: 1 Col: 7 Unexpected start tag (table). Expected DOCTYPE.
+Line: 1 Col: 18 Unexpected start tag (plaintext) in table context caused voodoo mode.
+Line: 1 Col: 22 Unexpected end of file. Expected table content.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "<td>"
+|     <table>
+
+#data
+<plaintext></plaintext>
+#errors
+Line: 1 Col: 11 Unexpected start tag (plaintext). Expected DOCTYPE.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <plaintext>
+|       "</plaintext>"
+
+#data
+<!DOCTYPE html><table><tr>TEST
+#errors
+Line: 1 Col: 30 Unexpected non-space characters in table context caused voodoo mode.
+Line: 1 Col: 30 Unexpected end of file. Expected table content.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "TEST"
+|     <table>
+|       <tbody>
+|         <tr>
+
+#data
+<!DOCTYPE html><body t1=1><body t2=2><body t3=3 t4=4>
+#errors
+Line: 1 Col: 37 Unexpected start tag (body).
+Line: 1 Col: 53 Unexpected start tag (body).
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     t1="1"
+|     t2="2"
+|     t3="3"
+|     t4="4"
+
+#data
+</b test
+#errors
+Line: 1 Col: 8 Unexpected end of file in attribute name.
+Line: 1 Col: 8 End tag contains unexpected attributes.
+Line: 1 Col: 8 Unexpected end tag (b). Expected DOCTYPE.
+Line: 1 Col: 8 Unexpected end tag (b) after the (implied) root element.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html></b test<b &=&>X
+#errors
+Line: 1 Col: 32 Named entity didn't end with ';'.
+Line: 1 Col: 33 End tag contains unexpected attributes.
+Line: 1 Col: 33 Unexpected end tag (b) after the (implied) root element.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "X"
+
+#data
+<!doctypehtml><scrIPt type=text/x-foobar;baz>X</SCRipt
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 54 Unexpected end of file in the tag name.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|     <script>
+|       type="text/x-foobar;baz"
+|       "X</SCRipt"
+|   <body>
+
+#data
+&
+#errors
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&#
+#errors
+Line: 1 Col: 1 Numeric entity expected. Got end of file instead.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#"
+
+#data
+&#X
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#X"
+
+#data
+&#x
+#errors
+Line: 1 Col: 3 Numeric entity expected but none found.
+Line: 1 Col: 3 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&#x"
+
+#data
+-
+#errors
+Line: 1 Col: 4 Numeric entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "-"
+
+#data
+&x-test
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&x-test"
+
+#data
+<!doctypehtml><p><li>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <li>
+
+#data
+<!doctypehtml><p><dt>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dt>
+
+#data
+<!doctypehtml><p><dd>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <dd>
+
+#data
+<!doctypehtml><p><form>
+#errors
+Line: 1 Col: 9 No space after literal string 'DOCTYPE'.
+Line: 1 Col: 23 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     <form>
+
+#data
+<!DOCTYPE html><p></P>X
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <p>
+|     "X"
+
+#data
+&
+#errors
+Line: 1 Col: 4 Named entity didn't end with ';'.
+Line: 1 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&"
+
+#data
+&AMp;
+#errors
+Line: 1 Col: 1 Named entity expected. Got none.
+Line: 1 Col: 1 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "&AMp;"
+
+#data
+<!DOCTYPE html><html><head></head><body><thisISasillyTESTelementNameToMakeSureCrazyTagNamesArePARSEDcorrectLY>
+#errors
+Line: 1 Col: 110 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <thisisasillytestelementnametomakesurecrazytagnamesareparsedcorrectly>
+
+#data
+<!DOCTYPE html>X</body>X
+#errors
+Line: 1 Col: 24 Unexpected non-space characters in the after body phase.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     "XX"
+
+#data
+<!DOCTYPE html><!-- X
+#errors
+Line: 1 Col: 21 Unexpected end of file in comment.
+#document
+| <!DOCTYPE html>
+| <!--  X -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><table><caption>test TEST</caption><td>test
+#errors
+Line: 1 Col: 54 Unexpected table cell start tag (td) in the table body phase.
+Line: 1 Col: 58 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <table>
+|       <caption>
+|         "test TEST"
+|       <tbody>
+|         <tr>
+|           <td>
+|             "test"
+
+#data
+<!DOCTYPE html><select><option><optgroup>
+#errors
+Line: 1 Col: 41 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><select><optgroup><option></optgroup><option><select><option>
+#errors
+Line: 1 Col: 68 Unexpected select start tag in the select phase treated as select end tag.
+Line: 1 Col: 76 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <option>
+|     <option>
+
+#data
+<!DOCTYPE html><select><optgroup><option><optgroup>
+#errors
+Line: 1 Col: 51 Expected closing tag. Unexpected end of file.
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+|         <option>
+|       <optgroup>
+
+#data
+<!DOCTYPE html><datalist><option>foo</datalist>bar
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <datalist>
+|       <option>
+|         "foo"
+|     "bar"
+
+#data
+<!DOCTYPE html><font><input><input></font>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <font>
+|       <input>
+|       <input>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX
+#errors
+Line: 1 Col: 29 Unexpected end of file in comment (-)
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><!-- XXX - XXX - XXX -->
+#errors
+#document
+| <!DOCTYPE html>
+| <!--  XXX - XXX - XXX  -->
+| <html>
+|   <head>
+|   <body>
+
+#data
+<isindex test=x name=x>
+#errors
+Line: 1 Col: 23 Unexpected start tag (isindex). Expected DOCTYPE.
+Line: 1 Col: 23 Unexpected start tag isindex. Don't use it!
+#document
+| <html>
+|   <head>
+|   <body>
+|     <form>
+|       <hr>
+|       <label>
+|         "This is a searchable index. Enter search keywords: "
+|         <input>
+|           name="isindex"
+|           test="x"
+|       <hr>
+
+#data
+test
+test
+#errors
+Line: 2 Col: 4 Unexpected non-space characters. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+|     "test
+test"
+
+#data
+<!DOCTYPE html><body><title>test</body>
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "test</body>"
+
+#data
+<!DOCTYPE html><body><title>X
+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       "X"
+|     <meta>
+|       name="z"
+|     <link>
+|       rel="foo"
+|     <style>
+|       "
+x { content:"</style" } "
+
+#data
+<!DOCTYPE html><select><optgroup></optgroup></select>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+|     <select>
+|       <optgroup>
+
+#data
+ 
+ 
+#errors
+Line: 2 Col: 1 Unexpected End of file. Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html>  <html>
+#errors
+#document
+| <!DOCTYPE html>
+| <html>
+|   <head>
+|   <body>
+
+#data
+<!DOCTYPE html><script>
+</script>  <title>x  
+#errors
+#document
+| 
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 21 Unexpected start tag (script) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+Line: 1 Col: 28 Unexpected start tag (style) that can be in head. Moved.
+#document
+| 
+|   
+|     
+#errors
+Line: 1 Col: 6 Unexpected start tag (head). Expected DOCTYPE.
+#document
+| 
+|   
+|   
+|   
+|     "x"
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+Line: 1 Col: 22 Unexpected end of file. Expected end tag (style).
+#document
+| 
+|   
+|      --> x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|     x
+#errors
+Line: 1 Col: 7 Unexpected start tag (style). Expected DOCTYPE.
+#document
+| 
+|   
+|      

+#errors
+#document
+| 
+| 
+|   
+|   
+|     
+|       ddd
+#errors
+#document
+| 
+|   
+|     
+#errors
+#document
+| 
+|   
+|   
+|     
+|     
  • 
+|       
+|