From 85cafef129c3826b0c5e290c89cfc7251fba43d5 Mon Sep 17 00:00:00 2001 From: Ondřej Surý Date: Fri, 8 Jul 2011 09:16:22 +0200 Subject: Imported Upstream version 2011.07.07 --- AUTHORS | 2 + CONTRIBUTORS | 6 +- doc/devel/release.html | 72 ++ doc/devel/weekly.html | 108 +- doc/go_tutorial.html | 1049 +++++++++--------- doc/go_tutorial.txt | 108 +- doc/htmlgen.go | 167 ++- doc/install.html | 6 +- doc/makehtml | 16 +- doc/prog.sh | 72 -- doc/progs/file_windows.go | 89 ++ doc/progs/run | 7 +- lib/codereview/codereview.py | 14 +- misc/dashboard/builder/http.go | 7 +- misc/dashboard/builder/main.go | 21 +- misc/dashboard/builder/package.go | 48 +- misc/dashboard/godashboard/auth.py | 13 + misc/dashboard/godashboard/gobuild.py | 7 +- misc/dashboard/godashboard/index.yaml | 1 - misc/dashboard/godashboard/package.html | 30 +- misc/dashboard/godashboard/package.py | 38 +- misc/dashboard/godashboard/static/style.css | 13 +- misc/emacs/go-mode.el | 4 +- misc/vim/ftdetect/gofiletype.vim | 2 +- src/cmd/5c/gc.h | 2 +- src/cmd/5l/asm.c | 31 +- src/cmd/5l/mkenam | 30 +- src/cmd/6c/gc.h | 1 + src/cmd/6l/asm.c | 29 +- src/cmd/6l/mkenam | 30 +- src/cmd/8a/a.h | 4 +- src/cmd/8a/a.y | 2 + src/cmd/8a/lex.c | 3 +- src/cmd/8c/gc.h | 2 + src/cmd/8c/swt.c | 4 +- src/cmd/8l/asm.c | 29 +- src/cmd/8l/mkenam | 30 +- src/cmd/cc/acid.c | 1 + src/cmd/cc/bits.c | 1 + src/cmd/cc/cc.h | 4 +- src/cmd/cc/cc.y | 1 + src/cmd/cc/com.c | 3 +- src/cmd/cc/com64.c | 1 + src/cmd/cc/dcl.c | 1 + src/cmd/cc/dpchk.c | 129 ++- src/cmd/cc/funct.c | 1 + src/cmd/cc/godefs.c | 3 +- src/cmd/cc/lex.c | 8 +- src/cmd/cc/lexbody | 6 +- src/cmd/cc/mac.c | 2 + src/cmd/cc/macbody | 6 +- src/cmd/cc/omachcap.c | 2 + src/cmd/cc/pgen.c | 2 +- src/cmd/cc/scon.c | 1 + src/cmd/cc/sub.c | 1 + src/cmd/cgo/gcc.go | 10 +- src/cmd/cgo/out.go | 2 +- src/cmd/ebnflint/ebnflint.go | 29 +- src/cmd/gc/go.h | 2 + src/cmd/gc/go.y | 12 +- src/cmd/gc/subr.c | 11 +- src/cmd/godoc/codewalk.go | 10 +- src/cmd/godoc/dirtrees.go | 9 +- src/cmd/godoc/godoc.go | 15 +- src/cmd/godoc/index.go | 4 +- src/cmd/godoc/main.go | 6 +- src/cmd/godoc/mapping.go | 14 +- src/cmd/godoc/utils.go | 2 +- src/cmd/gofix/Makefile | 14 +- src/cmd/gofix/filepath.go | 53 + src/cmd/gofix/filepath_test.go | 33 + src/cmd/gofix/fix.go | 159 ++- src/cmd/gofix/httpfs.go | 63 ++ src/cmd/gofix/httpfs_test.go | 47 + src/cmd/gofix/main.go | 2 +- src/cmd/gofix/signal.go | 49 + src/cmd/gofix/signal_test.go | 96 ++ src/cmd/gofix/sorthelpers.go | 47 + src/cmd/gofix/sorthelpers_test.go | 45 + src/cmd/gofix/sortslice.go | 50 + src/cmd/gofix/sortslice_test.go | 35 + src/cmd/gofix/stringssplit.go | 71 ++ src/cmd/gofix/stringssplit_test.go | 51 + src/cmd/gofix/testdata/reflect.template.go.in | 4 +- src/cmd/gofix/testdata/reflect.template.go.out | 4 +- src/cmd/gofmt/gofmt_test.go | 4 +- src/cmd/gofmt/rewrite.go | 2 +- src/cmd/goinstall/doc.go | 66 +- src/cmd/goinstall/download.go | 170 ++- src/cmd/goinstall/main.go | 6 +- src/cmd/gotest/doc.go | 11 +- src/cmd/gotest/flag.go | 4 + src/cmd/govet/Makefile | 3 + src/cmd/govet/govet.go | 62 +- src/cmd/goyacc/goyacc.go | 2 +- src/cmd/hgpatch/main.go | 4 +- src/cmd/ld/data.c | 39 +- src/cmd/ld/lib.c | 2 +- src/cmd/ld/lib.h | 2 + src/cmd/ld/symtab.c | 4 +- src/env.bash | 3 +- src/lib9/Makefile | 3 +- src/libmach/darwin.c | 13 +- src/pkg/Makefile | 3 +- src/pkg/asn1/asn1.go | 47 +- src/pkg/asn1/asn1_test.go | 6 +- src/pkg/asn1/common.go | 3 +- src/pkg/asn1/marshal.go | 2 +- src/pkg/asn1/marshal_test.go | 10 +- src/pkg/big/int.go | 58 +- src/pkg/big/int_test.go | 31 +- src/pkg/bufio/bufio.go | 22 +- src/pkg/bufio/bufio_test.go | 10 +- src/pkg/bytes/bytes.go | 26 +- src/pkg/bytes/bytes_test.go | 19 +- src/pkg/compress/lzw/reader_test.go | 2 +- src/pkg/crypto/aes/cipher.go | 4 +- src/pkg/crypto/blowfish/cipher.go | 4 +- src/pkg/crypto/ocsp/ocsp.go | 20 +- src/pkg/crypto/openpgp/keys.go | 67 ++ src/pkg/crypto/openpgp/packet/public_key.go | 10 +- src/pkg/crypto/openpgp/packet/signature.go | 62 +- src/pkg/crypto/openpgp/packet/signature_test.go | 22 +- src/pkg/crypto/openpgp/read.go | 5 +- src/pkg/crypto/openpgp/read_test.go | 23 + src/pkg/crypto/rand/rand_windows.go | 2 +- src/pkg/crypto/tls/generate_cert.go | 8 +- src/pkg/crypto/twofish/twofish.go | 2 +- src/pkg/crypto/x509/pkix/pkix.go | 12 +- src/pkg/crypto/x509/verify.go | 10 +- src/pkg/crypto/x509/verify_test.go | 11 +- src/pkg/crypto/x509/x509.go | 32 +- src/pkg/crypto/xtea/cipher.go | 4 +- src/pkg/csv/Makefile | 12 + src/pkg/csv/reader.go | 373 +++++++ src/pkg/csv/reader_test.go | 265 +++++ src/pkg/csv/writer.go | 123 +++ src/pkg/csv/writer_test.go | 44 + src/pkg/debug/proc/proc_linux.go | 2 +- src/pkg/exec/exec_test.go | 2 +- src/pkg/exec/lp_plan9.go | 2 +- src/pkg/exec/lp_unix.go | 2 +- src/pkg/exec/lp_windows.go | 4 +- src/pkg/exp/ogle/cmd.go | 2 +- src/pkg/exp/regexp/syntax/Makefile | 3 + src/pkg/exp/regexp/syntax/compile.go | 264 +++++ src/pkg/exp/regexp/syntax/parse.go | 754 +++++++++++-- src/pkg/exp/regexp/syntax/parse_test.go | 310 ++++-- src/pkg/exp/regexp/syntax/prog.go | 182 ++++ src/pkg/exp/regexp/syntax/prog_test.go | 91 ++ src/pkg/exp/regexp/syntax/regexp.go | 78 +- src/pkg/exp/regexp/syntax/simplify.go | 151 +++ src/pkg/exp/regexp/syntax/simplify_test.go | 151 +++ src/pkg/exp/template/Makefile | 5 +- src/pkg/exp/template/exec.go | 508 +++++++++ src/pkg/exp/template/exec_test.go | 342 ++++++ src/pkg/exp/template/funcs.go | 294 +++++ src/pkg/exp/template/lex.go | 180 +-- src/pkg/exp/template/lex_test.go | 44 +- src/pkg/exp/template/parse.go | 467 ++++++-- src/pkg/exp/template/parse_test.go | 127 ++- src/pkg/exp/template/set.go | 115 ++ src/pkg/exp/template/set_test.go | 101 ++ src/pkg/fmt/print.go | 33 +- src/pkg/fmt/scan.go | 12 + src/pkg/go/ast/print_test.go | 2 +- src/pkg/go/build/dir.go | 2 +- src/pkg/go/build/path.go | 3 + src/pkg/go/doc/comment.go | 2 +- src/pkg/go/doc/doc.go | 2 +- src/pkg/go/types/testdata/exports.go | 2 +- src/pkg/gob/decode.go | 4 +- src/pkg/gob/doc.go | 34 +- src/pkg/gob/encode.go | 15 +- src/pkg/gob/type.go | 5 - src/pkg/html/parse.go | 6 +- src/pkg/html/token_test.go | 2 +- src/pkg/http/cgi/host.go | 26 +- src/pkg/http/cgi/host_test.go | 78 +- src/pkg/http/cgi/testdata/test.cgi | 51 +- src/pkg/http/chunked.go | 11 + src/pkg/http/cookie.go | 4 +- src/pkg/http/fs.go | 68 +- src/pkg/http/fs_test.go | 66 ++ src/pkg/http/header.go | 2 +- src/pkg/http/persist.go | 6 + src/pkg/http/readrequest_test.go | 71 +- src/pkg/http/request.go | 19 +- src/pkg/http/requestwrite_test.go | 119 +- src/pkg/http/response.go | 2 +- src/pkg/http/reverseproxy_test.go | 3 + src/pkg/http/serve_test.go | 76 +- src/pkg/http/server.go | 23 +- src/pkg/http/spdy/read.go | 2 +- src/pkg/http/transfer.go | 56 +- src/pkg/http/transport.go | 4 +- src/pkg/http/url.go | 4 +- src/pkg/image/draw/draw_test.go | 172 ++- src/pkg/image/image.go | 17 +- src/pkg/image/image_test.go | 5 + src/pkg/index/suffixarray/suffixarray.go | 4 +- src/pkg/index/suffixarray/suffixarray_test.go | 2 +- src/pkg/io/io.go | 8 + src/pkg/json/decode.go | 2 +- src/pkg/json/decode_test.go | 9 +- src/pkg/json/encode.go | 16 +- src/pkg/json/scanner_test.go | 5 +- src/pkg/mail/message.go | 2 +- src/pkg/mime/mediatype.go | 13 +- src/pkg/mime/mediatype_test.go | 3 +- src/pkg/mime/multipart/multipart.go | 32 +- src/pkg/mime/multipart/multipart_test.go | 32 +- src/pkg/net/dial.go | 86 +- src/pkg/net/dnsmsg.go | 46 +- src/pkg/net/fd_windows.go | 48 +- src/pkg/net/hosts_test.go | 2 +- src/pkg/net/interface_windows.go | 4 +- src/pkg/net/ipsock.go | 12 +- src/pkg/net/sendfile_windows.go | 6 +- src/pkg/net/sock.go | 23 +- src/pkg/net/sock_windows.go | 2 +- src/pkg/os/Makefile | 2 +- src/pkg/os/env_windows.go | 2 +- src/pkg/os/exec_posix.go | 11 +- src/pkg/os/exec_windows.go | 8 +- src/pkg/os/file.go | 28 +- src/pkg/os/file_plan9.go | 26 + src/pkg/os/file_posix.go | 22 - src/pkg/os/file_unix.go | 45 + src/pkg/os/file_windows.go | 63 +- src/pkg/os/mkunixsignals.sh | 2 +- src/pkg/os/types.go | 2 +- src/pkg/patch/patch.go | 2 +- src/pkg/path/filepath/match.go | 2 +- src/pkg/path/filepath/path.go | 2 +- src/pkg/path/filepath/path_test.go | 4 - src/pkg/reflect/all_test.go | 76 +- src/pkg/reflect/type.go | 130 ++- src/pkg/reflect/value.go | 21 +- src/pkg/regexp/regexp.go | 1 + src/pkg/rpc/jsonrpc/all_test.go | 6 +- src/pkg/rpc/jsonrpc/client.go | 12 +- src/pkg/rpc/jsonrpc/server.go | 12 +- src/pkg/rpc/server.go | 2 +- src/pkg/runtime/386/atomic.c | 12 + src/pkg/runtime/Makefile | 1 + src/pkg/runtime/amd64/atomic.c | 12 + src/pkg/runtime/arm/atomic.c | 12 + src/pkg/runtime/cgo/darwin_386.c | 7 +- src/pkg/runtime/cgo/darwin_amd64.c | 7 +- src/pkg/runtime/cgo/freebsd_386.c | 7 +- src/pkg/runtime/cgo/freebsd_amd64.c | 7 +- src/pkg/runtime/cgo/linux_386.c | 7 +- src/pkg/runtime/cgo/linux_amd64.c | 7 +- src/pkg/runtime/cgo/windows_amd64.c | 18 +- src/pkg/runtime/debug/stack.go | 2 +- src/pkg/runtime/debug/stack_test.go | 2 +- src/pkg/runtime/mkasmh.sh | 25 +- src/pkg/runtime/proc.c | 2 +- src/pkg/runtime/runtime.h | 8 + src/pkg/runtime/sema.goc | 187 ++-- src/pkg/runtime/sema_test.go | 100 ++ src/pkg/runtime/windows/amd64/defs.h | 40 + src/pkg/runtime/windows/amd64/rt0.s | 10 + src/pkg/runtime/windows/amd64/signal.c | 20 + src/pkg/runtime/windows/amd64/sys.s | 129 +++ src/pkg/runtime/windows/mem.c | 10 +- src/pkg/runtime/windows/os.h | 3 + src/pkg/runtime/windows/thread.c | 18 +- src/pkg/smtp/smtp.go | 6 +- src/pkg/smtp/smtp_test.go | 4 +- src/pkg/sort/sort.go | 12 +- src/pkg/sort/sort_test.go | 24 +- src/pkg/strconv/fp_test.go | 6 +- src/pkg/strings/strings.go | 26 +- src/pkg/strings/strings_test.go | 22 +- src/pkg/sync/mutex.go | 63 +- src/pkg/sync/mutex_test.go | 99 +- src/pkg/sync/once.go | 14 +- src/pkg/sync/once_test.go | 25 + src/pkg/syscall/Makefile | 2 + src/pkg/syscall/asm_windows_386.s | 2 +- src/pkg/syscall/asm_windows_amd64.s | 7 + src/pkg/syscall/exec_unix.go | 31 + src/pkg/syscall/exec_windows.go | 16 +- src/pkg/syscall/mkall.sh | 11 +- src/pkg/syscall/mkerrors.sh | 1 + src/pkg/syscall/syscall_windows.go | 224 ++-- src/pkg/syscall/syscall_windows_386.go | 2 - src/pkg/syscall/syscall_windows_amd64.go | 5 + src/pkg/syscall/zerrors_darwin_386.go | 79 ++ src/pkg/syscall/zerrors_darwin_amd64.go | 79 ++ src/pkg/syscall/zerrors_freebsd_386.go | 72 ++ src/pkg/syscall/zerrors_freebsd_amd64.go | 72 ++ src/pkg/syscall/zerrors_linux_386.go | 63 ++ src/pkg/syscall/zerrors_linux_amd64.go | 63 ++ src/pkg/syscall/zerrors_linux_arm.go | 59 + src/pkg/syscall/zerrors_windows.go | 283 +++++ src/pkg/syscall/zerrors_windows_386.go | 284 +---- src/pkg/syscall/zerrors_windows_amd64.go | 5 + src/pkg/syscall/zsyscall_windows_386.go | 132 +-- src/pkg/syscall/zsyscall_windows_amd64.go | 1323 +++++++++++++++++++++++ src/pkg/syscall/zsysnum_windows_amd64.go | 3 + src/pkg/syscall/ztypes_windows.go | 656 +++++++++++ src/pkg/syscall/ztypes_windows_386.go | 659 +---------- src/pkg/syscall/ztypes_windows_amd64.go | 5 + src/pkg/template/execute.go | 2 +- src/pkg/template/parse.go | 2 +- src/pkg/testing/benchmark.go | 54 +- src/pkg/testing/iotest/reader.go | 21 +- src/pkg/testing/testing.go | 72 +- src/pkg/time/format.go | 2 +- src/pkg/time/sleep_test.go | 2 +- src/pkg/time/time_test.go | 17 +- src/pkg/unicode/maketables.go | 52 +- src/pkg/xml/Makefile | 1 + src/pkg/xml/atom_test.go | 50 + src/pkg/xml/embed_test.go | 10 +- src/pkg/xml/marshal.go | 228 ++++ src/pkg/xml/marshal_test.go | 299 +++++ src/pkg/xml/read.go | 41 +- src/pkg/xml/read_test.go | 42 +- src/run.bash | 2 +- src/version.bash | 9 +- test/fixedbugs/bug345.dir/io.go | 15 + test/fixedbugs/bug345.dir/main.go | 28 + test/fixedbugs/bug345.go | 7 + 327 files changed, 13435 insertions(+), 3614 deletions(-) delete mode 100755 doc/prog.sh create mode 100644 doc/progs/file_windows.go create mode 100644 misc/dashboard/godashboard/auth.py create mode 100644 src/cmd/gofix/filepath.go create mode 100644 src/cmd/gofix/filepath_test.go create mode 100644 src/cmd/gofix/httpfs.go create mode 100644 src/cmd/gofix/httpfs_test.go create mode 100644 src/cmd/gofix/signal.go create mode 100644 src/cmd/gofix/signal_test.go create mode 100644 src/cmd/gofix/sorthelpers.go create mode 100644 src/cmd/gofix/sorthelpers_test.go create mode 100644 src/cmd/gofix/sortslice.go create mode 100644 src/cmd/gofix/sortslice_test.go create mode 100644 src/cmd/gofix/stringssplit.go create mode 100644 src/cmd/gofix/stringssplit_test.go create mode 100644 src/pkg/csv/Makefile create mode 100644 src/pkg/csv/reader.go create mode 100644 src/pkg/csv/reader_test.go create mode 100644 src/pkg/csv/writer.go create mode 100644 src/pkg/csv/writer_test.go create mode 100644 src/pkg/exp/regexp/syntax/compile.go create mode 100644 src/pkg/exp/regexp/syntax/prog.go create mode 100644 src/pkg/exp/regexp/syntax/prog_test.go create mode 100644 src/pkg/exp/regexp/syntax/simplify.go create mode 100644 src/pkg/exp/regexp/syntax/simplify_test.go create mode 100644 src/pkg/exp/template/exec.go create mode 100644 src/pkg/exp/template/exec_test.go create mode 100644 src/pkg/exp/template/funcs.go create mode 100644 src/pkg/exp/template/set.go create mode 100644 src/pkg/exp/template/set_test.go create mode 100644 src/pkg/runtime/386/atomic.c create mode 100644 src/pkg/runtime/amd64/atomic.c create mode 100644 src/pkg/runtime/arm/atomic.c create mode 100644 src/pkg/runtime/sema_test.go create mode 100644 src/pkg/runtime/windows/amd64/defs.h create mode 100644 src/pkg/runtime/windows/amd64/rt0.s create mode 100644 src/pkg/runtime/windows/amd64/signal.c create mode 100644 src/pkg/runtime/windows/amd64/sys.s create mode 100644 src/pkg/syscall/asm_windows_amd64.s create mode 100644 src/pkg/syscall/syscall_windows_amd64.go create mode 100644 src/pkg/syscall/zerrors_windows.go create mode 100644 src/pkg/syscall/zerrors_windows_amd64.go create mode 100644 src/pkg/syscall/zsyscall_windows_amd64.go create mode 100644 src/pkg/syscall/zsysnum_windows_amd64.go create mode 100644 src/pkg/syscall/ztypes_windows.go create mode 100644 src/pkg/syscall/ztypes_windows_amd64.go create mode 100644 src/pkg/xml/atom_test.go create mode 100644 src/pkg/xml/marshal.go create mode 100644 src/pkg/xml/marshal_test.go create mode 100644 test/fixedbugs/bug345.dir/io.go create mode 100644 test/fixedbugs/bug345.dir/main.go create mode 100644 test/fixedbugs/bug345.go diff --git a/AUTHORS b/AUTHORS index 5471a8f80..f9af1a777 100644 --- a/AUTHORS +++ b/AUTHORS @@ -16,6 +16,7 @@ Alexander Orlov Alexey Borzenkov Amrut Joshi Andrei Vieru +Andrew Balholm Andrew Skiba Andrey Mirtchovski Andy Davis @@ -60,6 +61,7 @@ Fazlul Shahriar Firmansyah Adiputra Florian Uekermann Gary Burd +Gideon Jan-Wessel Redelinghuys Giles Lean Google Inc. Graham Miller diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e5037753a..8bb57f9ab 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -40,6 +40,7 @@ Alexander Orlov Alexey Borzenkov Amrut Joshi Andrei Vieru +Andrew Balholm Andrew Gerrand Andrew Skiba Andrey Mirtchovski @@ -100,6 +101,7 @@ Firmansyah Adiputra Florian Uekermann Fumitoshi Ukai Gary Burd +Gideon Jan-Wessel Redelinghuys Giles Lean Graham Miller Gustavo Niemeyer @@ -141,12 +143,13 @@ Ken Thompson Kevin Ballard Kirklin McDonald Kyle Consalus -Kyle Lemons +Kyle Lemons Larry Hosken Lorenzo Stoakes Lucio De Re Luit van Drongelen Luuk van Dijk +Marcel van Lohuizen Mark Zavislak Markus Duft Martin Neubauer @@ -171,6 +174,7 @@ Padraig Kitterick Paolo Giarrusso Pascal S. de Kloe Patrick Gavlin +Paul Borman Petar Maymounkov Peter Froehlich Peter McKenzie diff --git a/doc/devel/release.html b/doc/devel/release.html index 84ca622fa..d632200d3 100644 --- a/doc/devel/release.html +++ b/doc/devel/release.html @@ -14,6 +14,78 @@ hg pull hg update release.rNN +

r58 (released 2011/06/29)

+ +

+The r58 release corresponds to +weekly.2011-06-09 +with additional bug fixes. +This section highlights the most significant changes in this release. +For a more detailed summary, see the +weekly release notes. +For complete information, see the +Mercurial change list. +

+ +

Language

+ +

+This release fixes a use of uninitialized memory in programs that misuse goto. +

+ +

Packages

+ +

+As usual, gofix will handle the bulk of the rewrites +necessary for these changes to package APIs. +

+ +

+Package http drops the finalURL return +value from the Client.Get method. The value +is now available via the new Request field on http.Response. +Most instances of the type map[string][]string in have been +replaced with the new Values type. +

+ +

+Package exec has been redesigned with a more +convenient and succinct API. +

+ +

+Package strconv's Quote +function now escapes only those Unicode code points not classified as printable +by unicode.IsPrint. +Previously Quote would escape all non-ASCII characters. +This also affects the fmt package's "%q" +formatting directive. The previous quoting behavior is still available via +strconv's new QuoteToASCII function. +

+ +

+Package os/signal's +Signal and +UnixSignal types have been moved to the +os package. +

+ +

+Package image/draw is the new name for +exp/draw. The GUI-related code from exp/draw is now +located in the exp/gui package. +

+ +

Tools

+ +

+Goinstall now observes the GOPATH environment +variable to build and install your own code and external libraries outside of +the Go tree (and avoid writing Makefiles). +

+ +

r57 (released 2011/05/03)

diff --git a/doc/devel/weekly.html b/doc/devel/weekly.html index 0a043d410..bf16c8370 100644 --- a/doc/devel/weekly.html +++ b/doc/devel/weekly.html @@ -14,6 +14,112 @@ hg pull hg update weekly.YYYY-MM-DD +

2011-07-07

+ +
+This weekly snapshot includes changes to the strings, http, reflect, json, and
+xml packages. Code that uses these packages will need changes. Most of these
+changes can be made automatically with gofix.
+
+The strings package's Split function has itself been split into Split and
+SplitN. SplitN is the same as the old Split. The new Split is equivalent to
+SplitN with a final argument of -1.
+
+The http package has a new FileSystem interface that provides access to files.
+The FileServer helper now takes a FileSystem argument instead of an explicit
+file system root. By implementing your own FileSystem you can use the
+FileServer to serve arbitrary data.
+
+The reflect package supports a new struct tag scheme that enables sharing of
+struct tags between multiple packages.
+In this scheme, the tags must be of the form:
+        key:"value" key2:"value2"
+reflect.StructField's Tag field now has type StructTag (a string type), which
+has method Get(key string) string that returns the associated value.
+Clients of json and xml will need to be updated. Code that says
+        type T struct {
+                X int "name"
+        }
+should become
+        type T struct {
+                X int `json:"name"`  // or `xml:"name"`
+        }
+Use govet to identify struct tags that need to be changed to use the new syntax.
+
+Other changes:
+* 5l, 6l, 8l: drop use of ed during build.
+* asn1: support T61 and UTF8 string.
+* bufio: do not cache Read errors (thanks Graham Miller).
+* build: make version.bash aware of branches.
+* cgi: don't depend on CGI.pm for tests.
+* codereview: make --ignore_hgpatch_failure work again,
+	restrict sync to default branch.
+* crypto/openpgp: add ability to reserialize keys,
+	bug fix (thanks Gideon Jan-Wessel Redelinghuys).
+* crypto/tls: fix generate_cert.go.
+* crypto/x509: prevent chain cycles in Verify.
+* csv: new package.
+* doc: remove ed from apt-get package list.
+* docs: fold the prog.sh scripting from makehtml into htmlgen itself.
+* ebnflint: better handling of stdin.
+* exp/regexp/syntax: new experimental RE2-based regexp implementation.
+* exp/template: a new experimental templating package.
+* fmt: add SkipSpace to fmt's ScanState interface.
+* fmt: rename errno and error to err for doc consistency.
+* gc: avoid package name ambiguity in error messages,
+	fix package quoting logic,
+	fixes for Plan 9 (thanks Lucio De Re).
+* go/build: evaluate symlinks before comparing path to GOPATH.
+* gob: use exported fields in structs in the package documentation.
+* godoc: ignore directories that begin with '.',
+	search GOPATH for documentation.
+* gofix: os/signal, path/filepath, and sort fixes (thanks Robert Hencke),
+* goinstall: add support for generic hosts (thanks Julian Phillips),
+	only report successfully-installed packages to the dashboard,
+	try to access via https (thanks Yasuhiro Matsumoto).
+* gotest: add -test.benchtime and -test.cpu flags.
+* html: fixes and improvements (thanks Yasuhiro Matsumoto).
+* http/cgi: add Handler.Dir to specify working directory (thanks Yasuhiro Matsumoto).
+* http: add StripPrefix handler wrapper,
+	assume ContentLength 0 on GET requests,
+	better handling of 0-length Request.Body,
+	do TLS handshake explicitly before copying TLS state,
+	document that ServerConn and ClientConn are low-level,
+	make NewChunkedReader public (thanks Andrew Balholm),
+	respect Handlers setting Connection: close in their response.
+* image: more tests, Paletted.Opaque optimization.
+* io.WriteString: if the object has a WriteString method, use it (thanks Evan Shaw).
+* ld: elide the Go symbol table when using -s (thanks Anthony Martin).
+* ld: fix ELF strip by removing overlap of sections (thanks Gustavo Niemeyer).
+* mime/multipart: parse LF-delimited messages, not just CRLF.
+* mime: permit lower-case media type parameters (thanks Pascal S. de Kloe).
+* misc/dashboard: new features and improvements (not yet deployed).
+* misc/emacs: update list of builtins (thanks Quan Yong Zhai).
+* misc/vim: allow only utf-8 for file encoding (thanks Yasuhiro Matsumoto).
+* os: fix documentation for FileInfo.Name,
+	simplify WriteString,
+	use a different symbol from syscall in mkunixsignals.sh.
+* path/filepath: enable TestWalk to run on windows (thanks Alex Brainman).
+* reflect: add MethodByName,
+	allow Len on String values.
+* regexp: document that Regexp is thread-safe.
+* runtime/cgo: check for errors from pthread_create (thanks Albert Strasheim).
+* runtime: add Semacquire/Semrelease benchmarks,
+	improved Semacquire/Semrelease implementation,
+	windows/amd64 port (thanks Wei Guangjing).
+* sync: add fast path to Once,
+	improve Mutex to allow successive acquisitions,
+	new and improved benchmarks.
+* syscall: regenerate zerrors for darwin/linux/freebsd,
+	support for tty options in StartProcess (thanks Ken Rockot).
+* testing: make ResetTimer not start/stop the timer,
+	scale benchmark precision to 0.01ns if needed.
+* time: zero-pad two-digit years.
+* unicode/maketables: update debugging data.
+* windows: define and use syscall.Handle (thanks Wei Guangjing).
+* xml: add Marshal and MarshalIndent.
+
+

2011-06-23

@@ -128,7 +234,7 @@ Other changes:
 * xml: handle non-string attribute fields (thanks Maxim Ushakov).
 
-

2011-06-09

+

2011-06-09 (base for r58)

 This release includes changes to the strconv, http, and exp/draw packages.
diff --git a/doc/go_tutorial.html b/doc/go_tutorial.html
index 4f3f6b94b..822f9626e 100644
--- a/doc/go_tutorial.html
+++ b/doc/go_tutorial.html
@@ -26,14 +26,14 @@ cleanliness, blank lines remain blank.
 

Let's start in the usual way:

-

 
-05    package main
+
package main
 
-07    import fmt "fmt"  // Package implementing formatted I/O.
+import fmt "fmt"  // Package implementing formatted I/O.
 
-09    func main() {
-10        fmt.Printf("Hello, world; or Καλημέρα κόσμε; or こんにちは 世界\n")
-11    }
+func main() {
+    fmt.Printf("Hello, world; or Καλημέρα κόσμε; or こんにちは 世界\n")
+}
 

Every Go source file declares, using a package statement, which package it's part of. @@ -51,8 +51,8 @@ String constants can contain Unicode characters, encoded in UTF-8. The comment convention is the same as in C++:

-    /* ... */
-    // ...
+/* ... */
+// ...
 

Later we'll have much more to say about printing. @@ -96,67 +96,67 @@ a more robust run-time system although gccgo is catching up. Here's how to compile and run our program. With 6g, say,

-    $ 6g helloworld.go  # compile; object goes into helloworld.6
-    $ 6l helloworld.6   # link; output goes into 6.out
-    $ 6.out
-    Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
-    $
+$ 6g helloworld.go  # compile; object goes into helloworld.6
+$ 6l helloworld.6   # link; output goes into 6.out
+$ 6.out
+Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
+$
 

With gccgo it looks a little more traditional.

-    $ gccgo helloworld.go
-    $ a.out
-    Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
-    $
+$ gccgo helloworld.go
+$ a.out
+Hello, world; or Καλημέρα κόσμε; or こんにちは 世界
+$
 

Echo

Next up, here's a version of the Unix utility echo(1):

-

 
-05    package main
+
package main
 
-07    import (
-08        "os"
-09        "flag"  // command line option parser
-10    )
+import (
+    "os"
+    "flag"  // command line option parser
+)
 
-12    var omitNewline = flag.Bool("n", false, "don't print final newline")
+var omitNewline = flag.Bool("n", false, "don't print final newline")
 
-14    const (
-15        Space = " "
-16        Newline = "\n"
-17    )
+const (
+    Space = " "
+    Newline = "\n"
+)
 
-19    func main() {
-20        flag.Parse()   // Scans the arg list and sets up flags
-21        var s string = ""
-22        for i := 0; i < flag.NArg(); i++ {
-23            if i > 0 {
-24                s += Space
-25            }
-26            s += flag.Arg(i)
-27        }
-28        if !*omitNewline {
-29            s += Newline
-30        }
-31        os.Stdout.WriteString(s)
-32    }
+func main() {
+    flag.Parse()   // Scans the arg list and sets up flags
+    var s string = ""
+    for i := 0; i < flag.NArg(); i++ {
+        if i > 0 {
+            s += Space
+        }
+        s += flag.Arg(i)
+    }
+    if !*omitNewline {
+        s += Newline
+    }
+    os.Stdout.WriteString(s)
+}
 

This program is small but it's doing a number of new things. In the last example, we saw func introduce a function. The keywords var, const, and type (not used yet) also introduce declarations, as does import. Notice that we can group declarations of the same sort into -parenthesized lists, one item per line, as on lines 7-10 and 14-17. +parenthesized lists, one item per line, as in the import and const clauses here. But it's not necessary to do so; we could have said

-    const Space = " "
-    const Newline = "\n"
+const Space = " "
+const Newline = "\n"
 

This program imports the "os" package to access its Stdout variable, of type @@ -186,7 +186,7 @@ string variable we will use to build the output. The declaration statement has the form

-    var s string = ""
+var s string = ""
 

This is the var keyword, followed by the name of the variable, followed by @@ -197,20 +197,20 @@ string constant is of type string, we don't have to tell the compiler that. We could write

-    var s = ""
+var s = ""
 

or we could go even shorter and write the idiom

-    s := ""
+s := ""
 

The := operator is used a lot in Go to represent an initializing declaration. There's one in the for clause on the next line:

-

 
-22        for i := 0; i < flag.NArg(); i++ {
+
    for i := 0; i < flag.NArg(); i++ {
 

The flag package has parsed the arguments and left the non-flag arguments @@ -231,7 +231,7 @@ It's defined that way. Falling off the end of main.main means ''success''; if you want to signal an erroneous return, call

-    os.Exit(1)
+os.Exit(1)
 

The os package contains other essentials for getting @@ -259,20 +259,20 @@ Once you've built a string value, you can't change it, although of course you can change a string variable simply by reassigning it. This snippet from strings.go is legal code:

-

 
-10        s := "hello"
-11        if s[1] != 'e' { os.Exit(1) }
-12        s = "good bye"
-13        var p *string = &s
-14        *p = "ciao"
+
    s := "hello"
+    if s[1] != 'e' { os.Exit(1) }
+    s = "good bye"
+    var p *string = &s
+    *p = "ciao"
 

However the following statements are illegal because they would modify a string value:

-    s[0] = 'x'
-    (*p)[1] = 'y'
+s[0] = 'x'
+(*p)[1] = 'y'
 

In C++ terms, Go strings are a bit like const strings, while pointers @@ -284,7 +284,7 @@ read on. Arrays are declared like this:

-    var arrayOfInt [10]int
+var arrayOfInt [10]int
 

Arrays, like strings, are values, but they are mutable. This differs @@ -315,7 +315,7 @@ expression formed from a type followed by a brace-bounded expression like this:

-    [3]int{1,2,3}
+[3]int{1,2,3}
 

In this case the constructor builds an array of 3 ints. @@ -330,14 +330,14 @@ will slice the whole array.

Using slices one can write this function (from sum.go):

-

 
-09    func sum(a []int) int { // returns an int
-10        s := 0
-11        for i := 0; i < len(a); i++ {
-12            s += a[i]
-13        }
-14        return s
-15    }
+
func sum(a []int) int { // returns an int
+    s := 0
+    for i := 0; i < len(a); i++ {
+        s += a[i]
+    }
+    return s
+}
 

Note how the return type (int) is defined for sum by stating it @@ -348,14 +348,14 @@ a simpler way in a moment) constructs an array and slices it:

-    s := sum([3]int{1,2,3}[:])
+s := sum([3]int{1,2,3}[:])
 

If you are creating a regular array but want the compiler to count the elements for you, use ... as the array size:

-    s := sum([...]int{1,2,3}[:])
+s := sum([...]int{1,2,3}[:])
 

That's fussier than necessary, though. @@ -363,13 +363,13 @@ In practice, unless you're meticulous about storage layout within a data structure, a slice itself—using empty brackets with no size—is all you need:

-    s := sum([]int{1,2,3})
+s := sum([]int{1,2,3})
 

There are also maps, which you can initialize like this:

-    m := map[string]int{"one":1 , "two":2}
+m := map[string]int{"one":1 , "two":2}
 

The built-in function len, which returns number of elements, @@ -380,13 +380,13 @@ By the way, another thing that works on strings, arrays, slices, maps and channels is the range clause on for loops. Instead of writing

-    for i := 0; i < len(a); i++ { ... }
+for i := 0; i < len(a); i++ { ... }
 

to loop over the elements of a slice (or map or ...) , we could write

-    for i, v := range a { ... }
+for i, v := range a { ... }
 

This assigns i to the index and v to the value of the successive @@ -404,14 +404,14 @@ To allocate a new variable, use the built-in function new, which returns a pointer to the allocated storage.

-    type T struct { a, b int }
-    var t *T = new(T)
+type T struct { a, b int }
+var t *T = new(T)
 

or the more idiomatic

-    t := new(T)
+t := new(T)
 

Some types—maps, slices, and channels (see below)—have reference semantics. @@ -420,14 +420,14 @@ referencing the same underlying data will see the modification. For these three types you want to use the built-in function make:

-    m := make(map[string]int)
+m := make(map[string]int)
 

This statement initializes a new map ready to store entries. If you just declare the map, as in

-    var m map[string]int
+var m map[string]int
 

it creates a nil reference that cannot hold anything. To use the map, @@ -448,20 +448,20 @@ can overflow only when they are assigned to an integer variable with too little precision to represent the value.

-    const hardEight = (1 << 100) >> 97  // legal
+const hardEight = (1 << 100) >> 97  // legal
 

There are nuances that deserve redirection to the legalese of the language specification but here are some illustrative examples:

-    var a uint64 = 0  // a has type uint64, value 0
-    a := uint64(0)    // equivalent; uses a "conversion"
-    i := 0x1234       // i gets default type: int
-    var j int = 1e6   // legal - 1000000 is representable in an int
-    x := 1.5          // a float64, the default type for floating constants
-    i3div2 := 3/2     // integer division - result is 1
-    f3div2 := 3./2.   // floating-point division - result is 1.5
+var a uint64 = 0  // a has type uint64, value 0
+a := uint64(0)    // equivalent; uses a "conversion"
+i := 0x1234       // i gets default type: int
+var j int = 1e6   // legal - 1000000 is representable in an int
+x := 1.5          // a float64, the default type for floating constants
+i3div2 := 3/2     // integer division - result is 1
+f3div2 := 3./2.   // floating-point division - result is 1.5
 

Conversions only work for simple cases such as converting ints of one @@ -476,18 +476,18 @@ assigned to a variable. Next we'll look at a simple package for doing file I/O with an open/close/read/write interface. Here's the start of file.go:

-

 
-05    package file
+
package file
 
-07    import (
-08        "os"
-09        "syscall"
-10    )
+import (
+    "os"
+    "syscall"
+)
 
-12    type File struct {
-13        fd   int    // file descriptor number
-14        name string // file name at Open time
-15    }
+type File struct {
+    fd   int    // file descriptor number
+    name string // file name at Open time
+}
 

The first few lines declare the name of the @@ -518,13 +518,13 @@ will soon give it some exported, upper-case methods.

First, though, here is a factory to create a File:

-

 
-17    func newFile(fd int, name string) *File {
-18        if fd < 0 {
-19            return nil
-20        }
-21        return &File{fd, name}
-22    }
+
func newFile(fd int, name string) *File {
+    if fd < 0 {
+        return nil
+    }
+    return &File{fd, name}
+}
 

This returns a pointer to a new File structure with the file descriptor and name @@ -533,10 +533,10 @@ the ones used to build maps and arrays, to construct a new heap-allocated object. We could write

-    n := new(File)
-    n.fd = fd
-    n.name = name
-    return n
+n := new(File)
+n.fd = fd
+n.name = name
+return n
 

but for simple structures like File it's easier to return the address of a @@ -544,25 +544,26 @@ composite literal, as is done here on line 21.

We can use the factory to construct some familiar, exported variables of type *File:

-

 
-24    var (
-25        Stdin  = newFile(syscall.Stdin, "/dev/stdin")
-26        Stdout = newFile(syscall.Stdout, "/dev/stdout")
-27        Stderr = newFile(syscall.Stderr, "/dev/stderr")
-28    )
+
var (
+    Stdin  = newFile(syscall.Stdin, "/dev/stdin")
+    Stdout = newFile(syscall.Stdout, "/dev/stdout")
+    Stderr = newFile(syscall.Stderr, "/dev/stderr")
+)
+
 

The newFile function was not exported because it's internal. The proper, exported factory to use is OpenFile (we'll explain that name in a moment):

-

 
-30    func OpenFile(name string, mode int, perm uint32) (file *File, err os.Error) {
-31        r, e := syscall.Open(name, mode, perm)
-32        if e != 0 {
-33            err = os.Errno(e)
-34        }
-35        return newFile(r, name), err
-36    }
+
func OpenFile(name string, mode int, perm uint32) (file *File, err os.Error) {
+    r, e := syscall.Open(name, mode, perm)
+    if e != 0 {
+        err = os.Errno(e)
+    }
+    return newFile(r, name), err
+}
 

There are a number of new things in these few lines. First, OpenFile returns @@ -593,23 +594,23 @@ the implementation of our Open and Create; they're tri wrappers that eliminate common errors by capturing the tricky standard arguments to open and, especially, to create a file:

-

 
-38    const (
-39        O_RDONLY = syscall.O_RDONLY
-40        O_RDWR   = syscall.O_RDWR
-41        O_CREATE = syscall.O_CREAT
-42        O_TRUNC  = syscall.O_TRUNC
-43    )
+
const (
+    O_RDONLY = syscall.O_RDONLY
+    O_RDWR   = syscall.O_RDWR
+    O_CREATE = syscall.O_CREAT
+    O_TRUNC  = syscall.O_TRUNC
+)
 
-45    func Open(name string) (file *File, err os.Error) {
-46        return OpenFile(name, O_RDONLY, 0)
-47    }
+func Open(name string) (file *File, err os.Error) {
+    return OpenFile(name, O_RDONLY, 0)
+}
 

-

 
-49    func Create(name string) (file *File, err os.Error) {
-50        return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
-51    }
+
func Create(name string) (file *File, err os.Error) {
+    return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
+}
 

Back to our main story. @@ -619,44 +620,44 @@ of that type, placed in parentheses before the function name. Here are some methods for *File, each of which declares a receiver variable file.

-

 
-53    func (file *File) Close() os.Error {
-54        if file == nil {
-55            return os.EINVAL
-56        }
-57        e := syscall.Close(file.fd)
-58        file.fd = -1 // so it can't be closed again
-59        if e != 0 {
-60            return os.Errno(e)
-61        }
-62        return nil
-63    }
+
func (file *File) Close() os.Error {
+    if file == nil {
+        return os.EINVAL
+    }
+    e := syscall.Close(file.fd)
+    file.fd = -1 // so it can't be closed again
+    if e != 0 {
+        return os.Errno(e)
+    }
+    return nil
+}
 
-65    func (file *File) Read(b []byte) (ret int, err os.Error) {
-66        if file == nil {
-67            return -1, os.EINVAL
-68        }
-69        r, e := syscall.Read(file.fd, b)
-70        if e != 0 {
-71            err = os.Errno(e)
-72        }
-73        return int(r), err
-74    }
+func (file *File) Read(b []byte) (ret int, err os.Error) {
+    if file == nil {
+        return -1, os.EINVAL
+    }
+    r, e := syscall.Read(file.fd, b)
+    if e != 0 {
+        err = os.Errno(e)
+    }
+    return int(r), err
+}
 
-76    func (file *File) Write(b []byte) (ret int, err os.Error) {
-77        if file == nil {
-78            return -1, os.EINVAL
-79        }
-80        r, e := syscall.Write(file.fd, b)
-81        if e != 0 {
-82            err = os.Errno(e)
-83        }
-84        return int(r), err
-85    }
+func (file *File) Write(b []byte) (ret int, err os.Error) {
+    if file == nil {
+        return -1, os.EINVAL
+    }
+    r, e := syscall.Write(file.fd, b)
+    if e != 0 {
+        err = os.Errno(e)
+    }
+    return int(r), err
+}
 
-87    func (file *File) String() string {
-88        return file.name
-89    }
+func (file *File) String() string {
+    return file.name
+}
 

There is no implicit this and the receiver variable must be used to access @@ -674,24 +675,24 @@ set of such error values.

We can now use our new package:

-

 
-05    package main
+
package main
 
-07    import (
-08        "./file"
-09        "fmt"
-10        "os"
-11    )
+import (
+    "./file"
+    "fmt"
+    "os"
+)
 
-13    func main() {
-14        hello := []byte("hello, world\n")
-15        file.Stdout.Write(hello)
-16        f, err := file.Open("/does/not/exist")
-17        if f == nil {
-18            fmt.Printf("can't open file; err=%s\n",  err.String())
-19            os.Exit(1)
-20        }
-21    }
+func main() {
+    hello := []byte("hello, world\n")
+    file.Stdout.Write(hello)
+    f, err := file.Open("/does/not/exist")
+    if f == nil {
+        fmt.Printf("can't open file; err=%s\n",  err.String())
+        os.Exit(1)
+    }
+}
 

The ''./'' in the import of ''./file'' tells the compiler @@ -703,13 +704,13 @@ package.) Now we can compile and run the program. On Unix, this would be the result:

-    $ 6g file.go                       # compile file package
-    $ 6g helloworld3.go                # compile main package
-    $ 6l -o helloworld3 helloworld3.6  # link - no need to mention "file"
-    $ helloworld3
-    hello, world
-    can't open file; err=No such file or directory
-    $
+$ 6g file.go                       # compile file package
+$ 6g helloworld3.go                # compile main package
+$ 6l -o helloworld3 helloworld3.6  # link - no need to mention "file"
+$ helloworld3
+hello, world
+can't open file; err=No such file or directory
+$
 

Rotting cats

@@ -717,56 +718,56 @@ Now we can compile and run the program. On Unix, this would be the result: Building on the file package, here's a simple version of the Unix utility cat(1), progs/cat.go:

-

 
-05    package main
+
package main
 
-07    import (
-08        "./file"
-09        "flag"
-10        "fmt"
-11        "os"
-12    )
+import (
+    "./file"
+    "flag"
+    "fmt"
+    "os"
+)
 
-14    func cat(f *file.File) {
-15        const NBUF = 512
-16        var buf [NBUF]byte
-17        for {
-18            switch nr, er := f.Read(buf[:]); true {
-19            case nr < 0:
-20                fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", f.String(), er.String())
-21                os.Exit(1)
-22            case nr == 0: // EOF
-23                return
-24            case nr > 0:
-25                if nw, ew := file.Stdout.Write(buf[0:nr]); nw != nr {
-26                    fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", f.String(), ew.String())
-27                    os.Exit(1)
-28                }
-29            }
-30        }
-31    }
+func cat(f *file.File) {
+    const NBUF = 512
+    var buf [NBUF]byte
+    for {
+        switch nr, er := f.Read(buf[:]); true {
+        case nr < 0:
+            fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", f.String(), er.String())
+            os.Exit(1)
+        case nr == 0: // EOF
+            return
+        case nr > 0:
+            if nw, ew := file.Stdout.Write(buf[0:nr]); nw != nr {
+                fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", f.String(), ew.String())
+                os.Exit(1)
+            }
+        }
+    }
+}
 
-33    func main() {
-34        flag.Parse() // Scans the arg list and sets up flags
-35        if flag.NArg() == 0 {
-36            cat(file.Stdin)
-37        }
-38        for i := 0; i < flag.NArg(); i++ {
-39            f, err := file.Open(flag.Arg(i))
-40            if f == nil {
-41                fmt.Fprintf(os.Stderr, "cat: can't open %s: error %s\n", flag.Arg(i), err)
-42                os.Exit(1)
-43            }
-44            cat(f)
-45            f.Close()
-46        }
-47    }
+func main() {
+    flag.Parse() // Scans the arg list and sets up flags
+    if flag.NArg() == 0 {
+        cat(file.Stdin)
+    }
+    for i := 0; i < flag.NArg(); i++ {
+        f, err := file.Open(flag.Arg(i))
+        if f == nil {
+            fmt.Fprintf(os.Stderr, "cat: can't open %s: error %s\n", flag.Arg(i), err)
+            os.Exit(1)
+        }
+        cat(f)
+        f.Close()
+    }
+}
 

By now this should be easy to follow, but the switch statement introduces some new features. Like a for loop, an if or switch can include an -initialization statement. The switch on line 18 uses one to create variables -nr and er to hold the return values from the call to f.Read. (The if on line 25 +initialization statement. The switch statement in cat uses one to create variables +nr and er to hold the return values from the call to f.Read. (The if a few lines later has the same idea.) The switch statement is general: it evaluates the cases from top to bottom looking for the first case that matches the value; the case expressions don't need to be constants or even integers, as long as @@ -778,7 +779,7 @@ in a for statement, a missing value means true. In fa is a form of if-else chain. While we're here, it should be mentioned that in switch statements each case has an implicit break.

-Line 25 calls Write by slicing the incoming buffer, which is itself a slice. +The argument to file.Stdout.Write is created by slicing the array buf. Slices provide the standard Go way to handle I/O buffers.

Now let's make a variant of cat that optionally does rot13 on its input. @@ -789,11 +790,11 @@ The cat subroutine uses only two methods of f: R so let's start by defining an interface that has exactly those two methods. Here is code from progs/cat_rot13.go:

-

 
-26    type reader interface {
-27        Read(b []byte) (ret int, err os.Error)
-28        String() string
-29    }
+
type reader interface {
+    Read(b []byte) (ret int, err os.Error)
+    String() string
+}
 

Any type that has the two methods of reader—regardless of whatever @@ -806,68 +807,68 @@ existing reader and does rot13 on the data. To do this the type and implement the methods and with no other bookkeeping, we have a second implementation of the reader interface.

-

 
-31    type rotate13 struct {
-32        source reader
-33    }
+
type rotate13 struct {
+    source reader
+}
 
-35    func newRotate13(source reader) *rotate13 {
-36        return &rotate13{source}
-37    }
+func newRotate13(source reader) *rotate13 {
+    return &rotate13{source}
+}
 
-39    func (r13 *rotate13) Read(b []byte) (ret int, err os.Error) {
-40        r, e := r13.source.Read(b)
-41        for i := 0; i < r; i++ {
-42            b[i] = rot13(b[i])
-43        }
-44        return r, e
-45    }
+func (r13 *rotate13) Read(b []byte) (ret int, err os.Error) {
+    r, e := r13.source.Read(b)
+    for i := 0; i < r; i++ {
+        b[i] = rot13(b[i])
+    }
+    return r, e
+}
 
-47    func (r13 *rotate13) String() string {
-48        return r13.source.String()
-49    }
-50    // end of rotate13 implementation
+func (r13 *rotate13) String() string {
+    return r13.source.String()
+}
+// end of rotate13 implementation
 

-(The rot13 function called on line 42 is trivial and not worth reproducing here.) +(The rot13 function called in Read is trivial and not worth reproducing here.)

To use the new feature, we define a flag:

-

 
-14    var rot13Flag = flag.Bool("rot13", false, "rot13 the input")
+
var rot13Flag = flag.Bool("rot13", false, "rot13 the input")
 

and use it from within a mostly unchanged cat function:

-

 
-52    func cat(r reader) {
-53        const NBUF = 512
-54        var buf [NBUF]byte
+
func cat(r reader) {
+    const NBUF = 512
+    var buf [NBUF]byte
 
-56        if *rot13Flag {
-57            r = newRotate13(r)
-58        }
-59        for {
-60            switch nr, er := r.Read(buf[:]); {
-61            case nr < 0:
-62                fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", r.String(), er.String())
-63                os.Exit(1)
-64            case nr == 0: // EOF
-65                return
-66            case nr > 0:
-67                nw, ew := file.Stdout.Write(buf[0:nr])
-68                if nw != nr {
-69                    fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", r.String(), ew.String())
-70                    os.Exit(1)
-71                }
-72            }
-73        }
-74    }
+    if *rot13Flag {
+        r = newRotate13(r)
+    }
+    for {
+        switch nr, er := r.Read(buf[:]); {
+        case nr < 0:
+            fmt.Fprintf(os.Stderr, "cat: error reading from %s: %s\n", r.String(), er.String())
+            os.Exit(1)
+        case nr == 0: // EOF
+            return
+        case nr > 0:
+            nw, ew := file.Stdout.Write(buf[0:nr])
+            if nw != nr {
+                fmt.Fprintf(os.Stderr, "cat: error writing from %s: %s\n", r.String(), ew.String())
+                os.Exit(1)
+            }
+        }
+    }
+}
 

(We could also do the wrapping in main and leave cat mostly alone, except for changing the type of the argument; consider that an exercise.) -Lines 56 through 58 set it all up: If the rot13 flag is true, wrap the reader +The if at the top of cat sets it all up: If the rot13 flag is true, wrap the reader we received into a rotate13 and proceed. Note that the interface variables are values, not pointers: the argument is of type reader, not *reader, even though under the covers it holds a pointer to a struct. @@ -875,11 +876,11 @@ even though under the covers it holds a pointer to a struct. Here it is in action:

-    $ echo abcdefghijklmnopqrstuvwxyz | ./cat
-    abcdefghijklmnopqrstuvwxyz
-    $ echo abcdefghijklmnopqrstuvwxyz | ./cat --rot13
-    nopqrstuvwxyzabcdefghijklm
-    $
+$ echo abcdefghijklmnopqrstuvwxyz | ./cat
+abcdefghijklmnopqrstuvwxyz
+$ echo abcdefghijklmnopqrstuvwxyz | ./cat --rot13
+nopqrstuvwxyzabcdefghijklm
+$
 

Fans of dependency injection may take cheer from how easily interfaces @@ -895,7 +896,7 @@ implement a writer, or any other interface built from its methods t fits the current situation. Consider the empty interface

-    type Empty interface {}
+type Empty interface {}
 

Every type implements the empty interface, which makes it @@ -910,36 +911,36 @@ same interface variable.

As an example, consider this simple sort algorithm taken from progs/sort.go:

-

 
-13    func Sort(data Interface) {
-14        for i := 1; i < data.Len(); i++ {
-15            for j := i; j > 0 && data.Less(j, j-1); j-- {
-16                data.Swap(j, j-1)
-17            }
-18        }
-19    }
+
func Sort(data Interface) {
+    for i := 1; i < data.Len(); i++ {
+        for j := i; j > 0 && data.Less(j, j-1); j-- {
+            data.Swap(j, j-1)
+        }
+    }
+}
 

The code needs only three methods, which we wrap into sort's Interface:

-

 
-07    type Interface interface {
-08        Len() int
-09        Less(i, j int) bool
-10        Swap(i, j int)
-11    }
+
type Interface interface {
+    Len() int
+    Less(i, j int) bool
+    Swap(i, j int)
+}
 

We can apply Sort to any type that implements Len, Less, and Swap. The sort package includes the necessary methods to allow sorting of arrays of integers, strings, etc.; here's the code for arrays of int

-

 
-33    type IntSlice []int
+
type IntSlice []int
 
-35    func (p IntSlice) Len() int            { return len(p) }
-36    func (p IntSlice) Less(i, j int) bool  { return p[i] < p[j] }
-37    func (p IntSlice) Swap(i, j int)       { p[i], p[j] = p[j], p[i] }
+func (p IntSlice) Len() int            { return len(p) }
+func (p IntSlice) Less(i, j int) bool  { return p[i] < p[j] }
+func (p IntSlice) Swap(i, j int)       { p[i], p[j] = p[j], p[i] }
 

Here we see methods defined for non-struct types. You can define methods @@ -949,34 +950,34 @@ And now a routine to test it out, from progs/sortmain.go. This uses a function in the sort package, omitted here for brevity, to test that the result is sorted.

-

 
-12    func ints() {
-13        data := []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586}
-14        a := sort.IntSlice(data)
-15        sort.Sort(a)
-16        if !sort.IsSorted(a) {
-17            panic("fail")
-18        }
-19    }
+
func ints() {
+    data := []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586}
+    a := sort.IntSlice(data)
+    sort.Sort(a)
+    if !sort.IsSorted(a) {
+        panic("fail")
+    }
+}
 

If we have a new type we want to be able to sort, all we need to do is to implement the three methods for that type, like this:

-

 
-30    type day struct {
-31        num        int
-32        shortName  string
-33        longName   string
-34    }
+
type day struct {
+    num        int
+    shortName  string
+    longName   string
+}
 
-36    type dayArray struct {
-37        data []*day
-38    }
+type dayArray struct {
+    data []*day
+}
 
-40    func (p *dayArray) Len() int            { return len(p.data) }
-41    func (p *dayArray) Less(i, j int) bool  { return p.data[i].num < p.data[j].num }
-42    func (p *dayArray) Swap(i, j int)       { p.data[i], p.data[j] = p.data[j], p.data[i] }
+func (p *dayArray) Len() int            { return len(p.data) }
+func (p *dayArray) Less(i, j int) bool  { return p.data[i].num < p.data[j].num }
+func (p *dayArray) Swap(i, j int)       { p.data[i], p.data[j] = p.data[j], p.data[i] }
 

@@ -990,7 +991,7 @@ implements Printf, Fprintf, and so on. Within the fmt package, Printf is declared with this signature:

-    Printf(format string, v ...interface{}) (n int, errno os.Error)
+Printf(format string, v ...interface{}) (n int, errno os.Error)
 

The token ... introduces a variable-length argument list that in C would @@ -1011,34 +1012,34 @@ argument. It's easier in many cases in Go. Instead of %llud you can just say %d; Printf knows the size and signedness of the integer and can do the right thing for you. The snippet

-

 
-10        var u64 uint64 = 1<<64-1
-11        fmt.Printf("%d %d\n", u64, int64(u64))
+
    var u64 uint64 = 1<<64-1
+    fmt.Printf("%d %d\n", u64, int64(u64))
 

prints

-    18446744073709551615 -1
+18446744073709551615 -1
 

In fact, if you're lazy the format %v will print, in a simple appropriate style, any value, even an array or structure. The output of

-

 
-14        type T struct {
-15            a int
-16            b string
-17        }
-18        t := T{77, "Sunset Strip"}
-19        a := []int{1, 2, 3, 4}
-20        fmt.Printf("%v %v %v\n", u64, t, a)
+
    type T struct {
+        a int
+        b string
+    }
+    t := T{77, "Sunset Strip"}
+    a := []int{1, 2, 3, 4}
+    fmt.Printf("%v %v %v\n", u64, t, a)
 

is

-    18446744073709551615 {77 Sunset Strip} [1 2 3 4]
+18446744073709551615 {77 Sunset Strip} [1 2 3 4]
 

You can drop the formatting altogether if you use Print or Println @@ -1048,9 +1049,9 @@ of %v while Println inserts spaces between arguments and adds a newline. The output of each of these two lines is identical to that of the Printf call above.

-

 
-21        fmt.Print(u64, " ", t, " ", a, "\n")
-22        fmt.Println(u64, t, a)
+
    fmt.Print(u64, " ", t, " ", a, "\n")
+    fmt.Println(u64, t, a)
 

If you have your own type you'd like Printf or Print to format, @@ -1059,27 +1060,27 @@ routines will examine the value to inquire whether it implements the method and if so, use it rather than some other formatting. Here's a simple example.

-

 
-09    type testType struct {
-10        a int
-11        b string
-12    }
+
type testType struct {
+    a int
+    b string
+}
 
-14    func (t *testType) String() string {
-15        return fmt.Sprint(t.a) + " " + t.b
-16    }
+func (t *testType) String() string {
+    return fmt.Sprint(t.a) + " " + t.b
+}
 
-18    func main() {
-19        t := &testType{77, "Sunset Strip"}
-20        fmt.Println(t)
-21    }
+func main() {
+    t := &testType{77, "Sunset Strip"}
+    fmt.Println(t)
+}
 

Since *testType has a String method, the default formatter for that type will use it and produce the output

-    77 Sunset Strip
+77 Sunset Strip
 

Observe that the String method calls Sprint (the obvious Go @@ -1101,18 +1102,18 @@ Schematically, given a value v, it does this:

-    type Stringer interface {
-        String() string
-    }
+type Stringer interface {
+    String() string
+}
 

-    s, ok := v.(Stringer)  // Test whether v implements "String()"
-    if ok {
-        result = s.String()
-    } else {
-        result = defaultOutput(v)
-    }
+s, ok := v.(Stringer)  // Test whether v implements "String()"
+if ok {
+    result = s.String()
+} else {
+    result = defaultOutput(v)
+}
 

The code uses a ``type assertion'' (v.(Stringer)) to test if the value stored in @@ -1133,9 +1134,9 @@ not a file. Instead, it is a variable of type io.Writer, which is interface type defined in the io library:

-    type Writer interface {
-        Write(p []byte) (n int, err os.Error)
-    }
+type Writer interface {
+    Write(p []byte) (n int, err os.Error)
+}
 

(This interface is another conventional name, this time for Write; there are also @@ -1178,13 +1179,13 @@ coordinates the communication; as with maps and slices, use

Here is the first function in progs/sieve.go:

-

 
-09    // Send the sequence 2, 3, 4, ... to channel 'ch'.
-10    func generate(ch chan int) {
-11        for i := 2; ; i++ {
-12            ch <- i  // Send 'i' to channel 'ch'.
-13        }
-14    }
+
// Send the sequence 2, 3, 4, ... to channel 'ch'.
+func generate(ch chan int) {
+    for i := 2; ; i++ {
+        ch <- i  // Send 'i' to channel 'ch'.
+    }
+}
 

The generate function sends the sequence 2, 3, 4, 5, ... to its @@ -1197,17 +1198,17 @@ channel, and a prime number. It copies values from the input to the output, discarding anything divisible by the prime. The unary communications operator <- (receive) retrieves the next value on the channel.

-

 
-16    // Copy the values from channel 'in' to channel 'out',
-17    // removing those divisible by 'prime'.
-18    func filter(in, out chan int, prime int) {
-19        for {
-20            i := <-in  // Receive value of new variable 'i' from 'in'.
-21            if i % prime != 0 {
-22                out <- i  // Send 'i' to channel 'out'.
-23            }
-24        }
-25    }
+
// Copy the values from channel 'in' to channel 'out',
+// removing those divisible by 'prime'.
+func filter(in, out chan int, prime int) {
+    for {
+        i := <-in  // Receive value of new variable 'i' from 'in'.
+        if i % prime != 0 {
+            out <- i  // Send 'i' to channel 'out'.
+        }
+    }
+}
 

The generator and filters execute concurrently. Go has @@ -1219,37 +1220,37 @@ this starts the function running in parallel with the current computation but in the same address space:

-    go sum(hugeArray) // calculate sum in the background
+go sum(hugeArray) // calculate sum in the background
 

If you want to know when the calculation is done, pass a channel on which it can report back:

-    ch := make(chan int)
-    go sum(hugeArray, ch)
-    // ... do something else for a while
-    result := <-ch  // wait for, and retrieve, result
+ch := make(chan int)
+go sum(hugeArray, ch)
+// ... do something else for a while
+result := <-ch  // wait for, and retrieve, result
 

Back to our prime sieve. Here's how the sieve pipeline is stitched together:

-

 
-28    func main() {
-29        ch := make(chan int)  // Create a new channel.
-30        go generate(ch)  // Start generate() as a goroutine.
-31        for i := 0; i < 100; i++ { // Print the first hundred primes.
-32            prime := <-ch
-33            fmt.Println(prime)
-34            ch1 := make(chan int)
-35            go filter(ch, ch1, prime)
-36            ch = ch1
-37        }
-38    }
-
-

-Line 29 creates the initial channel to pass to generate, which it +

func main() {
+    ch := make(chan int)  // Create a new channel.
+    go generate(ch)  // Start generate() as a goroutine.
+    for i := 0; i < 100; i++ { // Print the first hundred primes.
+        prime := <-ch
+        fmt.Println(prime)
+        ch1 := make(chan int)
+        go filter(ch, ch1, prime)
+        ch = ch1
+    }
+}
+
+

+The first line of main creates the initial channel to pass to generate, which it then starts up. As each prime pops out of the channel, a new filter is added to the pipeline and its output becomes the new value of ch. @@ -1258,16 +1259,16 @@ The sieve program can be tweaked to use a pattern common in this style of programming. Here is a variant version of generate, from progs/sieve1.go:

-

 
-10    func generate() chan int {
-11        ch := make(chan int)
-12        go func(){
-13            for i := 2; ; i++ {
-14                ch <- i
-15            }
-16        }()
-17        return ch
-18    }
+
func generate() chan int {
+    ch := make(chan int)
+    go func(){
+        for i := 2; ; i++ {
+            ch <- i
+        }
+    }()
+    return ch
+}
 

This version does all the setup internally. It creates the output @@ -1275,54 +1276,54 @@ channel, launches a goroutine running a function literal, and returns the channel to the caller. It is a factory for concurrent execution, starting the goroutine and returning its connection.

-The function literal notation (lines 12-16) allows us to construct an +The function literal notation used in the go statement allows us to construct an anonymous function and invoke it on the spot. Notice that the local variable ch is available to the function literal and lives on even after generate returns.

The same change can be made to filter:

-

 
-21    func filter(in chan int, prime int) chan int {
-22        out := make(chan int)
-23        go func() {
-24            for {
-25                if i := <-in; i % prime != 0 {
-26                    out <- i
-27                }
-28            }
-29        }()
-30        return out
-31    }
+
func filter(in chan int, prime int) chan int {
+    out := make(chan int)
+    go func() {
+        for {
+            if i := <-in; i % prime != 0 {
+                out <- i
+            }
+        }
+    }()
+    return out
+}
 

The sieve function's main loop becomes simpler and clearer as a result, and while we're at it let's turn it into a factory too:

-

 
-33    func sieve() chan int {
-34        out := make(chan int)
-35        go func() {
-36            ch := generate()
-37            for {
-38                prime := <-ch
-39                out <- prime
-40                ch = filter(ch, prime)
-41            }
-42        }()
-43        return out
-44    }
+
func sieve() chan int {
+    out := make(chan int)
+    go func() {
+        ch := generate()
+        for {
+            prime := <-ch
+            out <- prime
+            ch = filter(ch, prime)
+        }
+    }()
+    return out
+}
 

Now main's interface to the prime sieve is a channel of primes:

-

 
-46    func main() {
-47        primes := sieve()
-48        for i := 0; i < 100; i++ { // Print the first hundred primes.
-49            fmt.Println(<-primes)
-50        }
-51    }
+
func main() {
+    primes := sieve()
+    for i := 0; i < 100; i++ { // Print the first hundred primes.
+        fmt.Println(<-primes)
+    }
+}
 

Multiplexing

@@ -1334,102 +1335,102 @@ A realistic client-server program is a lot of code, so here is a very simple sub to illustrate the idea. It starts by defining a request type, which embeds a channel that will be used for the reply.

-

 
-09    type request struct {
-10        a, b    int
-11        replyc  chan int
-12    }
+
type request struct {
+    a, b    int
+    replyc  chan int
+}
 

The server will be trivial: it will do simple binary operations on integers. Here's the code that invokes the operation and responds to the request:

-

 
-14    type binOp func(a, b int) int
+
type binOp func(a, b int) int
 
-16    func run(op binOp, req *request) {
-17        reply := op(req.a, req.b)
-18        req.replyc <- reply
-19    }
+func run(op binOp, req *request) {
+    reply := op(req.a, req.b)
+    req.replyc <- reply
+}
 

-Line 14 defines the name binOp to be a function taking two integers and +The type declaration makes binOp represent a function taking two integers and returning a third.

The server routine loops forever, receiving requests and, to avoid blocking due to a long-running operation, starting a goroutine to do the actual work.

-

 
-21    func server(op binOp, service chan *request) {
-22        for {
-23            req := <-service
-24            go run(op, req)  // don't wait for it
-25        }
-26    }
+
func server(op binOp, service chan *request) {
+    for {
+        req := <-service
+        go run(op, req)  // don't wait for it
+    }
+}
 

We construct a server in a familiar way, starting it and returning a channel connected to it:

-

 
-28    func startServer(op binOp) chan *request {
-29        req := make(chan *request)
-30        go server(op, req)
-31        return req
-32    }
+
func startServer(op binOp) chan *request {
+    req := make(chan *request)
+    go server(op, req)
+    return req
+}
 

Here's a simple test. It starts a server with an addition operator and sends out N requests without waiting for the replies. Only after all the requests are sent does it check the results.

-

 
-34    func main() {
-35        adder := startServer(func(a, b int) int { return a + b })
-36        const N = 100
-37        var reqs [N]request
-38        for i := 0; i < N; i++ {
-39            req := &reqs[i]
-40            req.a = i
-41            req.b = i + N
-42            req.replyc = make(chan int)
-43            adder <- req
-44        }
-45        for i := N-1; i >= 0; i-- {   // doesn't matter what order
-46            if <-reqs[i].replyc != N + 2*i {
-47                fmt.Println("fail at", i)
-48            }
-49        }
-50        fmt.Println("done")
-51    }
+
func main() {
+    adder := startServer(func(a, b int) int { return a + b })
+    const N = 100
+    var reqs [N]request
+    for i := 0; i < N; i++ {
+        req := &reqs[i]
+        req.a = i
+        req.b = i + N
+        req.replyc = make(chan int)
+        adder <- req
+    }
+    for i := N-1; i >= 0; i-- {   // doesn't matter what order
+        if <-reqs[i].replyc != N + 2*i {
+            fmt.Println("fail at", i)
+        }
+    }
+    fmt.Println("done")
+}
 

One annoyance with this program is that it doesn't shut down the server cleanly; when main returns there are a number of lingering goroutines blocked on communication. To solve this, we can provide a second, quit channel to the server:

-

 
-32    func startServer(op binOp) (service chan *request, quit chan bool) {
-33        service = make(chan *request)
-34        quit = make(chan bool)
-35        go server(op, service, quit)
-36        return service, quit
-37    }
+
func startServer(op binOp) (service chan *request, quit chan bool) {
+    service = make(chan *request)
+    quit = make(chan bool)
+    go server(op, service, quit)
+    return service, quit
+}
 

It passes the quit channel to the server function, which uses it like this:

-

 
-21    func server(op binOp, service chan *request, quit chan bool) {
-22        for {
-23            select {
-24            case req := <-service:
-25                go run(op, req)  // don't wait for it
-26            case <-quit:
-27                return
-28            }
-29        }
-30    }
+
func server(op binOp, service chan *request, quit chan bool) {
+    for {
+        select {
+        case req := <-service:
+            go run(op, req)  // don't wait for it
+        case <-quit:
+            return
+        }
+    }
+}
 

Inside server, the select statement chooses which of the multiple communications @@ -1442,12 +1443,12 @@ returns, terminating its execution. All that's left is to strobe the quit channel at the end of main:

-

 
-40        adder, quit := startServer(func(a, b int) int { return a + b })
+
    adder, quit := startServer(func(a, b int) int { return a + b })
 
... -
 
-55        quit <- true
+
    quit <- true
 

There's a lot more to Go programming and concurrent programming in general but this diff --git a/doc/go_tutorial.txt b/doc/go_tutorial.txt index 7e2bc7c4b..17ef6eee9 100644 --- a/doc/go_tutorial.txt +++ b/doc/go_tutorial.txt @@ -28,7 +28,7 @@ Hello, World Let's start in the usual way: ---PROG progs/helloworld.go /package/ END +!src progs/helloworld.go /package/ $ Every Go source file declares, using a "package" statement, which package it's part of. It may also import other packages to use their facilities. @@ -107,13 +107,13 @@ Echo Next up, here's a version of the Unix utility "echo(1)": ---PROG progs/echo.go /package/ END +!src progs/echo.go /package/ $ This program is small but it's doing a number of new things. In the last example, we saw "func" introduce a function. The keywords "var", "const", and "type" (not used yet) also introduce declarations, as does "import". Notice that we can group declarations of the same sort into -parenthesized lists, one item per line, as on lines 7-10 and 14-17. +parenthesized lists, one item per line, as in the "import" and "const" clauses here. But it's not necessary to do so; we could have said const Space = " " @@ -163,7 +163,7 @@ or we could go even shorter and write the idiom The ":=" operator is used a lot in Go to represent an initializing declaration. There's one in the "for" clause on the next line: ---PROG progs/echo.go /for/ +!src progs/echo.go /for/ The "flag" package has parsed the arguments and left the non-flag arguments in a list that can be iterated over in the obvious way. @@ -210,7 +210,7 @@ Once you've built a string value, you can't change it, although of course you can change a string variable simply by reassigning it. This snippet from "strings.go" is legal code: ---PROG progs/strings.go /hello/ /ciao/ +!src progs/strings.go /hello/ /ciao/ However the following statements are illegal because they would modify a "string" value: @@ -269,7 +269,7 @@ will slice the whole array. Using slices one can write this function (from "sum.go"): ---PROG progs/sum.go /sum/ /^}/ +!src progs/sum.go /sum/ /^}/ Note how the return type ("int") is defined for "sum" by stating it after the parameter list. @@ -386,7 +386,7 @@ An I/O Package Next we'll look at a simple package for doing file I/O with an open/close/read/write interface. Here's the start of "file.go": ---PROG progs/file.go /package/ /^}/ +!src progs/file.go /package/ /^}/ The first few lines declare the name of the package—"file"—and then import two packages. The "os" @@ -416,7 +416,7 @@ will soon give it some exported, upper-case methods. First, though, here is a factory to create a "File": ---PROG progs/file.go /newFile/ /^}/ +!src progs/file.go /newFile/ /^}/ This returns a pointer to a new "File" structure with the file descriptor and name filled in. This code uses Go's notion of a ''composite literal'', analogous to @@ -433,12 +433,12 @@ composite literal, as is done here on line 21. We can use the factory to construct some familiar, exported variables of type "*File": ---PROG progs/file.go /var/ /^.$/ +!src progs/file.go /var/ /^.$/ The "newFile" function was not exported because it's internal. The proper, exported factory to use is "OpenFile" (we'll explain that name in a moment): ---PROG progs/file.go /func.OpenFile/ /^}/ +!src progs/file.go /func.OpenFile/ /^}/ There are a number of new things in these few lines. First, "OpenFile" returns multiple values, a "File" and an error (more about errors in a moment). @@ -468,9 +468,9 @@ the implementation of our "Open" and "Create"; they're trivial wrappers that eliminate common errors by capturing the tricky standard arguments to open and, especially, to create a file: ---PROG progs/file.go /^const/ /^}/ +!src progs/file.go /^const/ /^}/ ---PROG progs/file.go /func.Create/ /^}/ +!src progs/file.go /func.Create/ /^}/ Back to our main story. Now that we can build "Files", we can write methods for them. To declare @@ -479,7 +479,7 @@ of that type, placed in parentheses before the function name. Here are some methods for "*File", each of which declares a receiver variable "file". ---PROG progs/file.go /Close/ END +!src progs/file.go /Close/ $ There is no implicit "this" and the receiver variable must be used to access members of the structure. Methods are not declared within @@ -496,7 +496,7 @@ set of such error values. We can now use our new package: ---PROG progs/helloworld3.go /package/ END +!src progs/helloworld3.go /package/ $ The ''"./"'' in the import of ''"./file"'' tells the compiler to use our own package rather than @@ -520,12 +520,12 @@ Rotting cats Building on the "file" package, here's a simple version of the Unix utility "cat(1)", "progs/cat.go": ---PROG progs/cat.go /package/ END +!src progs/cat.go /package/ $ By now this should be easy to follow, but the "switch" statement introduces some new features. Like a "for" loop, an "if" or "switch" can include an -initialization statement. The "switch" on line 18 uses one to create variables -"nr" and "er" to hold the return values from the call to "f.Read". (The "if" on line 25 +initialization statement. The "switch" statement in "cat" uses one to create variables +"nr" and "er" to hold the return values from the call to "f.Read". (The "if" a few lines later has the same idea.) The "switch" statement is general: it evaluates the cases from top to bottom looking for the first case that matches the value; the case expressions don't need to be constants or even integers, as long as @@ -537,7 +537,7 @@ in a "for" statement, a missing value means "true". In fact, such a "switch" is a form of "if-else" chain. While we're here, it should be mentioned that in "switch" statements each "case" has an implicit "break". -Line 25 calls "Write" by slicing the incoming buffer, which is itself a slice. +The argument to "file.Stdout.Write" is created by slicing the array "buf". Slices provide the standard Go way to handle I/O buffers. Now let's make a variant of "cat" that optionally does "rot13" on its input. @@ -548,7 +548,7 @@ The "cat" subroutine uses only two methods of "f": "Read" and "String", so let's start by defining an interface that has exactly those two methods. Here is code from "progs/cat_rot13.go": ---PROG progs/cat_rot13.go /type.reader/ /^}/ +!src progs/cat_rot13.go /type.reader/ /^}/ Any type that has the two methods of "reader"—regardless of whatever other methods the type may also have—is said to implement the @@ -560,34 +560,32 @@ existing "reader" and does "rot13" on the data. To do this, we just define the type and implement the methods and with no other bookkeeping, we have a second implementation of the "reader" interface. ---PROG progs/cat_rot13.go /type.rotate13/ /end.of.rotate13/ +!src progs/cat_rot13.go /type.rotate13/ /end.of.rotate13/ -(The "rot13" function called on line 42 is trivial and not worth reproducing here.) +(The "rot13" function called in "Read" is trivial and not worth reproducing here.) To use the new feature, we define a flag: ---PROG progs/cat_rot13.go /rot13Flag/ +!src progs/cat_rot13.go /rot13Flag/ and use it from within a mostly unchanged "cat" function: ---PROG progs/cat_rot13.go /func.cat/ /^}/ +!src progs/cat_rot13.go /func.cat/ /^}/ (We could also do the wrapping in "main" and leave "cat" mostly alone, except for changing the type of the argument; consider that an exercise.) -Lines 56 through 58 set it all up: If the "rot13" flag is true, wrap the "reader" +The "if" at the top of "cat" sets it all up: If the "rot13" flag is true, wrap the "reader" we received into a "rotate13" and proceed. Note that the interface variables are values, not pointers: the argument is of type "reader", not "*reader", even though under the covers it holds a pointer to a "struct". Here it is in action: -

 	$ echo abcdefghijklmnopqrstuvwxyz | ./cat
 	abcdefghijklmnopqrstuvwxyz
 	$ echo abcdefghijklmnopqrstuvwxyz | ./cat --rot13
 	nopqrstuvwxyzabcdefghijklm
 	$
-
Fans of dependency injection may take cheer from how easily interfaces allow us to substitute the implementation of a file descriptor. @@ -601,9 +599,7 @@ as we saw with "rot13". The type "file.File" implements "reader"; it could also implement a "writer", or any other interface built from its methods that fits the current situation. Consider the empty interface -
 	type Empty interface {}
-
Every type implements the empty interface, which makes it useful for things like containers. @@ -618,17 +614,17 @@ same interface variable. As an example, consider this simple sort algorithm taken from "progs/sort.go": ---PROG progs/sort.go /func.Sort/ /^}/ +!src progs/sort.go /func.Sort/ /^}/ The code needs only three methods, which we wrap into sort's "Interface": ---PROG progs/sort.go /interface/ /^}/ +!src progs/sort.go /interface/ /^}/ We can apply "Sort" to any type that implements "Len", "Less", and "Swap". The "sort" package includes the necessary methods to allow sorting of arrays of integers, strings, etc.; here's the code for arrays of "int" ---PROG progs/sort.go /type.*IntSlice/ /Swap/ +!src progs/sort.go /type.*IntSlice/ /Swap/ Here we see methods defined for non-"struct" types. You can define methods for any type you define and name in your package. @@ -637,12 +633,12 @@ And now a routine to test it out, from "progs/sortmain.go". This uses a function in the "sort" package, omitted here for brevity, to test that the result is sorted. ---PROG progs/sortmain.go /func.ints/ /^}/ +!src progs/sortmain.go /func.ints/ /^}/ If we have a new type we want to be able to sort, all we need to do is to implement the three methods for that type, like this: ---PROG progs/sortmain.go /type.day/ /Swap/ +!src progs/sortmain.go /type.day/ /Swap/ Printing @@ -675,7 +671,7 @@ argument. It's easier in many cases in Go. Instead of "%llud" you can just say "%d"; "Printf" knows the size and signedness of the integer and can do the right thing for you. The snippet ---PROG progs/print.go 'NR==10' 'NR==11' +!src progs/print.go 10 11 prints @@ -684,7 +680,7 @@ prints In fact, if you're lazy the format "%v" will print, in a simple appropriate style, any value, even an array or structure. The output of ---PROG progs/print.go 'NR==14' 'NR==20' +!src progs/print.go 14 20 is @@ -697,7 +693,7 @@ of "%v" while "Println" inserts spaces between arguments and adds a newline. The output of each of these two lines is identical to that of the "Printf" call above. ---PROG progs/print.go 'NR==21' 'NR==22' +!src progs/print.go 21 22 If you have your own type you'd like "Printf" or "Print" to format, just give it a "String" method that returns a string. The print @@ -705,7 +701,7 @@ routines will examine the value to inquire whether it implements the method and if so, use it rather than some other formatting. Here's a simple example. ---PROG progs/print_string.go 'NR==9' END +!src progs/print_string.go 9 $ Since "*testType" has a "String" method, the default formatter for that type will use it and produce the output @@ -803,7 +799,7 @@ coordinates the communication; as with maps and slices, use Here is the first function in "progs/sieve.go": ---PROG progs/sieve.go /Send/ /^}/ +!src progs/sieve.go /Send/ /^}/ The "generate" function sends the sequence 2, 3, 4, 5, ... to its argument channel, "ch", using the binary communications operator "<-". @@ -815,7 +811,7 @@ channel, and a prime number. It copies values from the input to the output, discarding anything divisible by the prime. The unary communications operator "<-" (receive) retrieves the next value on the channel. ---PROG progs/sieve.go /Copy.the/ /^}/ +!src progs/sieve.go /Copy.the/ /^}/ The generator and filters execute concurrently. Go has its own model of process/threads/light-weight processes/coroutines, @@ -838,9 +834,9 @@ on which it can report back: Back to our prime sieve. Here's how the sieve pipeline is stitched together: ---PROG progs/sieve.go /func.main/ /^}/ +!src progs/sieve.go /func.main/ /^}/ -Line 29 creates the initial channel to pass to "generate", which it +The first line of "main" creates the initial channel to pass to "generate", which it then starts up. As each prime pops out of the channel, a new "filter" is added to the pipeline and its output becomes the new value of "ch". @@ -849,30 +845,30 @@ The sieve program can be tweaked to use a pattern common in this style of programming. Here is a variant version of "generate", from "progs/sieve1.go": ---PROG progs/sieve1.go /func.generate/ /^}/ +!src progs/sieve1.go /func.generate/ /^}/ This version does all the setup internally. It creates the output channel, launches a goroutine running a function literal, and returns the channel to the caller. It is a factory for concurrent execution, starting the goroutine and returning its connection. -The function literal notation (lines 12-16) allows us to construct an +The function literal notation used in the "go" statement allows us to construct an anonymous function and invoke it on the spot. Notice that the local variable "ch" is available to the function literal and lives on even after "generate" returns. The same change can be made to "filter": ---PROG progs/sieve1.go /func.filter/ /^}/ +!src progs/sieve1.go /func.filter/ /^}/ The "sieve" function's main loop becomes simpler and clearer as a result, and while we're at it let's turn it into a factory too: ---PROG progs/sieve1.go /func.sieve/ /^}/ +!src progs/sieve1.go /func.sieve/ /^}/ Now "main"'s interface to the prime sieve is a channel of primes: ---PROG progs/sieve1.go /func.main/ /^}/ +!src progs/sieve1.go /func.main/ /^}/ Multiplexing ---- @@ -884,41 +880,41 @@ A realistic client-server program is a lot of code, so here is a very simple sub to illustrate the idea. It starts by defining a "request" type, which embeds a channel that will be used for the reply. ---PROG progs/server.go /type.request/ /^}/ +!src progs/server.go /type.request/ /^}/ The server will be trivial: it will do simple binary operations on integers. Here's the code that invokes the operation and responds to the request: ---PROG progs/server.go /type.binOp/ /^}/ +!src progs/server.go /type.binOp/ /^}/ -Line 14 defines the name "binOp" to be a function taking two integers and +The type declaration makes "binOp" represent a function taking two integers and returning a third. The "server" routine loops forever, receiving requests and, to avoid blocking due to a long-running operation, starting a goroutine to do the actual work. ---PROG progs/server.go /func.server/ /^}/ +!src progs/server.go /func.server/ /^}/ We construct a server in a familiar way, starting it and returning a channel connected to it: ---PROG progs/server.go /func.startServer/ /^}/ +!src progs/server.go /func.startServer/ /^}/ Here's a simple test. It starts a server with an addition operator and sends out "N" requests without waiting for the replies. Only after all the requests are sent does it check the results. ---PROG progs/server.go /func.main/ /^}/ +!src progs/server.go /func.main/ /^}/ One annoyance with this program is that it doesn't shut down the server cleanly; when "main" returns there are a number of lingering goroutines blocked on communication. To solve this, we can provide a second, "quit" channel to the server: ---PROG progs/server1.go /func.startServer/ /^}/ +!src progs/server1.go /func.startServer/ /^}/ It passes the quit channel to the "server" function, which uses it like this: ---PROG progs/server1.go /func.server/ /^}/ +!src progs/server1.go /func.server/ /^}/ Inside "server", the "select" statement chooses which of the multiple communications listed by its cases can proceed. If all are blocked, it waits until one can proceed; if @@ -930,9 +926,9 @@ returns, terminating its execution. All that's left is to strobe the "quit" channel at the end of main: ---PROG progs/server1.go /adder,.quit/ +!src progs/server1.go /adder,.quit/ ... ---PROG progs/server1.go /quit....true/ +!src progs/server1.go /quit....true/ There's a lot more to Go programming and concurrent programming in general but this quick tour should give you some of the basics. diff --git a/doc/htmlgen.go b/doc/htmlgen.go index 3a8feb8bc..5318a07dc 100644 --- a/doc/htmlgen.go +++ b/doc/htmlgen.go @@ -2,46 +2,80 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Process plain text into HTML. +// If --html is set, process plain text into HTML. // - h2's are made from lines followed by a line "----\n" -// - tab-indented blocks become
 blocks
+//	- tab-indented blocks become 
 blocks with the first tab deleted
 //	- blank lines become 

marks (except inside

 tags)
 //	- "quoted strings" become quoted strings
 
+// Lines beginning !src define pieces of program source to be
+// extracted from other files and injected as 
 blocks.
+// The syntax is simple: 1, 2, or 3 space-separated arguments:
+//
+// Whole file:
+//	!src foo.go
+// One line (here the signature of main):
+//	!src foo.go /^func.main/
+// Block of text, determined by start and end (here the body of main):
+// !src foo.go /^func.main/ /^}/
+//
+// Patterns can be /regular.expression/, a decimal number, or $
+// to signify the end of the file.
+// TODO: the regular expression cannot contain spaces; does this matter?
+
 package main
 
 import (
 	"bufio"
 	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"template"
 )
 
 var (
-	lines = make([][]byte, 0, 2000) // probably big enough; grows if not
+	html = flag.Bool("html", true, "process text into HTML")
+)
+
+var (
+	// lines holds the input and is reworked in place during processing.
+	lines = make([][]byte, 0, 20000)
 
 	empty   = []byte("")
 	newline = []byte("\n")
 	tab     = []byte("\t")
 	quote   = []byte(`"`)
-	indent  = []byte{' ', ' ', ' ', ' '}
+	indent  = []byte("    ")
 
 	sectionMarker = []byte("----\n")
 	preStart      = []byte("
")
 	preEnd        = []byte("
\n") pp = []byte("

\n") + + srcPrefix = []byte("!src") ) func main() { + flag.Parse() read() - headings() - coalesce(preStart, foldPre) - coalesce(tab, foldTabs) - paragraphs() - quotes() + programs() + if *html { + headings() + coalesce(preStart, foldPre) + coalesce(tab, foldTabs) + paragraphs() + quotes() + } write() } +// read turns standard input into a slice of lines. func read() { b := bufio.NewReader(os.Stdin) for { @@ -56,6 +90,7 @@ func read() { } } +// write puts the result on standard output. func write() { b := bufio.NewWriter(os.Stdout) for _, line := range lines { @@ -64,8 +99,104 @@ func write() { b.Flush() } -// each time prefix is found on a line, call fold and replace -// line with return value from fold. +// programs injects source code from !src invocations. +func programs() { + nlines := make([][]byte, 0, len(lines)*3/2) + for _, line := range lines { + if bytes.HasPrefix(line, srcPrefix) { + line = trim(line)[len(srcPrefix):] + prog := srcCommand(string(line)) + if *html { + nlines = append(nlines, []byte(fmt.Sprintf("

", line)))
+			}
+			for _, l := range prog {
+				nlines = append(nlines, htmlEscape(l))
+			}
+			if *html {
+				nlines = append(nlines, preEnd)
+			}
+		} else {
+			nlines = append(nlines, line)
+		}
+	}
+	lines = nlines
+}
+
+// srcCommand processes one !src invocation.
+func srcCommand(command string) [][]byte {
+	// TODO: quoted args so we can have 'a b'?
+	args := strings.Fields(command)
+	if len(args) == 0 || len(args) > 3 {
+		log.Fatal("bad syntax for src command: %s", command)
+	}
+	file := args[0]
+	lines := bytes.SplitAfter(readFile(file), newline)
+	// File plus zero args: whole file:
+	//	!src file.go
+	if len(args) == 1 {
+		return lines
+	}
+	start := match(file, 0, lines, string(args[1]))
+	// File plus one arg: one line:
+	//	!src file.go /foo/
+	if len(args) == 2 {
+		return [][]byte{lines[start]}
+	}
+	// File plus two args: range:
+	//	!src file.go /foo/ /^}/
+	end := match(file, start, lines, string(args[2]))
+	return lines[start : end+1] // +1 to include matched line.
+}
+
+// htmlEscape makes sure input is HTML clean, if necessary.
+func htmlEscape(input []byte) []byte {
+	if !*html || bytes.IndexAny(input, `&"<>`) < 0 {
+		return input
+	}
+	var b bytes.Buffer
+	template.HTMLEscape(&b, input)
+	return b.Bytes()
+}
+
+// readFile reads and returns a file as part of !src processing.
+func readFile(name string) []byte {
+	file, err := ioutil.ReadFile(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return file
+}
+
+// match identifies the input line that matches the pattern in a !src invocation.
+// If start>0, match lines starting there rather than at the beginning.
+func match(file string, start int, lines [][]byte, pattern string) int {
+	// $ matches the end of the file.
+	if pattern == "$" {
+		return len(lines) - 1
+	}
+	// Number matches the line.
+	if i, err := strconv.Atoi(pattern); err == nil {
+		return i - 1 // Lines are 1-indexed.
+	}
+	// /regexp/ matches the line that matches the regexp.
+	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
+		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
+		if err != nil {
+			log.Fatal(err)
+		}
+		for i := start; i < len(lines); i++ {
+			if re.Match(lines[i]) {
+				return i
+			}
+		}
+		log.Fatalf("%s: no match for %s", file, pattern)
+	}
+	log.Fatalf("unrecognized pattern: %s", pattern)
+	return 0
+}
+
+// coalesce combines lines. Each time prefix is found on a line,
+// it calls fold and replaces the line with return value from fold.
 func coalesce(prefix []byte, fold func(i int) (n int, line []byte)) {
 	j := 0 // output line number goes up by one each loop
 	for i := 0; i < len(lines); {
@@ -82,7 +213,7 @@ func coalesce(prefix []byte, fold func(i int) (n int, line []byte)) {
 	lines = lines[0:j]
 }
 
-// return the 
 block as a single slice
+// foldPre returns the 
 block as a single slice.
 func foldPre(i int) (n int, line []byte) {
 	buf := new(bytes.Buffer)
 	for i < len(lines) {
@@ -96,7 +227,7 @@ func foldPre(i int) (n int, line []byte) {
 	return n, buf.Bytes()
 }
 
-// return the tab-indented block as a single 
-bounded slice
+// foldTabs returns the tab-indented block as a single 
-bounded slice.
 func foldTabs(i int) (n int, line []byte) {
 	buf := new(bytes.Buffer)
 	buf.WriteString("
\n")
@@ -104,7 +235,7 @@ func foldTabs(i int) (n int, line []byte) {
 		if !bytes.HasPrefix(lines[i], tab) {
 			break
 		}
-		buf.Write(lines[i])
+		buf.Write(lines[i][1:]) // delete leading tab.
 		n++
 		i++
 	}
@@ -112,6 +243,7 @@ func foldTabs(i int) (n int, line []byte) {
 	return n, buf.Bytes()
 }
 
+// headings turns sections into HTML sections.
 func headings() {
 	b := bufio.NewWriter(os.Stdout)
 	for i, l := range lines {
@@ -123,6 +255,7 @@ func headings() {
 	b.Flush()
 }
 
+// paragraphs turns blank lines into paragraph marks.
 func paragraphs() {
 	for i, l := range lines {
 		if bytes.Equal(l, newline) {
@@ -131,12 +264,14 @@ func paragraphs() {
 	}
 }
 
+// quotes turns "x" in the file into x.
 func quotes() {
 	for i, l := range lines {
 		lines[i] = codeQuotes(l)
 	}
 }
 
+// quotes turns "x" in the line into x.
 func codeQuotes(l []byte) []byte {
 	if bytes.HasPrefix(l, preStart) {
 		return l
@@ -162,7 +297,7 @@ func codeQuotes(l []byte) []byte {
 	return buf.Bytes()
 }
 
-// drop trailing newline
+// trim drops the trailing newline, if present.
 func trim(l []byte) []byte {
 	n := len(l)
 	if n > 0 && l[n-1] == '\n' {
@@ -171,7 +306,7 @@ func trim(l []byte) []byte {
 	return l
 }
 
-// expand tabs to spaces. don't worry about columns.
+// expandTabs expands tabs to spaces. It doesn't worry about columns.
 func expandTabs(l []byte) []byte {
 	return bytes.Replace(l, tab, indent, -1)
 }
diff --git a/doc/install.html b/doc/install.html
index 2256123ec..a1bc89982 100644
--- a/doc/install.html
+++ b/doc/install.html
@@ -81,8 +81,8 @@ To build it, you need these programs installed:
 
  • the standard C libraries,
  • the parser generator Bison,
  • GNU make (version 3.81 or later), -
  • awk, and -
  • the text editor ed. +and +
  • awk.

    @@ -91,7 +91,7 @@ installed as part of Xcode.

    -

    On Ubuntu/Debian, use sudo apt-get install bison ed gawk gcc libc6-dev +

    On Ubuntu/Debian, use sudo apt-get install bison gawk gcc libc6-dev make. If you want to build 32-bit binaries on a 64-bit system you'll also need the libc6-dev-i386 package.

    diff --git a/doc/makehtml b/doc/makehtml index c9ac0c8e8..1b8caed69 100755 --- a/doc/makehtml +++ b/doc/makehtml @@ -7,7 +7,6 @@ set -e TXT=${1:-go_tutorial.txt} # input file HTML=$(basename $TXT .txt).html # output file (basename) -TMP=TEMP.txt # input to htmlgen if ! test -w $HTML then @@ -15,17 +14,4 @@ then exit 1 fi -if grep -q '^--PROG' $TXT -then - echo >&2 makehtml: processing PROG sections - <$TXT >$TMP awk ' - /^--PROG/ { system("sh ./prog.sh "$2" "$3" "$4" "); getline } - /^/ {print} - ' -else - cp $TXT $TMP -fi - -make htmlgen && ./htmlgen < $TMP > $HTML - -rm -f $TMP +make htmlgen && ./htmlgen < $TXT > $HTML diff --git a/doc/prog.sh b/doc/prog.sh deleted file mode 100755 index 6a540980a..000000000 --- a/doc/prog.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# generate HTML for a program excerpt. -# first arg is file name -# second arg is awk pattern to match start line -# third arg is awk pattern to stop processing -# -# missing third arg means print one line -# third arg "END" means proces rest of file -# missing second arg means process whole file -# -# examples: -# -# prog.sh foo.go # whole file -# prog.sh foo.go "/^func.main/" # signature of main -# prog.sh foo.go "/^func.main/" "/^}/ # body of main -# -# non-blank lines are annotated with line number in file - -# line numbers are printed %.2d to make them equal-width for nice formatting. -# the format gives a leading 0. the format %2d gives a leading space but -# that appears to confuse sanjay's makehtml formatter into bungling quotes -# because it makes some lines look indented. - -echo "
     "
    -
    -case $# in
    -3)
    -	if test "$3" = "END"  # $2 to end of file
    -	then
    -		awk '
    -			function LINE() { printf("%.2d\t%s\n", NR, $0) }
    -			BEGIN { printing = 0 }
    -			'$2' { printing = 1; LINE(); getline }
    -			printing { if($0 ~ /./) { LINE() } else { print "" } }
    -		'
    -	else	# $2 through $3
    -		awk '
    -			function LINE() { printf("%.2d\t%s\n", NR, $0) }
    -			BEGIN { printing = 0 }
    -			'$2' { printing = 1; LINE(); getline }
    -			'$3' && printing { if(printing) {printing = 0; LINE(); exit} }
    -			printing { if($0 ~ /./) { LINE() } else { print "" } }
    -		'
    -	fi
    -	;;
    -2)	# one line
    -	awk '
    -		function LINE() { printf("%.2d\t%s\n", NR, $0) }
    -		'$2' { LINE(); getline; exit }
    -	'
    -	;;
    -1)	# whole file
    -	awk '
    -		function LINE() { printf("%.2d\t%s\n", NR, $0) }
    -		{ if($0 ~ /./) { LINE() } else { print "" } }
    -	'
    -	;;
    -*)
    -	echo >&2 usage: prog.sh file.go /func.main/ /^}/
    -esac <$1 |
    -sed '
    -	s/&/\&/g
    -	s/"/\"/g
    -	s//\>/g
    -'
    -
    -echo '
    ' diff --git a/doc/progs/file_windows.go b/doc/progs/file_windows.go new file mode 100644 index 000000000..d5e7c00d3 --- /dev/null +++ b/doc/progs/file_windows.go @@ -0,0 +1,89 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package file + +import ( + "os" + "syscall" +) + +type File struct { + fd syscall.Handle // file descriptor number + name string // file name at Open time +} + +func newFile(fd syscall.Handle, name string) *File { + if fd < 0 { + return nil + } + return &File{fd, name} +} + +var ( + Stdin = newFile(syscall.Stdin, "/dev/stdin") + Stdout = newFile(syscall.Stdout, "/dev/stdout") + Stderr = newFile(syscall.Stderr, "/dev/stderr") +) + +func OpenFile(name string, mode int, perm uint32) (file *File, err os.Error) { + r, e := syscall.Open(name, mode, perm) + if e != 0 { + err = os.Errno(e) + } + return newFile(r, name), err +} + +const ( + O_RDONLY = syscall.O_RDONLY + O_RDWR = syscall.O_RDWR + O_CREATE = syscall.O_CREAT + O_TRUNC = syscall.O_TRUNC +) + +func Open(name string) (file *File, err os.Error) { + return OpenFile(name, O_RDONLY, 0) +} + +func Create(name string) (file *File, err os.Error) { + return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) +} + +func (file *File) Close() os.Error { + if file == nil { + return os.EINVAL + } + e := syscall.Close(file.fd) + file.fd = syscall.InvalidHandle // so it can't be closed again + if e != 0 { + return os.Errno(e) + } + return nil +} + +func (file *File) Read(b []byte) (ret int, err os.Error) { + if file == nil { + return -1, os.EINVAL + } + r, e := syscall.Read(file.fd, b) + if e != 0 { + err = os.Errno(e) + } + return int(r), err +} + +func (file *File) Write(b []byte) (ret int, err os.Error) { + if file == nil { + return -1, os.EINVAL + } + r, e := syscall.Write(file.fd, b) + if e != 0 { + err = os.Errno(e) + } + return int(r), err +} + +func (file *File) String() string { + return file.name +} diff --git a/doc/progs/run b/doc/progs/run index 241e65dfa..81781c9d2 100755 --- a/doc/progs/run +++ b/doc/progs/run @@ -14,8 +14,13 @@ fi rm -f *.$O +if [ "$GOOS" = "windows" ];then + $GC -o file.8 file_windows.go +else + $GC file.go +fi + for i in \ - file.go \ helloworld.go \ helloworld3.go \ echo.go \ diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py index a222919d8..385ac2c06 100644 --- a/lib/codereview/codereview.py +++ b/lib/codereview/codereview.py @@ -1316,7 +1316,7 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): # Create fresh CL and start with patch that would reverse the change. vers = short(rev.node()) cl = CL("new") - desc = rev.description() + desc = str(rev.description()) if mode == "undo": cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter else: @@ -1352,10 +1352,12 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): repo[vers].description() except: return "local repository is out of date; sync to get %s" % (vers) - patch, err = portPatch(repo, patch, vers, id) + patch1, err = portPatch(repo, patch, vers, id) if err != "": - return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id) - + if not opts["ignore_hgpatch_failure"]: + return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id) + else: + patch = patch1 argv = ["hgpatch"] if opts["no_incoming"] or mode == "backport": argv += ["--checksync=false"] @@ -1369,7 +1371,7 @@ def clpatch_or_undo(ui, repo, clname, opts, mode): return "hgpatch failed" cl.local = True cl.files = out.strip().split() - if not cl.files: + if not cl.files and not opts["ignore_hgpatch_failure"]: return "codereview issue %s has no changed files" % clname files = ChangedFiles(ui, repo, [], opts) extra = Sub(cl.files, files) @@ -1781,7 +1783,7 @@ def sync(ui, repo, **opts): err = commands.postincoming(ui, repo, modheads, True, "tip") if err: return err - commands.update(ui, repo) + commands.update(ui, repo, rev="default") sync_changes(ui, repo) def sync_note(msg): diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index 5e1da0c87..98400c51a 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -112,16 +112,15 @@ func packages() (pkgs []string, err os.Error) { return } -// updatePackage sends package build results and info to the dashboard -func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error { +// updatePackage sends package build results and info dashboard +func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) os.Error { return dash("POST", "package", nil, param{ "builder": b.name, "key": b.key, "path": pkg, - "state": strconv.Btoa(state), + "ok": strconv.Btoa(ok), "log": buildLog, "info": info, - "go_rev": hash[:12], }) } diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index 9a714fe79..989965bc4 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -60,8 +60,9 @@ var ( ) var ( - goroot string - releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`) + goroot string + binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) + releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) ) func main() { @@ -161,7 +162,7 @@ func NewBuilder(builder string) (*Builder, os.Error) { b := &Builder{name: builder} // get goos/goarch from builder string - s := strings.Split(builder, "-", 3) + s := strings.SplitN(builder, "-", 3) if len(s) >= 2 { b.goos, b.goarch = s[0], s[1] } else { @@ -177,7 +178,7 @@ func NewBuilder(builder string) (*Builder, os.Error) { if err != nil { return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err) } - v := strings.Split(string(c), "\n", -1) + v := strings.Split(string(c), "\n") b.key = v[0] if len(v) >= 3 { b.codeUsername, b.codePassword = v[1], v[2] @@ -200,7 +201,7 @@ func (b *Builder) buildExternal() { log.Println("hg pull failed:", err) continue } - hash, tag, err := firstTag(releaseRegexp) + hash, tag, err := firstTag(releaseRe) if err != nil { log.Println(err) continue @@ -321,7 +322,7 @@ func (b *Builder) buildHash(hash string) (err os.Error) { } // if this is a release, create tgz and upload to google code - releaseHash, release, err := firstTag(releaseRegexp) + releaseHash, release, err := firstTag(binaryTagRe) if hash == releaseHash { // clean out build state err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") @@ -392,7 +393,7 @@ func (b *Builder) envvWindows() []string { skip[name] = true } for _, kv := range os.Environ() { - s := strings.Split(kv, "=", 2) + s := strings.SplitN(kv, "=", 2) name := strings.ToUpper(s[0]) switch { case name == "": @@ -591,7 +592,7 @@ func fullHash(rev string) (hash string, err os.Error) { if s == "" { return "", fmt.Errorf("cannot find revision") } - if len(s) != 20 { + if len(s) != 40 { return "", fmt.Errorf("hg returned invalid hash " + s) } return s, nil @@ -602,7 +603,7 @@ var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) // firstTag returns the hash and tag of the most recent tag matching re. func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n", -1) { + for _, l := range strings.Split(o, "\n") { if l == "" { continue } @@ -615,7 +616,7 @@ func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { continue } tag = s[1] - hash, err = fullHash(s[3]) + hash, err = fullHash(s[2]) return } err = os.NewError("no matching tag found") diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go index ee65d7669..b6674428d 100644 --- a/misc/dashboard/builder/package.go +++ b/misc/dashboard/builder/package.go @@ -10,35 +10,47 @@ import ( "go/token" "log" "os" - "path" + "path/filepath" + "strings" ) +const MaxCommentLength = 500 // App Engine won't store more in a StringProperty. + func (b *Builder) buildPackages(workpath string, hash string) os.Error { pkgs, err := packages() if err != nil { return err } for _, p := range pkgs { - goroot := path.Join(workpath, "go") - goinstall := path.Join(goroot, "bin", "goinstall") + goroot := filepath.Join(workpath, "go") + gobin := filepath.Join(goroot, "bin") + goinstall := filepath.Join(gobin, "goinstall") envv := append(b.envv(), "GOROOT="+goroot) + // add GOBIN to path + for i, v := range envv { + if strings.HasPrefix(v, "PATH=") { + p := filepath.SplitList(v[5:]) + p = append([]string{gobin}, p...) + s := strings.Join(p, string(filepath.ListSeparator)) + envv[i] = "PATH=" + s + } + } + // goinstall - buildLog, code, err := runLog(envv, "", goroot, goinstall, p) + buildLog, code, err := runLog(envv, "", goroot, goinstall, "-log=false", p) if err != nil { log.Printf("goinstall %v: %v", p, err) - continue } - built := code != 0 // get doc comment from package source - info, err := packageComment(p, path.Join(goroot, "pkg", p)) + info, err := packageComment(p, filepath.Join(goroot, "src", "pkg", p)) if err != nil { - log.Printf("goinstall %v: %v", p, err) + log.Printf("packageComment %v: %v", p, err) } // update dashboard with build state + info - err = b.updatePackage(p, built, buildLog, info, hash) + err = b.updatePackage(p, code == 0, buildLog, info) if err != nil { log.Printf("updatePackage %v: %v", p, err) } @@ -46,9 +58,15 @@ func (b *Builder) buildPackages(workpath string, hash string) os.Error { return nil } +func isGoFile(fi *os.FileInfo) bool { + return fi.IsRegular() && // exclude directories + !strings.HasPrefix(fi.Name, ".") && // ignore .files + filepath.Ext(fi.Name) == ".go" +} + func packageComment(pkg, pkgpath string) (info string, err os.Error) { fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments) + pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments) if err != nil { return } @@ -62,5 +80,15 @@ func packageComment(pkg, pkgpath string) (info string, err os.Error) { pdoc := doc.NewPackageDoc(pkgs[name], pkg) info = pdoc.Doc } + // grab only first paragraph + if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 { + info = parts[0] + } + // replace newlines with spaces + info = strings.Replace(info, "\n", " ", -1) + // truncate + if len(info) > MaxCommentLength { + info = info[:MaxCommentLength] + } return } diff --git a/misc/dashboard/godashboard/auth.py b/misc/dashboard/godashboard/auth.py new file mode 100644 index 000000000..73a54c0d4 --- /dev/null +++ b/misc/dashboard/godashboard/auth.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import hmac + +# local imports +import key + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 5678f2e1b..685dc83a9 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -14,14 +14,13 @@ from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app import datetime import hashlib -import hmac import logging import os import re import bz2 # local imports -import key +from auth import auth import const # The majority of our state are commit objects. One of these exists for each of @@ -142,10 +141,6 @@ class DashboardHandler(webapp.RequestHandler): simplejson.dump(obj, self.response.out) return -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - # Todo serves /todo. It tells the builder which commits need to be built. class Todo(DashboardHandler): def get(self): diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml index 4a00c4a6f..f39299d5d 100644 --- a/misc/dashboard/godashboard/index.yaml +++ b/misc/dashboard/godashboard/index.yaml @@ -49,4 +49,3 @@ indexes: # manually, move them above the marker line. The index.yaml file is # automatically uploaded to the admin console when you next deploy # your application using appcfg.py. - diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html index 9332b5a79..043080b5b 100644 --- a/misc/dashboard/godashboard/package.html +++ b/misc/dashboard/godashboard/package.html @@ -19,37 +19,43 @@ Packages listed on this page are written by third parties and may or may not build or be safe to use.

    + +

    + An "ok" in the build column indicates that the package is + goinstallable + with the latest + release of Go. +

    + +

    + The info column shows the first paragraph from the + package doc comment. +

    Recently Installed Packages

    - + {% for r in by_time %} + - + {% endfor %}
    last installcountpathproject
    last installcountbuildpathinfo
    {{r.last_install|date:"Y-M-d H:i"}} {{r.count}}{% if r.ok %}ok{% else %} {% endif %} {{r.path}} - {% for p in r.project_set %} - {{p.name}} - {{p.descr}} - {% endfor %} - {% if r.info %}{{r.info|escape}}{% endif %}

    Most Installed Packages

    - + {% for r in by_count %} + - + {% endfor %}
    last installcountpathproject
    last installcountbuildpathinfo
    {{r.last_install|date:"Y-M-d H:i"}} {{r.count}}{% if r.ok %}ok{% else %} {% endif %} {{r.path}} - {% for p in r.project_set %} - {{p.name}} - {{p.descr}} - {% endfor %} - {% if r.info %}{{r.info|escape}}{% endif %}
    diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py index dd09593ac..316f3867f 100644 --- a/misc/dashboard/godashboard/package.py +++ b/misc/dashboard/godashboard/package.py @@ -23,6 +23,7 @@ import sets # local imports import toutf8 import const +from auth import auth template.register_template_library('toutf8') @@ -34,6 +35,11 @@ class Package(db.Model): count = db.IntegerProperty() last_install = db.DateTimeProperty() + # data contributed by gobuilder + info = db.StringProperty() + ok = db.BooleanProperty() + last_ok = db.DateTimeProperty() + class Project(db.Model): name = db.StringProperty(indexed=True) descr = db.StringProperty() @@ -43,22 +49,25 @@ class Project(db.Model): tags = db.ListProperty(str) approved = db.BooleanProperty(indexed=True) -re_bitbucket = re.compile(r'^bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$') -re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)$') +re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') +re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)(/[a-z0-9A-Z_.\-/]+)?$') re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') - def vc_to_web(path): if re_bitbucket.match(path): - check_url = 'http://' + path + '/?cmd=heads' - web = 'http://' + path + '/' + m = re_bitbucket.match(path) + check_url = 'http://' + m.group(1) + '/?cmd=heads' + web = 'http://' + m.group(1) + '/' elif re_github.match(path): m = re_github_web.match(path) check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' - web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' elif re_googlecode.match(path): + m = re_googlecode.match(path) check_url = 'http://'+path + if not m.group(2): # append / after bare '/hg' + check_url += '/' web = 'http://code.google.com/p/' + path[:path.index('.')] elif re_launchpad.match(path): check_url = web = 'https://'+path @@ -142,8 +151,7 @@ class PackagePage(webapp.RequestHandler): def can_get_url(self, url): try: - req = urllib2.Request(url) - response = urllib2.urlopen(req) + urllib2.urlopen(urllib2.Request(url)) return True except: return False @@ -173,15 +181,23 @@ class PackagePage(webapp.RequestHandler): return False p = Package(key_name = key, path = path, count = 0, web_url = web) + # is this the builder updating package metadata? + if auth(self.request): + p.info = self.request.get('info') + p.ok = self.request.get('ok') == "true" + if p.ok: + p.last_ok = datetime.datetime.utcnow() + else: + p.count += 1 + p.last_install = datetime.datetime.utcnow() + # update package object - p.count += 1 - p.last_install = datetime.datetime.utcnow() p.put() return True def post(self): path = self.request.get('path') - ok = self.record_pkg(path) + ok = db.run_in_transaction(self.record_pkg, path) if ok: self.response.set_status(200) self.response.out.write('ok') diff --git a/misc/dashboard/godashboard/static/style.css b/misc/dashboard/godashboard/static/style.css index 481af36d7..a7e61dda5 100644 --- a/misc/dashboard/godashboard/static/style.css +++ b/misc/dashboard/godashboard/static/style.css @@ -52,7 +52,7 @@ table.alternate tr td:last-child { padding-right: 0; } table.alternate tr:nth-child(2n) { - background-color: #f8f8f8; + background-color: #f0f0f0; } span.hash { font-family: monospace; @@ -62,10 +62,19 @@ span.hash { td.date { color: #aaa; } -td.result { +td.ok { text-align: center; + color: #060; + font-weight: bold; +} +td.ok a { + cursor: help; +} +th { + text-align: left; } th.builder { + text-align: center; font-weight: bold; } a.fail { diff --git a/misc/emacs/go-mode.el b/misc/emacs/go-mode.el index 03f0a2a8b..ba7f72397 100644 --- a/misc/emacs/go-mode.el +++ b/misc/emacs/go-mode.el @@ -69,8 +69,8 @@ some syntax analysis.") (defvar go-mode-font-lock-keywords - (let ((builtins '("cap" "close" "closed" "len" "make" "new" - "panic" "panicln" "print" "println")) + (let ((builtins '("append" "cap" "close" "complex" "copy" "imag" "len" + "make" "new" "panic" "print" "println" "real" "recover")) (constants '("nil" "true" "false" "iota")) (type-name "\\s *\\(?:[*(]\\s *\\)*\\(?:\\w+\\s *\\.\\s *\\)?\\(\\w+\\)") ) diff --git a/misc/vim/ftdetect/gofiletype.vim b/misc/vim/ftdetect/gofiletype.vim index 884312160..f03a1d8dc 100644 --- a/misc/vim/ftdetect/gofiletype.vim +++ b/misc/vim/ftdetect/gofiletype.vim @@ -1 +1 @@ -au BufRead,BufNewFile *.go set filetype=go +au BufReadPre,BufNewFile *.go set filetype=go fileencoding=utf-8 fileencodings=utf-8 diff --git a/src/cmd/5c/gc.h b/src/cmd/5c/gc.h index 549e0c88a..ff6d51916 100644 --- a/src/cmd/5c/gc.h +++ b/src/cmd/5c/gc.h @@ -28,7 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. - +#include #include "../cc/cc.h" #include "../5l/5.out.h" diff --git a/src/cmd/5l/asm.c b/src/cmd/5l/asm.c index 4afed2b80..2c9e50d00 100644 --- a/src/cmd/5l/asm.c +++ b/src/cmd/5l/asm.c @@ -68,9 +68,6 @@ enum { ElfStrText, ElfStrData, ElfStrBss, - ElfStrGosymcounts, - ElfStrGosymtab, - ElfStrGopclntab, ElfStrSymtab, ElfStrStrtab, ElfStrShstrtab, @@ -160,12 +157,11 @@ doelf(void) elfstr[ElfStrEmpty] = addstring(shstrtab, ""); elfstr[ElfStrText] = addstring(shstrtab, ".text"); elfstr[ElfStrData] = addstring(shstrtab, ".data"); - addstring(shstrtab, ".rodata"); elfstr[ElfStrBss] = addstring(shstrtab, ".bss"); + addstring(shstrtab, ".rodata"); + addstring(shstrtab, ".gosymtab"); + addstring(shstrtab, ".gopclntab"); if(!debug['s']) { - elfstr[ElfStrGosymcounts] = addstring(shstrtab, ".gosymcounts"); - elfstr[ElfStrGosymtab] = addstring(shstrtab, ".gosymtab"); - elfstr[ElfStrGopclntab] = addstring(shstrtab, ".gopclntab"); elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); } @@ -307,10 +303,11 @@ asmb(void) seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); codeblk(sect->vaddr, sect->len); - /* output read-only data in text segment */ - sect = segtext.sect->next; - seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); - datblk(sect->vaddr, sect->len); + /* output read-only data in text segment (rodata, gosymtab and pclntab) */ + for(sect = sect->next; sect != nil; sect = sect->next) { + seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); + datblk(sect->vaddr, sect->len); + } if(debug['v']) Bprint(&bso, "%5.2f datblk\n", cputime()); @@ -572,18 +569,6 @@ asmb(void) elfshbits(sect); if (!debug['s']) { - sh = newElfShdr(elfstr[ElfStrGosymtab]); - sh->type = SHT_PROGBITS; - sh->flags = SHF_ALLOC; - sh->addralign = 1; - shsym(sh, lookup("symtab", 0)); - - sh = newElfShdr(elfstr[ElfStrGopclntab]); - sh->type = SHT_PROGBITS; - sh->flags = SHF_ALLOC; - sh->addralign = 1; - shsym(sh, lookup("pclntab", 0)); - sh = newElfShdr(elfstr[ElfStrSymtab]); sh->type = SHT_SYMTAB; sh->off = symo; diff --git a/src/cmd/5l/mkenam b/src/cmd/5l/mkenam index 265cb9988..6cccb0263 100644 --- a/src/cmd/5l/mkenam +++ b/src/cmd/5l/mkenam @@ -28,18 +28,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -ed - ../5l/5.out.h <<'!' -v/^ A/d -,s/^ A/ "/ -g/ .*$/s/// -,s/,*$/",/ -1i -char* anames[] = -{ -. -$a -}; -. -w enam.c -Q -! +awk ' +BEGIN { + print "char* anames[] =" + print "{" +} + +/^ A/ { + name=$1 + sub(/,/, "", name) + sub(/^A/, "", name) + print "\t\"" name "\"," +} + +END { print "};" } +' ../5l/5.out.h >enam.c diff --git a/src/cmd/6c/gc.h b/src/cmd/6c/gc.h index 735cd8909..775d97281 100644 --- a/src/cmd/6c/gc.h +++ b/src/cmd/6c/gc.h @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "../cc/cc.h" #include "../6l/6.out.h" diff --git a/src/cmd/6l/asm.c b/src/cmd/6l/asm.c index 4c04112b7..9136e0379 100644 --- a/src/cmd/6l/asm.c +++ b/src/cmd/6l/asm.c @@ -87,9 +87,6 @@ enum { ElfStrText, ElfStrData, ElfStrBss, - ElfStrGosymcounts, - ElfStrGosymtab, - ElfStrGopclntab, ElfStrShstrtab, ElfStrSymtab, ElfStrStrtab, @@ -571,10 +568,9 @@ doelf(void) elfstr[ElfStrBss] = addstring(shstrtab, ".bss"); addstring(shstrtab, ".elfdata"); addstring(shstrtab, ".rodata"); + addstring(shstrtab, ".gosymtab"); + addstring(shstrtab, ".gopclntab"); if(!debug['s']) { - elfstr[ElfStrGosymcounts] = addstring(shstrtab, ".gosymcounts"); - elfstr[ElfStrGosymtab] = addstring(shstrtab, ".gosymtab"); - elfstr[ElfStrGopclntab] = addstring(shstrtab, ".gopclntab"); elfstr[ElfStrSymtab] = addstring(shstrtab, ".symtab"); elfstr[ElfStrStrtab] = addstring(shstrtab, ".strtab"); dwarfaddshstrings(shstrtab); @@ -718,10 +714,11 @@ asmb(void) seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); codeblk(sect->vaddr, sect->len); - /* output read-only data in text segment */ - sect = segtext.sect->next; - seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); - datblk(sect->vaddr, sect->len); + /* output read-only data in text segment (rodata, gosymtab and pclntab) */ + for(sect = sect->next; sect != nil; sect = sect->next) { + seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); + datblk(sect->vaddr, sect->len); + } if(debug['v']) Bprint(&bso, "%5.2f datblk\n", cputime()); @@ -1013,18 +1010,6 @@ asmb(void) elfshbits(sect); if (!debug['s']) { - sh = newElfShdr(elfstr[ElfStrGosymtab]); - sh->type = SHT_PROGBITS; - sh->flags = SHF_ALLOC; - sh->addralign = 1; - shsym(sh, lookup("symtab", 0)); - - sh = newElfShdr(elfstr[ElfStrGopclntab]); - sh->type = SHT_PROGBITS; - sh->flags = SHF_ALLOC; - sh->addralign = 1; - shsym(sh, lookup("pclntab", 0)); - sh = newElfShdr(elfstr[ElfStrSymtab]); sh->type = SHT_SYMTAB; sh->off = symo; diff --git a/src/cmd/6l/mkenam b/src/cmd/6l/mkenam index 5cabb2633..3001dbe93 100644 --- a/src/cmd/6l/mkenam +++ b/src/cmd/6l/mkenam @@ -28,18 +28,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -/bin/ed - ../6l/6.out.h <<'!' -v/^ A/d -,s/^ A/ "/ -g/ .*$/s/// -,s/,*$/",/ -1i -char* anames[] = -{ -. -$a -}; -. -w enam.c -Q -! +awk ' +BEGIN { + print "char* anames[] =" + print "{" +} + +/^ A/ { + name=$1 + sub(/,/, "", name) + sub(/^A/, "", name) + print "\t\"" name "\"," +} + +END { print "};" } +' ../6l/6.out.h >enam.c diff --git a/src/cmd/8a/a.h b/src/cmd/8a/a.h index 3cb30f4c2..c5c22d7ba 100644 --- a/src/cmd/8a/a.h +++ b/src/cmd/8a/a.h @@ -28,8 +28,6 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -#include -#include #include #include "../8l/8.out.h" @@ -57,7 +55,9 @@ typedef struct Gen2 Gen2; #define NSYMB 500 #define BUFSIZ 8192 #define HISTSZ 20 +#ifndef EOF #define EOF (-1) +#endif #define IGN (-2) #define GETC() ((--fi.c < 0)? filbuf(): *fi.p++ & 0xff) #define NHASH 503 diff --git a/src/cmd/8a/a.y b/src/cmd/8a/a.y index 04662f83d..a8ac773da 100644 --- a/src/cmd/8a/a.y +++ b/src/cmd/8a/a.y @@ -29,7 +29,9 @@ // THE SOFTWARE. %{ +#include #include /* if we don't, bison will, and a.h re-#defines getc */ +#include #include "a.h" %} %union { diff --git a/src/cmd/8a/lex.c b/src/cmd/8a/lex.c index 078861877..ab4de417a 100644 --- a/src/cmd/8a/lex.c +++ b/src/cmd/8a/lex.c @@ -29,9 +29,10 @@ // THE SOFTWARE. #define EXTERN +#include +#include #include "a.h" #include "y.tab.h" -#include enum { diff --git a/src/cmd/8c/gc.h b/src/cmd/8c/gc.h index 9fead60e4..32b80e995 100644 --- a/src/cmd/8c/gc.h +++ b/src/cmd/8c/gc.h @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "../cc/cc.h" #include "../8l/8.out.h" @@ -400,6 +401,7 @@ void shiftit(Type*, Node*, Node*); #pragma varargck type "A" int #pragma varargck type "B" Bits #pragma varargck type "D" Adr* +#pragma varargck type "lD" Adr* #pragma varargck type "P" Prog* #pragma varargck type "R" int #pragma varargck type "S" char* diff --git a/src/cmd/8c/swt.c b/src/cmd/8c/swt.c index d07a5439c..769ef2c66 100644 --- a/src/cmd/8c/swt.c +++ b/src/cmd/8c/swt.c @@ -237,10 +237,10 @@ outcode(void) Bprint(&b, "\n"); Bprint(&b, "$$ // exports\n\n"); Bprint(&b, "$$ // local types\n\n"); - Bprint(&b, "$$ // dynimport\n", thestring); + Bprint(&b, "$$ // dynimport\n"); for(i=0; ivaddr - segtext.vaddr + segtext.fileoff, 0); codeblk(sect->vaddr, sect->len); - /* output read-only data in text segment */ - sect = segtext.sect->next; - seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); - datblk(sect->vaddr, sect->len); + /* output read-only data in text segment (rodata, gosymtab and pclntab) */ + for(sect = sect->next; sect != nil; sect = sect->next) { + seek(cout, sect->vaddr - segtext.vaddr + segtext.fileoff, 0); + datblk(sect->vaddr, sect->len); + } if(debug['v']) Bprint(&bso, "%5.2f datblk\n", cputime()); @@ -1083,18 +1080,6 @@ asmb(void) elfshbits(sect); if (!debug['s']) { - sh = newElfShdr(elfstr[ElfStrGosymtab]); - sh->type = SHT_PROGBITS; - sh->flags = SHF_ALLOC; - sh->addralign = 1; - shsym(sh, lookup("symtab", 0)); - - sh = newElfShdr(elfstr[ElfStrGopclntab]); - sh->type = SHT_PROGBITS; - sh->flags = SHF_ALLOC; - sh->addralign = 1; - shsym(sh, lookup("pclntab", 0)); - sh = newElfShdr(elfstr[ElfStrSymtab]); sh->type = SHT_SYMTAB; sh->off = symo; diff --git a/src/cmd/8l/mkenam b/src/cmd/8l/mkenam index b33fec7cc..992aa3160 100644 --- a/src/cmd/8l/mkenam +++ b/src/cmd/8l/mkenam @@ -28,18 +28,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -ed - ../8l/8.out.h <<'!' -v/^ A/d -,s/^ A/ "/ -g/ .*$/s/// -,s/,*$/",/ -1i -char* anames[] = -{ -. -$a -}; -. -w enam.c -Q -! +awk ' +BEGIN { + print "char* anames[] =" + print "{" +} + +/^ A/ { + name=$1 + sub(/,/, "", name) + sub(/^A/, "", name) + print "\t\"" name "\"," +} + +END { print "};" } +' ../8l/8.out.h >enam.c diff --git a/src/cmd/cc/acid.c b/src/cmd/cc/acid.c index c6a6722bd..23147e519 100644 --- a/src/cmd/cc/acid.c +++ b/src/cmd/cc/acid.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" static char *kwd[] = diff --git a/src/cmd/cc/bits.c b/src/cmd/cc/bits.c index aef4449e8..4496d65e7 100644 --- a/src/cmd/cc/bits.c +++ b/src/cmd/cc/bits.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" Bits diff --git a/src/cmd/cc/cc.h b/src/cmd/cc/cc.h index 8e8f6af44..a38e658ce 100644 --- a/src/cmd/cc/cc.h +++ b/src/cmd/cc/cc.h @@ -28,10 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -#include #include #include -#include #pragma lib "../cc/cc.a$O" @@ -816,7 +814,9 @@ int machcap(Node*); #pragma varargck type "L" int32 #pragma varargck type "Q" int32 #pragma varargck type "O" int +#pragma varargck type "O" uint #pragma varargck type "T" Type* +#pragma varargck type "U" char* #pragma varargck type "|" int enum diff --git a/src/cmd/cc/cc.y b/src/cmd/cc/cc.y index 470fdae26..515a80372 100644 --- a/src/cmd/cc/cc.y +++ b/src/cmd/cc/cc.y @@ -29,6 +29,7 @@ // THE SOFTWARE. %{ +#include #include /* if we don't, bison will, and cc.h re-#defines getc */ #include "cc.h" %} diff --git a/src/cmd/cc/com.c b/src/cmd/cc/com.c index b1a8a4704..6e470ee64 100644 --- a/src/cmd/cc/com.c +++ b/src/cmd/cc/com.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" int compar(Node*, int); @@ -127,7 +128,7 @@ tcomo(Node *n, int f) case ORETURN: if(l == Z) { if(n->type->etype != TVOID) - warn(n, "null return of a typed function"); + diag(n, "null return of a typed function"); break; } if(tcom(l)) diff --git a/src/cmd/cc/com64.c b/src/cmd/cc/com64.c index 8d6e07d1b..fb7a3f750 100644 --- a/src/cmd/cc/com64.c +++ b/src/cmd/cc/com64.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" /* diff --git a/src/cmd/cc/dcl.c b/src/cmd/cc/dcl.c index 6f1b8a9a9..d624bf247 100644 --- a/src/cmd/cc/dcl.c +++ b/src/cmd/cc/dcl.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" Node* diff --git a/src/cmd/cc/dpchk.c b/src/cmd/cc/dpchk.c index 0e51101f1..42c245b56 100644 --- a/src/cmd/cc/dpchk.c +++ b/src/cmd/cc/dpchk.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "cc.h" #include "y.tab.h" @@ -56,7 +58,9 @@ struct Tname { char* name; int param; + int count; Tname* link; + Tprot* prot; }; static Type* indchar; @@ -131,8 +135,8 @@ getflag(char *s) return flag; } -void -newprot(Sym *m, Type *t, char *s) +static void +newprot(Sym *m, Type *t, char *s, Tprot **prot) { Bits flag; Tprot *l; @@ -142,32 +146,37 @@ newprot(Sym *m, Type *t, char *s) return; } flag = getflag(s); - for(l=tprot; l; l=l->link) + for(l=*prot; l; l=l->link) if(beq(flag, l->flag) && sametype(t, l->type)) return; l = alloc(sizeof(*l)); l->type = t; l->flag = flag; - l->link = tprot; - tprot = l; + l->link = *prot; + *prot = l; } -void -newname(char *s, int p) +static Tname* +newname(char *s, int p, int count) { Tname *l; for(l=tname; l; l=l->link) if(strcmp(l->name, s) == 0) { - if(l->param != p) + if(p >= 0 && l->param != p) yyerror("vargck %s already defined\n", s); - return; + return l; } + if(p < 0) + return nil; + l = alloc(sizeof(*l)); l->name = s; l->param = p; l->link = tname; + l->count = count; tname = l; + return l; } void @@ -234,6 +243,7 @@ pragvararg(void) int n, c; char *t; Type *ty; + Tname *l; if(!debug['F']) goto out; @@ -244,6 +254,8 @@ pragvararg(void) goto cktype; if(s && strcmp(s->name, "flag") == 0) goto ckflag; + if(s && strcmp(s->name, "countpos") == 0) + goto ckcount; yyerror("syntax in #pragma varargck"); goto out; @@ -255,7 +267,18 @@ ckpos: n = getnsn(); if(n < 0) goto bad; - newname(s->name, n); + newname(s->name, n, 0); + goto out; + +ckcount: +/*#pragma varargck countpos name 2*/ + s = getsym(); + if(s == S) + goto bad; + n = getnsn(); + if(n < 0) + goto bad; + newname(s->name, 0, n); goto out; ckflag: @@ -276,6 +299,25 @@ ckflag: goto out; cktype: + c = getnsc(); + unget(c); + if(c != '"') { +/*#pragma varargck type name int*/ + s = getsym(); + if(s == S) + goto bad; + l = newname(s->name, -1, -1); + s = getsym(); + if(s == S) + goto bad; + ty = s->type; + while((c = getnsc()) == '*') + ty = typ(TIND, ty); + unget(c); + newprot(s, ty, "a", &l->prot); + goto out; + } + /*#pragma varargck type O int*/ t = getquoted(); if(t == nil) @@ -287,7 +329,7 @@ cktype: while((c = getnsc()) == '*') ty = typ(TIND, ty); unget(c); - newprot(s, ty, t); + newprot(s, ty, t, &tprot); goto out; bad: @@ -384,7 +426,8 @@ dpcheck(Node *n) char *s; Node *a, *b; Tname *l; - int i; + Tprot *tl; + int i, j; if(n == Z) return; @@ -398,20 +441,76 @@ dpcheck(Node *n) if(l == 0) return; + if(l->count > 0) { + // fetch count, then check remaining length + i = l->count; + a = nil; + b = n->right; + while(i > 0) { + b = nextarg(b, &a); + i--; + } + if(a == Z) { + diag(n, "can't find count arg"); + return; + } + if(a->op != OCONST || !typechl[a->type->etype]) { + diag(n, "count is invalid constant"); + return; + } + j = a->vconst; + i = 0; + while(b != Z) { + b = nextarg(b, &a); + i++; + } + if(i != j) + diag(n, "found %d argument%s after count %d", i, i == 1 ? "" : "s", j); + } + + if(l->prot != nil) { + // check that all arguments after param or count + // are listed in type list. + i = l->count; + if(i == 0) + i = l->param; + if(i == 0) + return; + a = nil; + b = n->right; + while(i > 0) { + b = nextarg(b, &a); + i--; + } + if(a == Z) { + diag(n, "can't find count/param arg"); + return; + } + while(b != Z) { + b = nextarg(b, &a); + for(tl=l->prot; tl; tl=tl->link) + if(sametype(a->type, tl->type)) + break; + if(tl == nil) + diag(a, "invalid type %T in call to %s", a->type, s); + } + } + + if(l->param <= 0) + return; i = l->param; a = nil; b = n->right; - a = Z; while(i > 0) { b = nextarg(b, &a); i--; } if(a == Z) { - warn(n, "cant find format arg"); + diag(n, "can't find format arg"); return; } if(!sametype(indchar, a->type)) { - warn(n, "format arg type %T", a->type); + diag(n, "format arg type %T", a->type); return; } if(a->op != OADDR || a->left->op != ONAME || a->left->sym != symstring) { diff --git a/src/cmd/cc/funct.c b/src/cmd/cc/funct.c index 21d86258f..99477b2b2 100644 --- a/src/cmd/cc/funct.c +++ b/src/cmd/cc/funct.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" typedef struct Ftab Ftab; diff --git a/src/cmd/cc/godefs.c b/src/cmd/cc/godefs.c index 9503cb2f2..3ba979c8a 100644 --- a/src/cmd/cc/godefs.c +++ b/src/cmd/cc/godefs.c @@ -29,6 +29,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" static int upper; @@ -238,7 +239,7 @@ printtypename(Type *t) Bprint(&outbuf, "%U", n); break; case TFUNC: - Bprint(&outbuf, "func(", t); + Bprint(&outbuf, "func("); for(t1 = t->down; t1 != T; t1 = t1->down) { if(t1->etype == TVOID) break; diff --git a/src/cmd/cc/lex.c b/src/cmd/cc/lex.c index 71cc89bf0..15f2d374d 100644 --- a/src/cmd/cc/lex.c +++ b/src/cmd/cc/lex.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "cc.h" #include "y.tab.h" @@ -384,7 +386,7 @@ lookup(void) }else *w++ = *r; } - *w++ = '\0'; + *w = '\0'; h = 0; for(p=symb; *p;) { @@ -1524,7 +1526,7 @@ alloc(int32 n) p = malloc(n); if(p == nil) { print("alloc out of mem\n"); - exit(1); + exits("alloc: out of mem"); } memset(p, 0, n); return p; @@ -1538,7 +1540,7 @@ allocn(void *p, int32 n, int32 d) p = realloc(p, n+d); if(p == nil) { print("allocn out of mem\n"); - exit(1); + exits("allocn: out of mem"); } if(d > 0) memset((char*)p+n, 0, d); diff --git a/src/cmd/cc/lexbody b/src/cmd/cc/lexbody index 24f9bdc85..f4cc19c2e 100644 --- a/src/cmd/cc/lexbody +++ b/src/cmd/cc/lexbody @@ -96,7 +96,7 @@ alloc(int32 n) p = malloc(n); if(p == nil) { print("alloc out of mem\n"); - exit(1); + exits("alloc: out of mem"); } memset(p, 0, n); return p; @@ -110,7 +110,7 @@ allocn(void *p, int32 n, int32 d) p = realloc(p, n+d); if(p == nil) { print("allocn out of mem\n"); - exit(1); + exits("allocn: out of mem"); } if(d > 0) memset((char*)p+n, 0, d); @@ -245,7 +245,7 @@ lookup(void) }else *w++ = *r; } - *w++ = '\0'; + *w = '\0'; h = 0; for(p=symb; c = *p; p++) diff --git a/src/cmd/cc/mac.c b/src/cmd/cc/mac.c index c08cd9c97..43ae214d7 100644 --- a/src/cmd/cc/mac.c +++ b/src/cmd/cc/mac.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "cc.h" #include "macbody" diff --git a/src/cmd/cc/macbody b/src/cmd/cc/macbody index ca8a54c0b..ed66361f1 100644 --- a/src/cmd/cc/macbody +++ b/src/cmd/cc/macbody @@ -830,11 +830,11 @@ linehist(char *f, int offset) if(debug['f']) if(f) { if(offset) - print("%4ld: %s (#line %d)\n", lineno, f, offset); + print("%4d: %s (#line %d)\n", lineno, f, offset); else - print("%4ld: %s\n", lineno, f); + print("%4d: %s\n", lineno, f); } else - print("%4ld: \n", lineno); + print("%4d: \n", lineno); newflag = 0; h = alloc(sizeof(Hist)); diff --git a/src/cmd/cc/omachcap.c b/src/cmd/cc/omachcap.c index ec5aa86e9..f8fc1d88b 100644 --- a/src/cmd/cc/omachcap.c +++ b/src/cmd/cc/omachcap.c @@ -28,11 +28,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" /* default, like old cc */ int machcap(Node *n) { + USED(n); return 0; } diff --git a/src/cmd/cc/pgen.c b/src/cmd/cc/pgen.c index 5d17cafc9..0e5e8c059 100644 --- a/src/cmd/cc/pgen.c +++ b/src/cmd/cc/pgen.c @@ -112,7 +112,7 @@ codgen(Node *n, Node *nn) warnreach = 1; gen(n); if(canreach && thisfn->link->etype != TVOID) - warn(Z, "no return at end of function: %s", n1->sym->name); + diag(Z, "no return at end of function: %s", n1->sym->name); noretval(3); gbranch(ORETURN); diff --git a/src/cmd/cc/scon.c b/src/cmd/cc/scon.c index 3047ca44f..193331f77 100644 --- a/src/cmd/cc/scon.c +++ b/src/cmd/cc/scon.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" static Node* diff --git a/src/cmd/cc/sub.c b/src/cmd/cc/sub.c index e0d5df719..e5992e213 100644 --- a/src/cmd/cc/sub.c +++ b/src/cmd/cc/sub.c @@ -28,6 +28,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include #include "cc.h" Node* diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index 6b930f151..a4d83f1e7 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -66,7 +66,7 @@ func cname(s string) string { // preamble. Multiple occurrences are concatenated with a separating space, // even across files. func (p *Package) ParseFlags(f *File, srcfile string) { - linesIn := strings.Split(f.Preamble, "\n", -1) + linesIn := strings.Split(f.Preamble, "\n") linesOut := make([]string, 0, len(linesIn)) NextLine: @@ -78,7 +78,7 @@ NextLine: } l = strings.TrimSpace(l[4:]) - fields := strings.Split(l, ":", 2) + fields := strings.SplitN(l, ":", 2) if len(fields) != 2 { fatalf("%s: bad #cgo line: %s", srcfile, line) } @@ -275,7 +275,7 @@ func (p *Package) loadDefines(f *File) { b.WriteString(f.Preamble) stdout := p.gccDefines(b.Bytes()) - for _, line := range strings.Split(stdout, "\n", -1) { + for _, line := range strings.Split(stdout, "\n") { if len(line) < 9 || line[0:7] != "#define" { continue } @@ -397,7 +397,7 @@ func (p *Package) guessKinds(f *File) []*Name { isConst[i] = true // until proven otherwise } - for _, line := range strings.Split(stderr, "\n", -1) { + for _, line := range strings.Split(stderr, "\n") { if len(line) < 9 || line[0:9] != "cgo-test:" { // the user will see any compiler errors when the code is compiled later. continue @@ -1188,7 +1188,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { if ss, ok := dwarfToName[s]; ok { s = ss } - s = strings.Join(strings.Split(s, " ", -1), "") // strip spaces + s = strings.Join(strings.Split(s, " "), "") // strip spaces name := c.Ident("_Ctype_" + s) typedef[name.Name] = t.Go t.Go = name diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 7eecb3437..6802dd1cf 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -148,7 +148,7 @@ func dynimport(obj string) { fatalf("cannot load imported symbols from PE file %s: %v", obj, err) } for _, s := range sym { - ss := strings.Split(s, ":", -1) + ss := strings.Split(s, ":") fmt.Printf("#pragma dynimport %s %s %q\n", ss[0], ss[0], strings.ToLower(ss[1])) } return diff --git a/src/cmd/ebnflint/ebnflint.go b/src/cmd/ebnflint/ebnflint.go index cac39179f..0b0443156 100644 --- a/src/cmd/ebnflint/ebnflint.go +++ b/src/cmd/ebnflint/ebnflint.go @@ -35,6 +35,12 @@ var ( ) +func report(err os.Error) { + scanner.PrintError(os.Stderr, err) + os.Exit(1) +} + + func extractEBNF(src []byte) []byte { var buf bytes.Buffer @@ -75,34 +81,35 @@ func extractEBNF(src []byte) []byte { func main() { flag.Parse() - var filename string + var ( + filename string + src []byte + err os.Error + ) switch flag.NArg() { case 0: - filename = "/dev/stdin" + filename = "" + src, err = ioutil.ReadAll(os.Stdin) case 1: filename = flag.Arg(0) + src, err = ioutil.ReadFile(filename) default: usage() } - - src, err := ioutil.ReadFile(filename) if err != nil { - scanner.PrintError(os.Stderr, err) - os.Exit(1) + report(err) } - if filepath.Ext(filename) == ".html" { + if filepath.Ext(filename) == ".html" || bytes.Index(src, open) >= 0 { src = extractEBNF(src) } grammar, err := ebnf.Parse(fset, filename, src) if err != nil { - scanner.PrintError(os.Stderr, err) - os.Exit(1) + report(err) } if err = ebnf.Verify(fset, grammar, *start); err != nil { - scanner.PrintError(os.Stderr, err) - os.Exit(1) + report(err) } } diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h index b68768165..8ca086ee0 100644 --- a/src/cmd/gc/go.h +++ b/src/cmd/gc/go.h @@ -302,6 +302,7 @@ struct Sym uchar flags; uchar sym; // huffman encoding in object file Sym* link; + int32 npkg; // number of imported packages with this name // saved and restored by dcopy Pkg* pkg; @@ -777,6 +778,7 @@ EXTERN int32 nhunk; EXTERN int32 thunk; EXTERN int exporting; +EXTERN int erroring; EXTERN int noargnames; EXTERN int funcdepth; diff --git a/src/cmd/gc/go.y b/src/cmd/gc/go.y index 5d28c0e3b..01a4e822f 100644 --- a/src/cmd/gc/go.y +++ b/src/cmd/gc/go.y @@ -237,7 +237,11 @@ import_here: import_package: LPACKAGE sym import_safety ';' { - importpkg->name = $2->name; + if(importpkg->name == nil) { + importpkg->name = $2->name; + pkglookup($2->name, nil)->npkg++; + } else if(strcmp(importpkg->name, $2->name) != 0) + yyerror("conflicting names %s and %s for package %Z", importpkg->name, $2->name, importpkg->path); importpkg->direct = 1; if(safemode && !curio.importsafe) @@ -1657,7 +1661,11 @@ hidden_import: Pkg *p; p = mkpkg($3.u.sval); - p->name = $2->name; + if(p->name == nil) { + p->name = $2->name; + pkglookup($2->name, nil)->npkg++; + } else if(strcmp(p->name, $2->name) != 0) + yyerror("conflicting names %s and %s for package %Z", p->name, $2->name, p->path); } | LVAR hidden_pkg_importsym hidden_type ';' { diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c index 8eb60de31..40b0c4fd1 100644 --- a/src/cmd/gc/subr.c +++ b/src/cmd/gc/subr.c @@ -45,10 +45,12 @@ adderr(int line, char *fmt, va_list arg) Fmt f; Error *p; + erroring++; fmtstrinit(&f); fmtprint(&f, "%L: ", line); fmtvprint(&f, fmt, arg); fmtprint(&f, "\n"); + erroring--; if(nerr >= merr) { if(merr == 0) @@ -1122,7 +1124,14 @@ Sconv(Fmt *fp) return 0; } - if(s->pkg != localpkg || longsymnames || (fp->flags & FmtLong)) { + if(s->pkg && s->pkg != localpkg || longsymnames || (fp->flags & FmtLong)) { + // This one is for the user. If the package name + // was used by multiple packages, give the full + // import path to disambiguate. + if(erroring && pkglookup(s->pkg->name, nil)->npkg > 1) { + fmtprint(fp, "\"%Z\".%s", s->pkg->path, s->name); + return 0; + } fmtprint(fp, "%s.%s", s->pkg->name, s->name); return 0; } diff --git a/src/cmd/godoc/codewalk.go b/src/cmd/godoc/codewalk.go index 54bebe854..50043e2ab 100644 --- a/src/cmd/godoc/codewalk.go +++ b/src/cmd/godoc/codewalk.go @@ -74,7 +74,7 @@ func codewalk(w http.ResponseWriter, r *http.Request) { // A Codewalk represents a single codewalk read from an XML file. type Codewalk struct { - Title string "attr" + Title string `xml:"attr"` File []string Step []*Codestep } @@ -83,9 +83,9 @@ type Codewalk struct { // A Codestep is a single step in a codewalk. type Codestep struct { // Filled in from XML - Src string "attr" - Title string "attr" - XML string "innerxml" + Src string `xml:"attr"` + Title string `xml:"attr"` + XML string `xml:"innerxml"` // Derived from Src; not in XML. Err os.Error @@ -168,7 +168,7 @@ func loadCodewalk(filename string) (*Codewalk, os.Error) { cw.File[i] = f i++ } - sort.SortStrings(cw.File) + sort.Strings(cw.File) return cw, nil } diff --git a/src/cmd/godoc/dirtrees.go b/src/cmd/godoc/dirtrees.go index af44fa16a..e98e93a46 100644 --- a/src/cmd/godoc/dirtrees.go +++ b/src/cmd/godoc/dirtrees.go @@ -30,7 +30,7 @@ type Directory struct { func isGoFile(fi FileInfo) bool { name := fi.Name() return fi.IsRegular() && - !strings.HasPrefix(name, ".") && // ignore .files + len(name) > 0 && name[0] != '.' && // ignore .files filepath.Ext(name) == ".go" } @@ -43,7 +43,8 @@ func isPkgFile(fi FileInfo) bool { func isPkgDir(fi FileInfo) bool { name := fi.Name() - return fi.IsDirectory() && len(name) > 0 && name[0] != '_' + return fi.IsDirectory() && len(name) > 0 && + name[0] != '_' && name[0] != '.' // ignore _files and .files } @@ -267,8 +268,8 @@ func (dir *Directory) lookupLocal(name string) *Directory { // lookup looks for the *Directory for a given path, relative to dir. func (dir *Directory) lookup(path string) *Directory { - d := strings.Split(dir.Path, string(filepath.Separator), -1) - p := strings.Split(path, string(filepath.Separator), -1) + d := strings.Split(dir.Path, string(filepath.Separator)) + p := strings.Split(path, string(filepath.Separator)) i := 0 for i < len(d) { if i >= len(p) || d[i] != p[i] { diff --git a/src/cmd/godoc/godoc.go b/src/cmd/godoc/godoc.go index 6987d911b..20ebd3183 100644 --- a/src/cmd/godoc/godoc.go +++ b/src/cmd/godoc/godoc.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "go/ast" + "go/build" "go/doc" "go/printer" "go/token" @@ -83,8 +84,16 @@ var ( func initHandlers() { - fsMap.Init(*pkgPath) - fileServer = http.FileServer(*goroot, "") + paths := filepath.SplitList(*pkgPath) + for _, t := range build.Path { + if t.Goroot { + continue + } + paths = append(paths, t.SrcDir()) + } + fsMap.Init(paths) + + fileServer = http.FileServer(http.Dir(*goroot)) cmdHandler = httpHandler{"/cmd/", filepath.Join(*goroot, "src", "cmd"), false} pkgHandler = httpHandler{"/pkg/", filepath.Join(*goroot, "src", "pkg"), true} } @@ -160,7 +169,7 @@ func readDirList(filename string) ([]string, os.Error) { } return e == nil && isPkgDir(d) } - list := canonicalizePaths(strings.Split(string(contents), "\n", -1), filter) + list := canonicalizePaths(strings.Split(string(contents), "\n"), filter) // for each parent path, remove all it's children q // (requirement for binary search to work when filtering) i := 0 diff --git a/src/cmd/godoc/index.go b/src/cmd/godoc/index.go index 61caee101..e0c89e794 100644 --- a/src/cmd/godoc/index.go +++ b/src/cmd/godoc/index.go @@ -901,7 +901,7 @@ func isIdentifier(s string) bool { // identifier, Lookup returns a LookupResult, and a list of alternative // spellings, if any. If the query syntax is wrong, an error is reported. func (x *Index) Lookup(query string) (match *LookupResult, alt *AltWords, err os.Error) { - ss := strings.Split(query, ".", -1) + ss := strings.Split(query, ".") // check query syntax for _, s := range ss { @@ -954,7 +954,7 @@ func (list positionList) Swap(i, j int) { list[i], list[j] = list[j], list[ // unique returns the list sorted and with duplicate entries removed func unique(list []int) []int { - sort.SortInts(list) + sort.Ints(list) var last int i := 0 for _, x := range list { diff --git a/src/cmd/godoc/main.go b/src/cmd/godoc/main.go index 55f6031bc..51fcf8dd0 100644 --- a/src/cmd/godoc/main.go +++ b/src/cmd/godoc/main.go @@ -31,6 +31,7 @@ import ( "flag" "fmt" "go/ast" + "go/build" "http" _ "http/pprof" // to serve /debug/pprof/* "io" @@ -332,7 +333,10 @@ func main() { } relpath := path abspath := path - if !filepath.IsAbs(path) { + if t, pkg, err := build.FindTree(path); err == nil { + relpath = pkg + abspath = filepath.Join(t.SrcDir(), pkg) + } else if !filepath.IsAbs(path) { abspath = absolutePath(path, pkgHandler.fsRoot) } else { relpath = relativeURL(path) diff --git a/src/cmd/godoc/mapping.go b/src/cmd/godoc/mapping.go index 73f1881a2..92614e83e 100644 --- a/src/cmd/godoc/mapping.go +++ b/src/cmd/godoc/mapping.go @@ -59,10 +59,10 @@ type mapping struct { } -// Init initializes the Mapping from a list of paths separated by -// filepath.ListSeparator. Empty paths are ignored; relative paths -// are assumed to be relative to the current working directory and -// converted to absolute paths. For each path of the form: +// Init initializes the Mapping from a list of paths. +// Empty paths are ignored; relative paths are assumed to be relative to +// the current working directory and converted to absolute paths. +// For each path of the form: // // dirname/localname // @@ -80,8 +80,8 @@ type mapping struct { // user -> /home/user // public -> /home/build/public // -func (m *Mapping) Init(paths string) { - pathlist := canonicalizePaths(filepath.SplitList(paths), nil) +func (m *Mapping) Init(paths []string) { + pathlist := canonicalizePaths(paths, nil) list := make([]mapping, len(pathlist)) // create mapping list @@ -120,7 +120,7 @@ func (m *Mapping) PrefixList() []string { } // sort the list and remove duplicate entries - sort.SortStrings(list) + sort.Strings(list) i := 0 prev := "" for _, path := range list { diff --git a/src/cmd/godoc/utils.go b/src/cmd/godoc/utils.go index 660bf6d04..e2637ab3d 100644 --- a/src/cmd/godoc/utils.go +++ b/src/cmd/godoc/utils.go @@ -80,7 +80,7 @@ func canonicalizePaths(list []string, filter func(path string) bool) []string { list = list[0:i] // sort the list and remove duplicate entries - sort.SortStrings(list) + sort.Strings(list) i = 0 prev := "" for _, path := range list { diff --git a/src/cmd/gofix/Makefile b/src/cmd/gofix/Makefile index b157649e8..7ce21e8aa 100644 --- a/src/cmd/gofix/Makefile +++ b/src/cmd/gofix/Makefile @@ -6,16 +6,22 @@ include ../../Make.inc TARG=gofix GOFILES=\ + filepath.go\ fix.go\ - netdial.go\ - main.go\ - oserrorstring.go\ - osopen.go\ httpfinalurl.go\ + httpfs.go\ httpheaders.go\ httpserver.go\ + main.go\ + netdial.go\ + oserrorstring.go\ + osopen.go\ procattr.go\ reflect.go\ + signal.go\ + sorthelpers.go\ + sortslice.go\ + stringssplit.go\ typecheck.go\ include ../../Make.cmd diff --git a/src/cmd/gofix/filepath.go b/src/cmd/gofix/filepath.go new file mode 100644 index 000000000..1d0ad6879 --- /dev/null +++ b/src/cmd/gofix/filepath.go @@ -0,0 +1,53 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +func init() { + register(fix{ + "filepath", + filepathFunc, + `Adapt code from filepath.[List]SeparatorString to string(filepath.[List]Separator). + +http://codereview.appspot.com/4527090 +`, + }) +} + +func filepathFunc(f *ast.File) (fixed bool) { + if !imports(f, "path/filepath") { + return + } + + walk(f, func(n interface{}) { + e, ok := n.(*ast.Expr) + if !ok { + return + } + + var ident string + switch { + case isPkgDot(*e, "filepath", "SeparatorString"): + ident = "filepath.Separator" + case isPkgDot(*e, "filepath", "ListSeparatorString"): + ident = "filepath.ListSeparator" + default: + return + } + + // string(filepath.[List]Separator) + *e = &ast.CallExpr{ + Fun: ast.NewIdent("string"), + Args: []ast.Expr{ast.NewIdent(ident)}, + } + + fixed = true + }) + + return +} diff --git a/src/cmd/gofix/filepath_test.go b/src/cmd/gofix/filepath_test.go new file mode 100644 index 000000000..d170c3ae3 --- /dev/null +++ b/src/cmd/gofix/filepath_test.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(filepathTests) +} + +var filepathTests = []testCase{ + { + Name: "filepath.0", + In: `package main + +import ( + "path/filepath" +) + +var _ = filepath.SeparatorString +var _ = filepath.ListSeparatorString +`, + Out: `package main + +import ( + "path/filepath" +) + +var _ = string(filepath.Separator) +var _ = string(filepath.ListSeparator) +`, + }, +} diff --git a/src/cmd/gofix/fix.go b/src/cmd/gofix/fix.go index 0852ce21e..c1c5a746c 100644 --- a/src/cmd/gofix/fix.go +++ b/src/cmd/gofix/fix.go @@ -10,6 +10,7 @@ import ( "go/token" "os" "strconv" + "strings" ) type fix struct { @@ -258,13 +259,28 @@ func walkBeforeAfter(x interface{}, before, after func(interface{})) { // imports returns true if f imports path. func imports(f *ast.File, path string) bool { + return importSpec(f, path) != nil +} + +// importSpec returns the import spec if f imports path, +// or nil otherwise. +func importSpec(f *ast.File, path string) *ast.ImportSpec { for _, s := range f.Imports { - t, err := strconv.Unquote(s.Path.Value) - if err == nil && t == path { - return true + if importPath(s) == path { + return s } } - return false + return nil +} + +// importPath returns the unquoted import path of s, +// or "" if the path is not properly quoted. +func importPath(s *ast.ImportSpec) string { + t, err := strconv.Unquote(s.Path.Value) + if err == nil { + return t + } + return "" } // isPkgDot returns true if t is the expression "pkg.name" @@ -420,3 +436,138 @@ func newPkgDot(pos token.Pos, pkg, name string) ast.Expr { }, } } + +// addImport adds the import path to the file f, if absent. +func addImport(f *ast.File, path string) { + if imports(f, path) { + return + } + + newImport := &ast.ImportSpec{ + Path: &ast.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(path), + }, + } + + var impdecl *ast.GenDecl + + // Find an import decl to add to. + for _, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + + if ok && gen.Tok == token.IMPORT { + impdecl = gen + break + } + } + + // No import decl found. Add one. + if impdecl == nil { + impdecl = &ast.GenDecl{ + Tok: token.IMPORT, + } + f.Decls = append(f.Decls, nil) + copy(f.Decls[1:], f.Decls) + f.Decls[0] = impdecl + } + + // Ensure the import decl has parentheses, if needed. + if len(impdecl.Specs) > 0 && !impdecl.Lparen.IsValid() { + impdecl.Lparen = impdecl.Pos() + } + + // Assume the import paths are alphabetically ordered. + // If they are not, the result is ugly, but legal. + insertAt := len(impdecl.Specs) // default to end of specs + for i, spec := range impdecl.Specs { + impspec := spec.(*ast.ImportSpec) + if importPath(impspec) > path { + insertAt = i + break + } + } + + impdecl.Specs = append(impdecl.Specs, nil) + copy(impdecl.Specs[insertAt+1:], impdecl.Specs[insertAt:]) + impdecl.Specs[insertAt] = newImport + + f.Imports = append(f.Imports, newImport) +} + +// deleteImport deletes the import path from the file f, if present. +func deleteImport(f *ast.File, path string) { + oldImport := importSpec(f, path) + + // Find the import node that imports path, if any. + for i, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.IMPORT { + continue + } + for j, spec := range gen.Specs { + impspec := spec.(*ast.ImportSpec) + + if oldImport != impspec { + continue + } + + // We found an import spec that imports path. + // Delete it. + copy(gen.Specs[j:], gen.Specs[j+1:]) + gen.Specs = gen.Specs[:len(gen.Specs)-1] + + // If this was the last import spec in this decl, + // delete the decl, too. + if len(gen.Specs) == 0 { + copy(f.Decls[i:], f.Decls[i+1:]) + f.Decls = f.Decls[:len(f.Decls)-1] + } else if len(gen.Specs) == 1 { + gen.Lparen = token.NoPos // drop parens + } + + break + } + } + + // Delete it from f.Imports. + for i, imp := range f.Imports { + if imp == oldImport { + copy(f.Imports[i:], f.Imports[i+1:]) + f.Imports = f.Imports[:len(f.Imports)-1] + break + } + } +} + +func usesImport(f *ast.File, path string) (used bool) { + spec := importSpec(f, path) + if spec == nil { + return + } + + name := spec.Name.String() + switch name { + case "": + // If the package name is not explicitly specified, + // make an educated guess. This is not guaranteed to be correct. + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 { + name = path + } else { + name = path[lastSlash+1:] + } + case "_", ".": + // Not sure if this import is used - err on the side of caution. + return true + } + + walk(f, func(n interface{}) { + sel, ok := n.(*ast.SelectorExpr) + if ok && isTopName(sel.X, name) { + used = true + } + }) + + return +} diff --git a/src/cmd/gofix/httpfs.go b/src/cmd/gofix/httpfs.go new file mode 100644 index 000000000..7f2765680 --- /dev/null +++ b/src/cmd/gofix/httpfs.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +var httpFileSystemFix = fix{ + "httpfs", + httpfs, + `Adapt http FileServer to take a FileSystem. + +http://codereview.appspot.com/4629047 http FileSystem interface +`, +} + +func init() { + register(httpFileSystemFix) +} + +func httpfs(f *ast.File) bool { + if !imports(f, "http") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + if !ok || !isPkgDot(call.Fun, "http", "FileServer") { + return + } + if len(call.Args) != 2 { + return + } + dir, prefix := call.Args[0], call.Args[1] + call.Args = []ast.Expr{&ast.CallExpr{ + Fun: &ast.SelectorExpr{ast.NewIdent("http"), ast.NewIdent("Dir")}, + Args: []ast.Expr{dir}, + }} + wrapInStripHandler := true + if prefixLit, ok := prefix.(*ast.BasicLit); ok { + if prefixLit.Kind == token.STRING && (prefixLit.Value == `"/"` || prefixLit.Value == `""`) { + wrapInStripHandler = false + } + } + if wrapInStripHandler { + call.Fun.(*ast.SelectorExpr).Sel = ast.NewIdent("StripPrefix") + call.Args = []ast.Expr{ + prefix, + &ast.CallExpr{ + Fun: &ast.SelectorExpr{ast.NewIdent("http"), ast.NewIdent("FileServer")}, + Args: call.Args, + }, + } + } + fixed = true + }) + return fixed +} diff --git a/src/cmd/gofix/httpfs_test.go b/src/cmd/gofix/httpfs_test.go new file mode 100644 index 000000000..d1804e93b --- /dev/null +++ b/src/cmd/gofix/httpfs_test.go @@ -0,0 +1,47 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(httpFileSystemTests) +} + +var httpFileSystemTests = []testCase{ + { + Name: "httpfs.0", + In: `package httpfs + +import ( + "http" +) + +func f() { + _ = http.FileServer("/var/www/foo", "/") + _ = http.FileServer("/var/www/foo", "") + _ = http.FileServer("/var/www/foo/bar", "/bar") + s := "/foo" + _ = http.FileServer(s, "/") + prefix := "/p" + _ = http.FileServer(s, prefix) +} +`, + Out: `package httpfs + +import ( + "http" +) + +func f() { + _ = http.FileServer(http.Dir("/var/www/foo")) + _ = http.FileServer(http.Dir("/var/www/foo")) + _ = http.StripPrefix("/bar", http.FileServer(http.Dir("/var/www/foo/bar"))) + s := "/foo" + _ = http.FileServer(http.Dir(s)) + prefix := "/p" + _ = http.StripPrefix(prefix, http.FileServer(http.Dir(s))) +} +`, + }, +} diff --git a/src/cmd/gofix/main.go b/src/cmd/gofix/main.go index 05495bc0d..e7e7013c5 100644 --- a/src/cmd/gofix/main.go +++ b/src/cmd/gofix/main.go @@ -53,7 +53,7 @@ func main() { if *allowedRewrites != "" { allowed = make(map[string]bool) - for _, f := range strings.Split(*allowedRewrites, ",", -1) { + for _, f := range strings.Split(*allowedRewrites, ",") { allowed[f] = true } } diff --git a/src/cmd/gofix/signal.go b/src/cmd/gofix/signal.go new file mode 100644 index 000000000..53c338851 --- /dev/null +++ b/src/cmd/gofix/signal.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "strings" +) + +func init() { + register(fix{ + "signal", + signal, + `Adapt code to types moved from os/signal to signal. + +http://codereview.appspot.com/4437091 +`, + }) +} + +func signal(f *ast.File) (fixed bool) { + if !imports(f, "os/signal") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + + if !ok || !isTopName(s.X, "signal") { + return + } + + sel := s.Sel.String() + if sel == "Signal" || sel == "UnixSignal" || strings.HasPrefix(sel, "SIG") { + s.X = &ast.Ident{Name: "os"} + fixed = true + } + }) + + if fixed { + addImport(f, "os") + if !usesImport(f, "os/signal") { + deleteImport(f, "os/signal") + } + } + return +} diff --git a/src/cmd/gofix/signal_test.go b/src/cmd/gofix/signal_test.go new file mode 100644 index 000000000..2500e9cee --- /dev/null +++ b/src/cmd/gofix/signal_test.go @@ -0,0 +1,96 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(signalTests) +} + +var signalTests = []testCase{ + { + Name: "signal.0", + In: `package main + +import ( + _ "a" + "os/signal" + _ "z" +) + +type T1 signal.UnixSignal +type T2 signal.Signal + +func f() { + _ = signal.SIGHUP + _ = signal.Incoming +} +`, + Out: `package main + +import ( + _ "a" + "os" + "os/signal" + _ "z" +) + +type T1 os.UnixSignal +type T2 os.Signal + +func f() { + _ = os.SIGHUP + _ = signal.Incoming +} +`, + }, + { + Name: "signal.1", + In: `package main + +import ( + "os" + "os/signal" +) + +func f() { + var _ os.Error + _ = signal.SIGHUP +} +`, + Out: `package main + +import "os" + + +func f() { + var _ os.Error + _ = os.SIGHUP +} +`, + }, + { + Name: "signal.2", + In: `package main + +import "os" +import "os/signal" + +func f() { + var _ os.Error + _ = signal.SIGHUP +} +`, + Out: `package main + +import "os" + + +func f() { + var _ os.Error + _ = os.SIGHUP +} +`, + }, +} diff --git a/src/cmd/gofix/sorthelpers.go b/src/cmd/gofix/sorthelpers.go new file mode 100644 index 000000000..4d0bee6e7 --- /dev/null +++ b/src/cmd/gofix/sorthelpers.go @@ -0,0 +1,47 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +func init() { + register(fix{ + "sorthelpers", + sorthelpers, + `Adapt code from sort.Sort[Ints|Float64s|Strings] to sort.[Ints|Float64s|Strings]. +`, + }) +} + + +func sorthelpers(f *ast.File) (fixed bool) { + if !imports(f, "sort") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + if !ok || !isTopName(s.X, "sort") { + return + } + + switch s.Sel.String() { + case "SortFloat64s": + s.Sel.Name = "Float64s" + case "SortInts": + s.Sel.Name = "Ints" + case "SortStrings": + s.Sel.Name = "Strings" + default: + return + } + + fixed = true + }) + + return +} diff --git a/src/cmd/gofix/sorthelpers_test.go b/src/cmd/gofix/sorthelpers_test.go new file mode 100644 index 000000000..6c37858fd --- /dev/null +++ b/src/cmd/gofix/sorthelpers_test.go @@ -0,0 +1,45 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(sorthelpersTests) +} + +var sorthelpersTests = []testCase{ + { + Name: "sortslice.0", + In: `package main + +import ( + "sort" +) + +func main() { + var s []string + sort.SortStrings(s) + var i []ints + sort.SortInts(i) + var f []float64 + sort.SortFloat64s(f) +} +`, + Out: `package main + +import ( + "sort" +) + +func main() { + var s []string + sort.Strings(s) + var i []ints + sort.Ints(i) + var f []float64 + sort.Float64s(f) +} +`, + }, +} diff --git a/src/cmd/gofix/sortslice.go b/src/cmd/gofix/sortslice.go new file mode 100644 index 000000000..b9c108b5a --- /dev/null +++ b/src/cmd/gofix/sortslice.go @@ -0,0 +1,50 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +func init() { + register(fix{ + "sortslice", + sortslice, + `Adapt code from sort.[Float64|Int|String]Array to sort.[Float64|Int|String]Slice. + +http://codereview.appspot.com/4602054 +http://codereview.appspot.com/4639041 +`, + }) +} + + +func sortslice(f *ast.File) (fixed bool) { + if !imports(f, "sort") { + return + } + + walk(f, func(n interface{}) { + s, ok := n.(*ast.SelectorExpr) + if !ok || !isTopName(s.X, "sort") { + return + } + + switch s.Sel.String() { + case "Float64Array": + s.Sel.Name = "Float64Slice" + case "IntArray": + s.Sel.Name = "IntSlice" + case "StringArray": + s.Sel.Name = "StringSlice" + default: + return + } + + fixed = true + }) + + return +} diff --git a/src/cmd/gofix/sortslice_test.go b/src/cmd/gofix/sortslice_test.go new file mode 100644 index 000000000..404feb26f --- /dev/null +++ b/src/cmd/gofix/sortslice_test.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(sortsliceTests) +} + +var sortsliceTests = []testCase{ + { + Name: "sortslice.0", + In: `package main + +import ( + "sort" +) + +var _ = sort.Float64Array +var _ = sort.IntArray +var _ = sort.StringArray +`, + Out: `package main + +import ( + "sort" +) + +var _ = sort.Float64Slice +var _ = sort.IntSlice +var _ = sort.StringSlice +`, + }, +} diff --git a/src/cmd/gofix/stringssplit.go b/src/cmd/gofix/stringssplit.go new file mode 100644 index 000000000..4a1fe93d3 --- /dev/null +++ b/src/cmd/gofix/stringssplit.go @@ -0,0 +1,71 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +var stringssplitFix = fix{ + "stringssplit", + stringssplit, + `Restore strings.Split to its original meaning and add strings.SplitN. Bytes too. + +http://codereview.appspot.com/4661051 +`, +} + +func init() { + register(stringssplitFix) +} + +func stringssplit(f *ast.File) bool { + if !imports(f, "bytes") && !imports(f, "strings") { + return false + } + + fixed := false + walk(f, func(n interface{}) { + call, ok := n.(*ast.CallExpr) + // func Split(s, sep string, n int) []string + // func SplitAfter(s, sep string, n int) []string + if !ok || len(call.Args) != 3 { + return + } + // Is this our function? + switch { + case isPkgDot(call.Fun, "bytes", "Split"): + case isPkgDot(call.Fun, "bytes", "SplitAfter"): + case isPkgDot(call.Fun, "strings", "Split"): + case isPkgDot(call.Fun, "strings", "SplitAfter"): + default: + return + } + + sel := call.Fun.(*ast.SelectorExpr) + args := call.Args + fixed = true // We're committed. + + // Is the last argument -1? If so, drop the arg. + // (Actually we just look for a negative integer literal.) + // Otherwise, Split->SplitN and keep the arg. + final := args[2] + if unary, ok := final.(*ast.UnaryExpr); ok && unary.Op == token.SUB { + if lit, ok := unary.X.(*ast.BasicLit); ok { + // Is it an integer? If so, it's a negative integer and that's what we're after. + if lit.Kind == token.INT { + // drop the last arg. + call.Args = args[0:2] + return + } + } + } + + // If not, rename and keep the argument list. + sel.Sel.Name += "N" + }) + return fixed +} diff --git a/src/cmd/gofix/stringssplit_test.go b/src/cmd/gofix/stringssplit_test.go new file mode 100644 index 000000000..b925722af --- /dev/null +++ b/src/cmd/gofix/stringssplit_test.go @@ -0,0 +1,51 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(stringssplitTests) +} + +var stringssplitTests = []testCase{ + { + Name: "stringssplit.0", + In: `package main + +import ( + "bytes" + "strings" +) + +func f() { + bytes.Split(a, b, c) + bytes.Split(a, b, -1) + bytes.SplitAfter(a, b, c) + bytes.SplitAfter(a, b, -1) + strings.Split(a, b, c) + strings.Split(a, b, -1) + strings.SplitAfter(a, b, c) + strings.SplitAfter(a, b, -1) +} +`, + Out: `package main + +import ( + "bytes" + "strings" +) + +func f() { + bytes.SplitN(a, b, c) + bytes.Split(a, b) + bytes.SplitAfterN(a, b, c) + bytes.SplitAfter(a, b) + strings.SplitN(a, b, c) + strings.Split(a, b) + strings.SplitAfterN(a, b, c) + strings.SplitAfter(a, b) +} +`, + }, +} diff --git a/src/cmd/gofix/testdata/reflect.template.go.in b/src/cmd/gofix/testdata/reflect.template.go.in index ba06de4e3..1f5a8128f 100644 --- a/src/cmd/gofix/testdata/reflect.template.go.in +++ b/src/cmd/gofix/testdata/reflect.template.go.in @@ -444,7 +444,7 @@ func (t *Template) newVariable(words []string) *variableElement { bar := strings.IndexRune(lastWord, '|') if bar >= 0 { words[len(words)-1] = lastWord[0:bar] - formatters = strings.Split(lastWord[bar+1:], "|", -1) + formatters = strings.Split(lastWord[bar+1:], "|") } // We could remember the function address here and avoid the lookup later, @@ -705,7 +705,7 @@ func (t *Template) findVar(st *state, s string) reflect.Value { if s == "@" { return indirectPtr(data, numStars) } - for _, elem := range strings.Split(s, ".", -1) { + for _, elem := range strings.Split(s, ".") { // Look up field; data must be a struct or map. data = t.lookup(st, data, elem) if data == nil { diff --git a/src/cmd/gofix/testdata/reflect.template.go.out b/src/cmd/gofix/testdata/reflect.template.go.out index c36288455..f2f56ef3c 100644 --- a/src/cmd/gofix/testdata/reflect.template.go.out +++ b/src/cmd/gofix/testdata/reflect.template.go.out @@ -444,7 +444,7 @@ func (t *Template) newVariable(words []string) *variableElement { bar := strings.IndexRune(lastWord, '|') if bar >= 0 { words[len(words)-1] = lastWord[0:bar] - formatters = strings.Split(lastWord[bar+1:], "|", -1) + formatters = strings.Split(lastWord[bar+1:], "|") } // We could remember the function address here and avoid the lookup later, @@ -705,7 +705,7 @@ func (t *Template) findVar(st *state, s string) reflect.Value { if s == "@" { return indirectPtr(data, numStars) } - for _, elem := range strings.Split(s, ".", -1) { + for _, elem := range strings.Split(s, ".") { // Look up field; data must be a struct or map. data = t.lookup(st, data, elem) if !data.IsValid() { diff --git a/src/cmd/gofmt/gofmt_test.go b/src/cmd/gofmt/gofmt_test.go index a72530307..70700554b 100644 --- a/src/cmd/gofmt/gofmt_test.go +++ b/src/cmd/gofmt/gofmt_test.go @@ -20,8 +20,8 @@ func runTest(t *testing.T, dirname, in, out, flags string) { // process flags *simplifyAST = false *rewriteRule = "" - for _, flag := range strings.Split(flags, " ", -1) { - elts := strings.Split(flag, "=", 2) + for _, flag := range strings.Split(flags, " ") { + elts := strings.SplitN(flag, "=", 2) name := elts[0] value := "" if len(elts) == 2 { diff --git a/src/cmd/gofmt/rewrite.go b/src/cmd/gofmt/rewrite.go index 4c24282f3..f7f1fe824 100644 --- a/src/cmd/gofmt/rewrite.go +++ b/src/cmd/gofmt/rewrite.go @@ -22,7 +22,7 @@ func initRewrite() { rewrite = nil // disable any previous rewrite return } - f := strings.Split(*rewriteRule, "->", -1) + f := strings.Split(*rewriteRule, "->") if len(f) != 2 { fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n") os.Exit(2) diff --git a/src/cmd/goinstall/doc.go b/src/cmd/goinstall/doc.go index 52b09d37e..a5df7b3bd 100644 --- a/src/cmd/goinstall/doc.go +++ b/src/cmd/goinstall/doc.go @@ -5,7 +5,8 @@ /* Goinstall is an experiment in automatic package installation. It installs packages, possibly downloading them from the internet. -It maintains a list of public Go packages at http://godashboard.appspot.com/package. +It maintains a list of public Go packages at +http://godashboard.appspot.com/package. Usage: goinstall [flags] importpath... @@ -41,9 +42,22 @@ Another common idiom is to use to update, recompile, and reinstall all goinstalled packages. The source code for a package with import path foo/bar is expected -to be in the directory $GOROOT/src/pkg/foo/bar/. If the import -path refers to a code hosting site, goinstall will download the code -if necessary. The recognized code hosting sites are: +to be in the directory $GOROOT/src/pkg/foo/bar/ or $GOPATH/src/foo/bar/. +See "The GOPATH Environment Variable" for more about GOPATH. + +By default, goinstall prints output only when it encounters an error. +The -v flag causes goinstall to print information about packages +being considered and installed. + +Goinstall ignores Makefiles. + + +Remote Repositories + +If a package import path refers to a remote repository, goinstall will +download the code if necessary. + +Goinstall recognizes packages from a few common code hosting sites: BitBucket (Mercurial) @@ -72,7 +86,6 @@ if necessary. The recognized code hosting sites are: import "launchpad.net/~user/project/branch" import "launchpad.net/~user/project/branch/sub/directory" - If the destination directory (e.g., $GOROOT/src/pkg/bitbucket.org/user/project) already exists and contains an appropriate checkout, goinstall will not attempt to fetch updates. The -u flag changes this behavior, @@ -84,19 +97,42 @@ named "release". If there is one, it uses that version of the code. Otherwise it uses the default version selected by the version control system, typically HEAD for git, tip for Mercurial. -After a successful download and installation of a publicly accessible -remote package, goinstall reports the installation to godashboard.appspot.com, -which increments a count associated with the package and the time -of its most recent installation. This mechanism powers the package list -at http://godashboard.appspot.com/package, allowing Go programmers -to learn about popular packages that might be worth looking at. +After a successful download and installation of one of these import paths, +goinstall reports the installation to godashboard.appspot.com, which +increments a count associated with the package and the time of its most +recent installation. This mechanism powers the package list at +http://godashboard.appspot.com/package, allowing Go programmers to learn about +popular packages that might be worth looking at. The -dashboard=false flag disables this reporting. -By default, goinstall prints output only when it encounters an error. -The -v flag causes goinstall to print information about packages -being considered and installed. +For code hosted on other servers, goinstall recognizes the general form + + repository.vcs/path + +as denoting the given repository, with or without the .vcs suffix, using +the named version control system, and then the path inside that repository. +The supported version control systems are: + + Bazaar .bzr + Git .git + Mercurial .hg + Subversion .svn + +For example, + + import "example.org/user/foo.hg" + +denotes the root directory of the Mercurial repository at example.org/user/foo +or foo.hg, and + + import "example.org/repo.git/foo/bar" + +denotes the foo/bar directory of the Git repository at example.com/repo or +repo.git. -Goinstall does not use make. Makefiles are ignored by goinstall. +When a version control system supports multiple protocols, goinstall tries each +in turn. +For example, for Git it tries git://, then https://, then http://. The GOPATH Environment Variable diff --git a/src/cmd/goinstall/download.go b/src/cmd/goinstall/download.go index d209fa82b..da892a69d 100644 --- a/src/cmd/goinstall/download.go +++ b/src/cmd/goinstall/download.go @@ -7,6 +7,8 @@ package main import ( + "exec" + "fmt" "http" "os" "path/filepath" @@ -31,11 +33,6 @@ func maybeReportToDashboard(path string) { } } -type host struct { - pattern *regexp.Regexp - protocol string -} - // a vcs represents a version control system // like Mercurial, Git, or Subversion. type vcs struct { @@ -57,9 +54,10 @@ type vcs struct { defaultHosts []host } -type vcsMatch struct { - *vcs - prefix, repo string +type host struct { + pattern *regexp.Regexp + protocol string + suffix string } var hg = vcs{ @@ -75,10 +73,11 @@ var hg = vcs{ logLimitFlag: "-l1", logReleaseFlag: "-rrelease", check: "identify", - protocols: []string{"http"}, + protocols: []string{"https", "http"}, + suffix: ".hg", defaultHosts: []host{ - {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/hg)(/[a-z0-9A-Z_.\-/]*)?$`), "https"}, - {regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http"}, + {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/hg)(/[a-z0-9A-Z_.\-/]*)?$`), "https", ""}, + {regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http", ""}, }, } @@ -94,11 +93,11 @@ var git = vcs{ log: "show-ref", logLimitFlag: "", logReleaseFlag: "release", - check: "peek-remote", - protocols: []string{"git", "http"}, + check: "ls-remote", + protocols: []string{"git", "https", "http"}, suffix: ".git", defaultHosts: []host{ - {regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http"}, + {regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http", ".git"}, }, } @@ -114,9 +113,10 @@ var svn = vcs{ logLimitFlag: "-l1", logReleaseFlag: "release", check: "info", - protocols: []string{"http", "svn"}, + protocols: []string{"https", "http", "svn"}, + suffix: ".svn", defaultHosts: []host{ - {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/svn)(/[a-z0-9A-Z_.\-/]*)?$`), "https"}, + {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/svn)(/[a-z0-9A-Z_.\-/]*)?$`), "https", ""}, }, } @@ -134,23 +134,79 @@ var bzr = vcs{ logLimitFlag: "-l1", logReleaseFlag: "-rrelease", check: "info", - protocols: []string{"http", "bzr"}, + protocols: []string{"https", "http", "bzr"}, + suffix: ".bzr", defaultHosts: []host{ - {regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`), "https"}, + {regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`), "https", ""}, }, } var vcsList = []*vcs{&git, &hg, &bzr, &svn} +type vcsMatch struct { + *vcs + prefix, repo string +} + +// findHostedRepo checks whether pkg is located at one of +// the supported code hosting sites and, if so, returns a match. +func findHostedRepo(pkg string) (*vcsMatch, os.Error) { + for _, v := range vcsList { + for _, host := range v.defaultHosts { + if hm := host.pattern.FindStringSubmatch(pkg); hm != nil { + if host.suffix != "" && strings.HasSuffix(hm[1], host.suffix) { + return nil, os.NewError("repository " + pkg + " should not have " + v.suffix + " suffix") + } + repo := host.protocol + "://" + hm[1] + host.suffix + return &vcsMatch{v, hm[1], repo}, nil + } + } + } + return nil, nil +} + +// findAnyRepo looks for a vcs suffix in pkg (.git, etc) and returns a match. +func findAnyRepo(pkg string) (*vcsMatch, os.Error) { + for _, v := range vcsList { + i := strings.Index(pkg+"/", v.suffix+"/") + if i < 0 { + continue + } + if !strings.Contains(pkg[:i], "/") { + continue // don't match vcs suffix in the host name + } + if m := v.find(pkg[:i]); m != nil { + return m, nil + } + return nil, fmt.Errorf("couldn't find %s repository", v.name) + } + return nil, nil +} + +func (v *vcs) find(pkg string) *vcsMatch { + for _, proto := range v.protocols { + for _, suffix := range []string{"", v.suffix} { + repo := proto + "://" + pkg + suffix + out, err := exec.Command(v.cmd, v.check, repo).CombinedOutput() + if err == nil { + printf("find %s: found %s\n", pkg, repo) + return &vcsMatch{v, pkg + v.suffix, repo} + } + printf("find %s: %s %s %s: %v\n%s\n", pkg, v.cmd, v.check, repo, err, out) + } + } + return nil +} + // isRemote returns true if the first part of the package name looks like a // hostname - i.e. contains at least one '.' and the last part is at least 2 // characters. func isRemote(pkg string) bool { - parts := strings.Split(pkg, "/", 2) + parts := strings.SplitN(pkg, "/", 2) if len(parts) != 2 { return false } - parts = strings.Split(parts[0], ".", -1) + parts = strings.Split(parts[0], ".") if len(parts) < 2 || len(parts[len(parts)-1]) < 2 { return false } @@ -158,26 +214,35 @@ func isRemote(pkg string) bool { } // download checks out or updates pkg from the remote server. -func download(pkg, srcDir string) os.Error { +func download(pkg, srcDir string) (dashReport bool, err os.Error) { if strings.Contains(pkg, "..") { - return os.NewError("invalid path (contains ..)") + err = os.NewError("invalid path (contains ..)") + return } - var m *vcsMatch - for _, v := range vcsList { - for _, host := range v.defaultHosts { - if hm := host.pattern.FindStringSubmatch(pkg); hm != nil { - if v.suffix != "" && strings.HasSuffix(hm[1], v.suffix) { - return os.NewError("repository " + pkg + " should not have " + v.suffix + " suffix") - } - repo := host.protocol + "://" + hm[1] + v.suffix - m = &vcsMatch{v, hm[1], repo} - } + m, err := findHostedRepo(pkg) + if err != nil { + return + } + if m != nil { + dashReport = true // only report public code hosting sites + } else { + m, err = findAnyRepo(pkg) + if err != nil { + return } } if m == nil { - return os.NewError("cannot download: " + pkg) + err = os.NewError("cannot download: " + pkg) + return + } + installed, err := m.checkoutRepo(srcDir, m.prefix, m.repo) + if err != nil { + return + } + if !installed { + dashReport = false } - return vcsCheckout(m.vcs, srcDir, m.prefix, m.repo, pkg) + return } // Try to detect if a "release" tag exists. If it does, update @@ -196,47 +261,46 @@ func (v *vcs) updateRepo(dst string) os.Error { return nil } -// vcsCheckout checks out repo into dst using vcs. +// checkoutRepo checks out repo into dst using vcs. // It tries to check out (or update, if the dst already // exists and -u was specified on the command line) // the repository at tag/branch "release". If there is no // such tag or branch, it falls back to the repository tip. -func vcsCheckout(vcs *vcs, srcDir, pkgprefix, repo, dashpath string) os.Error { +func (vcs *vcs) checkoutRepo(srcDir, pkgprefix, repo string) (installed bool, err os.Error) { dst := filepath.Join(srcDir, filepath.FromSlash(pkgprefix)) dir, err := os.Stat(filepath.Join(dst, vcs.metadir)) if err == nil && !dir.IsDirectory() { - return os.NewError("not a directory: " + dst) + err = os.NewError("not a directory: " + dst) + return } if err != nil { parent, _ := filepath.Split(dst) - if err := os.MkdirAll(parent, 0777); err != nil { - return err + if err = os.MkdirAll(parent, 0777); err != nil { + return } - if err := run(string(filepath.Separator), nil, vcs.cmd, vcs.clone, repo, dst); err != nil { - return err + if err = run(string(filepath.Separator), nil, vcs.cmd, vcs.clone, repo, dst); err != nil { + return } - if err := vcs.updateRepo(dst); err != nil { - return err + if err = vcs.updateRepo(dst); err != nil { + return } - // success on first installation - report - maybeReportToDashboard(dashpath) + installed = true } else if *update { // Retrieve new revisions from the remote branch, if the VCS // supports this operation independently (e.g. svn doesn't) if vcs.pull != "" { if vcs.pullForceFlag != "" { - if err := run(dst, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil { - return err + if err = run(dst, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil { + return } - } else if err := run(dst, nil, vcs.cmd, vcs.pull); err != nil { - return err + } else if err = run(dst, nil, vcs.cmd, vcs.pull); err != nil { + return } } - // Update to release or latest revision - if err := vcs.updateRepo(dst); err != nil { - return err + if err = vcs.updateRepo(dst); err != nil { + return } } - return nil + return } diff --git a/src/cmd/goinstall/main.go b/src/cmd/goinstall/main.go index bdf8469a0..5cdf0f18e 100644 --- a/src/cmd/goinstall/main.go +++ b/src/cmd/goinstall/main.go @@ -182,9 +182,10 @@ func install(pkg, parent string) { } // Download remote packages if not found or forced with -u flag. remote := isRemote(pkg) + dashReport := false if remote && (err == build.ErrNotFound || (err == nil && *update)) { printf("%s: download\n", pkg) - err = download(pkg, tree.SrcDir()) + dashReport, err = download(pkg, tree.SrcDir()) } if err != nil { errorf("%s: %v\n", pkg, err) @@ -243,6 +244,9 @@ func install(pkg, parent string) { } } } + if dashReport { + maybeReportToDashboard(pkg) + } if remote { // mark package as installed in $GOROOT/goinstall.log logPackage(pkg) diff --git a/src/cmd/gotest/doc.go b/src/cmd/gotest/doc.go index 9dba390c1..5be06f817 100644 --- a/src/cmd/gotest/doc.go +++ b/src/cmd/gotest/doc.go @@ -53,7 +53,9 @@ The resulting test binary, called (for amd64) 6.out, has several flags. Usage: 6.out [-test.v] [-test.run pattern] [-test.bench pattern] \ [-test.cpuprofile=cpu.out] \ - [-test.memprofile=mem.out] [-test.memprofilerate=1] + [-test.memprofile=mem.out] [-test.memprofilerate=1] \ + [-test.timeout=10] [-test.short] \ + [-test.benchtime=3] [-test.cpu=1,2,3,4] The -test.v flag causes the tests to be logged as they run. The -test.run flag causes only those tests whose names match the regular @@ -93,6 +95,13 @@ The -test.timeout flag sets a timeout for the test in seconds. If the test runs for longer than that, it will panic, dumping a stack trace of all existing goroutines. +The -test.benchtime flag specifies the number of seconds to run each benchmark. +The default is one second. + +The -test.cpu flag specifies a list of GOMAXPROCS values for which +the tests or benchmarks are executed. The default is the current +value of GOMAXPROCS. + For convenience, each of these -test.X flags of the test binary is also available as the flag -X in gotest itself. Flags not listed here are unaffected. For instance, the command diff --git a/src/cmd/gotest/flag.go b/src/cmd/gotest/flag.go index 780c78b9c..c3a28f9a3 100644 --- a/src/cmd/gotest/flag.go +++ b/src/cmd/gotest/flag.go @@ -23,6 +23,8 @@ var usageMessage = `Usage of %s: // These flags can be passed with or without a "test." prefix: -v or -test.v. -bench="": passes -test.bench to test + -benchtime=1: passes -test.benchtime to test + -cpu="": passes -test.cpu to test -cpuprofile="": passes -test.cpuprofile to test -memprofile="": passes -test.memprofile to test -memprofilerate=0: passes -test.memprofilerate to test @@ -56,6 +58,8 @@ var flagDefn = []*flagSpec{ // passed to 6.out, adding a "test." prefix to the name if necessary: -v becomes -test.v. &flagSpec{name: "bench", passToTest: true}, + &flagSpec{name: "benchtime", passToTest: true}, + &flagSpec{name: "cpu", passToTest: true}, &flagSpec{name: "cpuprofile", passToTest: true}, &flagSpec{name: "memprofile", passToTest: true}, &flagSpec{name: "memprofilerate", passToTest: true}, diff --git a/src/cmd/govet/Makefile b/src/cmd/govet/Makefile index 291b27197..f565b78f5 100644 --- a/src/cmd/govet/Makefile +++ b/src/cmd/govet/Makefile @@ -9,3 +9,6 @@ GOFILES=\ govet.go\ include ../../Make.cmd + +test testshort: $(TARG) + ../../../test/errchk $(TARG) -printfuncs='Warn:1,Warnf:1' govet.go diff --git a/src/cmd/govet/govet.go b/src/cmd/govet/govet.go index b811c61a2..5b24d2ff0 100644 --- a/src/cmd/govet/govet.go +++ b/src/cmd/govet/govet.go @@ -15,6 +15,7 @@ import ( "go/token" "os" "path/filepath" + "reflect" "strconv" "strings" "utf8" @@ -50,7 +51,7 @@ func main() { flag.Parse() if *printfuncs != "" { - for _, name := range strings.Split(*printfuncs, ",", -1) { + for _, name := range strings.Split(*printfuncs, ",") { if len(name) == 0 { flag.Usage() } @@ -59,7 +60,7 @@ func main() { var err os.Error skip, err = strconv.Atoi(name[colon+1:]) if err != nil { - error(`illegal format for "Func:N" argument %q; %s`, name, err) + errorf(`illegal format for "Func:N" argument %q; %s`, name, err) } name = name[:colon] } @@ -93,7 +94,7 @@ func doFile(name string, reader io.Reader) { fs := token.NewFileSet() parsedFile, err := parser.ParseFile(fs, name, reader, 0) if err != nil { - error("%s: %s", name, err) + errorf("%s: %s", name, err) return } file := &File{fs.File(parsedFile.Pos())} @@ -121,7 +122,7 @@ func walkDir(root string) { done := make(chan bool) go func() { for e := range errors { - error("walk error: %s", e) + errorf("walk error: %s", e) } done <- true }() @@ -132,7 +133,7 @@ func walkDir(root string) { // error formats the error to standard error, adding program // identification and a newline -func error(format string, args ...interface{}) { +func errorf(format string, args ...interface{}) { fmt.Fprintf(os.Stderr, "govet: "+format+"\n", args...) setExit(2) } @@ -185,15 +186,35 @@ func (f *File) checkFile(name string, file *ast.File) { // Visit implements the ast.Visitor interface. func (f *File) Visit(node ast.Node) ast.Visitor { - // TODO: could return nil for nodes that cannot contain a CallExpr - - // will shortcut traversal. Worthwhile? switch n := node.(type) { case *ast.CallExpr: f.checkCallExpr(n) + case *ast.Field: + f.checkFieldTag(n) } return f } +// checkField checks a struct field tag. +func (f *File) checkFieldTag(field *ast.Field) { + if field.Tag == nil { + return + } + + tag, err := strconv.Unquote(field.Tag.Value) + if err != nil { + f.Warnf(field.Pos(), "unable to read struct tag %s", field.Tag.Value) + return + } + + // Check tag for validity by appending + // new key:value to end and checking that + // the tag parsing code can find it. + if reflect.StructTag(tag+` _gofix:"_magic"`).Get("_gofix") != "_magic" { + f.Warnf(field.Pos(), "struct field tag %s not compatible with reflect.StructTag.Get", field.Tag.Value) + return + } +} // checkCallExpr checks a call expression. func (f *File) checkCallExpr(call *ast.CallExpr) { @@ -358,19 +379,24 @@ func (f *File) checkPrint(call *ast.CallExpr, name string, skip int) { } // This function never executes, but it serves as a simple test for the program. -// Test with govet -printfuncs="Bad:1,Badf:1,Warn:1,Warnf:1" govet.go +// Test with make test. func BadFunctionUsedInTests() { - fmt.Println() // niladic call - fmt.Println("%s", "hi") // % in call to Println - fmt.Printf("%s", "hi", 3) // wrong # percents - fmt.Printf("%s%%%d", "hi", 3) // right # percents - fmt.Printf("%.*d", 3, 3) // right # percents, with a * - fmt.Printf("%.*d", 3, 3, 3) // wrong # percents, with a * - printf("now is the time", "buddy") // no %s - Printf("now is the time", "buddy") // no %s + fmt.Println() // not an error + fmt.Println("%s", "hi") // ERROR "possible formatting directive in Println call" + fmt.Printf("%s", "hi", 3) // ERROR "wrong number of args in Printf call" + fmt.Printf("%s%%%d", "hi", 3) // correct + fmt.Printf("%.*d", 3, 3) // correct + fmt.Printf("%.*d", 3, 3, 3) // ERROR "wrong number of args in Printf call" + printf("now is the time", "buddy") // ERROR "no formatting directive" + Printf("now is the time", "buddy") // ERROR "no formatting directive" + Printf("hi") // ok f := new(File) - f.Warn(0, "%s", "hello", 3) // % in call to added function - f.Warnf(0, "%s", "hello", 3) // wrong # %s in call to added function + f.Warn(0, "%s", "hello", 3) // ERROR "possible formatting directive in Warn call" + f.Warnf(0, "%s", "hello", 3) // ERROR "wrong number of args in Warnf call" +} + +type BadTypeUsedInTests struct { + X int "hello" // ERROR "struct field tag" } // printf is used by the test. diff --git a/src/cmd/goyacc/goyacc.go b/src/cmd/goyacc/goyacc.go index 220c99492..543f8b1e8 100644 --- a/src/cmd/goyacc/goyacc.go +++ b/src/cmd/goyacc/goyacc.go @@ -2834,7 +2834,7 @@ func others() { // copy yaccpar fmt.Fprintf(ftable, "\n//line yaccpar:1\n") - parts := strings.Split(yaccpar, prefix+"run()", 2) + parts := strings.SplitN(yaccpar, prefix+"run()", 2) fmt.Fprintf(ftable, "%v", parts[0]) ftable.Write(fcode.Bytes()) fmt.Fprintf(ftable, "%v", parts[1]) diff --git a/src/cmd/hgpatch/main.go b/src/cmd/hgpatch/main.go index 1f3e5e736..4f7aec22b 100644 --- a/src/cmd/hgpatch/main.go +++ b/src/cmd/hgpatch/main.go @@ -176,7 +176,7 @@ func main() { list[i] = f i++ } - sort.SortStrings(list) + sort.Strings(list) for _, f := range list { fmt.Printf("%s\n", f) } @@ -282,7 +282,7 @@ func hgModified() ([]string, os.Error) { if err != nil { return nil, err } - return strings.Split(strings.TrimSpace(out), "\n", -1), nil + return strings.Split(strings.TrimSpace(out), "\n"), nil } // hgAdd adds name to the repository. diff --git a/src/cmd/ld/data.c b/src/cmd/ld/data.c index bdad58ff9..f1132fc8b 100644 --- a/src/cmd/ld/data.c +++ b/src/cmd/ld/data.c @@ -789,14 +789,34 @@ dodata(void) sect->vaddr = 0; datsize = 0; s = datap; - for(; s != nil && s->type < SDATA; s = s->next) { + for(; s != nil && s->type < SSYMTAB; s = s->next) { s->type = SRODATA; t = rnd(s->size, PtrSize); s->value = datsize; datsize += t; } sect->len = datsize - sect->vaddr; - + + /* gosymtab */ + sect = addsection(&segtext, ".gosymtab", 04); + sect->vaddr = datsize; + for(; s != nil && s->type < SPCLNTAB; s = s->next) { + s->type = SRODATA; + s->value = datsize; + datsize += s->size; + } + sect->len = datsize - sect->vaddr; + + /* gopclntab */ + sect = addsection(&segtext, ".gopclntab", 04); + sect->vaddr = datsize; + for(; s != nil && s->type < SDATA; s = s->next) { + s->type = SRODATA; + s->value = datsize; + datsize += s->size; + } + sect->len = datsize - sect->vaddr; + /* data */ datsize = 0; sect = addsection(&segdata, ".data", 06); @@ -890,7 +910,7 @@ textaddress(void) void address(void) { - Section *s, *text, *data, *rodata; + Section *s, *text, *data, *rodata, *symtab, *pclntab; Sym *sym, *sub; uvlong va; @@ -921,7 +941,9 @@ address(void) segdata.filelen = segdata.sect->len; // assume .data is first text = segtext.sect; - rodata = segtext.sect->next; + rodata = text->next; + symtab = rodata->next; + pclntab = symtab->next; data = segdata.sect; for(sym = datap; sym != nil; sym = sym->next) { @@ -938,12 +960,11 @@ address(void) xdefine("etext", STEXT, text->vaddr + text->len); xdefine("rodata", SRODATA, rodata->vaddr); xdefine("erodata", SRODATA, rodata->vaddr + rodata->len); + xdefine("symtab", SRODATA, symtab->vaddr); + xdefine("esymtab", SRODATA, symtab->vaddr + symtab->len); + xdefine("pclntab", SRODATA, pclntab->vaddr); + xdefine("epclntab", SRODATA, pclntab->vaddr + pclntab->len); xdefine("data", SBSS, data->vaddr); xdefine("edata", SBSS, data->vaddr + data->len); xdefine("end", SBSS, segdata.vaddr + segdata.len); - - sym = lookup("pclntab", 0); - xdefine("epclntab", SRODATA, sym->value + sym->size); - sym = lookup("symtab", 0); - xdefine("esymtab", SRODATA, sym->value + sym->size); } diff --git a/src/cmd/ld/lib.c b/src/cmd/ld/lib.c index 04ee790a4..77a62f5de 100644 --- a/src/cmd/ld/lib.c +++ b/src/cmd/ld/lib.c @@ -956,7 +956,7 @@ pclntab(void) uchar *bp; sym = lookup("pclntab", 0); - sym->type = SRODATA; + sym->type = SPCLNTAB; sym->reachable = 1; if(debug['s']) return; diff --git a/src/cmd/ld/lib.h b/src/cmd/ld/lib.h index 463713143..347987195 100644 --- a/src/cmd/ld/lib.h +++ b/src/cmd/ld/lib.h @@ -40,6 +40,8 @@ enum SSTRING, SGOSTRING, SRODATA, + SSYMTAB, + SPCLNTAB, SDATA, SMACHO, /* Mach-O __nl_symbol_ptr */ SMACHOGOT, diff --git a/src/cmd/ld/symtab.c b/src/cmd/ld/symtab.c index c66eca148..60e146b35 100644 --- a/src/cmd/ld/symtab.c +++ b/src/cmd/ld/symtab.c @@ -351,7 +351,7 @@ symtab(void) s->reachable = 1; symt = lookup("symtab", 0); - symt->type = SRODATA; + symt->type = SSYMTAB; symt->size = 0; symt->reachable = 1; @@ -372,5 +372,7 @@ symtab(void) } } + if(debug['s']) + return; genasmsym(putsymb); } diff --git a/src/env.bash b/src/env.bash index 19402f306..f83012a26 100644 --- a/src/env.bash +++ b/src/env.bash @@ -55,7 +55,6 @@ PROGS=" cp cut echo - ed egrep gcc grep @@ -74,7 +73,7 @@ PROGS=" uniq " -for i in bison ed awk gcc $MAKE; do +for i in $PROGS; do if ! which $i >/dev/null 2>&1; then echo "Cannot find '$i' on search path." 1>&2 echo "See http://golang.org/doc/install.html#ctools" 1>&2 diff --git a/src/lib9/Makefile b/src/lib9/Makefile index d222e2f53..28c97c9b4 100644 --- a/src/lib9/Makefile +++ b/src/lib9/Makefile @@ -116,5 +116,6 @@ GOROOT_FINAL?=$(GOROOT) $(HOST_CC) -c $(HOST_CFLAGS) $< goos.$O: goos.c - $(HOST_CC) -c $(HOST_CFLAGS) -DGOOS='"$(GOOS)"' -DGOARCH='"$(GOARCH)"' -DGOROOT='"$(GOROOT_FINAL)"' -DGOVERSION='"'"$$(../version.bash)"'"' $< + GOVERSION=`../version.bash` && \ + $(HOST_CC) -c $(HOST_CFLAGS) -DGOOS='"$(GOOS)"' -DGOARCH='"$(GOARCH)"' -DGOROOT='"$(GOROOT_FINAL)"' -DGOVERSION='"'"$$GOVERSION"'"' $< diff --git a/src/libmach/darwin.c b/src/libmach/darwin.c index c443a4fba..63abde313 100644 --- a/src/libmach/darwin.c +++ b/src/libmach/darwin.c @@ -222,12 +222,21 @@ addpid(int pid, int force) // The excthread reads that port and signals // us if we are waiting on that thread. pthread_t p; + int err; excport = mach_reply_port(); pthread_mutex_init(&mu, nil); pthread_cond_init(&cond, nil); - pthread_create(&p, nil, excthread, nil); - pthread_create(&p, nil, waitthread, (void*)(uintptr)pid); + err = pthread_create(&p, nil, excthread, nil); + if (err != 0) { + fprint(2, "pthread_create failed: %s\n", strerror(err)); + abort(); + } + err = pthread_create(&p, nil, waitthread, (void*)(uintptr)pid); + if (err != 0) { + fprint(2, "pthread_create failed: %s\n", strerror(err)); + abort(); + } first = 0; } diff --git a/src/pkg/Makefile b/src/pkg/Makefile index f18dc1f9b..7338399c2 100644 --- a/src/pkg/Makefile +++ b/src/pkg/Makefile @@ -62,6 +62,7 @@ DIRS=\ crypto/x509\ crypto/x509/pkix\ crypto/xtea\ + csv\ debug/dwarf\ debug/macho\ debug/elf\ @@ -82,6 +83,7 @@ DIRS=\ exp/gui\ exp/gui/x11\ exp/regexp/syntax\ + exp/template\ expvar\ flag\ fmt\ @@ -240,7 +242,6 @@ NOTEST+=\ ../cmd/godoc\ ../cmd/goinstall\ ../cmd/gotest\ - ../cmd/govet\ ../cmd/goyacc\ ../cmd/hgpatch\ diff --git a/src/pkg/asn1/asn1.go b/src/pkg/asn1/asn1.go index 2650ef2a2..655772931 100644 --- a/src/pkg/asn1/asn1.go +++ b/src/pkg/asn1/asn1.go @@ -149,7 +149,7 @@ func (b BitString) RightAlign() []byte { return a } -// parseBitString parses an ASN.1 bit string from the given byte array and returns it. +// parseBitString parses an ASN.1 bit string from the given byte slice and returns it. func parseBitString(bytes []byte) (ret BitString, err os.Error) { if len(bytes) == 0 { err = SyntaxError{"zero length BIT STRING"} @@ -227,7 +227,7 @@ type Enumerated int type Flag bool // parseBase128Int parses a base-128 encoded int from the given offset in the -// given byte array. It returns the value and the new offset. +// given byte slice. It returns the value and the new offset. func parseBase128Int(bytes []byte, initOffset int) (ret, offset int, err os.Error) { offset = initOffset for shifted := 0; offset < len(bytes); shifted++ { @@ -259,7 +259,7 @@ func parseUTCTime(bytes []byte) (ret *time.Time, err os.Error) { return } -// parseGeneralizedTime parses the GeneralizedTime from the given byte array +// parseGeneralizedTime parses the GeneralizedTime from the given byte slice // and returns the resulting time. func parseGeneralizedTime(bytes []byte) (ret *time.Time, err os.Error) { return time.Parse("20060102150405Z0700", string(bytes)) @@ -300,7 +300,7 @@ func isPrintable(b byte) bool { // IA5String // parseIA5String parses a ASN.1 IA5String (ASCII string) from the given -// byte array and returns it. +// byte slice and returns it. func parseIA5String(bytes []byte) (ret string, err os.Error) { for _, b := range bytes { if b >= 0x80 { @@ -315,11 +315,19 @@ func parseIA5String(bytes []byte) (ret string, err os.Error) { // T61String // parseT61String parses a ASN.1 T61String (8-bit clean string) from the given -// byte array and returns it. +// byte slice and returns it. func parseT61String(bytes []byte) (ret string, err os.Error) { return string(bytes), nil } +// UTF8String + +// parseUTF8String parses a ASN.1 UTF8String (raw UTF-8) from the given byte +// array and returns it. +func parseUTF8String(bytes []byte) (ret string, err os.Error) { + return string(bytes), nil +} + // A RawValue represents an undecoded ASN.1 object. type RawValue struct { Class, Tag int @@ -336,7 +344,7 @@ type RawContent []byte // Tagging // parseTagAndLength parses an ASN.1 tag and length pair from the given offset -// into a byte array. It returns the parsed data and the new offset. SET and +// into a byte slice. It returns the parsed data and the new offset. SET and // SET OF (tag 17) are mapped to SEQUENCE and SEQUENCE OF (tag 16) since we // don't distinguish between ordered and unordered objects in this code. func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset int, err os.Error) { @@ -393,7 +401,7 @@ func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset i } // parseSequenceOf is used for SEQUENCE OF and SET OF values. It tries to parse -// a number of ASN.1 values from the given byte array and returns them as a +// a number of ASN.1 values from the given byte slice and returns them as a // slice of Go values of the given type. func parseSequenceOf(bytes []byte, sliceType reflect.Type, elemType reflect.Type) (ret reflect.Value, err os.Error) { expectedTag, compoundType, ok := getUniversalType(elemType) @@ -456,7 +464,7 @@ func invalidLength(offset, length, sliceLength int) bool { return offset+length < offset || offset+length > sliceLength } -// parseField is the main parsing function. Given a byte array and an offset +// parseField is the main parsing function. Given a byte slice and an offset // into the array, it will try to parse a suitable ASN.1 value out and store it // in the given Value. func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParameters) (offset int, err os.Error) { @@ -573,16 +581,15 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam } } - // Special case for strings: PrintableString and IA5String both map to - // the Go type string. getUniversalType returns the tag for - // PrintableString when it sees a string so, if we see an IA5String on - // the wire, we change the universal type to match. - if universalTag == tagPrintableString && t.tag == tagIA5String { - universalTag = tagIA5String - } - // Likewise for GeneralString - if universalTag == tagPrintableString && t.tag == tagGeneralString { - universalTag = tagGeneralString + // Special case for strings: all the ASN.1 string types map to the Go + // type string. getUniversalType returns the tag for PrintableString + // when it sees a string, so if we see a different string type on the + // wire, we change the universal type to match. + if universalTag == tagPrintableString { + switch t.tag { + case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: + universalTag = t.tag + } } // Special case for time: UTCTime and GeneralizedTime both map to the @@ -707,7 +714,7 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam if i == 0 && field.Type == rawContentsType { continue } - innerOffset, err = parseField(val.Field(i), innerBytes, innerOffset, parseFieldParameters(field.Tag)) + innerOffset, err = parseField(val.Field(i), innerBytes, innerOffset, parseFieldParameters(field.Tag.Get("asn1"))) if err != nil { return } @@ -738,6 +745,8 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam v, err = parseIA5String(innerBytes) case tagT61String: v, err = parseT61String(innerBytes) + case tagUTF8String: + v, err = parseUTF8String(innerBytes) case tagGeneralString: // GeneralString is specified in ISO-2022/ECMA-35, // A brief review suggests that it includes structures diff --git a/src/pkg/asn1/asn1_test.go b/src/pkg/asn1/asn1_test.go index 463dbe026..3c9478618 100644 --- a/src/pkg/asn1/asn1_test.go +++ b/src/pkg/asn1/asn1_test.go @@ -299,11 +299,11 @@ type TestObjectIdentifierStruct struct { } type TestContextSpecificTags struct { - A int "tag:1" + A int `asn1:"tag:1"` } type TestContextSpecificTags2 struct { - A int "explicit,tag:1" + A int `asn1:"explicit,tag:1"` B int } @@ -353,7 +353,7 @@ type Certificate struct { } type TBSCertificate struct { - Version int "optional,explicit,default:0,tag:0" + Version int `asn1:"optional,explicit,default:0,tag:0"` SerialNumber RawValue SignatureAlgorithm AlgorithmIdentifier Issuer RDNSequence diff --git a/src/pkg/asn1/common.go b/src/pkg/asn1/common.go index 9db887e25..01f4f7b6e 100644 --- a/src/pkg/asn1/common.go +++ b/src/pkg/asn1/common.go @@ -25,6 +25,7 @@ const ( tagOctetString = 4 tagOID = 6 tagEnum = 10 + tagUTF8String = 12 tagSequence = 16 tagSet = 17 tagPrintableString = 19 @@ -83,7 +84,7 @@ type fieldParameters struct { // parseFieldParameters will parse it into a fieldParameters structure, // ignoring unknown parts of the string. func parseFieldParameters(str string) (ret fieldParameters) { - for _, part := range strings.Split(str, ",", -1) { + for _, part := range strings.Split(str, ",") { switch { case part == "optional": ret.optional = true diff --git a/src/pkg/asn1/marshal.go b/src/pkg/asn1/marshal.go index 7212c91ef..d7eb63bf8 100644 --- a/src/pkg/asn1/marshal.go +++ b/src/pkg/asn1/marshal.go @@ -413,7 +413,7 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter for i := startingField; i < t.NumField(); i++ { var pre *forkableWriter pre, out = out.fork() - err = marshalField(pre, v.Field(i), parseFieldParameters(t.Field(i).Tag)) + err = marshalField(pre, v.Field(i), parseFieldParameters(t.Field(i).Tag.Get("asn1"))) if err != nil { return } diff --git a/src/pkg/asn1/marshal_test.go b/src/pkg/asn1/marshal_test.go index a9517634d..03df5f1e1 100644 --- a/src/pkg/asn1/marshal_test.go +++ b/src/pkg/asn1/marshal_test.go @@ -30,23 +30,23 @@ type rawContentsStruct struct { } type implicitTagTest struct { - A int "implicit,tag:5" + A int `asn1:"implicit,tag:5"` } type explicitTagTest struct { - A int "explicit,tag:5" + A int `asn1:"explicit,tag:5"` } type ia5StringTest struct { - A string "ia5" + A string `asn1:"ia5"` } type printableStringTest struct { - A string "printable" + A string `asn1:"printable"` } type optionalRawValueTest struct { - A RawValue "optional" + A RawValue `asn1:"optional"` } type testSET []int diff --git a/src/pkg/big/int.go b/src/pkg/big/int.go index 4d47a82d5..0948919cd 100755 --- a/src/pkg/big/int.go +++ b/src/pkg/big/int.go @@ -368,11 +368,60 @@ func (x *Int) Format(s fmt.State, ch int) { format = "0X%s" } } - if x.neg { - format = "-" + format + t := fmt.Sprintf(format, x.abs.string(cs)) + + // insert spaces in hexadecimal formats if needed + if len(t) > 0 && s.Flag(' ') && (ch == 'x' || ch == 'X') { + spaces := (len(t)+1)/2 - 1 + spaced := make([]byte, len(t)+spaces) + var i, j int + spaced[i] = t[j] + i++ + j++ + if len(t)&1 == 0 { + spaced[i] = t[j] + i++ + j++ + } + for j < len(t) { + spaced[i] = ' ' + i++ + spaced[i] = t[j] + i++ + j++ + spaced[i] = t[j] + i++ + j++ + } + t = string(spaced) + } + + // determine sign prefix + prefix := "" + switch { + case x.neg: + prefix = "-" + case s.Flag('+'): + prefix = "+" + case s.Flag(' ') && ch != 'x' && ch != 'X': + prefix = " " + } + + // fill to minimum width and prepend sign prefix + if width, ok := s.Width(); ok && len(t)+len(prefix) < width { + if s.Flag('0') { + t = fmt.Sprintf("%s%0*d%s", prefix, width-len(t)-len(prefix), 0, t) + } else { + if s.Flag('-') { + width = -width + } + t = fmt.Sprintf("%*s", width, prefix+t) + } + } else if prefix != "" { + t = prefix + t } - fmt.Fprintf(s, format, x.abs.string(cs)) + fmt.Fprint(s, t) } @@ -417,6 +466,7 @@ func (z *Int) scan(r io.RuneScanner, base int) (*Int, int, os.Error) { // the scanned number. It accepts the formats 'b' (binary), 'o' (octal), // 'd' (decimal), 'x' (lowercase hexadecimal), and 'X' (uppercase hexadecimal). func (z *Int) Scan(s fmt.ScanState, ch int) os.Error { + s.SkipSpace() // skip leading space characters base := 0 switch ch { case 'b': @@ -585,7 +635,7 @@ func ProbablyPrime(z *Int, n int) bool { } -// Rand sets z to a pseudo-random number in [0, n) and returns z. +// Rand sets z to a pseudo-random number in [0, n) and returns z. func (z *Int) Rand(rnd *rand.Rand, n *Int) *Int { z.neg = false if n.neg == true || len(n.abs) == 0 { diff --git a/src/pkg/big/int_test.go b/src/pkg/big/int_test.go index 58a55030d..7f33c9522 100755 --- a/src/pkg/big/int_test.go +++ b/src/pkg/big/int_test.go @@ -376,6 +376,35 @@ var formatTests = []struct { {"-10", "%#X", "-0XA"}, {"10", "%#y", "%!y(big.Int=10)"}, {"-10", "%#y", "%!y(big.Int=-10)"}, + + {"1234", "%d", "1234"}, + {"1234", "%3d", "1234"}, + {"1234", "%4d", "1234"}, + {"-1234", "%d", "-1234"}, + {"1234", "% 5d", " 1234"}, + {"1234", "%+5d", "+1234"}, + {"1234", "%-5d", "1234 "}, + {"1234", "%x", "4d2"}, + {"1234", "%X", "4D2"}, + {"1234", "% x", "4 d2"}, + {"-1234", "%3x", "-4d2"}, + {"-1234", "%4x", "-4d2"}, + {"-1234", "%5x", " -4d2"}, + {"-1234", "%-5x", "-4d2 "}, + {"-1234", "% x", "-4 d2"}, + {"1234", "%03d", "1234"}, + {"1234", "%04d", "1234"}, + {"1234", "%05d", "01234"}, + {"1234", "%06d", "001234"}, + {"-1234", "%06d", "-01234"}, + {"1234", "%+06d", "+01234"}, + {"1234", "% 06d", " 01234"}, + {"1234", "%-6d", "1234 "}, + {"1234", "%-06d", "001234"}, + {"-1234", "%-06d", "-01234"}, + {"10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", // 10**100 + "% x", + "12 49 ad 25 94 c3 7c eb 0b 27 84 c4 ce 0b f3 8a ce 40 8e 21 1a 7c aa b2 43 08 a8 2e 8f 10 00 00 00 00 00 00 00 00 00 00 00 00"}, } @@ -391,7 +420,7 @@ func TestFormat(t *testing.T) { } output := fmt.Sprintf(test.format, x) if output != test.output { - t.Errorf("#%d got %s; want %s", i, output, test.output) + t.Errorf("#%d got %q; want %q", i, output, test.output) } } } diff --git a/src/pkg/bufio/bufio.go b/src/pkg/bufio/bufio.go index 497e770fb..cb2667b28 100644 --- a/src/pkg/bufio/bufio.go +++ b/src/pkg/bufio/bufio.go @@ -103,6 +103,12 @@ func (b *Reader) fill() { } } +func (b *Reader) readErr() os.Error { + err := b.err + b.err = nil + return err +} + // Peek returns the next n bytes without advancing the reader. The bytes stop // being valid at the next read call. If Peek returns fewer than n bytes, it // also returns an error explaining why the read is short. The error is @@ -121,7 +127,7 @@ func (b *Reader) Peek(n int) ([]byte, os.Error) { if m > n { m = n } - err := b.err + err := b.readErr() if m < n && err == nil { err = ErrBufferFull } @@ -136,11 +142,11 @@ func (b *Reader) Peek(n int) ([]byte, os.Error) { func (b *Reader) Read(p []byte) (n int, err os.Error) { n = len(p) if n == 0 { - return 0, b.err + return 0, b.readErr() } if b.w == b.r { if b.err != nil { - return 0, b.err + return 0, b.readErr() } if len(p) >= len(b.buf) { // Large read, empty buffer. @@ -150,11 +156,11 @@ func (b *Reader) Read(p []byte) (n int, err os.Error) { b.lastByte = int(p[n-1]) b.lastRuneSize = -1 } - return n, b.err + return n, b.readErr() } b.fill() if b.w == b.r { - return 0, b.err + return 0, b.readErr() } } @@ -174,7 +180,7 @@ func (b *Reader) ReadByte() (c byte, err os.Error) { b.lastRuneSize = -1 for b.w == b.r { if b.err != nil { - return 0, b.err + return 0, b.readErr() } b.fill() } @@ -210,7 +216,7 @@ func (b *Reader) ReadRune() (rune int, size int, err os.Error) { } b.lastRuneSize = -1 if b.r == b.w { - return 0, 0, b.err + return 0, 0, b.readErr() } rune, size = int(b.buf[b.r]), 1 if rune >= 0x80 { @@ -262,7 +268,7 @@ func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) { if b.err != nil { line := b.buf[b.r:b.w] b.r = b.w - return line, b.err + return line, b.readErr() } n := b.Buffered() diff --git a/src/pkg/bufio/bufio_test.go b/src/pkg/bufio/bufio_test.go index 123adac29..5709213c8 100644 --- a/src/pkg/bufio/bufio_test.go +++ b/src/pkg/bufio/bufio_test.go @@ -53,11 +53,12 @@ func readBytes(buf *Reader) string { if e == os.EOF { break } - if e != nil { + if e == nil { + b[nb] = c + nb++ + } else if e != iotest.ErrTimeout { panic("Data: " + e.String()) } - b[nb] = c - nb++ } return string(b[0:nb]) } @@ -86,6 +87,7 @@ var readMakers = []readMaker{ {"byte", iotest.OneByteReader}, {"half", iotest.HalfReader}, {"data+err", iotest.DataErrReader}, + {"timeout", iotest.TimeoutReader}, } // Call ReadString (which ends up calling everything else) @@ -97,7 +99,7 @@ func readLines(b *Reader) string { if e == os.EOF { break } - if e != nil { + if e != nil && e != iotest.ErrTimeout { panic("GetLines: " + e.String()) } s += s1 diff --git a/src/pkg/bytes/bytes.go b/src/pkg/bytes/bytes.go index 0f9ac9863..3cec60f96 100644 --- a/src/pkg/bytes/bytes.go +++ b/src/pkg/bytes/bytes.go @@ -212,26 +212,40 @@ func genSplit(s, sep []byte, sepSave, n int) [][]byte { return a[0 : na+1] } -// Split slices s into subslices separated by sep and returns a slice of +// SplitN slices s into subslices separated by sep and returns a slice of // the subslices between those separators. -// If sep is empty, Split splits after each UTF-8 sequence. +// If sep is empty, SplitN splits after each UTF-8 sequence. // The count determines the number of subslices to return: // n > 0: at most n subslices; the last subslice will be the unsplit remainder. // n == 0: the result is nil (zero subslices) // n < 0: all subslices -func Split(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } +func SplitN(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } -// SplitAfter slices s into subslices after each instance of sep and +// SplitAfterN slices s into subslices after each instance of sep and // returns a slice of those subslices. -// If sep is empty, Split splits after each UTF-8 sequence. +// If sep is empty, SplitAfterN splits after each UTF-8 sequence. // The count determines the number of subslices to return: // n > 0: at most n subslices; the last subslice will be the unsplit remainder. // n == 0: the result is nil (zero subslices) // n < 0: all subslices -func SplitAfter(s, sep []byte, n int) [][]byte { +func SplitAfterN(s, sep []byte, n int) [][]byte { return genSplit(s, sep, len(sep), n) } +// Split slices s into all subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, Split splits after each UTF-8 sequence. +// It is equivalent to SplitN with a count of -1. +func Split(s, sep []byte) [][]byte { return genSplit(s, sep, 0, -1) } + +// SplitAfter slices s into all subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfter splits after each UTF-8 sequence. +// It is equivalent to SplitAfterN with a count of -1. +func SplitAfter(s, sep []byte) [][]byte { + return genSplit(s, sep, len(sep), -1) +} + // Fields splits the array s around each instance of one or more consecutive white space // characters, returning a slice of subarrays of s or an empty list if s contains only white space. func Fields(s []byte) [][]byte { diff --git a/src/pkg/bytes/bytes_test.go b/src/pkg/bytes/bytes_test.go index 4ce291a4f..753935309 100644 --- a/src/pkg/bytes/bytes_test.go +++ b/src/pkg/bytes/bytes_test.go @@ -6,6 +6,7 @@ package bytes_test import ( . "bytes" + "reflect" "testing" "unicode" "utf8" @@ -315,7 +316,7 @@ var explodetests = []ExplodeTest{ func TestExplode(t *testing.T) { for _, tt := range explodetests { - a := Split([]byte(tt.s), nil, tt.n) + a := SplitN([]byte(tt.s), nil, tt.n) result := arrayOfString(a) if !eq(result, tt.a) { t.Errorf(`Explode("%s", %d) = %v; want %v`, tt.s, tt.n, result, tt.a) @@ -354,7 +355,7 @@ var splittests = []SplitTest{ func TestSplit(t *testing.T) { for _, tt := range splittests { - a := Split([]byte(tt.s), []byte(tt.sep), tt.n) + a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n) result := arrayOfString(a) if !eq(result, tt.a) { t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) @@ -367,6 +368,12 @@ func TestSplit(t *testing.T) { if string(s) != tt.s { t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) } + if tt.n < 0 { + b := Split([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("Split disagrees withSplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } } } @@ -388,7 +395,7 @@ var splitaftertests = []SplitTest{ func TestSplitAfter(t *testing.T) { for _, tt := range splitaftertests { - a := SplitAfter([]byte(tt.s), []byte(tt.sep), tt.n) + a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n) result := arrayOfString(a) if !eq(result, tt.a) { t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) @@ -398,6 +405,12 @@ func TestSplitAfter(t *testing.T) { if string(s) != tt.s { t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) } + if tt.n < 0 { + b := SplitAfter([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("SplitAfter disagrees withSplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } } } diff --git a/src/pkg/compress/lzw/reader_test.go b/src/pkg/compress/lzw/reader_test.go index 72121a6b5..f8042b0d1 100644 --- a/src/pkg/compress/lzw/reader_test.go +++ b/src/pkg/compress/lzw/reader_test.go @@ -84,7 +84,7 @@ var lzwTests = []lzwTest{ func TestReader(t *testing.T) { b := bytes.NewBuffer(nil) for _, tt := range lzwTests { - d := strings.Split(tt.desc, ";", -1) + d := strings.Split(tt.desc, ";") var order Order switch d[1] { case "LSB": diff --git a/src/pkg/crypto/aes/cipher.go b/src/pkg/crypto/aes/cipher.go index 3a9d02318..73223531e 100644 --- a/src/pkg/crypto/aes/cipher.go +++ b/src/pkg/crypto/aes/cipher.go @@ -45,14 +45,14 @@ func NewCipher(key []byte) (*Cipher, os.Error) { // BlockSize returns the AES block size, 16 bytes. // It is necessary to satisfy the Cipher interface in the -// package "crypto/block". +// package "crypto/cipher". func (c *Cipher) BlockSize() int { return BlockSize } // Encrypt encrypts the 16-byte buffer src using the key k // and stores the result in dst. // Note that for amounts of data larger than a block, // it is not safe to just call Encrypt on successive blocks; -// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c.enc, dst, src) } // Decrypt decrypts the 16-byte buffer src using the key k diff --git a/src/pkg/crypto/blowfish/cipher.go b/src/pkg/crypto/blowfish/cipher.go index f3c5175ac..6c37dfe94 100644 --- a/src/pkg/crypto/blowfish/cipher.go +++ b/src/pkg/crypto/blowfish/cipher.go @@ -42,14 +42,14 @@ func NewCipher(key []byte) (*Cipher, os.Error) { // BlockSize returns the Blowfish block size, 8 bytes. // It is necessary to satisfy the Cipher interface in the -// package "crypto/block". +// package "crypto/cipher". func (c *Cipher) BlockSize() int { return BlockSize } // Encrypt encrypts the 8-byte buffer src using the key k // and stores the result in dst. // Note that for amounts of data larger than a block, // it is not safe to just call Encrypt on successive blocks; -// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). func (c *Cipher) Encrypt(dst, src []byte) { l := uint32(src[0])<<24 | uint32(src[1])<<16 | uint32(src[2])<<8 | uint32(src[3]) r := uint32(src[4])<<24 | uint32(src[5])<<16 | uint32(src[6])<<8 | uint32(src[7]) diff --git a/src/pkg/crypto/ocsp/ocsp.go b/src/pkg/crypto/ocsp/ocsp.go index 57dbe7d2d..e725bded8 100644 --- a/src/pkg/crypto/ocsp/ocsp.go +++ b/src/pkg/crypto/ocsp/ocsp.go @@ -43,7 +43,7 @@ type certID struct { type responseASN1 struct { Status asn1.Enumerated - Response responseBytes "explicit,tag:0" + Response responseBytes `asn1:"explicit,tag:0"` } type responseBytes struct { @@ -55,30 +55,30 @@ type basicResponse struct { TBSResponseData responseData SignatureAlgorithm pkix.AlgorithmIdentifier Signature asn1.BitString - Certificates []asn1.RawValue "explicit,tag:0,optional" + Certificates []asn1.RawValue `asn1:"explicit,tag:0,optional"` } type responseData struct { Raw asn1.RawContent - Version int "optional,default:1,explicit,tag:0" - RequestorName pkix.RDNSequence "optional,explicit,tag:1" - KeyHash []byte "optional,explicit,tag:2" + Version int `asn1:"optional,default:1,explicit,tag:0"` + RequestorName pkix.RDNSequence `asn1:"optional,explicit,tag:1"` + KeyHash []byte `asn1:"optional,explicit,tag:2"` ProducedAt *time.Time Responses []singleResponse } type singleResponse struct { CertID certID - Good asn1.Flag "explicit,tag:0,optional" - Revoked revokedInfo "explicit,tag:1,optional" - Unknown asn1.Flag "explicit,tag:2,optional" + Good asn1.Flag `asn1:"explicit,tag:0,optional"` + Revoked revokedInfo `asn1:"explicit,tag:1,optional"` + Unknown asn1.Flag `asn1:"explicit,tag:2,optional"` ThisUpdate *time.Time - NextUpdate *time.Time "explicit,tag:0,optional" + NextUpdate *time.Time `asn1:"explicit,tag:0,optional"` } type revokedInfo struct { RevocationTime *time.Time - Reason int "explicit,tag:0,optional" + Reason int `asn1:"explicit,tag:0,optional"` } // This is the exposed reflection of the internal OCSP structures. diff --git a/src/pkg/crypto/openpgp/keys.go b/src/pkg/crypto/openpgp/keys.go index d12d07d7e..c70fb7927 100644 --- a/src/pkg/crypto/openpgp/keys.go +++ b/src/pkg/crypto/openpgp/keys.go @@ -12,6 +12,7 @@ import ( "crypto/rsa" "io" "os" + "time" ) // PublicKeyType is the armor type for a PGP public key. @@ -476,3 +477,69 @@ func (e *Entity) SerializePrivate(w io.Writer) (err os.Error) { } return nil } + +// Serialize writes the public part of the given Entity to w. (No private +// key material will be output). +func (e *Entity) Serialize(w io.Writer) os.Error { + err := e.PrimaryKey.Serialize(w) + if err != nil { + return err + } + for _, ident := range e.Identities { + err = ident.UserId.Serialize(w) + if err != nil { + return err + } + err = ident.SelfSignature.Serialize(w) + if err != nil { + return err + } + for _, sig := range ident.Signatures { + err = sig.Serialize(w) + if err != nil { + return err + } + } + } + for _, subkey := range e.Subkeys { + err = subkey.PublicKey.Serialize(w) + if err != nil { + return err + } + err = subkey.Sig.Serialize(w) + if err != nil { + return err + } + } + return nil +} + +// SignIdentity adds a signature to e, from signer, attesting that identity is +// associated with e. The provided identity must already be an element of +// e.Identities and the private key of signer must have been decrypted if +// necessary. +func (e *Entity) SignIdentity(identity string, signer *Entity) os.Error { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing Entity must have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing Entity's private key must be decrypted") + } + ident, ok := e.Identities[identity] + if !ok { + return error.InvalidArgumentError("given identity string not found in Entity") + } + + sig := &packet.Signature{ + SigType: packet.SigTypeGenericCert, + PubKeyAlgo: signer.PrivateKey.PubKeyAlgo, + Hash: crypto.SHA256, + CreationTime: uint32(time.Seconds()), + IssuerKeyId: &signer.PrivateKey.KeyId, + } + if err := sig.SignKey(e.PrimaryKey, signer.PrivateKey); err != nil { + return err + } + ident.Signatures = append(ident.Signatures, sig) + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/public_key.go b/src/pkg/crypto/openpgp/packet/public_key.go index ba4d481f0..e6b0ae5f3 100644 --- a/src/pkg/crypto/openpgp/packet/public_key.go +++ b/src/pkg/crypto/openpgp/packet/public_key.go @@ -219,7 +219,11 @@ func (pk *PublicKey) Serialize(w io.Writer) (err os.Error) { panic("unknown public key algorithm") } - err = serializeHeader(w, packetTypePublicKey, length) + packetType := packetTypePublicKey + if pk.IsSubkey { + packetType = packetTypePublicSubkey + } + err = serializeHeader(w, packetType, length) if err != nil { return } @@ -279,14 +283,14 @@ func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err os.E switch pk.PubKeyAlgo { case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: rsaPublicKey, _ := pk.PublicKey.(*rsa.PublicKey) - err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.RSASignature) + err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.RSASignature.bytes) if err != nil { return error.SignatureError("RSA verification failure") } return nil case PubKeyAlgoDSA: dsaPublicKey, _ := pk.PublicKey.(*dsa.PublicKey) - if !dsa.Verify(dsaPublicKey, hashBytes, sig.DSASigR, sig.DSASigS) { + if !dsa.Verify(dsaPublicKey, hashBytes, new(big.Int).SetBytes(sig.DSASigR.bytes), new(big.Int).SetBytes(sig.DSASigS.bytes)) { return error.SignatureError("DSA verification failure") } return nil diff --git a/src/pkg/crypto/openpgp/packet/signature.go b/src/pkg/crypto/openpgp/packet/signature.go index 123c99fb2..7577e2875 100644 --- a/src/pkg/crypto/openpgp/packet/signature.go +++ b/src/pkg/crypto/openpgp/packet/signature.go @@ -5,7 +5,6 @@ package packet import ( - "big" "crypto" "crypto/dsa" "crypto/openpgp/error" @@ -32,8 +31,11 @@ type Signature struct { HashTag [2]byte CreationTime uint32 // Unix epoch time - RSASignature []byte - DSASigR, DSASigS *big.Int + RSASignature parsedMPI + DSASigR, DSASigS parsedMPI + + // rawSubpackets contains the unparsed subpackets, in order. + rawSubpackets []outputSubpacket // The following are optional so are nil when not included in the // signature. @@ -128,14 +130,11 @@ func (sig *Signature) parse(r io.Reader) (err os.Error) { switch sig.PubKeyAlgo { case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: - sig.RSASignature, _, err = readMPI(r) + sig.RSASignature.bytes, sig.RSASignature.bitLength, err = readMPI(r) case PubKeyAlgoDSA: - var rBytes, sBytes []byte - rBytes, _, err = readMPI(r) - sig.DSASigR = new(big.Int).SetBytes(rBytes) + sig.DSASigR.bytes, sig.DSASigR.bitLength, err = readMPI(r) if err == nil { - sBytes, _, err = readMPI(r) - sig.DSASigS = new(big.Int).SetBytes(sBytes) + sig.DSASigS.bytes, sig.DSASigS.bitLength, err = readMPI(r) } default: panic("unreachable") @@ -179,7 +178,7 @@ func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (r // RFC 4880, section 5.2.3.1 var ( length uint32 - packetType byte + packetType signatureSubpacketType isCritical bool ) switch { @@ -211,10 +210,11 @@ func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (r err = error.StructuralError("zero length signature subpacket") return } - packetType = subpacket[0] & 0x7f + packetType = signatureSubpacketType(subpacket[0] & 0x7f) isCritical = subpacket[0]&0x80 == 0x80 subpacket = subpacket[1:] - switch signatureSubpacketType(packetType) { + sig.rawSubpackets = append(sig.rawSubpackets, outputSubpacket{isHashed, packetType, isCritical, subpacket}) + switch packetType { case creationTimeSubpacket: if !isHashed { err = error.StructuralError("signature creation time in non-hashed area") @@ -385,7 +385,6 @@ func serializeSubpackets(to []byte, subpackets []outputSubpacket, hashed bool) { // buildHashSuffix constructs the HashSuffix member of sig in preparation for signing. func (sig *Signature) buildHashSuffix() (err os.Error) { - sig.outSubpackets = sig.buildSubpackets() hashedSubpacketsLen := subpacketsLength(sig.outSubpackets, true) var ok bool @@ -428,6 +427,7 @@ func (sig *Signature) signPrepareHash(h hash.Hash) (digest []byte, err os.Error) // the hash of the message to be signed and will be mutated by this function. // On success, the signature is stored in sig. Call Serialize to write it out. func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey) (err os.Error) { + sig.outSubpackets = sig.buildSubpackets() digest, err := sig.signPrepareHash(h) if err != nil { return @@ -435,9 +435,16 @@ func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey) (err os.Error) { switch priv.PubKeyAlgo { case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: - sig.RSASignature, err = rsa.SignPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), sig.Hash, digest) + sig.RSASignature.bytes, err = rsa.SignPKCS1v15(rand.Reader, priv.PrivateKey.(*rsa.PrivateKey), sig.Hash, digest) + sig.RSASignature.bitLength = uint16(8 * len(sig.RSASignature.bytes)) case PubKeyAlgoDSA: - sig.DSASigR, sig.DSASigS, err = dsa.Sign(rand.Reader, priv.PrivateKey.(*dsa.PrivateKey), digest) + r, s, err := dsa.Sign(rand.Reader, priv.PrivateKey.(*dsa.PrivateKey), digest) + if err == nil { + sig.DSASigR.bytes = r.Bytes() + sig.DSASigR.bitLength = uint16(8 * len(sig.DSASigR.bytes)) + sig.DSASigS.bytes = s.Bytes() + sig.DSASigS.bitLength = uint16(8 * len(sig.DSASigS.bytes)) + } default: err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) } @@ -468,17 +475,20 @@ func (sig *Signature) SignKey(pub *PublicKey, priv *PrivateKey) os.Error { // Serialize marshals sig to w. SignRSA or SignDSA must have been called first. func (sig *Signature) Serialize(w io.Writer) (err os.Error) { - if sig.RSASignature == nil && sig.DSASigR == nil { + if len(sig.outSubpackets) == 0 { + sig.outSubpackets = sig.rawSubpackets + } + if sig.RSASignature.bytes == nil && sig.DSASigR.bytes == nil { return error.InvalidArgumentError("Signature: need to call SignRSA or SignDSA before Serialize") } sigLength := 0 switch sig.PubKeyAlgo { case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: - sigLength = len(sig.RSASignature) + sigLength = 2 + len(sig.RSASignature.bytes) case PubKeyAlgoDSA: - sigLength = mpiLength(sig.DSASigR) - sigLength += mpiLength(sig.DSASigS) + sigLength = 2 + len(sig.DSASigR.bytes) + sigLength += 2 + len(sig.DSASigS.bytes) default: panic("impossible") } @@ -486,7 +496,7 @@ func (sig *Signature) Serialize(w io.Writer) (err os.Error) { unhashedSubpacketsLen := subpacketsLength(sig.outSubpackets, false) length := len(sig.HashSuffix) - 6 /* trailer not included */ + 2 /* length of unhashed subpackets */ + unhashedSubpacketsLen + - 2 /* hash tag */ + 2 /* length of signature MPI */ + sigLength + 2 /* hash tag */ + sigLength err = serializeHeader(w, packetTypeSignature, length) if err != nil { return @@ -513,12 +523,9 @@ func (sig *Signature) Serialize(w io.Writer) (err os.Error) { switch sig.PubKeyAlgo { case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: - err = writeMPI(w, 8*uint16(len(sig.RSASignature)), sig.RSASignature) + err = writeMPIs(w, sig.RSASignature) case PubKeyAlgoDSA: - err = writeBig(w, sig.DSASigR) - if err == nil { - err = writeBig(w, sig.DSASigS) - } + err = writeMPIs(w, sig.DSASigR, sig.DSASigS) default: panic("impossible") } @@ -529,6 +536,7 @@ func (sig *Signature) Serialize(w io.Writer) (err os.Error) { type outputSubpacket struct { hashed bool // true if this subpacket is in the hashed area. subpacketType signatureSubpacketType + isCritical bool contents []byte } @@ -538,12 +546,12 @@ func (sig *Signature) buildSubpackets() (subpackets []outputSubpacket) { creationTime[1] = byte(sig.CreationTime >> 16) creationTime[2] = byte(sig.CreationTime >> 8) creationTime[3] = byte(sig.CreationTime) - subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, creationTime}) + subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, false, creationTime}) if sig.IssuerKeyId != nil { keyId := make([]byte, 8) binary.BigEndian.PutUint64(keyId, *sig.IssuerKeyId) - subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, keyId}) + subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, false, keyId}) } return diff --git a/src/pkg/crypto/openpgp/packet/signature_test.go b/src/pkg/crypto/openpgp/packet/signature_test.go index 1305548b2..c1bbde8b0 100644 --- a/src/pkg/crypto/openpgp/packet/signature_test.go +++ b/src/pkg/crypto/openpgp/packet/signature_test.go @@ -12,9 +12,7 @@ import ( ) func TestSignatureRead(t *testing.T) { - signatureData, _ := hex.DecodeString(signatureDataHex) - buf := bytes.NewBuffer(signatureData) - packet, err := Read(buf) + packet, err := Read(readerFromHex(signatureDataHex)) if err != nil { t.Error(err) return @@ -25,4 +23,20 @@ func TestSignatureRead(t *testing.T) { } } -const signatureDataHex = "89011c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e" +func TestSignatureReserialize(t *testing.T) { + packet, _ := Read(readerFromHex(signatureDataHex)) + sig := packet.(*Signature) + out := new(bytes.Buffer) + err := sig.Serialize(out) + if err != nil { + t.Errorf("error reserializing: %s", err) + return + } + + expected, _ := hex.DecodeString(signatureDataHex) + if !bytes.Equal(expected, out.Bytes()) { + t.Errorf("output doesn't match input (got vs expected):\n%s\n%s", hex.Dump(out.Bytes()), hex.Dump(expected)) + } +} + +const signatureDataHex = "c2c05c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e" diff --git a/src/pkg/crypto/openpgp/read.go b/src/pkg/crypto/openpgp/read.go index 683014752..d95f613c6 100644 --- a/src/pkg/crypto/openpgp/read.go +++ b/src/pkg/crypto/openpgp/read.go @@ -250,11 +250,12 @@ FindLiteralData: md.IsSigned = true md.SignedByKeyId = p.KeyId keys := keyring.KeysById(p.KeyId) - for _, key := range keys { + for i, key := range keys { if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { continue } - md.SignedBy = &key + md.SignedBy = &keys[i] + break } case *packet.LiteralData: md.LiteralData = p diff --git a/src/pkg/crypto/openpgp/read_test.go b/src/pkg/crypto/openpgp/read_test.go index 7182e94b5..4dc290ef2 100644 --- a/src/pkg/crypto/openpgp/read_test.go +++ b/src/pkg/crypto/openpgp/read_test.go @@ -33,6 +33,29 @@ func TestReadKeyRing(t *testing.T) { } } +func TestRereadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Errorf("error in initial parse: %s", err) + return + } + out := new(bytes.Buffer) + err = kring[0].Serialize(out) + if err != nil { + t.Errorf("error in serialization: %s", err) + return + } + kring, err = ReadKeyRing(out) + if err != nil { + t.Errorf("error in second parse: %s", err) + return + } + + if len(kring) != 1 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB { + t.Errorf("bad keyring: %#v", kring) + } +} + func TestReadPrivateKeyRing(t *testing.T) { kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) if err != nil { diff --git a/src/pkg/crypto/rand/rand_windows.go b/src/pkg/crypto/rand/rand_windows.go index 281d6dc6a..0eab6b213 100755 --- a/src/pkg/crypto/rand/rand_windows.go +++ b/src/pkg/crypto/rand/rand_windows.go @@ -19,7 +19,7 @@ func init() { Reader = &rngReader{} } // A rngReader satisfies reads by reading from the Windows CryptGenRandom API. type rngReader struct { - prov uint32 + prov syscall.Handle mu sync.Mutex } diff --git a/src/pkg/crypto/tls/generate_cert.go b/src/pkg/crypto/tls/generate_cert.go index f46188879..41206e276 100644 --- a/src/pkg/crypto/tls/generate_cert.go +++ b/src/pkg/crypto/tls/generate_cert.go @@ -8,8 +8,10 @@ package main import ( - "crypto/rsa" + "big" + "crypto/x509/pkix" "crypto/rand" + "crypto/rsa" "crypto/x509" "encoding/pem" "flag" @@ -32,8 +34,8 @@ func main() { now := time.Seconds() template := x509.Certificate{ - SerialNumber: []byte{0}, - Subject: x509.Name{ + SerialNumber: new(big.Int).SetInt64(0), + Subject: pkix.Name{ CommonName: *hostName, Organization: []string{"Acme Co"}, }, diff --git a/src/pkg/crypto/twofish/twofish.go b/src/pkg/crypto/twofish/twofish.go index 1a1aac9b9..2e537c606 100644 --- a/src/pkg/crypto/twofish/twofish.go +++ b/src/pkg/crypto/twofish/twofish.go @@ -269,7 +269,7 @@ func h(in, key []byte, offset int) uint32 { // Encrypt encrypts a 16-byte block from src to dst, which may overlap. // Note that for amounts of data larger than a block, // it is not safe to just call Encrypt on successive blocks; -// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). func (c *Cipher) Encrypt(dst, src []byte) { S1 := c.s[0] S2 := c.s[1] diff --git a/src/pkg/crypto/x509/pkix/pkix.go b/src/pkg/crypto/x509/pkix/pkix.go index 7806b2a2e..266fd557a 100644 --- a/src/pkg/crypto/x509/pkix/pkix.go +++ b/src/pkg/crypto/x509/pkix/pkix.go @@ -16,7 +16,7 @@ import ( // 5280, section 4.1.1.2. type AlgorithmIdentifier struct { Algorithm asn1.ObjectIdentifier - Parameters asn1.RawValue "optional" + Parameters asn1.RawValue `asn1:"optional"` } type RDNSequence []RelativeDistinguishedNameSET @@ -32,7 +32,7 @@ type AttributeTypeAndValue struct { // 5280, section 4.2. type Extension struct { Id asn1.ObjectIdentifier - Critical bool "optional" + Critical bool `asn1:"optional"` Value []byte } @@ -149,13 +149,13 @@ func (certList *CertificateList) HasExpired(currentTimeSeconds int64) bool { // 5280, section 5.1. type TBSCertificateList struct { Raw asn1.RawContent - Version int "optional,default:2" + Version int `asn1:"optional,default:2"` Signature AlgorithmIdentifier Issuer RDNSequence ThisUpdate *time.Time NextUpdate *time.Time - RevokedCertificates []RevokedCertificate "optional" - Extensions []Extension "tag:0,optional,explicit" + RevokedCertificates []RevokedCertificate `asn1:"optional"` + Extensions []Extension `asn1:"tag:0,optional,explicit"` } // RevokedCertificate represents the ASN.1 structure of the same name. See RFC @@ -163,5 +163,5 @@ type TBSCertificateList struct { type RevokedCertificate struct { SerialNumber *big.Int RevocationTime *time.Time - Extensions []Extension "optional" + Extensions []Extension `asn1:"optional"` } diff --git a/src/pkg/crypto/x509/verify.go b/src/pkg/crypto/x509/verify.go index 9145880a2..cad863db8 100644 --- a/src/pkg/crypto/x509/verify.go +++ b/src/pkg/crypto/x509/verify.go @@ -171,8 +171,14 @@ func (c *Certificate) buildChains(cache map[int][][]*Certificate, currentChain [ chains = append(chains, appendToFreshChain(currentChain, root)) } +nextIntermediate: for _, intermediateNum := range opts.Intermediates.findVerifiedParents(c) { intermediate := opts.Intermediates.certs[intermediateNum] + for _, cert := range currentChain { + if cert == intermediate { + continue nextIntermediate + } + } err = intermediate.isValid(intermediateCertificate, opts) if err != nil { continue @@ -202,8 +208,8 @@ func matchHostnames(pattern, host string) bool { return false } - patternParts := strings.Split(pattern, ".", -1) - hostParts := strings.Split(host, ".", -1) + patternParts := strings.Split(pattern, ".") + hostParts := strings.Split(host, ".") if len(patternParts) != len(hostParts) { return false diff --git a/src/pkg/crypto/x509/verify_test.go b/src/pkg/crypto/x509/verify_test.go index 7a631186a..111f60eb1 100644 --- a/src/pkg/crypto/x509/verify_test.go +++ b/src/pkg/crypto/x509/verify_test.go @@ -72,23 +72,24 @@ var verifyTests = []verifyTest{ }, }, { - leaf: googleLeaf, - intermediates: []string{verisignRoot, thawteIntermediate}, - roots: []string{verisignRoot}, + leaf: dnssecExpLeaf, + intermediates: []string{startComIntermediate}, + roots: []string{startComRoot}, currentTime: 1302726541, expectedChains: [][]string{ - []string{"Google", "Thawte", "VeriSign"}, + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, }, }, { leaf: dnssecExpLeaf, - intermediates: []string{startComIntermediate}, + intermediates: []string{startComIntermediate, startComRoot}, roots: []string{startComRoot}, currentTime: 1302726541, expectedChains: [][]string{ []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, }, }, } diff --git a/src/pkg/crypto/x509/x509.go b/src/pkg/crypto/x509/x509.go index 8bafeda5c..348727a26 100644 --- a/src/pkg/crypto/x509/x509.go +++ b/src/pkg/crypto/x509/x509.go @@ -30,11 +30,11 @@ type pkcs1PrivateKey struct { P *big.Int Q *big.Int // We ignore these values, if present, because rsa will calculate them. - Dp *big.Int "optional" - Dq *big.Int "optional" - Qinv *big.Int "optional" + Dp *big.Int `asn1:"optional"` + Dq *big.Int `asn1:"optional"` + Qinv *big.Int `asn1:"optional"` - AdditionalPrimes []pkcs1AdditionalRSAPrime "optional" + AdditionalPrimes []pkcs1AdditionalRSAPrime `asn1:"optional"` } type pkcs1AdditionalRSAPrime struct { @@ -136,16 +136,16 @@ type certificate struct { type tbsCertificate struct { Raw asn1.RawContent - Version int "optional,explicit,default:1,tag:0" + Version int `asn1:"optional,explicit,default:1,tag:0"` SerialNumber *big.Int SignatureAlgorithm pkix.AlgorithmIdentifier Issuer pkix.RDNSequence Validity validity Subject pkix.RDNSequence PublicKey publicKeyInfo - UniqueId asn1.BitString "optional,tag:1" - SubjectUniqueId asn1.BitString "optional,tag:2" - Extensions []pkix.Extension "optional,explicit,tag:3" + UniqueId asn1.BitString `asn1:"optional,tag:1"` + SubjectUniqueId asn1.BitString `asn1:"optional,tag:2"` + Extensions []pkix.Extension `asn1:"optional,explicit,tag:3"` } type dsaAlgorithmParameters struct { @@ -168,7 +168,7 @@ type publicKeyInfo struct { // RFC 5280, 4.2.1.1 type authKeyId struct { - Id []byte "optional,tag:0" + Id []byte `asn1:"optional,tag:0"` } type SignatureAlgorithm int @@ -480,8 +480,8 @@ func (h UnhandledCriticalExtension) String() string { } type basicConstraints struct { - IsCA bool "optional" - MaxPathLen int "optional" + IsCA bool `asn1:"optional"` + MaxPathLen int `asn1:"optional"` } type rsaPublicKey struct { @@ -497,14 +497,14 @@ type policyInformation struct { // RFC 5280, 4.2.1.10 type nameConstraints struct { - Permitted []generalSubtree "optional,tag:0" - Excluded []generalSubtree "optional,tag:1" + Permitted []generalSubtree `asn1:"optional,tag:0"` + Excluded []generalSubtree `asn1:"optional,tag:1"` } type generalSubtree struct { - Name string "tag:2,optional,ia5" - Min int "optional,tag:0" - Max int "optional,tag:1" + Name string `asn1:"tag:2,optional,ia5"` + Min int `asn1:"optional,tag:0"` + Max int `asn1:"optional,tag:1"` } func parsePublicKey(algo PublicKeyAlgorithm, keyData *publicKeyInfo) (interface{}, os.Error) { diff --git a/src/pkg/crypto/xtea/cipher.go b/src/pkg/crypto/xtea/cipher.go index f2a5da003..b3fba3c84 100644 --- a/src/pkg/crypto/xtea/cipher.go +++ b/src/pkg/crypto/xtea/cipher.go @@ -48,13 +48,13 @@ func NewCipher(key []byte) (*Cipher, os.Error) { // BlockSize returns the XTEA block size, 8 bytes. // It is necessary to satisfy the Cipher interface in the -// package "crypto/block". +// package "crypto/cipher". func (c *Cipher) BlockSize() int { return BlockSize } // Encrypt encrypts the 8 byte buffer src using the key and stores the result in dst. // Note that for amounts of data larger than a block, // it is not safe to just call Encrypt on successive blocks; -// instead, use an encryption mode like CBC (see crypto/block/cbc.go). +// instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). func (c *Cipher) Encrypt(dst, src []byte) { encryptBlock(c, dst, src) } // Decrypt decrypts the 8 byte buffer src using the key k and stores the result in dst. diff --git a/src/pkg/csv/Makefile b/src/pkg/csv/Makefile new file mode 100644 index 000000000..e364d51d2 --- /dev/null +++ b/src/pkg/csv/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=csv +GOFILES=\ + reader.go\ + writer.go\ + +include ../../Make.pkg diff --git a/src/pkg/csv/reader.go b/src/pkg/csv/reader.go new file mode 100644 index 000000000..1f4b61cf9 --- /dev/null +++ b/src/pkg/csv/reader.go @@ -0,0 +1,373 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package csv reads and writes comma-separated values (CSV) files. +// +// A csv file contains zero or more records of one or more fields per record. +// Each record is separated by the newline character. The final record may +// optionally be followed by a newline character. +// +// field1,field2,field3 +// +// White space is considered part of a field. +// +// Carriage returns before newline characters are silently removed. +// +// Blank lines are ignored. A line with only whitespace characters (excluding +// the ending newline character) is not considered a blank line. +// +// Fields which start and stop with the quote character " are called +// quoted-fields. The beginning and ending quote are not part of the +// field. +// +// The source: +// +// normal string,"quoted-field" +// +// results in the fields +// +// {`normal string`, `quoted-field`} +// +// Within a quoted-field a quote character followed by a second quote +// character is considered a single quote. +// +// "the ""word"" is true","a ""quoted-field""" +// +// results in +// +// {`the "word" is true`, `a "quoted-field"`} +// +// Newlines and commas may be included in a quoted-field +// +// "Multi-line +// field","comma is ," +// +// results in +// +// {`Multi-line +// field`, `comma is ,`} +package csv + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "unicode" +) + +// A ParseError is returned for parsing errors. +// The first line is 1. The first column is 0. +type ParseError struct { + Line int // Line where the error occurred + Column int // Column (rune index) where the error occurred + Error os.Error // The actual error +} + +func (e *ParseError) String() string { + return fmt.Sprintf("line %d, column %d: %s", e.Line, e.Column, e.Error) +} + +// These are the errors that can be returned in ParseError.Error +var ( + ErrTrailingComma = os.NewError("extra delimiter at end of line") + ErrBareQuote = os.NewError("bare \" in non-quoted-field") + ErrQuote = os.NewError("extraneous \" in field") + ErrFieldCount = os.NewError("wrong number of fields in line") +) + +// A Reader reads records from a CSV-encoded file. +// +// As returned by NewReader, a Reader expects input conforming to RFC 4180. +// The exported fields can be changed to customize the details before the +// first call to Read or ReadAll. +// +// Comma is the field delimiter. It defaults to ','. +// +// Comment, if not 0, is the comment character. Lines beginning with the +// Comment character is ignored. +// +// If FieldsPerRecord is positive, Read requires each record to +// have the given number of fields. If FieldsPerRecord is 0, Read sets it to +// the number of fields in the first record, so that future records must +// have the same field count. +// +// If LazyQuotes is true, a quote may appear in an unquoted field and a +// non-doubled quote may appear in a quoted field. +// +// If TrailingComma is true, the last field may be a unquoted empty field. +// +// If TrimLeadingSpace is true, leading white space in a field is ignored. +type Reader struct { + Comma int // Field delimiter (set to ',' by NewReader) + Comment int // Comment character for start of line + FieldsPerRecord int // Number of expected fields per record + LazyQuotes bool // Allow lazy quotes + TrailingComma bool // Allow trailing comma + TrimLeadingSpace bool // Trim leading space + line int + column int + r *bufio.Reader + field bytes.Buffer +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.Reader) *Reader { + return &Reader{ + Comma: ',', + r: bufio.NewReader(r), + } +} + +// error creates a new ParseError based on err. +func (r *Reader) error(err os.Error) os.Error { + return &ParseError{ + Line: r.line, + Column: r.column, + Error: err, + } +} + +// Read reads one record from r. The record is a slice of strings with each +// string representing one field. +func (r *Reader) Read() (record []string, err os.Error) { + for { + record, err = r.parseRecord() + if record != nil { + break + } + if err != nil { + return nil, err + } + } + + if r.FieldsPerRecord > 0 { + if len(record) != r.FieldsPerRecord { + r.column = 0 // report at start of record + return record, r.error(ErrFieldCount) + } + } else if r.FieldsPerRecord == 0 { + r.FieldsPerRecord = len(record) + } + return record, nil +} + +// ReadAll reads all the remaining records from r. +// Each record is a slice of fields. +func (r *Reader) ReadAll() (records [][]string, err os.Error) { + for { + record, err := r.Read() + if err == os.EOF { + return records, nil + } + if err != nil { + return nil, err + } + records = append(records, record) + } + panic("unreachable") +} + +// readRune reads one rune from r, folding \r\n to \n and keeping track +// of our far into the line we have read. r.column will point to the start +// of this rune, not the end of this rune. +func (r *Reader) readRune() (int, os.Error) { + rune, _, err := r.r.ReadRune() + + // Handle \r\n here. We make the simplifying assumption that + // anytime \r is followed by \n that it can be folded to \n. + // We will not detect files which contain both \r\n and bare \n. + if rune == '\r' { + rune, _, err = r.r.ReadRune() + if err == nil { + if rune != '\n' { + r.r.UnreadRune() + rune = '\r' + } + } + } + r.column++ + return rune, err +} + +// unreadRune puts the last rune read from r back. +func (r *Reader) unreadRune() { + r.r.UnreadRune() + r.column-- +} + +// skip reads runes up to and including the rune delim or until error. +func (r *Reader) skip(delim int) os.Error { + for { + rune, err := r.readRune() + if err != nil { + return err + } + if rune == delim { + return nil + } + } + panic("unreachable") +} + +// parseRecord reads and parses a single csv record from r. +func (r *Reader) parseRecord() (fields []string, err os.Error) { + // Each record starts on a new line. We increment our line + // number (lines start at 1, not 0) and set column to -1 + // so as we increment in readRune it points to the character we read. + r.line++ + r.column = -1 + + // Peek at the first rune. If it is an error we are done. + // If we are support comments and it is the comment character + // the skip to the end of line. + + rune, _, err := r.r.ReadRune() + if err != nil { + return nil, err + } + + if r.Comment != 0 && rune == r.Comment { + return nil, r.skip('\n') + } + r.r.UnreadRune() + + // At this point we have at least one field. + for { + haveField, delim, err := r.parseField() + if haveField { + fields = append(fields, r.field.String()) + } + if delim == '\n' || err == os.EOF { + return fields, err + } else if err != nil { + return nil, err + } + } + panic("unreachable") +} + + +// parseField parses the next field in the record. The read field is +// located in r.field. Delim is the first character not part of the field +// (r.Comma or '\n'). +func (r *Reader) parseField() (haveField bool, delim int, err os.Error) { + r.field.Reset() + + rune, err := r.readRune() + if err != nil { + // If we have EOF and are not at the start of a line + // then we return the empty field. We have already + // checked for trailing commas if needed. + if err == os.EOF && r.column != 0 { + return true, 0, err + } + return false, 0, err + } + + if r.TrimLeadingSpace { + for unicode.IsSpace(rune) { + rune, err = r.readRune() + if err != nil { + return false, 0, err + } + } + } + + switch rune { + case r.Comma: + // will check below + + case '\n': + // We are a trailing empty field or a blank linke + if r.column == 0 { + return false, rune, nil + } + return true, rune, nil + + case '"': + // quoted field + Quoted: + for { + rune, err = r.readRune() + if err != nil { + if err == os.EOF { + if r.LazyQuotes { + return true, 0, err + } + return false, 0, r.error(ErrQuote) + } + return false, 0, err + } + switch rune { + case '"': + rune, err = r.readRune() + if err != nil || rune == r.Comma { + break Quoted + } + if rune == '\n' { + return true, rune, nil + } + if rune != '"' { + if !r.LazyQuotes { + r.column-- + return false, 0, r.error(ErrQuote) + } + // accept the bare quote + r.field.WriteRune('"') + } + case '\n': + r.line++ + r.column = -1 + } + r.field.WriteRune(rune) + } + + default: + // unquoted field + for { + r.field.WriteRune(rune) + rune, err = r.readRune() + if err != nil || rune == r.Comma { + break + } + if rune == '\n' { + return true, rune, nil + } + if !r.LazyQuotes && rune == '"' { + return false, 0, r.error(ErrBareQuote) + } + } + } + + if err != nil { + if err == os.EOF { + return true, 0, err + } + return false, 0, err + } + + if !r.TrailingComma { + // We don't allow trailing commas. See if we + // are at the end of the line (being mindful + // of triming spaces + c := r.column + rune, err = r.readRune() + if r.TrimLeadingSpace { + for unicode.IsSpace(rune) { + rune, err = r.readRune() + if err != nil { + break + } + } + } + if err == os.EOF || rune == '\n' { + r.column = c // report the comma + return false, 0, r.error(ErrTrailingComma) + } + r.unreadRune() + } + return true, rune, nil +} diff --git a/src/pkg/csv/reader_test.go b/src/pkg/csv/reader_test.go new file mode 100644 index 000000000..0068bad1d --- /dev/null +++ b/src/pkg/csv/reader_test.go @@ -0,0 +1,265 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "reflect" + "strings" + "testing" +) + +var readTests = []struct { + Name string + Input string + Output [][]string + UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1 + + // These fields are copied into the Reader + Comma int + Comment int + FieldsPerRecord int + LazyQuotes bool + TrailingComma bool + TrimLeadingSpace bool + + Error string + Line int // Expected error line if != 0 + Column int // Expected error column if line != 0 +}{ + { + Name: "Simple", + Input: "a,b,c\n", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "CRLF", + Input: "a,b\r\nc,d\r\n", + Output: [][]string{{"a", "b"}, {"c", "d"}}, + }, + { + Name: "BareCR", + Input: "a,b\rc,d\r\n", + Output: [][]string{{"a", "b\rc", "d"}}, + }, + { + Name: "RFC4180test", + UseFieldsPerRecord: true, + Input: `#field1,field2,field3 +"aaa","bb +b","ccc" +"a,a","b""bb","ccc" +zzz,yyy,xxx +`, + Output: [][]string{ + {"#field1", "field2", "field3"}, + {"aaa", "bb\nb", "ccc"}, + {"a,a", `b"bb`, "ccc"}, + {"zzz", "yyy", "xxx"}, + }, + }, + { + Name: "NoEOLTest", + Input: "a,b,c", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "Semicolon", + Comma: ';', + Input: "a;b;c\n", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "MultiLine", + Input: `"two +line","one line","three +line +field"`, + Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}}, + }, + { + Name: "BlankLine", + Input: "a,b,c\n\nd,e,f\n\n", + Output: [][]string{ + {"a", "b", "c"}, + {"d", "e", "f"}, + }, + }, + { + Name: "TrimSpace", + Input: " a, b, c\n", + TrimLeadingSpace: true, + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "LeadingSpace", + Input: " a, b, c\n", + Output: [][]string{{" a", " b", " c"}}, + }, + { + Name: "Comment", + Comment: '#', + Input: "#1,2,3\na,b,c\n#comment", + Output: [][]string{{"a", "b", "c"}}, + }, + { + Name: "NoComment", + Input: "#1,2,3\na,b,c", + Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}}, + }, + { + Name: "LazyQuotes", + LazyQuotes: true, + Input: `a "word","1"2",a","b`, + Output: [][]string{{`a "word"`, `1"2`, `a"`, `b`}}, + }, + { + Name: "BareQuotes", + LazyQuotes: true, + Input: `a "word","1"2",a"`, + Output: [][]string{{`a "word"`, `1"2`, `a"`}}, + }, + { + Name: "BareDoubleQuotes", + LazyQuotes: true, + Input: `a""b,c`, + Output: [][]string{{`a""b`, `c`}}, + }, + { + Name: "BadDoubleQuotes", + Input: `a""b,c`, + Output: [][]string{{`a""b`, `c`}}, + Error: `bare " in non-quoted-field`, Line: 1, Column: 1, + }, + { + Name: "TrimQuote", + Input: ` "a"," b",c`, + TrimLeadingSpace: true, + Output: [][]string{{"a", " b", "c"}}, + }, + { + Name: "BadBareQuote", + Input: `a "word","b"`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 2, + }, + { + Name: "BadTrailingQuote", + Input: `"a word",b"`, + Error: `bare " in non-quoted-field`, Line: 1, Column: 10, + }, + { + Name: "ExtraneousQuote", + Input: `"a "word","b"`, + Error: `extraneous " in field`, Line: 1, Column: 3, + }, + { + Name: "BadFieldCount", + UseFieldsPerRecord: true, + Input: "a,b,c\nd,e", + Error: "wrong number of fields", Line: 2, + }, + { + Name: "BadFieldCount1", + UseFieldsPerRecord: true, + FieldsPerRecord: 2, + Input: `a,b,c`, + Error: "wrong number of fields", Line: 1, + }, + { + Name: "FieldCount", + Input: "a,b,c\nd,e", + Output: [][]string{{"a", "b", "c"}, {"d", "e"}}, + }, + { + Name: "BadTrailingCommaEOF", + Input: "a,b,c,", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaEOL", + Input: "a,b,c,\n", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaSpaceEOF", + TrimLeadingSpace: true, + Input: "a,b,c, ", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaSpaceEOL", + TrimLeadingSpace: true, + Input: "a,b,c, \n", + Error: "extra delimiter at end of line", Line: 1, Column: 5, + }, + { + Name: "BadTrailingCommaLine3", + TrimLeadingSpace: true, + Input: "a,b,c\nd,e,f\ng,hi,", + Error: "extra delimiter at end of line", Line: 3, Column: 4, + }, + { + Name: "NotTrailingComma3", + Input: "a,b,c, \n", + Output: [][]string{{"a", "b", "c", " "}}, + }, + { + Name: "CommaFieldTest", + TrailingComma: true, + Input: `x,y,z,w +x,y,z, +x,y,, +x,,, +,,, +"x","y","z","w" +"x","y","z","" +"x","y","","" +"x","","","" +"","","","" +`, + Output: [][]string{ + {"x", "y", "z", "w"}, + {"x", "y", "z", ""}, + {"x", "y", "", ""}, + {"x", "", "", ""}, + {"", "", "", ""}, + {"x", "y", "z", "w"}, + {"x", "y", "z", ""}, + {"x", "y", "", ""}, + {"x", "", "", ""}, + {"", "", "", ""}, + }, + }, +} + +func TestRead(t *testing.T) { + for _, tt := range readTests { + r := NewReader(strings.NewReader(tt.Input)) + r.Comment = tt.Comment + if tt.UseFieldsPerRecord { + r.FieldsPerRecord = tt.FieldsPerRecord + } else { + r.FieldsPerRecord = -1 + } + r.LazyQuotes = tt.LazyQuotes + r.TrailingComma = tt.TrailingComma + r.TrimLeadingSpace = tt.TrimLeadingSpace + if tt.Comma != 0 { + r.Comma = tt.Comma + } + out, err := r.ReadAll() + perr, _ := err.(*ParseError) + if tt.Error != "" { + if err == nil || !strings.Contains(err.String(), tt.Error) { + t.Errorf("%s: error %v, want error %q", tt.Name, err, tt.Error) + } else if tt.Line != 0 && (tt.Line != perr.Line || tt.Column != perr.Column) { + t.Errorf("%s: error at %d:%d expected %d:%d", tt.Name, perr.Line, perr.Column, tt.Line, tt.Column) + } + } else if err != nil { + t.Errorf("%s: unexpected error %v", tt.Name, err) + } else if !reflect.DeepEqual(out, tt.Output) { + t.Errorf("%s: out=%q want %q", tt.Name, out, tt.Output) + } + } +} diff --git a/src/pkg/csv/writer.go b/src/pkg/csv/writer.go new file mode 100644 index 000000000..01386da19 --- /dev/null +++ b/src/pkg/csv/writer.go @@ -0,0 +1,123 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "bufio" + "io" + "os" + "strings" + "unicode" + "utf8" +) + +// A Writer writes records to a CSV encoded file. +// +// As returned by NewWriter, a Writer writes records terminated by a +// newline and uses ',' as the field delimiter. The exported fields can be +// changed to customize the details before the first call to Write or WriteAll. +// +// Comma is the field delimiter. +// +// If UseCRLF is true, the Writer ends each record with \r\n instead of \n. +// just \n is written. +type Writer struct { + Comma int // Field delimiter (set to to ',' by NewWriter) + UseCRLF bool // True to use \r\n as the line terminator + w *bufio.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{ + Comma: ',', + w: bufio.NewWriter(w), + } +} + +// Writer writes a single CSV record to w along with any necessary quoting. +// A record is a slice of strings with each string being one field. +func (w *Writer) Write(record []string) (err os.Error) { + for n, field := range record { + if n > 0 { + if _, err = w.w.WriteRune(w.Comma); err != nil { + return + } + } + + // If we don't have to have a quoted field then just + // write out the field and continue to the next field. + if !w.fieldNeedsQuotes(field) { + if _, err = w.w.WriteString(field); err != nil { + return + } + continue + } + if err = w.w.WriteByte('"'); err != nil { + return + } + + for _, rune := range field { + switch rune { + case '"': + _, err = w.w.WriteString(`""`) + case '\r': + if !w.UseCRLF { + err = w.w.WriteByte('\r') + } + case '\n': + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } + default: + _, err = w.w.WriteRune(rune) + } + if err != nil { + return + } + } + + if err = w.w.WriteByte('"'); err != nil { + return + } + } + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } + return +} + +// Flush writes any buffered data to the underlying io.Writer. +func (w *Writer) Flush() { + w.w.Flush() +} + +// WriteAll writes multiple CSV records to w using Write and then calls Flush. +func (w *Writer) WriteAll(records [][]string) (err os.Error) { + for _, record := range records { + err = w.Write(record) + if err != nil { + break + } + } + w.Flush() + return nil +} + +// fieldNeedsQuotes returns true if our field must be enclosed in quotes. +// Empty fields, files with a Comma, fields with a quote or newline, and +// fields which start with a space must be enclosed in quotes. +func (w *Writer) fieldNeedsQuotes(field string) bool { + if len(field) == 0 || strings.IndexRune(field, w.Comma) >= 0 || strings.IndexAny(field, "\"\r\n") >= 0 { + return true + } + + rune, _ := utf8.DecodeRuneInString(field) + return unicode.IsSpace(rune) +} diff --git a/src/pkg/csv/writer_test.go b/src/pkg/csv/writer_test.go new file mode 100644 index 000000000..578959007 --- /dev/null +++ b/src/pkg/csv/writer_test.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package csv + +import ( + "bytes" + "testing" +) + +var writeTests = []struct { + Input [][]string + Output string + UseCRLF bool +}{ + {Input: [][]string{{"abc"}}, Output: "abc\n"}, + {Input: [][]string{{"abc"}}, Output: "abc\r\n", UseCRLF: true}, + {Input: [][]string{{`"abc"`}}, Output: `"""abc"""` + "\n"}, + {Input: [][]string{{`a"b`}}, Output: `"a""b"` + "\n"}, + {Input: [][]string{{`"a"b"`}}, Output: `"""a""b"""` + "\n"}, + {Input: [][]string{{" abc"}}, Output: `" abc"` + "\n"}, + {Input: [][]string{{"abc,def"}}, Output: `"abc,def"` + "\n"}, + {Input: [][]string{{"abc", "def"}}, Output: "abc,def\n"}, + {Input: [][]string{{"abc"}, {"def"}}, Output: "abc\ndef\n"}, + {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\ndef\"\n"}, + {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\r\ndef\"\r\n", UseCRLF: true}, +} + +func TestWrite(t *testing.T) { + for n, tt := range writeTests { + b := &bytes.Buffer{} + f := NewWriter(b) + f.UseCRLF = tt.UseCRLF + err := f.WriteAll(tt.Input) + if err != nil { + t.Errorf("Unexpected error: %s\n", err) + } + out := b.String() + if out != tt.Output { + t.Errorf("#%d: out=%q want %q", n, out, tt.Output) + } + } +} diff --git a/src/pkg/debug/proc/proc_linux.go b/src/pkg/debug/proc/proc_linux.go index 5831b0e97..7ec797114 100644 --- a/src/pkg/debug/proc/proc_linux.go +++ b/src/pkg/debug/proc/proc_linux.go @@ -1229,7 +1229,7 @@ func (p *process) attachAllThreads() os.Error { return err } - statParts := strings.Split(string(statFile), " ", 4) + statParts := strings.SplitN(string(statFile), " ", 4) if len(statParts) > 2 && statParts[2] == "Z" { // tid is a zombie p.logTrace("thread %d is a zombie", tid) diff --git a/src/pkg/exec/exec_test.go b/src/pkg/exec/exec_test.go index c45a7d70a..f6cebb905 100644 --- a/src/pkg/exec/exec_test.go +++ b/src/pkg/exec/exec_test.go @@ -55,7 +55,7 @@ func TestCatGoodAndBadFile(t *testing.T) { t.Errorf("expected Waitmsg from cat combined; got %T: %v", err, err) } s := string(bs) - sp := strings.Split(s, "\n", 2) + sp := strings.SplitN(s, "\n", 2) if len(sp) != 2 { t.Fatalf("expected two lines from cat; got %q", s) } diff --git a/src/pkg/exec/lp_plan9.go b/src/pkg/exec/lp_plan9.go index c4e2a7a0f..e4751a4df 100644 --- a/src/pkg/exec/lp_plan9.go +++ b/src/pkg/exec/lp_plan9.go @@ -42,7 +42,7 @@ func LookPath(file string) (string, os.Error) { } path := os.Getenv("path") - for _, dir := range strings.Split(path, "\000", -1) { + for _, dir := range strings.Split(path, "\000") { if err := findExecutable(dir + "/" + file); err == nil { return dir + "/" + file, nil } diff --git a/src/pkg/exec/lp_unix.go b/src/pkg/exec/lp_unix.go index cdf720768..008fb11a8 100644 --- a/src/pkg/exec/lp_unix.go +++ b/src/pkg/exec/lp_unix.go @@ -39,7 +39,7 @@ func LookPath(file string) (string, os.Error) { return "", &Error{file, err} } pathenv := os.Getenv("PATH") - for _, dir := range strings.Split(pathenv, ":", -1) { + for _, dir := range strings.Split(pathenv, ":") { if dir == "" { // Unix shell semantics: path element "" means "." dir = "." diff --git a/src/pkg/exec/lp_windows.go b/src/pkg/exec/lp_windows.go index 47763458f..7581088eb 100644 --- a/src/pkg/exec/lp_windows.go +++ b/src/pkg/exec/lp_windows.go @@ -47,7 +47,7 @@ func LookPath(file string) (f string, err os.Error) { x = `.COM;.EXE;.BAT;.CMD` } exts := []string{} - for _, e := range strings.Split(strings.ToLower(x), `;`, -1) { + for _, e := range strings.Split(strings.ToLower(x), `;`) { if e == "" { continue } @@ -67,7 +67,7 @@ func LookPath(file string) (f string, err os.Error) { return } } else { - for _, dir := range strings.Split(pathenv, `;`, -1) { + for _, dir := range strings.Split(pathenv, `;`) { if f, err = findExecutable(dir+`\`+file, exts); err == nil { return } diff --git a/src/pkg/exp/ogle/cmd.go b/src/pkg/exp/ogle/cmd.go index a8db523ea..ff0d24c69 100644 --- a/src/pkg/exp/ogle/cmd.go +++ b/src/pkg/exp/ogle/cmd.go @@ -154,7 +154,7 @@ func cmdLoad(args []byte) os.Error { } println("Attached to", pid) } else { - parts := strings.Split(path, " ", -1) + parts := strings.Split(path, " ") if len(parts) == 0 { fname = "" } else { diff --git a/src/pkg/exp/regexp/syntax/Makefile b/src/pkg/exp/regexp/syntax/Makefile index 8e0b4c1e6..97d4ad6ca 100644 --- a/src/pkg/exp/regexp/syntax/Makefile +++ b/src/pkg/exp/regexp/syntax/Makefile @@ -6,8 +6,11 @@ include ../../../../Make.inc TARG=exp/regexp/syntax GOFILES=\ + compile.go\ parse.go\ perl_groups.go\ + prog.go\ regexp.go\ + simplify.go\ include ../../../../Make.pkg diff --git a/src/pkg/exp/regexp/syntax/compile.go b/src/pkg/exp/regexp/syntax/compile.go new file mode 100644 index 000000000..ec9556fde --- /dev/null +++ b/src/pkg/exp/regexp/syntax/compile.go @@ -0,0 +1,264 @@ +package syntax + +import ( + "os" + "unicode" +) + +// A patchList is a list of instruction pointers that need to be filled in (patched). +// Because the pointers haven't been filled in yet, we can reuse their storage +// to hold the list. It's kind of sleazy, but works well in practice. +// See http://swtch.com/~rsc/regexp/regexp1.html for inspiration. +// +// These aren't really pointers: they're integers, so we can reinterpret them +// this way without using package unsafe. A value l denotes +// p.inst[l>>1].Out (l&1==0) or .Arg (l&1==1). +// l == 0 denotes the empty list, okay because we start every program +// with a fail instruction, so we'll never want to point at its output link. +type patchList uint32 + +func (l patchList) next(p *Prog) patchList { + i := &p.Inst[l>>1] + if l&1 == 0 { + return patchList(i.Out) + } + return patchList(i.Arg) +} + +func (l patchList) patch(p *Prog, val uint32) { + for l != 0 { + i := &p.Inst[l>>1] + if l&1 == 0 { + l = patchList(i.Out) + i.Out = val + } else { + l = patchList(i.Arg) + i.Arg = val + } + } +} + +func (l1 patchList) append(p *Prog, l2 patchList) patchList { + if l1 == 0 { + return l2 + } + if l2 == 0 { + return l1 + } + + last := l1 + for { + next := last.next(p) + if next == 0 { + break + } + last = next + } + + i := &p.Inst[last>>1] + if last&1 == 0 { + i.Out = uint32(l2) + } else { + i.Arg = uint32(l2) + } + return l1 +} + +// A frag represents a compiled program fragment. +type frag struct { + i uint32 // index of first instruction + out patchList // where to record end instruction +} + +type compiler struct { + p *Prog +} + +// Compile compiles the regexp into a program to be executed. +func Compile(re *Regexp) (*Prog, os.Error) { + var c compiler + c.init() + f := c.compile(re) + f.out.patch(c.p, c.inst(InstMatch).i) + c.p.Start = int(f.i) + return c.p, nil +} + +func (c *compiler) init() { + c.p = new(Prog) + c.inst(InstFail) +} + +var anyRuneNotNL = []int{0, '\n' - 1, '\n' - 1, unicode.MaxRune} +var anyRune = []int{0, unicode.MaxRune} + +func (c *compiler) compile(re *Regexp) frag { + switch re.Op { + case OpNoMatch: + return c.fail() + case OpEmptyMatch: + return c.nop() + case OpLiteral: + if len(re.Rune) == 0 { + return c.nop() + } + var f frag + for j := range re.Rune { + f1 := c.rune(re.Rune[j : j+1]) + if j == 0 { + f = f1 + } else { + f = c.cat(f, f1) + } + } + return f + case OpCharClass: + return c.rune(re.Rune) + case OpAnyCharNotNL: + return c.rune(anyRuneNotNL) + case OpAnyChar: + return c.rune(anyRune) + case OpBeginLine: + return c.empty(EmptyBeginLine) + case OpEndLine: + return c.empty(EmptyEndLine) + case OpBeginText: + return c.empty(EmptyBeginText) + case OpEndText: + return c.empty(EmptyEndText) + case OpWordBoundary: + return c.empty(EmptyWordBoundary) + case OpNoWordBoundary: + return c.empty(EmptyNoWordBoundary) + case OpCapture: + bra := c.cap(uint32(re.Cap << 1)) + sub := c.compile(re.Sub[0]) + ket := c.cap(uint32(re.Cap<<1 | 1)) + return c.cat(c.cat(bra, sub), ket) + case OpStar: + return c.star(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpPlus: + return c.plus(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpQuest: + return c.quest(c.compile(re.Sub[0]), re.Flags&NonGreedy != 0) + case OpConcat: + if len(re.Sub) == 0 { + return c.nop() + } + var f frag + for i, sub := range re.Sub { + if i == 0 { + f = c.compile(sub) + } else { + f = c.cat(f, c.compile(sub)) + } + } + return f + case OpAlternate: + var f frag + for _, sub := range re.Sub { + f = c.alt(f, c.compile(sub)) + } + return f + } + panic("regexp: unhandled case in compile") +} + +func (c *compiler) inst(op InstOp) frag { + // TODO: impose length limit + f := frag{i: uint32(len(c.p.Inst))} + c.p.Inst = append(c.p.Inst, Inst{Op: op}) + return f +} + +func (c *compiler) nop() frag { + f := c.inst(InstNop) + f.out = patchList(f.i << 1) + return f +} + +func (c *compiler) fail() frag { + return frag{} +} + +func (c *compiler) cap(arg uint32) frag { + f := c.inst(InstCapture) + f.out = patchList(f.i << 1) + c.p.Inst[f.i].Arg = arg + return f +} + +func (c *compiler) cat(f1, f2 frag) frag { + // concat of failure is failure + if f1.i == 0 || f2.i == 0 { + return frag{} + } + + // TODO: elide nop + + f1.out.patch(c.p, f2.i) + return frag{f1.i, f2.out} +} + +func (c *compiler) alt(f1, f2 frag) frag { + // alt of failure is other + if f1.i == 0 { + return f2 + } + if f2.i == 0 { + return f1 + } + + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + i.Out = f1.i + i.Arg = f2.i + f.out = f1.out.append(c.p, f2.out) + return f +} + +func (c *compiler) quest(f1 frag, nongreedy bool) frag { + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + if nongreedy { + i.Arg = f1.i + f.out = patchList(f.i << 1) + } else { + i.Out = f1.i + f.out = patchList(f.i<<1 | 1) + } + f.out = f.out.append(c.p, f1.out) + return f +} + +func (c *compiler) star(f1 frag, nongreedy bool) frag { + f := c.inst(InstAlt) + i := &c.p.Inst[f.i] + if nongreedy { + i.Arg = f1.i + f.out = patchList(f.i << 1) + } else { + i.Out = f1.i + f.out = patchList(f.i<<1 | 1) + } + f1.out.patch(c.p, f.i) + return f +} + +func (c *compiler) plus(f1 frag, nongreedy bool) frag { + return frag{f1.i, c.star(f1, nongreedy).out} +} + +func (c *compiler) empty(op EmptyOp) frag { + f := c.inst(InstEmptyWidth) + c.p.Inst[f.i].Arg = uint32(op) + f.out = patchList(f.i << 1) + return f +} + +func (c *compiler) rune(rune []int) frag { + f := c.inst(InstRune) + c.p.Inst[f.i].Rune = rune + f.out = patchList(f.i << 1) + return f +} diff --git a/src/pkg/exp/regexp/syntax/parse.go b/src/pkg/exp/regexp/syntax/parse.go index d04f25097..b6c91f7e1 100644 --- a/src/pkg/exp/regexp/syntax/parse.go +++ b/src/pkg/exp/regexp/syntax/parse.go @@ -79,28 +79,108 @@ const ( type parser struct { flags Flags // parse mode flags stack []*Regexp // stack of parsed expressions - numCap int // number of capturing groups seen + free *Regexp + numCap int // number of capturing groups seen wholeRegexp string + tmpClass []int // temporary char class work space +} + +func (p *parser) newRegexp(op Op) *Regexp { + re := p.free + if re != nil { + p.free = re.Sub0[0] + *re = Regexp{} + } else { + re = new(Regexp) + } + re.Op = op + return re +} + +func (p *parser) reuse(re *Regexp) { + re.Sub0[0] = p.free + p.free = re } // Parse stack manipulation. // push pushes the regexp re onto the parse stack and returns the regexp. func (p *parser) push(re *Regexp) *Regexp { - // TODO: automatic concatenation - // TODO: turn character class into literal - // TODO: compute simple + if re.Op == OpCharClass && len(re.Rune) == 2 && re.Rune[0] == re.Rune[1] { + // Single rune. + if p.maybeConcat(re.Rune[0], p.flags&^FoldCase) { + return nil + } + re.Op = OpLiteral + re.Rune = re.Rune[:1] + re.Flags = p.flags &^ FoldCase + } else if re.Op == OpCharClass && len(re.Rune) == 4 && + re.Rune[0] == re.Rune[1] && re.Rune[2] == re.Rune[3] && + unicode.SimpleFold(re.Rune[0]) == re.Rune[2] && + unicode.SimpleFold(re.Rune[2]) == re.Rune[0] || + re.Op == OpCharClass && len(re.Rune) == 2 && + re.Rune[0]+1 == re.Rune[1] && + unicode.SimpleFold(re.Rune[0]) == re.Rune[1] && + unicode.SimpleFold(re.Rune[1]) == re.Rune[0] { + // Case-insensitive rune like [Aa] or [Δδ]. + if p.maybeConcat(re.Rune[0], p.flags|FoldCase) { + return nil + } + + // Rewrite as (case-insensitive) literal. + re.Op = OpLiteral + re.Rune = re.Rune[:1] + re.Flags = p.flags | FoldCase + } else { + // Incremental concatenation. + p.maybeConcat(-1, 0) + } p.stack = append(p.stack, re) return re } -// newLiteral returns a new OpLiteral Regexp with the given flags -func newLiteral(r int, flags Flags) *Regexp { - re := &Regexp{ - Op: OpLiteral, - Flags: flags, +// maybeConcat implements incremental concatenation +// of literal runes into string nodes. The parser calls this +// before each push, so only the top fragment of the stack +// might need processing. Since this is called before a push, +// the topmost literal is no longer subject to operators like * +// (Otherwise ab* would turn into (ab)*.) +// If r >= 0 and there's a node left over, maybeConcat uses it +// to push r with the given flags. +// maybeConcat reports whether r was pushed. +func (p *parser) maybeConcat(r int, flags Flags) bool { + n := len(p.stack) + if n < 2 { + return false } + + re1 := p.stack[n-1] + re2 := p.stack[n-2] + if re1.Op != OpLiteral || re2.Op != OpLiteral || re1.Flags&FoldCase != re2.Flags&FoldCase { + return false + } + + // Push re1 into re2. + re2.Rune = append(re2.Rune, re1.Rune...) + + // Reuse re1 if possible. + if r >= 0 { + re1.Rune = re1.Rune0[:1] + re1.Rune[0] = r + re1.Flags = flags + return true + } + + p.stack = p.stack[:n-1] + p.reuse(re1) + return false // did not push r +} + +// newLiteral returns a new OpLiteral Regexp with the given flags +func (p *parser) newLiteral(r int, flags Flags) *Regexp { + re := p.newRegexp(OpLiteral) + re.Flags = flags re.Rune0[0] = r re.Rune = re.Rune0[:1] return re @@ -108,14 +188,16 @@ func newLiteral(r int, flags Flags) *Regexp { // literal pushes a literal regexp for the rune r on the stack // and returns that regexp. -func (p *parser) literal(r int) *Regexp { - return p.push(newLiteral(r, p.flags)) +func (p *parser) literal(r int) { + p.push(p.newLiteral(r, p.flags)) } // op pushes a regexp with the given op onto the stack // and returns that regexp. func (p *parser) op(op Op) *Regexp { - return p.push(&Regexp{Op: op, Flags: p.flags}) + re := p.newRegexp(op) + re.Flags = p.flags + return p.push(re) } // repeat replaces the top stack element with itself repeated @@ -139,12 +221,10 @@ func (p *parser) repeat(op Op, min, max int, opstr, t, lastRepeat string) (strin return "", &Error{ErrMissingRepeatArgument, opstr} } sub := p.stack[n-1] - re := &Regexp{ - Op: op, - Min: min, - Max: max, - Flags: flags, - } + re := p.newRegexp(op) + re.Min = min + re.Max = max + re.Flags = flags re.Sub = re.Sub0[:1] re.Sub[0] = sub p.stack[n-1] = re @@ -153,60 +233,385 @@ func (p *parser) repeat(op Op, min, max int, opstr, t, lastRepeat string) (strin // concat replaces the top of the stack (above the topmost '|' or '(') with its concatenation. func (p *parser) concat() *Regexp { - // TODO: Flatten concats. + p.maybeConcat(-1, 0) // Scan down to find pseudo-operator | or (. i := len(p.stack) for i > 0 && p.stack[i-1].Op < opPseudo { i-- } - sub := p.stack[i:] + subs := p.stack[i:] p.stack = p.stack[:i] - var re *Regexp - switch len(sub) { - case 0: - re = &Regexp{Op: OpEmptyMatch} - case 1: - re = sub[0] - default: - re = &Regexp{Op: OpConcat} - re.Sub = append(re.Sub0[:0], sub...) + // Empty concatenation is special case. + if len(subs) == 0 { + return p.push(p.newRegexp(OpEmptyMatch)) } - return p.push(re) + + return p.push(p.collapse(subs, OpConcat)) } // alternate replaces the top of the stack (above the topmost '(') with its alternation. func (p *parser) alternate() *Regexp { - // TODO: Flatten alternates. - // Scan down to find pseudo-operator (. // There are no | above (. i := len(p.stack) for i > 0 && p.stack[i-1].Op < opPseudo { i-- } - sub := p.stack[i:] + subs := p.stack[i:] p.stack = p.stack[:i] - var re *Regexp - switch len(sub) { - case 0: - re = &Regexp{Op: OpNoMatch} - case 1: - re = sub[0] - default: - re = &Regexp{Op: OpAlternate} - re.Sub = append(re.Sub0[:0], sub...) + // Make sure top class is clean. + // All the others already are (see swapVerticalBar). + if len(subs) > 0 { + cleanAlt(subs[len(subs)-1]) } - return p.push(re) + + // Empty alternate is special case + // (shouldn't happen but easy to handle). + if len(subs) == 0 { + return p.push(p.newRegexp(OpNoMatch)) + } + + return p.push(p.collapse(subs, OpAlternate)) } -func literalRegexp(s string, flags Flags) *Regexp { - re := &Regexp{ - Op: OpLiteral, - Flags: flags, +// cleanAlt cleans re for eventual inclusion in an alternation. +func cleanAlt(re *Regexp) { + switch re.Op { + case OpCharClass: + re.Rune = cleanClass(&re.Rune) + if len(re.Rune) == 2 && re.Rune[0] == 0 && re.Rune[1] == unicode.MaxRune { + re.Rune = nil + re.Op = OpAnyChar + return + } + if len(re.Rune) == 4 && re.Rune[0] == 0 && re.Rune[1] == '\n'-1 && re.Rune[2] == '\n'+1 && re.Rune[3] == unicode.MaxRune { + re.Rune = nil + re.Op = OpAnyCharNotNL + return + } + if cap(re.Rune)-len(re.Rune) > 100 { + // re.Rune will not grow any more. + // Make a copy or inline to reclaim storage. + re.Rune = append(re.Rune0[:0], re.Rune...) + } + } +} + +// collapse returns the result of applying op to sub. +// If sub contains op nodes, they all get hoisted up +// so that there is never a concat of a concat or an +// alternate of an alternate. +func (p *parser) collapse(subs []*Regexp, op Op) *Regexp { + if len(subs) == 1 { + return subs[0] + } + re := p.newRegexp(op) + re.Sub = re.Sub0[:0] + for _, sub := range subs { + if sub.Op == op { + re.Sub = append(re.Sub, sub.Sub...) + p.reuse(sub) + } else { + re.Sub = append(re.Sub, sub) + } + } + if op == OpAlternate { + re.Sub = p.factor(re.Sub, re.Flags) + if len(re.Sub) == 1 { + old := re + re = re.Sub[0] + p.reuse(old) + } + } + return re +} + +// factor factors common prefixes from the alternation list sub. +// It returns a replacement list that reuses the same storage and +// frees (passes to p.reuse) any removed *Regexps. +// +// For example, +// ABC|ABD|AEF|BCX|BCY +// simplifies by literal prefix extraction to +// A(B(C|D)|EF)|BC(X|Y) +// which simplifies by character class introduction to +// A(B[CD]|EF)|BC[XY] +// +func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { + if len(sub) < 2 { + return sub + } + + // Round 1: Factor out common literal prefixes. + var str []int + var strflags Flags + start := 0 + out := sub[:0] + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that all begin + // with str as modified by strflags. + var istr []int + var iflags Flags + if i < len(sub) { + istr, iflags = p.leadingString(sub[i]) + if iflags == strflags { + same := 0 + for same < len(str) && same < len(istr) && str[same] == istr[same] { + same++ + } + if same > 0 { + // Matches at least one rune in current range. + // Keep going around. + str = str[:same] + continue + } + } + } + + // Found end of a run with common leading literal string: + // sub[start:i] all begin with str[0:len(str)], but sub[i] + // does not even begin with str[0]. + // + // Factor out common string and append factored expression to out. + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + // Just one: don't bother factoring. + out = append(out, sub[start]) + } else { + // Construct factored form: prefix(suffix1|suffix2|...) + prefix := p.newRegexp(OpLiteral) + prefix.Flags = strflags + prefix.Rune = append(prefix.Rune[:0], str...) + + for j := start; j < i; j++ { + sub[j] = p.removeLeadingString(sub[j], len(str)) + } + suffix := p.collapse(sub[start:i], OpAlternate) // recurse + + re := p.newRegexp(OpConcat) + re.Sub = append(re.Sub[:0], prefix, suffix) + out = append(out, re) + } + + // Prepare for next iteration. + start = i + str = istr + strflags = iflags + } + sub = out + + // Round 2: Factor out common complex prefixes, + // just the first piece of each concatenation, + // whatever it is. This is good enough a lot of the time. + start = 0 + out = sub[:0] + var first *Regexp + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that all begin + // with str as modified by strflags. + var ifirst *Regexp + if i < len(sub) { + ifirst = p.leadingRegexp(sub[i]) + if first != nil && first.Equal(ifirst) { + continue + } + } + + // Found end of a run with common leading regexp: + // sub[start:i] all begin with first but sub[i] does not. + // + // Factor out common regexp and append factored expression to out. + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + // Just one: don't bother factoring. + out = append(out, sub[start]) + } else { + // Construct factored form: prefix(suffix1|suffix2|...) + prefix := first + + for j := start; j < i; j++ { + reuse := j != start // prefix came from sub[start] + sub[j] = p.removeLeadingRegexp(sub[j], reuse) + } + suffix := p.collapse(sub[start:i], OpAlternate) // recurse + + re := p.newRegexp(OpConcat) + re.Sub = append(re.Sub[:0], prefix, suffix) + out = append(out, re) + } + + // Prepare for next iteration. + start = i + first = ifirst + } + sub = out + + // Round 3: Collapse runs of single literals into character classes. + start = 0 + out = sub[:0] + for i := 0; i <= len(sub); i++ { + // Invariant: the Regexps that were in sub[0:start] have been + // used or marked for reuse, and the slice space has been reused + // for out (len(out) <= start). + // + // Invariant: sub[start:i] consists of regexps that are either + // literal runes or character classes. + if i < len(sub) && isCharClass(sub[i]) { + continue + } + + // sub[i] is not a char or char class; + // emit char class for sub[start:i]... + if i == start { + // Nothing to do - run of length 0. + } else if i == start+1 { + out = append(out, sub[start]) + } else { + // Make new char class. + // Start with most complex regexp in sub[start]. + max := start + for j := start + 1; j < i; j++ { + if sub[max].Op < sub[j].Op || sub[max].Op == sub[j].Op && len(sub[max].Rune) < len(sub[j].Rune) { + max = j + } + } + sub[start], sub[max] = sub[max], sub[start] + + for j := start + 1; j < i; j++ { + mergeCharClass(sub[start], sub[j]) + p.reuse(sub[j]) + } + cleanAlt(sub[start]) + out = append(out, sub[start]) + } + + // ... and then emit sub[i]. + if i < len(sub) { + out = append(out, sub[i]) + } + start = i + 1 + } + sub = out + + // Round 4: Collapse runs of empty matches into a single empty match. + start = 0 + out = sub[:0] + for i := range sub { + if i+1 < len(sub) && sub[i].Op == OpEmptyMatch && sub[i+1].Op == OpEmptyMatch { + continue + } + out = append(out, sub[i]) + } + sub = out + + return sub +} + +// leadingString returns the leading literal string that re begins with. +// The string refers to storage in re or its children. +func (p *parser) leadingString(re *Regexp) ([]int, Flags) { + if re.Op == OpConcat && len(re.Sub) > 0 { + re = re.Sub[0] + } + if re.Op != OpLiteral { + return nil, 0 + } + return re.Rune, re.Flags & FoldCase +} + +// removeLeadingString removes the first n leading runes +// from the beginning of re. It returns the replacement for re. +func (p *parser) removeLeadingString(re *Regexp, n int) *Regexp { + if re.Op == OpConcat && len(re.Sub) > 0 { + // Removing a leading string in a concatenation + // might simplify the concatenation. + sub := re.Sub[0] + sub = p.removeLeadingString(sub, n) + re.Sub[0] = sub + if sub.Op == OpEmptyMatch { + p.reuse(sub) + switch len(re.Sub) { + case 0, 1: + // Impossible but handle. + re.Op = OpEmptyMatch + re.Sub = nil + case 2: + old := re + re = re.Sub[1] + p.reuse(old) + default: + copy(re.Sub, re.Sub[1:]) + re.Sub = re.Sub[:len(re.Sub)-1] + } + } + return re } + + if re.Op == OpLiteral { + re.Rune = re.Rune[:copy(re.Rune, re.Rune[n:])] + if len(re.Rune) == 0 { + re.Op = OpEmptyMatch + } + } + return re +} + +// leadingRegexp returns the leading regexp that re begins with. +// The regexp refers to storage in re or its children. +func (p *parser) leadingRegexp(re *Regexp) *Regexp { + if re.Op == OpEmptyMatch { + return nil + } + if re.Op == OpConcat && len(re.Sub) > 0 { + sub := re.Sub[0] + if sub.Op == OpEmptyMatch { + return nil + } + return sub + } + return re +} + +// removeLeadingRegexp removes the leading regexp in re. +// It returns the replacement for re. +// If reuse is true, it passes the removed regexp (if no longer needed) to p.reuse. +func (p *parser) removeLeadingRegexp(re *Regexp, reuse bool) *Regexp { + if re.Op == OpConcat && len(re.Sub) > 0 { + if reuse { + p.reuse(re.Sub[0]) + } + re.Sub = re.Sub[:copy(re.Sub, re.Sub[1:])] + switch len(re.Sub) { + case 0: + re.Op = OpEmptyMatch + re.Sub = nil + case 1: + old := re + re = re.Sub[0] + p.reuse(old) + } + return re + } + re.Op = OpEmptyMatch + return re +} + +func literalRegexp(s string, flags Flags) *Regexp { + re := &Regexp{Op: OpLiteral} + re.Flags = flags re.Rune = re.Rune0[:0] // use local storage for small strings for _, c := range s { if len(re.Rune) >= cap(re.Rune) { @@ -264,7 +669,6 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { p.op(opLeftParen).Cap = p.numCap t = t[1:] case '|': - p.concat() if err = p.parseVerticalBar(); err != nil { return nil, err } @@ -360,7 +764,8 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { } } - re := &Regexp{Op: OpCharClass, Flags: p.flags} + re := p.newRegexp(OpCharClass) + re.Flags = p.flags // Look for Unicode character group like \p{Han} if len(t) >= 2 && (t[1] == 'p' || t[1] == 'P') { @@ -371,7 +776,6 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { if r != nil { re.Rune = r t = rest - // TODO: Handle FoldCase flag. p.push(re) break BigSwitch } @@ -381,12 +785,10 @@ func Parse(s string, flags Flags) (*Regexp, os.Error) { if r, rest := p.parsePerlClassEscape(t, re.Rune0[:0]); r != nil { re.Rune = r t = rest - // TODO: Handle FoldCase flag. p.push(re) break BigSwitch } - - // TODO: Give re back to parser's pool. + p.reuse(re) // Ordinary single-character escape. if c, t, err = p.parseEscape(t); err != nil { @@ -592,6 +994,35 @@ func (p *parser) parseInt(s string) (n int, rest string, ok bool) { return } +// can this be represented as a character class? +// single-rune literal string, char class, ., and .|\n. +func isCharClass(re *Regexp) bool { + return re.Op == OpLiteral && len(re.Rune) == 1 || + re.Op == OpCharClass || + re.Op == OpAnyCharNotNL || + re.Op == OpAnyChar +} + +// does re match r? +func matchRune(re *Regexp, r int) bool { + switch re.Op { + case OpLiteral: + return len(re.Rune) == 1 && re.Rune[0] == r + case OpCharClass: + for i := 0; i < len(re.Rune); i += 2 { + if re.Rune[i] <= r && r <= re.Rune[i+1] { + return true + } + } + return false + case OpAnyCharNotNL: + return r != '\n' + case OpAnyChar: + return true + } + return false +} + // parseVerticalBar handles a | in the input. func (p *parser) parseVerticalBar() os.Error { p.concat() @@ -607,14 +1038,66 @@ func (p *parser) parseVerticalBar() os.Error { return nil } +// mergeCharClass makes dst = dst|src. +// The caller must ensure that dst.Op >= src.Op, +// to reduce the amount of copying. +func mergeCharClass(dst, src *Regexp) { + switch dst.Op { + case OpAnyChar: + // src doesn't add anything. + case OpAnyCharNotNL: + // src might add \n + if matchRune(src, '\n') { + dst.Op = OpAnyChar + } + case OpCharClass: + // src is simpler, so either literal or char class + if src.Op == OpLiteral { + dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + } else { + dst.Rune = appendClass(dst.Rune, src.Rune) + } + case OpLiteral: + // both literal + if src.Rune[0] == dst.Rune[0] { + break + } + dst.Op = OpCharClass + dst.Rune = append(dst.Rune, dst.Rune[0]) + dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + } +} + // If the top of the stack is an element followed by an opVerticalBar // swapVerticalBar swaps the two and returns true. // Otherwise it returns false. func (p *parser) swapVerticalBar() bool { - if n := len(p.stack); n >= 2 { + // If above and below vertical bar are literal or char class, + // can merge into a single char class. + n := len(p.stack) + if n >= 3 && p.stack[n-2].Op == opVerticalBar && isCharClass(p.stack[n-1]) && isCharClass(p.stack[n-3]) { + re1 := p.stack[n-1] + re3 := p.stack[n-3] + // Make re3 the more complex of the two. + if re1.Op > re3.Op { + re1, re3 = re3, re1 + p.stack[n-3] = re3 + } + mergeCharClass(re3, re1) + p.reuse(re1) + p.stack = p.stack[:n-1] + return true + } + + if n >= 2 { re1 := p.stack[n-1] re2 := p.stack[n-2] if re2.Op == opVerticalBar { + if n >= 3 { + // Now out of reach. + // Clean opportunistically. + cleanAlt(p.stack[n-3]) + } p.stack[n-2] = re1 p.stack[n-1] = re2 return true @@ -729,6 +1212,7 @@ Switch: if r > unicode.MaxRune { break Switch } + nhex++ } if nhex == 0 { break Switch @@ -801,12 +1285,7 @@ func (p *parser) parsePerlClassEscape(s string, r []int) (out []int, rest string if g.sign == 0 { return } - if g.sign < 0 { - r = appendNegatedClass(r, g.class) - } else { - r = appendClass(r, g.class) - } - return r, s[2:] + return p.appendGroup(r, g), s[2:] } // parseNamedClass parses a leading POSIX named character class like [:alnum:] @@ -827,23 +1306,40 @@ func (p *parser) parseNamedClass(s string, r []int) (out []int, rest string, err if g.sign == 0 { return nil, "", &Error{ErrInvalidCharRange, name} } - if g.sign < 0 { - r = appendNegatedClass(r, g.class) + return p.appendGroup(r, g), s, nil +} + +func (p *parser) appendGroup(r []int, g charGroup) []int { + if p.flags&FoldCase == 0 { + if g.sign < 0 { + r = appendNegatedClass(r, g.class) + } else { + r = appendClass(r, g.class) + } } else { - r = appendClass(r, g.class) + tmp := p.tmpClass[:0] + tmp = appendFoldedClass(tmp, g.class) + p.tmpClass = tmp + tmp = cleanClass(&p.tmpClass) + if g.sign < 0 { + r = appendNegatedClass(r, tmp) + } else { + r = appendClass(r, tmp) + } } - return r, s, nil + return r } -// unicodeTable returns the unicode.RangeTable identified by name. -func unicodeTable(name string) *unicode.RangeTable { +// unicodeTable returns the unicode.RangeTable identified by name +// and the table of additional fold-equivalent code points. +func unicodeTable(name string) (*unicode.RangeTable, *unicode.RangeTable) { if t := unicode.Categories[name]; t != nil { - return t + return t, unicode.FoldCategory[name] } if t := unicode.Scripts[name]; t != nil { - return t + return t, unicode.FoldScript[name] } - return nil + return nil, nil } // parseUnicodeClass parses a leading Unicode character class like \p{Han} @@ -891,14 +1387,31 @@ func (p *parser) parseUnicodeClass(s string, r []int) (out []int, rest string, e name = name[1:] } - tab := unicodeTable(name) + tab, fold := unicodeTable(name) if tab == nil { return nil, "", &Error{ErrInvalidCharRange, seq} } - if sign > 0 { - r = appendTable(r, tab) + + if p.flags&FoldCase == 0 || fold == nil { + if sign > 0 { + r = appendTable(r, tab) + } else { + r = appendNegatedTable(r, tab) + } } else { - r = appendNegatedTable(r, tab) + // Merge and clean tab and fold in a temporary buffer. + // This is necessary for the negative case and just tidy + // for the positive case. + tmp := p.tmpClass[:0] + tmp = appendTable(tmp, tab) + tmp = appendTable(tmp, fold) + p.tmpClass = tmp + tmp = cleanClass(&p.tmpClass) + if sign > 0 { + r = appendClass(r, tmp) + } else { + r = appendNegatedClass(r, tmp) + } } return r, t, nil } @@ -907,7 +1420,8 @@ func (p *parser) parseUnicodeClass(s string, r []int) (out []int, rest string, e // and pushes it onto the parse stack. func (p *parser) parseClass(s string) (rest string, err os.Error) { t := s[1:] // chop [ - re := &Regexp{Op: OpCharClass, Flags: p.flags} + re := p.newRegexp(OpCharClass) + re.Flags = p.flags re.Rune = re.Rune0[:0] sign := +1 @@ -979,12 +1493,14 @@ func (p *parser) parseClass(s string) (rest string, err os.Error) { return "", &Error{Code: ErrInvalidCharRange, Expr: rng} } } - class = appendRange(class, lo, hi) + if p.flags&FoldCase == 0 { + class = appendRange(class, lo, hi) + } else { + class = appendFoldedRange(class, lo, hi) + } } t = t[1:] // chop ] - // TODO: Handle FoldCase flag. - // Use &re.Rune instead of &class to avoid allocation. re.Rune = class class = cleanClass(&re.Rune) @@ -999,10 +1515,15 @@ func (p *parser) parseClass(s string) (rest string, err os.Error) { // cleanClass sorts the ranges (pairs of elements of r), // merges them, and eliminates duplicates. func cleanClass(rp *[]int) []int { + // Sort by lo increasing, hi decreasing to break ties. sort.Sort(ranges{rp}) r := *rp + if len(r) < 2 { + return r + } + // Merge abutting, overlapping. w := 2 // write index for i := 2; i < len(r); i += 2 { @@ -1025,23 +1546,71 @@ func cleanClass(rp *[]int) []int { // appendRange returns the result of appending the range lo-hi to the class r. func appendRange(r []int, lo, hi int) []int { - // Expand last range if overlaps or abuts. - if n := len(r); n > 0 { - rlo, rhi := r[n-2], r[n-1] - if lo <= rhi+1 && rlo <= hi+1 { - if lo < rlo { - r[n-2] = lo - } - if hi > rhi { - r[n-1] = hi + // Expand last range or next to last range if it overlaps or abuts. + // Checking two ranges helps when appending case-folded + // alphabets, so that one range can be expanding A-Z and the + // other expanding a-z. + n := len(r) + for i := 2; i <= 4; i += 2 { // twice, using i=2, i=4 + if n >= i { + rlo, rhi := r[n-i], r[n-i+1] + if lo <= rhi+1 && rlo <= hi+1 { + if lo < rlo { + r[n-i] = lo + } + if hi > rhi { + r[n-i+1] = hi + } + return r } - return r } } return append(r, lo, hi) } +const ( + // minimum and maximum runes involved in folding. + // checked during test. + minFold = 0x0041 + maxFold = 0x1044f +) + +// appendFoldedRange returns the result of appending the range lo-hi +// and its case folding-equivalent runes to the class r. +func appendFoldedRange(r []int, lo, hi int) []int { + // Optimizations. + if lo <= minFold && hi >= maxFold { + // Range is full: folding can't add more. + return appendRange(r, lo, hi) + } + if hi < minFold || lo > maxFold { + // Range is outside folding possibilities. + return appendRange(r, lo, hi) + } + if lo < minFold { + // [lo, minFold-1] needs no folding. + r = appendRange(r, lo, minFold-1) + lo = minFold + } + if hi > maxFold { + // [maxFold+1, hi] needs no folding. + r = appendRange(r, maxFold+1, hi) + hi = maxFold + } + + // Brute force. Depend on appendRange to coalesce ranges on the fly. + for c := lo; c <= hi; c++ { + r = appendRange(r, c, c) + f := unicode.SimpleFold(c) + for f != c { + r = appendRange(r, f, f) + f = unicode.SimpleFold(f) + } + } + return r +} + // appendClass returns the result of appending the class x to the class r. // It assume x is clean. func appendClass(r []int, x []int) []int { @@ -1051,6 +1620,14 @@ func appendClass(r []int, x []int) []int { return r } +// appendFolded returns the result of appending the case folding of the class x to the class r. +func appendFoldedClass(r []int, x []int) []int { + for i := 0; i < len(x); i += 2 { + r = appendFoldedRange(r, x[i], x[i+1]) + } + return r +} + // appendNegatedClass returns the result of appending the negation of the class x to the class r. // It assumes x is clean. func appendNegatedClass(r []int, x []int) []int { @@ -1148,10 +1725,11 @@ func negateClass(r []int) []int { } nextLo = hi + 1 } + r = r[:w] if nextLo <= unicode.MaxRune { // It's possible for the negation to have one more // range - this one - than the original class, so use append. - r = append(r[:w], nextLo, unicode.MaxRune) + r = append(r, nextLo, unicode.MaxRune) } return r } diff --git a/src/pkg/exp/regexp/syntax/parse_test.go b/src/pkg/exp/regexp/syntax/parse_test.go index b52cab1a1..779b9afde 100644 --- a/src/pkg/exp/regexp/syntax/parse_test.go +++ b/src/pkg/exp/regexp/syntax/parse_test.go @@ -16,141 +16,152 @@ var parseTests = []struct { Dump string }{ // Base cases - {"a", "lit{a}"}, - {"a.", "cat{lit{a}dot{}}"}, - {"a.b", "cat{lit{a}dot{}lit{b}}"}, - // { "ab", "str{ab}" }, - {"ab", "cat{lit{a}lit{b}}"}, - {"a.b.c", "cat{lit{a}dot{}lit{b}dot{}lit{c}}"}, - // { "abc", "str{abc}" }, - {"abc", "cat{lit{a}lit{b}lit{c}}"}, - {"a|^", "alt{lit{a}bol{}}"}, - // { "a|b", "cc{0x61-0x62}" }, - {"a|b", "alt{lit{a}lit{b}}"}, - {"(a)", "cap{lit{a}}"}, - {"(a)|b", "alt{cap{lit{a}}lit{b}}"}, - {"a*", "star{lit{a}}"}, - {"a+", "plus{lit{a}}"}, - {"a?", "que{lit{a}}"}, - {"a{2}", "rep{2,2 lit{a}}"}, - {"a{2,3}", "rep{2,3 lit{a}}"}, - {"a{2,}", "rep{2,-1 lit{a}}"}, - {"a*?", "nstar{lit{a}}"}, - {"a+?", "nplus{lit{a}}"}, - {"a??", "nque{lit{a}}"}, - {"a{2}?", "nrep{2,2 lit{a}}"}, - {"a{2,3}?", "nrep{2,3 lit{a}}"}, - {"a{2,}?", "nrep{2,-1 lit{a}}"}, - {"", "emp{}"}, - // { "|", "emp{}" }, // alt{emp{}emp{}} but got factored - {"|", "alt{emp{}emp{}}"}, - {"|x|", "alt{emp{}lit{x}emp{}}"}, - {".", "dot{}"}, - {"^", "bol{}"}, - {"$", "eol{}"}, - {"\\|", "lit{|}"}, - {"\\(", "lit{(}"}, - {"\\)", "lit{)}"}, - {"\\*", "lit{*}"}, - {"\\+", "lit{+}"}, - {"\\?", "lit{?}"}, - {"{", "lit{{}"}, - {"}", "lit{}}"}, - {"\\.", "lit{.}"}, - {"\\^", "lit{^}"}, - {"\\$", "lit{$}"}, - {"\\\\", "lit{\\}"}, - {"[ace]", "cc{0x61 0x63 0x65}"}, - {"[abc]", "cc{0x61-0x63}"}, - {"[a-z]", "cc{0x61-0x7a}"}, - // { "[a]", "lit{a}" }, - {"[a]", "cc{0x61}"}, - {"\\-", "lit{-}"}, - {"-", "lit{-}"}, - {"\\_", "lit{_}"}, + {`a`, `lit{a}`}, + {`a.`, `cat{lit{a}dot{}}`}, + {`a.b`, `cat{lit{a}dot{}lit{b}}`}, + {`ab`, `str{ab}`}, + {`a.b.c`, `cat{lit{a}dot{}lit{b}dot{}lit{c}}`}, + {`abc`, `str{abc}`}, + {`a|^`, `alt{lit{a}bol{}}`}, + {`a|b`, `cc{0x61-0x62}`}, + {`(a)`, `cap{lit{a}}`}, + {`(a)|b`, `alt{cap{lit{a}}lit{b}}`}, + {`a*`, `star{lit{a}}`}, + {`a+`, `plus{lit{a}}`}, + {`a?`, `que{lit{a}}`}, + {`a{2}`, `rep{2,2 lit{a}}`}, + {`a{2,3}`, `rep{2,3 lit{a}}`}, + {`a{2,}`, `rep{2,-1 lit{a}}`}, + {`a*?`, `nstar{lit{a}}`}, + {`a+?`, `nplus{lit{a}}`}, + {`a??`, `nque{lit{a}}`}, + {`a{2}?`, `nrep{2,2 lit{a}}`}, + {`a{2,3}?`, `nrep{2,3 lit{a}}`}, + {`a{2,}?`, `nrep{2,-1 lit{a}}`}, + {``, `emp{}`}, + {`|`, `emp{}`}, // alt{emp{}emp{}} but got factored + {`|x|`, `alt{emp{}lit{x}emp{}}`}, + {`.`, `dot{}`}, + {`^`, `bol{}`}, + {`$`, `eol{}`}, + {`\|`, `lit{|}`}, + {`\(`, `lit{(}`}, + {`\)`, `lit{)}`}, + {`\*`, `lit{*}`}, + {`\+`, `lit{+}`}, + {`\?`, `lit{?}`}, + {`{`, `lit{{}`}, + {`}`, `lit{}}`}, + {`\.`, `lit{.}`}, + {`\^`, `lit{^}`}, + {`\$`, `lit{$}`}, + {`\\`, `lit{\}`}, + {`[ace]`, `cc{0x61 0x63 0x65}`}, + {`[abc]`, `cc{0x61-0x63}`}, + {`[a-z]`, `cc{0x61-0x7a}`}, + {`[a]`, `lit{a}`}, + {`\-`, `lit{-}`}, + {`-`, `lit{-}`}, + {`\_`, `lit{_}`}, + {`abc`, `str{abc}`}, + {`abc|def`, `alt{str{abc}str{def}}`}, + {`abc|def|ghi`, `alt{str{abc}str{def}str{ghi}}`}, // Posix and Perl extensions - {"[[:lower:]]", "cc{0x61-0x7a}"}, - {"[a-z]", "cc{0x61-0x7a}"}, - {"[^[:lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"}, - {"[[:^lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"}, - // { "(?i)[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, - // { "(?i)[a-z]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, - // { "(?i)[^[:lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, - // { "(?i)[[:^lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, - {"\\d", "cc{0x30-0x39}"}, - {"\\D", "cc{0x0-0x2f 0x3a-0x10ffff}"}, - {"\\s", "cc{0x9-0xa 0xc-0xd 0x20}"}, - {"\\S", "cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}"}, - {"\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}"}, - {"\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}"}, - // { "(?i)\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}" }, - // { "(?i)\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, - {"[^\\\\]", "cc{0x0-0x5b 0x5d-0x10ffff}"}, - // { "\\C", "byte{}" }, + {`[[:lower:]]`, `cc{0x61-0x7a}`}, + {`[a-z]`, `cc{0x61-0x7a}`}, + {`[^[:lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`}, + {`[[:^lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`}, + {`(?i)[[:lower:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)[a-z]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)[^[:lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`(?i)[[:^lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`\d`, `cc{0x30-0x39}`}, + {`\D`, `cc{0x0-0x2f 0x3a-0x10ffff}`}, + {`\s`, `cc{0x9-0xa 0xc-0xd 0x20}`}, + {`\S`, `cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}`}, + {`\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}`}, + {`\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}`}, + {`(?i)\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}`}, + {`(?i)\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`}, + {`[^\\]`, `cc{0x0-0x5b 0x5d-0x10ffff}`}, + // { `\C`, `byte{}` }, // probably never // Unicode, negatives, and a double negative. - {"\\p{Braille}", "cc{0x2800-0x28ff}"}, - {"\\P{Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, - {"\\p{^Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, - {"\\P{^Braille}", "cc{0x2800-0x28ff}"}, - {"\\pZ", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"}, - {"[\\p{Braille}]", "cc{0x2800-0x28ff}"}, - {"[\\P{Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, - {"[\\p{^Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"}, - {"[\\P{^Braille}]", "cc{0x2800-0x28ff}"}, - {"[\\pZ]", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"}, + {`\p{Braille}`, `cc{0x2800-0x28ff}`}, + {`\P{Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`\p{^Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`\P{^Braille}`, `cc{0x2800-0x28ff}`}, + {`\pZ`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, + {`[\p{Braille}]`, `cc{0x2800-0x28ff}`}, + {`[\P{Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`[\p{^Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`}, + {`[\P{^Braille}]`, `cc{0x2800-0x28ff}`}, + {`[\pZ]`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, + {`\p{Lu}`, mkCharClass(unicode.IsUpper)}, + {`[\p{Lu}]`, mkCharClass(unicode.IsUpper)}, + {`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)}, + + // Hex, octal. + {`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`}, + {`[\x{41}-\x7a]\x61`, `cat{cc{0x41-0x7a}lit{a}}`}, // More interesting regular expressions. - // { "a{,2}", "str{a{,2}}" }, - // { "\\.\\^\\$\\\\", "str{.^$\\}" }, - {"[a-zABC]", "cc{0x41-0x43 0x61-0x7a}"}, - {"[^a]", "cc{0x0-0x60 0x62-0x10ffff}"}, - {"[\xce\xb1-\xce\xb5\xe2\x98\xba]", "cc{0x3b1-0x3b5 0x263a}"}, // utf-8 - {"a*{", "cat{star{lit{a}}lit{{}}"}, + {`a{,2}`, `str{a{,2}}`}, + {`\.\^\$\\`, `str{.^$\}`}, + {`[a-zABC]`, `cc{0x41-0x43 0x61-0x7a}`}, + {`[^a]`, `cc{0x0-0x60 0x62-0x10ffff}`}, + {`[α-ε☺]`, `cc{0x3b1-0x3b5 0x263a}`}, // utf-8 + {`a*{`, `cat{star{lit{a}}lit{{}}`}, // Test precedences - // { "(?:ab)*", "star{str{ab}}" }, - // { "(ab)*", "star{cap{str{ab}}}" }, - // { "ab|cd", "alt{str{ab}str{cd}}" }, - // { "a(b|c)d", "cat{lit{a}cap{cc{0x62-0x63}}lit{d}}" }, - {"(?:ab)*", "star{cat{lit{a}lit{b}}}"}, - {"(ab)*", "star{cap{cat{lit{a}lit{b}}}}"}, - {"ab|cd", "alt{cat{lit{a}lit{b}}cat{lit{c}lit{d}}}"}, - {"a(b|c)d", "cat{lit{a}cap{alt{lit{b}lit{c}}}lit{d}}"}, + {`(?:ab)*`, `star{str{ab}}`}, + {`(ab)*`, `star{cap{str{ab}}}`}, + {`ab|cd`, `alt{str{ab}str{cd}}`}, + {`a(b|c)d`, `cat{lit{a}cap{cc{0x62-0x63}}lit{d}}`}, // Test flattening. - {"(?:a)", "lit{a}"}, - // { "(?:ab)(?:cd)", "str{abcd}" }, - // { "(?:a|b)|(?:c|d)", "cc{0x61-0x64}" }, - // { "a|.", "dot{}" }, - // { ".|a", "dot{}" }, + {`(?:a)`, `lit{a}`}, + {`(?:ab)(?:cd)`, `str{abcd}`}, + {`(?:a+b+)(?:c+d+)`, `cat{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`}, + {`(?:a+|b+)|(?:c+|d+)`, `alt{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`}, + {`(?:a|b)|(?:c|d)`, `cc{0x61-0x64}`}, + {`a|.`, `dot{}`}, + {`.|a`, `dot{}`}, + {`(?:[abc]|A|Z|hello|world)`, `alt{cc{0x41 0x5a 0x61-0x63}str{hello}str{world}}`}, + {`(?:[abc]|A|Z)`, `cc{0x41 0x5a 0x61-0x63}`}, // Test Perl quoted literals - {"\\Q+|*?{[\\E", "str{+|*?{[}"}, - {"\\Q+\\E+", "plus{lit{+}}"}, - {"\\Q\\\\E", "lit{\\}"}, - {"\\Q\\\\\\E", "str{\\\\}"}, + {`\Q+|*?{[\E`, `str{+|*?{[}`}, + {`\Q+\E+`, `plus{lit{+}}`}, + {`\Q\\E`, `lit{\}`}, + {`\Q\\\E`, `str{\\}`}, // Test Perl \A and \z - {"(?m)^", "bol{}"}, - {"(?m)$", "eol{}"}, - {"(?-m)^", "bot{}"}, - {"(?-m)$", "eot{}"}, - {"(?m)\\A", "bot{}"}, - {"(?m)\\z", "eot{\\z}"}, - {"(?-m)\\A", "bot{}"}, - {"(?-m)\\z", "eot{\\z}"}, + {`(?m)^`, `bol{}`}, + {`(?m)$`, `eol{}`}, + {`(?-m)^`, `bot{}`}, + {`(?-m)$`, `eot{}`}, + {`(?m)\A`, `bot{}`}, + {`(?m)\z`, `eot{\z}`}, + {`(?-m)\A`, `bot{}`}, + {`(?-m)\z`, `eot{\z}`}, // Test named captures - {"(?Pa)", "cap{name:lit{a}}"}, + {`(?Pa)`, `cap{name:lit{a}}`}, // Case-folded literals - // { "[Aa]", "litfold{a}" }, + {`[Aa]`, `litfold{A}`}, + {`[\x{100}\x{101}]`, `litfold{Ā}`}, + {`[Δδ]`, `litfold{Δ}`}, // Strings - // { "abcde", "str{abcde}" }, - // { "[Aa][Bb]cd", "cat{strfold{ab}str{cd}}" }, + {`abcde`, `str{abcde}`}, + {`[Aa][Bb]cd`, `cat{strfold{AB}str{cd}}`}, + + // Factoring. + {`abc|abd|aef|bcx|bcy`, `alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}cat{str{bc}cc{0x78-0x79}}}`}, + {`ax+y|ax+z|ay+w`, `cat{lit{a}alt{cat{plus{lit{x}}cc{0x79-0x7a}}cat{plus{lit{y}}lit{w}}}}`}, } const testFlags = MatchNL | PerlX | UnicodeGroups @@ -223,8 +234,9 @@ func dumpRegexp(b *bytes.Buffer, re *Regexp) { } if re.Flags&FoldCase != 0 { for _, r := range re.Rune { - if unicode.ToUpper(r) != r { + if unicode.SimpleFold(r) != r { b.WriteString("fold") + break } } } @@ -270,3 +282,69 @@ func dumpRegexp(b *bytes.Buffer, re *Regexp) { } b.WriteByte('}') } + +func mkCharClass(f func(int) bool) string { + re := &Regexp{Op: OpCharClass} + lo := -1 + for i := 0; i <= unicode.MaxRune; i++ { + if f(i) { + if lo < 0 { + lo = i + } + } else { + if lo >= 0 { + re.Rune = append(re.Rune, lo, i-1) + lo = -1 + } + } + } + if lo >= 0 { + re.Rune = append(re.Rune, lo, unicode.MaxRune) + } + return dump(re) +} + +func isUpperFold(rune int) bool { + if unicode.IsUpper(rune) { + return true + } + c := unicode.SimpleFold(rune) + for c != rune { + if unicode.IsUpper(c) { + return true + } + c = unicode.SimpleFold(c) + } + return false +} + +func TestFoldConstants(t *testing.T) { + last := -1 + for i := 0; i <= unicode.MaxRune; i++ { + if unicode.SimpleFold(i) == i { + continue + } + if last == -1 && minFold != i { + t.Errorf("minFold=%#U should be %#U", minFold, i) + } + last = i + } + if maxFold != last { + t.Errorf("maxFold=%#U should be %#U", maxFold, last) + } +} + +func TestAppendRangeCollapse(t *testing.T) { + // AppendRange should collapse each of the new ranges + // into the earlier ones (it looks back two ranges), so that + // the slice never grows very large. + // Note that we are not calling cleanClass. + var r []int + for i := 'A'; i <= 'Z'; i++ { + r = appendRange(r, i, i) + r = appendRange(r, i+'a'-'A', i+'a'-'A') + } + if string(r) != "AZaz" { + t.Errorf("appendRange interlaced A-Z a-z = %s, want AZaz", string(r)) + } +} diff --git a/src/pkg/exp/regexp/syntax/prog.go b/src/pkg/exp/regexp/syntax/prog.go new file mode 100644 index 000000000..6eeb3da0c --- /dev/null +++ b/src/pkg/exp/regexp/syntax/prog.go @@ -0,0 +1,182 @@ +package syntax + +import ( + "bytes" + "strconv" +) + +// Compiled program. +// May not belong in this package, but convenient for now. + +// A Prog is a compiled regular expression program. +type Prog struct { + Inst []Inst + Start int // index of start instruction +} + +// An InstOp is an instruction opcode. +type InstOp uint8 + +const ( + InstAlt InstOp = iota + InstAltMatch + InstCapture + InstEmptyWidth + InstMatch + InstFail + InstNop + InstRune +) + +// An EmptyOp specifies a kind or mixture of zero-width assertions. +type EmptyOp uint8 + +const ( + EmptyBeginLine EmptyOp = 1 << iota + EmptyEndLine + EmptyBeginText + EmptyEndText + EmptyWordBoundary + EmptyNoWordBoundary +) + +// An Inst is a single instruction in a regular expression program. +type Inst struct { + Op InstOp + Out uint32 // all but InstMatch, InstFail + Arg uint32 // InstAlt, InstAltMatch, InstCapture, InstEmptyWidth + Rune []int +} + +func (p *Prog) String() string { + var b bytes.Buffer + dumpProg(&b, p) + return b.String() +} + +// MatchRune returns true if the instruction matches (and consumes) r. +// It should only be called when i.Op == InstRune. +func (i *Inst) MatchRune(r int) bool { + rune := i.Rune + + // Special case: single-rune slice is from literal string, not char class. + // TODO: Case folding. + if len(rune) == 1 { + return r == rune[0] + } + + // Peek at the first few pairs. + // Should handle ASCII well. + for j := 0; j < len(rune) && j <= 8; j += 2 { + if r < rune[j] { + return false + } + if r <= rune[j+1] { + return true + } + } + + // Otherwise binary search. + lo := 0 + hi := len(rune) / 2 + for lo < hi { + m := lo + (hi-lo)/2 + if c := rune[2*m]; c <= r { + if r <= rune[2*m+1] { + return true + } + lo = m + 1 + } else { + hi = m + } + } + return false +} + +// As per re2's Prog::IsWordChar. Determines whether rune is an ASCII word char. +// Since we act on runes, it would be easy to support Unicode here. +func wordRune(rune int) bool { + return rune == '_' || + ('A' <= rune && rune <= 'Z') || + ('a' <= rune && rune <= 'z') || + ('0' <= rune && rune <= '9') +} + +// MatchEmptyWidth returns true if the instruction matches +// an empty string between the runes before and after. +// It should only be called when i.Op == InstEmptyWidth. +func (i *Inst) MatchEmptyWidth(before int, after int) bool { + switch EmptyOp(i.Arg) { + case EmptyBeginLine: + return before == '\n' || before == -1 + case EmptyEndLine: + return after == '\n' || after == -1 + case EmptyBeginText: + return before == -1 + case EmptyEndText: + return after == -1 + case EmptyWordBoundary: + return wordRune(before) != wordRune(after) + case EmptyNoWordBoundary: + return wordRune(before) == wordRune(after) + } + panic("unknown empty width arg") +} + + +func (i *Inst) String() string { + var b bytes.Buffer + dumpInst(&b, i) + return b.String() +} + +func bw(b *bytes.Buffer, args ...string) { + for _, s := range args { + b.WriteString(s) + } +} + +func dumpProg(b *bytes.Buffer, p *Prog) { + for j := range p.Inst { + i := &p.Inst[j] + pc := strconv.Itoa(j) + if len(pc) < 3 { + b.WriteString(" "[len(pc):]) + } + if j == p.Start { + pc += "*" + } + bw(b, pc, "\t") + dumpInst(b, i) + bw(b, "\n") + } +} + +func u32(i uint32) string { + return strconv.Uitoa64(uint64(i)) +} + +func dumpInst(b *bytes.Buffer, i *Inst) { + switch i.Op { + case InstAlt: + bw(b, "alt -> ", u32(i.Out), ", ", u32(i.Arg)) + case InstAltMatch: + bw(b, "altmatch -> ", u32(i.Out), ", ", u32(i.Arg)) + case InstCapture: + bw(b, "cap ", u32(i.Arg), " -> ", u32(i.Out)) + case InstEmptyWidth: + bw(b, "empty ", u32(i.Arg), " -> ", u32(i.Out)) + case InstMatch: + bw(b, "match") + case InstFail: + bw(b, "fail") + case InstNop: + bw(b, "nop -> ", u32(i.Out)) + case InstRune: + if i.Rune == nil { + // shouldn't happen + bw(b, "rune ") + } + bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune)), " -> ", u32(i.Out)) + } +} diff --git a/src/pkg/exp/regexp/syntax/prog_test.go b/src/pkg/exp/regexp/syntax/prog_test.go new file mode 100644 index 000000000..7be4281c2 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/prog_test.go @@ -0,0 +1,91 @@ +package syntax + +import ( + "testing" +) + +var compileTests = []struct { + Regexp string + Prog string +}{ + {"a", ` 0 fail + 1* rune "a" -> 2 + 2 match +`}, + {"[A-M][n-z]", ` 0 fail + 1* rune "AM" -> 2 + 2 rune "nz" -> 3 + 3 match +`}, + {"", ` 0 fail + 1* nop -> 2 + 2 match +`}, + {"a?", ` 0 fail + 1 rune "a" -> 3 + 2* alt -> 1, 3 + 3 match +`}, + {"a??", ` 0 fail + 1 rune "a" -> 3 + 2* alt -> 3, 1 + 3 match +`}, + {"a+", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 1, 3 + 3 match +`}, + {"a+?", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 3, 1 + 3 match +`}, + {"a*", ` 0 fail + 1 rune "a" -> 2 + 2* alt -> 1, 3 + 3 match +`}, + {"a*?", ` 0 fail + 1 rune "a" -> 2 + 2* alt -> 3, 1 + 3 match +`}, + {"a+b+", ` 0 fail + 1* rune "a" -> 2 + 2 alt -> 1, 3 + 3 rune "b" -> 4 + 4 alt -> 3, 5 + 5 match +`}, + {"(a+)(b+)", ` 0 fail + 1* cap 2 -> 2 + 2 rune "a" -> 3 + 3 alt -> 2, 4 + 4 cap 3 -> 5 + 5 cap 4 -> 6 + 6 rune "b" -> 7 + 7 alt -> 6, 8 + 8 cap 5 -> 9 + 9 match +`}, + {"a+|b+", ` 0 fail + 1 rune "a" -> 2 + 2 alt -> 1, 6 + 3 rune "b" -> 4 + 4 alt -> 3, 6 + 5* alt -> 1, 3 + 6 match +`}, +} + +func TestCompile(t *testing.T) { + for _, tt := range compileTests { + re, _ := Parse(tt.Regexp, Perl) + p, _ := Compile(re) + s := p.String() + if s != tt.Prog { + t.Errorf("compiled %#q:\n--- have\n%s---\n--- want\n%s---", tt.Regexp, s, tt.Prog) + } + } +} diff --git a/src/pkg/exp/regexp/syntax/regexp.go b/src/pkg/exp/regexp/syntax/regexp.go index a0c465967..00a4addef 100644 --- a/src/pkg/exp/regexp/syntax/regexp.go +++ b/src/pkg/exp/regexp/syntax/regexp.go @@ -33,6 +33,8 @@ type Regexp struct { type Op uint8 // Operators are listed in precedence order, tightest binding to weakest. +// Character class operators are listed simplest to most complex +// (OpLiteral, OpCharClass, OpAnyCharNotNL, OpAnyChar). const ( OpNoMatch Op = 1 + iota // matches no strings @@ -58,6 +60,59 @@ const ( const opPseudo Op = 128 // where pseudo-ops start +// Equal returns true if x and y have identical structure. +func (x *Regexp) Equal(y *Regexp) bool { + if x == nil || y == nil { + return x == y + } + if x.Op != y.Op { + return false + } + switch x.Op { + case OpEndText: + // The parse flags remember whether this is \z or \Z. + if x.Flags&WasDollar != y.Flags&WasDollar { + return false + } + + case OpLiteral, OpCharClass: + if len(x.Rune) != len(y.Rune) { + return false + } + for i, r := range x.Rune { + if r != y.Rune[i] { + return false + } + } + + case OpAlternate, OpConcat: + if len(x.Sub) != len(y.Sub) { + return false + } + for i, sub := range x.Sub { + if !sub.Equal(y.Sub[i]) { + return false + } + } + + case OpStar, OpPlus, OpQuest: + if x.Flags&NonGreedy != y.Flags&NonGreedy || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + + case OpRepeat: + if x.Flags&NonGreedy != y.Flags&NonGreedy || x.Min != y.Min || x.Max != y.Max || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + + case OpCapture: + if x.Cap != y.Cap || x.Name != y.Name || !x.Sub[0].Equal(y.Sub[0]) { + return false + } + } + return true +} + // writeRegexp writes the Perl syntax for the regular expression re to b. func writeRegexp(b *bytes.Buffer, re *Regexp) { switch re.Op { @@ -68,16 +123,24 @@ func writeRegexp(b *bytes.Buffer, re *Regexp) { case OpEmptyMatch: b.WriteString(`(?:)`) case OpLiteral: + if re.Flags&FoldCase != 0 { + b.WriteString(`(?i:`) + } for _, r := range re.Rune { escape(b, r, false) } + if re.Flags&FoldCase != 0 { + b.WriteString(`)`) + } case OpCharClass: if len(re.Rune)%2 != 0 { b.WriteString(`[invalid char class]`) break } b.WriteRune('[') - if len(re.Rune) > 0 && re.Rune[0] == 0 && re.Rune[len(re.Rune)-1] == unicode.MaxRune { + if len(re.Rune) == 0 { + b.WriteString(`^\x00-\x{10FFFF}`) + } else if re.Rune[0] == 0 && re.Rune[len(re.Rune)-1] == unicode.MaxRune { // Contains 0 and MaxRune. Probably a negated class. // Print the gaps. b.WriteRune('^') @@ -124,7 +187,9 @@ func writeRegexp(b *bytes.Buffer, re *Regexp) { } else { b.WriteRune('(') } - writeRegexp(b, re.Sub[0]) + if re.Sub[0].Op != OpEmptyMatch { + writeRegexp(b, re.Sub[0]) + } b.WriteRune(')') case OpStar, OpPlus, OpQuest, OpRepeat: if sub := re.Sub[0]; sub.Op > OpCapture { @@ -203,6 +268,15 @@ func escape(b *bytes.Buffer, r int, force bool) { case '\v': b.WriteString(`\v`) default: + if r < 0x100 { + b.WriteString(`\x`) + s := strconv.Itob(r, 16) + if len(s) == 1 { + b.WriteRune('0') + } + b.WriteString(s) + break + } b.WriteString(`\x{`) b.WriteString(strconv.Itob(r, 16)) b.WriteString(`}`) diff --git a/src/pkg/exp/regexp/syntax/simplify.go b/src/pkg/exp/regexp/syntax/simplify.go new file mode 100644 index 000000000..72390417b --- /dev/null +++ b/src/pkg/exp/regexp/syntax/simplify.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// Simplify returns a regexp equivalent to re but without counted repetitions +// and with various other simplifications, such as rewriting /(?:a+)+/ to /a+/. +// The resulting regexp will execute correctly but its string representation +// will not produce the same parse tree, because capturing parentheses +// may have been duplicated or removed. For example, the simplified form +// for /(x){1,2}/ is /(x)(x)?/ but both parentheses capture as $1. +// The returned regexp may share structure with or be the original. +func (re *Regexp) Simplify() *Regexp { + if re == nil { + return nil + } + switch re.Op { + case OpCapture, OpConcat, OpAlternate: + // Simplify children, building new Regexp if children change. + nre := re + for i, sub := range re.Sub { + nsub := sub.Simplify() + if nre == re && nsub != sub { + // Start a copy. + nre = new(Regexp) + *nre = *re + nre.Rune = nil + nre.Sub = append(nre.Sub0[:0], re.Sub[:i]...) + } + if nre != re { + nre.Sub = append(nre.Sub, nsub) + } + } + return nre + + case OpStar, OpPlus, OpQuest: + sub := re.Sub[0].Simplify() + return simplify1(re.Op, re.Flags, sub, re) + + case OpRepeat: + // Special special case: x{0} matches the empty string + // and doesn't even need to consider x. + if re.Min == 0 && re.Max == 0 { + return &Regexp{Op: OpEmptyMatch} + } + + // The fun begins. + sub := re.Sub[0].Simplify() + + // x{n,} means at least n matches of x. + if re.Max == -1 { + // Special case: x{0,} is x*. + if re.Min == 0 { + return simplify1(OpStar, re.Flags, sub, nil) + } + + // Special case: x{1,} is x+. + if re.Min == 1 { + return simplify1(OpPlus, re.Flags, sub, nil) + } + + // General case: x{4,} is xxxx+. + nre := &Regexp{Op: OpConcat} + nre.Sub = nre.Sub0[:0] + for i := 0; i < re.Min-1; i++ { + nre.Sub = append(nre.Sub, sub) + } + nre.Sub = append(nre.Sub, simplify1(OpPlus, re.Flags, sub, nil)) + return nre + } + + // Special case x{0} handled above. + + // Special case: x{1} is just x. + if re.Min == 1 && re.Max == 1 { + return sub + } + + // General case: x{n,m} means n copies of x and m copies of x? + // The machine will do less work if we nest the final m copies, + // so that x{2,5} = xx(x(x(x)?)?)? + + // Build leading prefix: xx. + var prefix *Regexp + if re.Min > 0 { + prefix = &Regexp{Op: OpConcat} + prefix.Sub = prefix.Sub0[:0] + for i := 0; i < re.Min; i++ { + prefix.Sub = append(prefix.Sub, sub) + } + } + + // Build and attach suffix: (x(x(x)?)?)? + if re.Max > re.Min { + suffix := simplify1(OpQuest, re.Flags, sub, nil) + for i := re.Min + 1; i < re.Max; i++ { + nre2 := &Regexp{Op: OpConcat} + nre2.Sub = append(nre2.Sub0[:0], sub, suffix) + suffix = simplify1(OpQuest, re.Flags, nre2, nil) + } + if prefix == nil { + return suffix + } + prefix.Sub = append(prefix.Sub, suffix) + } + if prefix != nil { + return prefix + } + + // Some degenerate case like min > max or min < max < 0. + // Handle as impossible match. + return &Regexp{Op: OpNoMatch} + } + + return re +} + +// simplify1 implements Simplify for the unary OpStar, +// OpPlus, and OpQuest operators. It returns the simple regexp +// equivalent to +// +// Regexp{Op: op, Flags: flags, Sub: {sub}} +// +// under the assumption that sub is already simple, and +// without first allocating that structure. If the regexp +// to be returned turns out to be equivalent to re, simplify1 +// returns re instead. +// +// simplify1 is factored out of Simplify because the implementation +// for other operators generates these unary expressions. +// Letting them call simplify1 makes sure the expressions they +// generate are simple. +func simplify1(op Op, flags Flags, sub, re *Regexp) *Regexp { + // Special case: repeat the empty string as much as + // you want, but it's still the empty string. + if sub.Op == OpEmptyMatch { + return sub + } + // The operators are idempotent if the flags match. + if op == sub.Op && flags&NonGreedy == sub.Flags&NonGreedy { + return sub + } + if re != nil && re.Op == op && re.Flags&NonGreedy == flags&NonGreedy && sub == re.Sub[0] { + return re + } + + re = &Regexp{Op: op, Flags: flags} + re.Sub = append(re.Sub0[:0], sub) + return re +} diff --git a/src/pkg/exp/regexp/syntax/simplify_test.go b/src/pkg/exp/regexp/syntax/simplify_test.go new file mode 100644 index 000000000..c8cec2183 --- /dev/null +++ b/src/pkg/exp/regexp/syntax/simplify_test.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import "testing" + +var simplifyTests = []struct { + Regexp string + Simple string +}{ + // Already-simple constructs + {`a`, `a`}, + {`ab`, `ab`}, + {`a|b`, `[a-b]`}, + {`ab|cd`, `ab|cd`}, + {`(ab)*`, `(ab)*`}, + {`(ab)+`, `(ab)+`}, + {`(ab)?`, `(ab)?`}, + {`.`, `.`}, + {`^`, `^`}, + {`$`, `$`}, + {`[ac]`, `[ac]`}, + {`[^ac]`, `[^ac]`}, + + // Posix character classes + {`[[:alnum:]]`, `[0-9A-Za-z]`}, + {`[[:alpha:]]`, `[A-Za-z]`}, + {`[[:blank:]]`, `[\t ]`}, + {`[[:cntrl:]]`, `[\x00-\x1f\x7f]`}, + {`[[:digit:]]`, `[0-9]`}, + {`[[:graph:]]`, `[!-~]`}, + {`[[:lower:]]`, `[a-z]`}, + {`[[:print:]]`, `[ -~]`}, + {`[[:punct:]]`, "[!-/:-@\\[-`\\{-~]"}, + {`[[:space:]]`, `[\t-\r ]`}, + {`[[:upper:]]`, `[A-Z]`}, + {`[[:xdigit:]]`, `[0-9A-Fa-f]`}, + + // Perl character classes + {`\d`, `[0-9]`}, + {`\s`, `[\t-\n\f-\r ]`}, + {`\w`, `[0-9A-Z_a-z]`}, + {`\D`, `[^0-9]`}, + {`\S`, `[^\t-\n\f-\r ]`}, + {`\W`, `[^0-9A-Z_a-z]`}, + {`[\d]`, `[0-9]`}, + {`[\s]`, `[\t-\n\f-\r ]`}, + {`[\w]`, `[0-9A-Z_a-z]`}, + {`[\D]`, `[^0-9]`}, + {`[\S]`, `[^\t-\n\f-\r ]`}, + {`[\W]`, `[^0-9A-Z_a-z]`}, + + // Posix repetitions + {`a{1}`, `a`}, + {`a{2}`, `aa`}, + {`a{5}`, `aaaaa`}, + {`a{0,1}`, `a?`}, + // The next three are illegible because Simplify inserts (?:) + // parens instead of () parens to avoid creating extra + // captured subexpressions. The comments show a version with fewer parens. + {`(a){0,2}`, `(?:(a)(a)?)?`}, // (aa?)? + {`(a){0,4}`, `(?:(a)(?:(a)(?:(a)(a)?)?)?)?`}, // (a(a(aa?)?)?)? + {`(a){2,6}`, `(a)(a)(?:(a)(?:(a)(?:(a)(a)?)?)?)?`}, // aa(a(a(aa?)?)?)? + {`a{0,2}`, `(?:aa?)?`}, // (aa?)? + {`a{0,4}`, `(?:a(?:a(?:aa?)?)?)?`}, // (a(a(aa?)?)?)? + {`a{2,6}`, `aa(?:a(?:a(?:aa?)?)?)?`}, // aa(a(a(aa?)?)?)? + {`a{0,}`, `a*`}, + {`a{1,}`, `a+`}, + {`a{2,}`, `aa+`}, + {`a{5,}`, `aaaaa+`}, + + // Test that operators simplify their arguments. + {`(?:a{1,}){1,}`, `a+`}, + {`(a{1,}b{1,})`, `(a+b+)`}, + {`a{1,}|b{1,}`, `a+|b+`}, + {`(?:a{1,})*`, `(?:a+)*`}, + {`(?:a{1,})+`, `a+`}, + {`(?:a{1,})?`, `(?:a+)?`}, + {``, `(?:)`}, + {`a{0}`, `(?:)`}, + + // Character class simplification + {`[ab]`, `[a-b]`}, + {`[a-za-za-z]`, `[a-z]`}, + {`[A-Za-zA-Za-z]`, `[A-Za-z]`}, + {`[ABCDEFGH]`, `[A-H]`}, + {`[AB-CD-EF-GH]`, `[A-H]`}, + {`[W-ZP-XE-R]`, `[E-Z]`}, + {`[a-ee-gg-m]`, `[a-m]`}, + {`[a-ea-ha-m]`, `[a-m]`}, + {`[a-ma-ha-e]`, `[a-m]`}, + {`[a-zA-Z0-9 -~]`, `[ -~]`}, + + // Empty character classes + {`[^[:cntrl:][:^cntrl:]]`, `[^\x00-\x{10FFFF}]`}, + + // Full character classes + {`[[:cntrl:][:^cntrl:]]`, `.`}, + + // Unicode case folding. + {`(?i)A`, `(?i:A)`}, + {`(?i)a`, `(?i:a)`}, + {`(?i)[A]`, `(?i:A)`}, + {`(?i)[a]`, `(?i:A)`}, + {`(?i)K`, `(?i:K)`}, + {`(?i)k`, `(?i:k)`}, + {`(?i)\x{212a}`, "(?i:\u212A)"}, + {`(?i)[K]`, "[Kk\u212A]"}, + {`(?i)[k]`, "[Kk\u212A]"}, + {`(?i)[\x{212a}]`, "[Kk\u212A]"}, + {`(?i)[a-z]`, "[A-Za-z\u017F\u212A]"}, + {`(?i)[\x00-\x{FFFD}]`, "[\\x00-\uFFFD]"}, + {`(?i)[\x00-\x{10FFFF}]`, `.`}, + + // Empty string as a regular expression. + // The empty string must be preserved inside parens in order + // to make submatches work right, so these tests are less + // interesting than they might otherwise be. String inserts + // explicit (?:) in place of non-parenthesized empty strings, + // to make them easier to spot for other parsers. + {`(a|b|)`, `([a-b]|(?:))`}, + {`(|)`, `()`}, + {`a()`, `a()`}, + {`(()|())`, `(()|())`}, + {`(a|)`, `(a|(?:))`}, + {`ab()cd()`, `ab()cd()`}, + {`()`, `()`}, + {`()*`, `()*`}, + {`()+`, `()+`}, + {`()?`, `()?`}, + {`(){0}`, `(?:)`}, + {`(){1}`, `()`}, + {`(){1,}`, `()+`}, + {`(){0,2}`, `(?:()()?)?`}, +} + +func TestSimplify(t *testing.T) { + for _, tt := range simplifyTests { + re, err := Parse(tt.Regexp, MatchNL|Perl&^OneLine) + if err != nil { + t.Errorf("Parse(%#q) = error %v", tt.Regexp, err) + continue + } + s := re.Simplify().String() + if s != tt.Simple { + t.Errorf("Simplify(%#q) = %#q, want %#q", tt.Regexp, s, tt.Simple) + } + } +} diff --git a/src/pkg/exp/template/Makefile b/src/pkg/exp/template/Makefile index ab9832f61..8550b0d52 100644 --- a/src/pkg/exp/template/Makefile +++ b/src/pkg/exp/template/Makefile @@ -4,9 +4,12 @@ include ../../../Make.inc -TARG=template +TARG=exp/template GOFILES=\ + exec.go\ + funcs.go\ lex.go\ parse.go\ + set.go\ include ../../../Make.pkg diff --git a/src/pkg/exp/template/exec.go b/src/pkg/exp/template/exec.go new file mode 100644 index 000000000..fb0a9e621 --- /dev/null +++ b/src/pkg/exp/template/exec.go @@ -0,0 +1,508 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package template + +import ( + "fmt" + "io" + "os" + "reflect" + "strings" + "unicode" + "utf8" +) + +// state represents the state of an execution. It's not part of the +// template so that multiple executions of the same template +// can execute in parallel. +type state struct { + tmpl *Template + wr io.Writer + set *Set + line int // line number for errors +} + +// errorf formats the error and terminates processing. +func (s *state) errorf(format string, args ...interface{}) { + format = fmt.Sprintf("template: %s:%d: %s", s.tmpl.name, s.line, format) + panic(fmt.Errorf(format, args...)) +} + +// error terminates processing. +func (s *state) error(err os.Error) { + s.errorf("%s", err) +} + +// Execute applies a parsed template to the specified data object, +// writing the output to wr. +func (t *Template) Execute(wr io.Writer, data interface{}) os.Error { + return t.ExecuteInSet(wr, data, nil) +} + +// ExecuteInSet applies a parsed template to the specified data object, +// writing the output to wr. Nested template invocations will be resolved +// from the specified set. +func (t *Template) ExecuteInSet(wr io.Writer, data interface{}, set *Set) (err os.Error) { + defer t.recover(&err) + state := &state{ + tmpl: t, + wr: wr, + set: set, + line: 1, + } + if t.root == nil { + state.errorf("must be parsed before execution") + } + state.walk(reflect.ValueOf(data), t.root) + return +} + +// Walk functions step through the major pieces of the template structure, +// generating output as they go. +func (s *state) walk(data reflect.Value, n node) { + switch n := n.(type) { + case *actionNode: + s.line = n.line + s.printValue(n, s.evalPipeline(data, n.pipeline)) + case *listNode: + for _, node := range n.nodes { + s.walk(data, node) + } + case *ifNode: + s.line = n.line + s.walkIfOrWith(nodeIf, data, n.pipeline, n.list, n.elseList) + case *rangeNode: + s.line = n.line + s.walkRange(data, n) + case *textNode: + if _, err := s.wr.Write(n.text); err != nil { + s.error(err) + } + case *templateNode: + s.line = n.line + s.walkTemplate(data, n) + case *withNode: + s.line = n.line + s.walkIfOrWith(nodeWith, data, n.pipeline, n.list, n.elseList) + default: + s.errorf("unknown node: %s", n) + } +} + +// walkIfOrWith walks an 'if' or 'with' node. The two control structures +// are identical in behavior except that 'with' sets dot. +func (s *state) walkIfOrWith(typ nodeType, data reflect.Value, pipe []*commandNode, list, elseList *listNode) { + val := s.evalPipeline(data, pipe) + truth, ok := isTrue(val) + if !ok { + s.errorf("if/with can't use value of type %T", val.Interface()) + } + if truth { + if typ == nodeWith { + data = val + } + s.walk(data, list) + } else if elseList != nil { + s.walk(data, elseList) + } +} + +// isTrue returns whether the value is 'true', in the sense of not the zero of its type, +// and whether the value has a meaningful truth value. +func isTrue(val reflect.Value) (truth, ok bool) { + switch val.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + truth = val.Len() > 0 + case reflect.Bool: + truth = val.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + truth = val.Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + truth = val.Uint() != 0 + case reflect.Float32, reflect.Float64: + truth = val.Float() != 0 + case reflect.Complex64, reflect.Complex128: + truth = val.Complex() != 0 + case reflect.Chan, reflect.Func, reflect.Ptr: + truth = !val.IsNil() + default: + return + } + return truth, true +} + +func (s *state) walkRange(data reflect.Value, r *rangeNode) { + val := s.evalPipeline(data, r.pipeline) + switch val.Kind() { + case reflect.Array, reflect.Slice: + if val.Len() == 0 { + break + } + for i := 0; i < val.Len(); i++ { + s.walk(val.Index(i), r.list) + } + return + case reflect.Map: + if val.Len() == 0 { + break + } + for _, key := range val.MapKeys() { + s.walk(val.MapIndex(key), r.list) + } + return + default: + s.errorf("range can't iterate over value of type %T", val.Interface()) + } + if r.elseList != nil { + s.walk(data, r.elseList) + } +} + +func (s *state) walkTemplate(data reflect.Value, t *templateNode) { + name := s.evalArg(data, reflect.TypeOf("string"), t.name).String() + if s.set == nil { + s.errorf("no set defined in which to invoke template named %q", name) + } + tmpl := s.set.tmpl[name] + if tmpl == nil { + s.errorf("template %q not in set", name) + } + data = s.evalPipeline(data, t.pipeline) + newState := *s + newState.tmpl = tmpl + newState.walk(data, tmpl.root) +} + +// Eval functions evaluate pipelines, commands, and their elements and extract +// values from the data structure by examining fields, calling methods, and so on. +// The printing of those values happens only through walk functions. + +func (s *state) evalPipeline(data reflect.Value, pipe []*commandNode) reflect.Value { + value := reflect.Value{} + for _, cmd := range pipe { + value = s.evalCommand(data, cmd, value) // previous value is this one's final arg. + // If the object has type interface{}, dig down one level to the thing inside. + if value.Kind() == reflect.Interface && value.Type().NumMethod() == 0 { + value = reflect.ValueOf(value.Interface()) // lovely! + } + } + return value +} + +func (s *state) evalCommand(data reflect.Value, cmd *commandNode, final reflect.Value) reflect.Value { + firstWord := cmd.args[0] + switch n := firstWord.(type) { + case *fieldNode: + return s.evalFieldNode(data, n, cmd.args, final) + case *identifierNode: + return s.evalFieldOrCall(data, n.ident, cmd.args, final) + } + if len(cmd.args) > 1 || final.IsValid() { + s.errorf("can't give argument to non-function %s", cmd.args[0]) + } + switch word := cmd.args[0].(type) { + case *dotNode: + return data + case *boolNode: + return reflect.ValueOf(word.true) + case *numberNode: + // These are ideal constants but we don't know the type + // and we have no context. (If it was a method argument, + // we'd know what we need.) The syntax guides us to some extent. + switch { + case word.isComplex: + return reflect.ValueOf(word.complex128) // incontrovertible. + case word.isFloat && strings.IndexAny(word.text, ".eE") >= 0: + return reflect.ValueOf(word.float64) + case word.isInt: + return reflect.ValueOf(word.int64) + case word.isUint: + return reflect.ValueOf(word.uint64) + } + case *stringNode: + return reflect.ValueOf(word.text) + } + s.errorf("can't handle command %q", firstWord) + panic("not reached") +} + +func (s *state) evalFieldNode(data reflect.Value, field *fieldNode, args []node, final reflect.Value) reflect.Value { + // Up to the last entry, it must be a field. + n := len(field.ident) + for i := 0; i < n-1; i++ { + data = s.evalField(data, field.ident[i]) + } + // Now it can be a field or method and if a method, gets arguments. + return s.evalFieldOrCall(data, field.ident[n-1], args, final) +} + +// Is this an exported - upper case - name? +func isExported(name string) bool { + rune, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(rune) +} + +func (s *state) evalField(data reflect.Value, fieldName string) reflect.Value { + var isNil bool + if data, isNil = indirect(data); isNil { + s.errorf("%s is nil pointer", fieldName) + } + switch data.Kind() { + case reflect.Struct: + // Is it a field? + field := data.FieldByName(fieldName) + // TODO: look higher up the tree if we can't find it here. Also unexported fields + // might succeed higher up, as map keys. + if field.IsValid() && isExported(fieldName) { // valid and exported + return field + } + s.errorf("%s has no exported field %q", data.Type(), fieldName) + default: + s.errorf("can't evaluate field %s of type %s", fieldName, data.Type()) + } + panic("not reached") +} + +func (s *state) evalFieldOrCall(data reflect.Value, fieldName string, args []node, final reflect.Value) reflect.Value { + // Is it a function? + if function, ok := findFunction(fieldName, s.tmpl, s.set); ok { + return s.evalCall(data, function, fieldName, false, args, final) + } + ptr := data + for data.Kind() == reflect.Ptr && !data.IsNil() { + ptr, data = data, reflect.Indirect(data) + } + // Is it a method? We use the pointer because it has value methods too. + if method, ok := methodByName(ptr.Type(), fieldName); ok { + return s.evalCall(ptr, method.Func, fieldName, true, args, final) + } + if len(args) > 1 || final.IsValid() { + s.errorf("%s is not a method but has arguments", fieldName) + } + switch data.Kind() { + case reflect.Struct: + return s.evalField(data, fieldName) + default: + s.errorf("can't handle evaluation of field %s of type %s", fieldName, data.Type()) + } + panic("not reached") +} + +// TODO: delete when reflect's own MethodByName is released. +func methodByName(typ reflect.Type, name string) (reflect.Method, bool) { + for i := 0; i < typ.NumMethod(); i++ { + if typ.Method(i).Name == name { + return typ.Method(i), true + } + } + return reflect.Method{}, false +} + +var ( + osErrorType = reflect.TypeOf(new(os.Error)).Elem() +) + +func (s *state) evalCall(v, fun reflect.Value, name string, isMethod bool, args []node, final reflect.Value) reflect.Value { + typ := fun.Type() + if !isMethod && len(args) > 0 { // Args will be nil if it's a niladic call in an argument list + args = args[1:] // first arg is name of function; not used in call. + } + numIn := len(args) + if final.IsValid() { + numIn++ + } + numFixed := len(args) + if typ.IsVariadic() { + numFixed = typ.NumIn() - 1 // last arg is the variadic one. + if numIn < numFixed { + s.errorf("wrong number of args for %s: want at least %d got %d", name, typ.NumIn()-1, len(args)) + } + } else if numIn < typ.NumIn()-1 || !typ.IsVariadic() && numIn != typ.NumIn() { + s.errorf("wrong number of args for %s: want %d got %d", name, typ.NumIn(), len(args)) + } + if !goodFunc(typ) { + s.errorf("can't handle multiple results from method/function %q", name) + } + // Build the arg list. + argv := make([]reflect.Value, numIn) + // First arg is the receiver. + i := 0 + if isMethod { + argv[0] = v + i++ + } + // Others must be evaluated. Fixed args first. + for ; i < numFixed; i++ { + argv[i] = s.evalArg(v, typ.In(i), args[i]) + } + // And now the ... args. + if typ.IsVariadic() { + argType := typ.In(typ.NumIn() - 1).Elem() // Argument is a slice. + for ; i < len(args); i++ { + argv[i] = s.evalArg(v, argType, args[i]) + } + } + // Add final value if necessary. + if final.IsValid() { + argv[len(args)] = final + } + result := fun.Call(argv) + // If we have an os.Error that is not nil, stop execution and return that error to the caller. + if len(result) == 2 && !result[1].IsNil() { + s.error(result[1].Interface().(os.Error)) + } + return result[0] +} + +func (s *state) evalArg(data reflect.Value, typ reflect.Type, n node) reflect.Value { + if field, ok := n.(*fieldNode); ok { + value := s.evalFieldNode(data, field, []node{n}, reflect.Value{}) + if !value.Type().AssignableTo(typ) { + s.errorf("wrong type for value; expected %s; got %s", typ, value.Type()) + } + return value + } + switch typ.Kind() { + case reflect.Bool: + return s.evalBool(typ, n) + case reflect.String: + return s.evalString(typ, n) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return s.evalInteger(typ, n) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return s.evalUnsignedInteger(typ, n) + case reflect.Float32, reflect.Float64: + return s.evalFloat(typ, n) + case reflect.Complex64, reflect.Complex128: + return s.evalComplex(typ, n) + case reflect.Interface: + if typ.NumMethod() == 0 { + return s.evalEmptyInterface(data, typ, n) + } + } + s.errorf("can't handle %s for arg of type %s", n, typ) + panic("not reached") +} + +func (s *state) evalBool(typ reflect.Type, n node) reflect.Value { + if n, ok := n.(*boolNode); ok { + value := reflect.New(typ).Elem() + value.SetBool(n.true) + return value + } + s.errorf("expected bool; found %s", n) + panic("not reached") +} + +func (s *state) evalString(typ reflect.Type, n node) reflect.Value { + if n, ok := n.(*stringNode); ok { + value := reflect.New(typ).Elem() + value.SetString(n.text) + return value + } + s.errorf("expected string; found %s", n) + panic("not reached") +} + +func (s *state) evalInteger(typ reflect.Type, n node) reflect.Value { + if n, ok := n.(*numberNode); ok && n.isInt { + value := reflect.New(typ).Elem() + value.SetInt(n.int64) + return value + } + s.errorf("expected integer; found %s", n) + panic("not reached") +} + +func (s *state) evalUnsignedInteger(typ reflect.Type, n node) reflect.Value { + if n, ok := n.(*numberNode); ok && n.isUint { + value := reflect.New(typ).Elem() + value.SetUint(n.uint64) + return value + } + s.errorf("expected unsigned integer; found %s", n) + panic("not reached") +} + +func (s *state) evalFloat(typ reflect.Type, n node) reflect.Value { + if n, ok := n.(*numberNode); ok && n.isFloat { + value := reflect.New(typ).Elem() + value.SetFloat(n.float64) + return value + } + s.errorf("expected float; found %s", n) + panic("not reached") +} + +func (s *state) evalComplex(typ reflect.Type, n node) reflect.Value { + if n, ok := n.(*numberNode); ok && n.isComplex { + value := reflect.New(typ).Elem() + value.SetComplex(n.complex128) + return value + } + s.errorf("expected complex; found %s", n) + panic("not reached") +} + +func (s *state) evalEmptyInterface(data reflect.Value, typ reflect.Type, n node) reflect.Value { + switch n := n.(type) { + case *boolNode: + return reflect.ValueOf(n.true) + case *dotNode: + return data + case *fieldNode: + return s.evalFieldNode(data, n, nil, reflect.Value{}) + case *identifierNode: + return s.evalFieldOrCall(data, n.ident, nil, reflect.Value{}) + case *numberNode: + if n.isComplex { + return reflect.ValueOf(n.complex128) + } + if n.isInt { + return reflect.ValueOf(n.int64) + } + if n.isUint { + return reflect.ValueOf(n.uint64) + } + if n.isFloat { + return reflect.ValueOf(n.float64) + } + case *stringNode: + return reflect.ValueOf(n.text) + } + s.errorf("can't handle assignment of %s to empty interface argument", n) + panic("not reached") +} + +// indirect returns the item at the end of indirection, and a bool to indicate if it's nil. +func indirect(v reflect.Value) (rv reflect.Value, isNil bool) { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return v, true + } + v = v.Elem() + } + return v, false +} + +// printValue writes the textual representation of the value to the output of +// the template. +func (s *state) printValue(n node, v reflect.Value) { + if !v.IsValid() { + fmt.Fprint(s.wr, "") + return + } + switch v.Kind() { + case reflect.Ptr: + var isNil bool + if v, isNil = indirect(v); isNil { + fmt.Fprint(s.wr, "") + return + } + case reflect.Chan, reflect.Func, reflect.Interface: + s.errorf("can't print %s of type %s", n, v.Type()) + } + fmt.Fprint(s.wr, v.Interface()) +} diff --git a/src/pkg/exp/template/exec_test.go b/src/pkg/exp/template/exec_test.go new file mode 100644 index 000000000..86b958e84 --- /dev/null +++ b/src/pkg/exp/template/exec_test.go @@ -0,0 +1,342 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package template + +import ( + "bytes" + "fmt" + "os" + "sort" + "strings" + "testing" +) + +// T has lots of interesting pieces to use to test execution. +type T struct { + // Basics + I int + U16 uint16 + X string + FloatZero float64 + ComplexZero float64 + // Nested structs. + U *U + // Slices + SI []int + SIEmpty []int + SB []bool + // Maps + MSI map[string]int + MSIone map[string]int // one element, for deterministic output + MSIEmpty map[string]int + SMSI []map[string]int + // Empty interfaces; used to see if we can dig inside one. + Empty0 interface{} // nil + Empty1 interface{} + Empty2 interface{} + Empty3 interface{} + Empty4 interface{} + // Pointers + PI *int + PSI *[]int + NIL *int +} + +var tVal = &T{ + I: 17, + U16: 16, + X: "x", + U: &U{"v"}, + SI: []int{3, 4, 5}, + SB: []bool{true, false}, + MSI: map[string]int{"one": 1, "two": 2, "three": 3}, + MSIone: map[string]int{"one": 1}, + SMSI: []map[string]int{ + {"one": 1, "two": 2}, + {"eleven": 11, "twelve": 12}, + }, + Empty1: 3, + Empty2: "empty2", + Empty3: []int{7, 8}, + Empty4: &U{"v"}, + PI: newInt(23), + PSI: newIntSlice(21, 22, 23), +} + +// Helpers for creation. +func newInt(n int) *int { + p := new(int) + *p = n + return p +} + +func newIntSlice(n ...int) *[]int { + p := new([]int) + *p = make([]int, len(n)) + copy(*p, n) + return p +} + +// Simple methods with and without arguments. +func (t *T) Method0() string { + return "resultOfMethod0" +} + +func (t *T) Method1(a int) int { + return a +} + +func (t *T) Method2(a uint16, b string) string { + return fmt.Sprintf("Method2: %d %s", a, b) +} + +func (t *T) MAdd(a int, b []int) []int { + v := make([]int, len(b)) + for i, x := range b { + v[i] = x + a + } + return v +} + +// MSort is used to sort map keys for stable output. (Nice trick!) +func (t *T) MSort(m map[string]int) []string { + keys := make([]string, len(m)) + i := 0 + for k := range m { + keys[i] = k + i++ + } + sort.Strings(keys) + return keys +} + +// EPERM returns a value and an os.Error according to its argument. +func (t *T) EPERM(error bool) (bool, os.Error) { + if error { + return true, os.EPERM + } + return false, nil +} + +type U struct { + V string +} + +type execTest struct { + name string + input string + output string + data interface{} + ok bool +} + +var execTests = []execTest{ + // Trivial cases. + {"empty", "", "", nil, true}, + {"text", "some text", "some text", nil, true}, + + // Fields of structs. + {".X", "-{{.X}}-", "-x-", tVal, true}, + {".U.V", "-{{.U.V}}-", "-v-", tVal, true}, + + // Dots of all kinds to test basic evaluation. + {"dot int", "<{{.}}>", "<13>", 13, true}, + {"dot uint", "<{{.}}>", "<14>", uint(14), true}, + {"dot float", "<{{.}}>", "<15.1>", 15.1, true}, + {"dot bool", "<{{.}}>", "", true, true}, + {"dot complex", "<{{.}}>", "<(16.2-17i)>", 16.2 - 17i, true}, + {"dot string", "<{{.}}>", "", "hello", true}, + {"dot slice", "<{{.}}>", "<[-1 -2 -3]>", []int{-1, -2, -3}, true}, + {"dot map", "<{{.}}>", "", map[string]int{"one": 11, "two": 22}, true}, + {"dot struct", "<{{.}}>", "<{7 seven}>", struct { + a int + b string + }{7, "seven"}, true}, + + // Pointers. + {"*int", "{{.PI}}", "23", tVal, true}, + {"*[]int", "{{.PSI}}", "[21 22 23]", tVal, true}, + {"*[]int[1]", "{{index .PSI 1}}", "22", tVal, true}, + {"NIL", "{{.NIL}}", "", tVal, true}, + + // Emtpy interfaces holding values. + {"empty nil", "{{.Empty0}}", "", tVal, true}, + {"empty with int", "{{.Empty1}}", "3", tVal, true}, + {"empty with string", "{{.Empty2}}", "empty2", tVal, true}, + {"empty with slice", "{{.Empty3}}", "[7 8]", tVal, true}, + {"empty with struct", "{{.Empty4}}", "{v}", tVal, true}, + + // Method calls. + {".Method0", "-{{.Method0}}-", "-resultOfMethod0-", tVal, true}, + {".Method1(1234)", "-{{.Method1 1234}}-", "-1234-", tVal, true}, + {".Method1(.I)", "-{{.Method1 .I}}-", "-17-", tVal, true}, + {".Method2(3, .X)", "-{{.Method2 3 .X}}-", "-Method2: 3 x-", tVal, true}, + {".Method2(.U16, `str`)", "-{{.Method2 .U16 `str`}}-", "-Method2: 16 str-", tVal, true}, + + // Pipelines. + {"pipeline", "-{{.Method0 | .Method2 .U16}}-", "-Method2: 16 resultOfMethod0-", tVal, true}, + + // If. + {"if true", "{{if true}}TRUE{{end}}", "TRUE", tVal, true}, + {"if false", "{{if false}}TRUE{{else}}FALSE{{end}}", "FALSE", tVal, true}, + {"if 1", "{{if 1}}NON-ZERO{{else}}ZERO{{end}}", "NON-ZERO", tVal, true}, + {"if 0", "{{if 0}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"if 1.5", "{{if 1.5}}NON-ZERO{{else}}ZERO{{end}}", "NON-ZERO", tVal, true}, + {"if 0.0", "{{if .FloatZero}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"if 1.5i", "{{if 1.5i}}NON-ZERO{{else}}ZERO{{end}}", "NON-ZERO", tVal, true}, + {"if 0.0i", "{{if .ComplexZero}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"if emptystring", "{{if ``}}NON-EMPTY{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"if string", "{{if `notempty`}}NON-EMPTY{{else}}EMPTY{{end}}", "NON-EMPTY", tVal, true}, + {"if emptyslice", "{{if .SIEmpty}}NON-EMPTY{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"if slice", "{{if .SI}}NON-EMPTY{{else}}EMPTY{{end}}", "NON-EMPTY", tVal, true}, + {"if emptymap", "{{if .MSIEmpty}}NON-EMPTY{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"if map", "{{if .MSI}}NON-EMPTY{{else}}EMPTY{{end}}", "NON-EMPTY", tVal, true}, + + // Printf. + {"printf", `{{printf "hello, printf"}}`, "hello, printf", tVal, true}, + {"printf int", `{{printf "%04x" 127}}`, "007f", tVal, true}, + {"printf float", `{{printf "%g" 3.5}}`, "3.5", tVal, true}, + {"printf complex", `{{printf "%g" 1+7i}}`, "(1+7i)", tVal, true}, + {"printf string", `{{printf "%s" "hello"}}`, "hello", tVal, true}, + {"printf function", `{{printf "%#q" gopher}}`, "`gopher`", tVal, true}, + {"printf field", `{{printf "%s" .U.V}}`, "v", tVal, true}, + {"printf method", `{{printf "%s" .Method0}}`, "resultOfMethod0", tVal, true}, + {"printf lots", `{{printf "%d %s %g %s" 127 "hello" 7-3i .Method0}}`, "127 hello (7-3i) resultOfMethod0", tVal, true}, + + // HTML. + {"html", `{{html ""}}`, + "<script>alert("XSS");</script>", nil, true}, + {"html pipeline", `{{printf "" | html}}`, + "<script>alert("XSS");</script>", nil, true}, + + // JavaScript. + {"js", `{{js .}}`, `It\'d be nice.`, `It'd be nice.`, true}, + + // Booleans + {"not", "{{not true}} {{not false}}", "false true", nil, true}, + {"and", "{{and 0 0}} {{and 1 0}} {{and 0 1}} {{and 1 1}}", "false false false true", nil, true}, + {"or", "{{or 0 0}} {{or 1 0}} {{or 0 1}} {{or 1 1}}", "false true true true", nil, true}, + {"boolean if", "{{if and true 1 `hi`}}TRUE{{else}}FALSE{{end}}", "TRUE", tVal, true}, + {"boolean if not", "{{if and true 1 `hi` | not}}TRUE{{else}}FALSE{{end}}", "FALSE", nil, true}, + + // Indexing. + {"slice[0]", "{{index .SI 0}}", "3", tVal, true}, + {"slice[1]", "{{index .SI 1}}", "4", tVal, true}, + {"slice[HUGE]", "{{index .SI 10}}", "", tVal, false}, + {"slice[WRONG]", "{{index .SI `hello`}}", "", tVal, false}, + {"map[one]", "{{index .MSI `one`}}", "1", tVal, true}, + {"map[two]", "{{index .MSI `two`}}", "2", tVal, true}, + {"map[NO]", "{{index .MSI `XXX`}}", "", tVal, false}, + {"map[WRONG]", "{{index .MSI 10}}", "", tVal, false}, + {"double index", "{{index .SMSI 1 `eleven`}}", "11", tVal, true}, + + // With. + {"with true", "{{with true}}{{.}}{{end}}", "true", tVal, true}, + {"with false", "{{with false}}{{.}}{{else}}FALSE{{end}}", "FALSE", tVal, true}, + {"with 1", "{{with 1}}{{.}}{{else}}ZERO{{end}}", "1", tVal, true}, + {"with 0", "{{with 0}}{{.}}{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"with 1.5", "{{with 1.5}}{{.}}{{else}}ZERO{{end}}", "1.5", tVal, true}, + {"with 0.0", "{{with .FloatZero}}{{.}}{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"with 1.5i", "{{with 1.5i}}{{.}}{{else}}ZERO{{end}}", "(0+1.5i)", tVal, true}, + {"with 0.0i", "{{with .ComplexZero}}{{.}}{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"with emptystring", "{{with ``}}{{.}}{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"with string", "{{with `notempty`}}{{.}}{{else}}EMPTY{{end}}", "notempty", tVal, true}, + {"with emptyslice", "{{with .SIEmpty}}{{.}}{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"with slice", "{{with .SI}}{{.}}{{else}}EMPTY{{end}}", "[3 4 5]", tVal, true}, + {"with emptymap", "{{with .MSIEmpty}}{{.}}{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"with map", "{{with .MSIone}}{{.}}{{else}}EMPTY{{end}}", "map[one:1]", tVal, true}, + {"with empty interface, struct field", "{{with .Empty4}}{{.V}}{{end}}", "v", tVal, true}, + + // Range. + {"range []int", "{{range .SI}}-{{.}}-{{end}}", "-3--4--5-", tVal, true}, + {"range empty no else", "{{range .SIEmpty}}-{{.}}-{{end}}", "", tVal, true}, + {"range []int else", "{{range .SI}}-{{.}}-{{else}}EMPTY{{end}}", "-3--4--5-", tVal, true}, + {"range empty else", "{{range .SIEmpty}}-{{.}}-{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"range []bool", "{{range .SB}}-{{.}}-{{end}}", "-true--false-", tVal, true}, + {"range []int method", "{{range .SI | .MAdd .I}}-{{.}}-{{end}}", "-20--21--22-", tVal, true}, + {"range map", "{{range .MSI | .MSort}}-{{.}}-{{end}}", "-one--three--two-", tVal, true}, + {"range empty map no else", "{{range .MSIEmpty}}-{{.}}-{{end}}", "", tVal, true}, + {"range map else", "{{range .MSI | .MSort}}-{{.}}-{{else}}EMPTY{{end}}", "-one--three--two-", tVal, true}, + {"range empty map else", "{{range .MSIEmpty}}-{{.}}-{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, + {"range empty interface", "{{range .Empty3}}-{{.}}-{{else}}EMPTY{{end}}", "-7--8-", tVal, true}, + + // Error handling. + {"error method, error", "{{.EPERM true}}", "", tVal, false}, + {"error method, no error", "{{.EPERM false}}", "false", tVal, true}, +} + +func gopher() string { + return "gopher" +} + +func testExecute(execTests []execTest, set *Set, t *testing.T) { + b := new(bytes.Buffer) + funcs := FuncMap{"gopher": gopher} + for _, test := range execTests { + tmpl := New(test.name).Funcs(funcs) + err := tmpl.Parse(test.input) + if err != nil { + t.Errorf("%s: parse error: %s", test.name, err) + continue + } + b.Reset() + err = tmpl.ExecuteInSet(b, test.data, set) + switch { + case !test.ok && err == nil: + t.Errorf("%s: expected error; got none", test.name) + continue + case test.ok && err != nil: + t.Errorf("%s: unexpected execute error: %s", test.name, err) + continue + case !test.ok && err != nil: + // expected error, got one + if *debug { + fmt.Printf("%s: %s\n\t%s\n", test.name, test.input, err) + } + } + result := b.String() + if result != test.output { + t.Errorf("%s: expected\n\t%q\ngot\n\t%q", test.name, test.output, result) + } + } +} + +func TestExecute(t *testing.T) { + testExecute(execTests, nil, t) +} + +// Check that an error from a method flows back to the top. +func TestExecuteError(t *testing.T) { + b := new(bytes.Buffer) + tmpl := New("error") + err := tmpl.Parse("{{.EPERM true}}") + if err != nil { + t.Fatalf("parse error: %s", err) + } + err = tmpl.Execute(b, tVal) + if err == nil { + t.Errorf("expected error; got none") + } else if !strings.Contains(err.String(), os.EPERM.String()) { + t.Errorf("expected os.EPERM; got %s", err) + } +} + +func TestJSEscaping(t *testing.T) { + testCases := []struct { + in, exp string + }{ + {`a`, `a`}, + {`'foo`, `\'foo`}, + {`Go "jump" \`, `Go \"jump\" \\`}, + {`Yukihiro says "今日は世界"`, `Yukihiro says \"今日は世界\"`}, + {"unprintable \uFDFF", `unprintable \uFDFF`}, + } + for _, tc := range testCases { + s := JSEscapeString(tc.in) + if s != tc.exp { + t.Errorf("JS escaping [%s] got [%s] want [%s]", tc.in, s, tc.exp) + } + } +} diff --git a/src/pkg/exp/template/funcs.go b/src/pkg/exp/template/funcs.go new file mode 100644 index 000000000..66be40fd4 --- /dev/null +++ b/src/pkg/exp/template/funcs.go @@ -0,0 +1,294 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package template + +import ( + "bytes" + "fmt" + "io" + "os" + "reflect" + "strings" + "unicode" + "utf8" +) + +// FuncMap is the type of the map defining the mapping from names to functions. +// Each function must have either a single return value, or two return values of +// which the second has type os.Error. +type FuncMap map[string]interface{} + +var funcs = map[string]reflect.Value{ + "and": reflect.ValueOf(and), + "html": reflect.ValueOf(HTMLEscaper), + "index": reflect.ValueOf(index), + "js": reflect.ValueOf(JSEscaper), + "not": reflect.ValueOf(not), + "or": reflect.ValueOf(or), + "printf": reflect.ValueOf(fmt.Sprintf), +} + +// addFuncs adds to values the functions in funcs, converting them to reflect.Values. +func addFuncs(values map[string]reflect.Value, funcMap FuncMap) { + for name, fn := range funcMap { + v := reflect.ValueOf(fn) + if v.Kind() != reflect.Func { + panic("value for " + name + " not a function") + } + if !goodFunc(v.Type()) { + panic(fmt.Errorf("can't handle multiple results from method/function %q", name)) + } + values[name] = v + } +} + +// goodFunc checks that the function or method has the right result signature. +func goodFunc(typ reflect.Type) bool { + // We allow functions with 1 result or 2 results where the second is an os.Error. + switch { + case typ.NumOut() == 1: + return true + case typ.NumOut() == 2 && typ.Out(1) == osErrorType: + return true + } + return false +} + +// findFunction looks for a function in the template, set, and global map. +func findFunction(name string, tmpl *Template, set *Set) (reflect.Value, bool) { + if tmpl != nil { + if fn := tmpl.funcs[name]; fn.IsValid() { + return fn, true + } + } + if set != nil { + if fn := set.funcs[name]; fn.IsValid() { + return fn, true + } + } + if fn := funcs[name]; fn.IsValid() { + return fn, true + } + return reflect.Value{}, false +} + +// Indexing. + +// index returns the result of indexing its first argument by the following +// arguments. Thus "index x 1 2 3" is, in Go syntax, x[1][2][3]. Each +// indexed item must be a map, slice, or array. +func index(item interface{}, indices ...interface{}) (interface{}, os.Error) { + v := reflect.ValueOf(item) + for _, i := range indices { + index := reflect.ValueOf(i) + var isNil bool + if v, isNil = indirect(v); isNil { + return nil, fmt.Errorf("index of nil pointer") + } + switch v.Kind() { + case reflect.Array, reflect.Slice: + var x int64 + switch index.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + x = index.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + x = int64(index.Uint()) + default: + return nil, fmt.Errorf("cannot index slice/array with type %s", index.Type()) + } + if x < 0 || x >= int64(v.Len()) { + return nil, fmt.Errorf("index out of range: %d", x) + } + v = v.Index(int(x)) + case reflect.Map: + if !index.Type().AssignableTo(v.Type().Key()) { + return nil, fmt.Errorf("%s is not index type for %s", index.Type(), v.Type()) + } + v = v.MapIndex(index) + if !v.IsValid() { + return nil, fmt.Errorf("index %v not present in map", index.Interface()) + } + default: + return nil, fmt.Errorf("can't index item of type %s", index.Type()) + } + } + return v.Interface(), nil +} + +// Boolean logic. + +// and returns the Boolean AND of its arguments. +func and(arg0 interface{}, args ...interface{}) (truth bool) { + truth, _ = isTrue(reflect.ValueOf(arg0)) + for i := 0; truth && i < len(args); i++ { + truth, _ = isTrue(reflect.ValueOf(args[i])) + } + return +} + +// or returns the Boolean OR of its arguments. +func or(arg0 interface{}, args ...interface{}) (truth bool) { + truth, _ = isTrue(reflect.ValueOf(arg0)) + for i := 0; !truth && i < len(args); i++ { + truth, _ = isTrue(reflect.ValueOf(args[i])) + } + return +} + +// not returns the Boolean negation of its argument. +func not(arg interface{}) (truth bool) { + truth, _ = isTrue(reflect.ValueOf(arg)) + return !truth +} + +// HTML escaping. + +var ( + htmlQuot = []byte(""") // shorter than """ + htmlApos = []byte("'") // shorter than "'" + htmlAmp = []byte("&") + htmlLt = []byte("<") + htmlGt = []byte(">") +) + +// HTMLEscape writes to w the escaped HTML equivalent of the plain text data b. +func HTMLEscape(w io.Writer, b []byte) { + last := 0 + for i, c := range b { + var html []byte + switch c { + case '"': + html = htmlQuot + case '\'': + html = htmlApos + case '&': + html = htmlAmp + case '<': + html = htmlLt + case '>': + html = htmlGt + default: + continue + } + w.Write(b[last:i]) + w.Write(html) + last = i + 1 + } + w.Write(b[last:]) +} + +// HTMLEscapeString returns the escaped HTML equivalent of the plain text data s. +func HTMLEscapeString(s string) string { + // Avoid allocation if we can. + if strings.IndexAny(s, `'"&<>`) < 0 { + return s + } + var b bytes.Buffer + HTMLEscape(&b, []byte(s)) + return b.String() +} + +// HTMLEscaper returns the escaped HTML equivalent of the textual +// representation of its arguments. +func HTMLEscaper(args ...interface{}) string { + ok := false + var s string + if len(args) == 1 { + s, ok = args[0].(string) + } + if !ok { + s = fmt.Sprint(args...) + } + return HTMLEscapeString(s) +} + +// JavaScript escaping. + +var ( + jsLowUni = []byte(`\u00`) + hex = []byte("0123456789ABCDEF") + + jsBackslash = []byte(`\\`) + jsApos = []byte(`\'`) + jsQuot = []byte(`\"`) +) + + +// JSEscape writes to w the escaped JavaScript equivalent of the plain text data b. +func JSEscape(w io.Writer, b []byte) { + last := 0 + for i := 0; i < len(b); i++ { + c := b[i] + + if ' ' <= c && c < utf8.RuneSelf && c != '\\' && c != '"' && c != '\'' { + // fast path: nothing to do + continue + } + w.Write(b[last:i]) + + if c < utf8.RuneSelf { + // Quotes and slashes get quoted. + // Control characters get written as \u00XX. + switch c { + case '\\': + w.Write(jsBackslash) + case '\'': + w.Write(jsApos) + case '"': + w.Write(jsQuot) + default: + w.Write(jsLowUni) + t, b := c>>4, c&0x0f + w.Write(hex[t : t+1]) + w.Write(hex[b : b+1]) + } + } else { + // Unicode rune. + rune, size := utf8.DecodeRune(b[i:]) + if unicode.IsPrint(rune) { + w.Write(b[i : i+size]) + } else { + // TODO(dsymonds): Do this without fmt? + fmt.Fprintf(w, "\\u%04X", rune) + } + i += size - 1 + } + last = i + 1 + } + w.Write(b[last:]) +} + +// JSEscapeString returns the escaped JavaScript equivalent of the plain text data s. +func JSEscapeString(s string) string { + // Avoid allocation if we can. + if strings.IndexFunc(s, jsIsSpecial) < 0 { + return s + } + var b bytes.Buffer + JSEscape(&b, []byte(s)) + return b.String() +} + +func jsIsSpecial(rune int) bool { + switch rune { + case '\\', '\'', '"': + return true + } + return rune < ' ' || utf8.RuneSelf <= rune +} + +// JSEscaper returns the escaped JavaScript equivalent of the textual +// representation of its arguments. +func JSEscaper(args ...interface{}) string { + ok := false + var s string + if len(args) == 1 { + s, ok = args[0].(string) + } + if !ok { + s = fmt.Sprint(args...) + } + return JSEscapeString(s) +} diff --git a/src/pkg/exp/template/lex.go b/src/pkg/exp/template/lex.go index 826d3eb88..d78152979 100644 --- a/src/pkg/exp/template/lex.go +++ b/src/pkg/exp/template/lex.go @@ -18,58 +18,71 @@ type item struct { } func (i item) String() string { - switch i.typ { - case itemEOF: + switch { + case i.typ == itemEOF: return "EOF" - case itemError: + case i.typ == itemError: return i.val - } - if len(i.val) > 10 { + case i.typ > itemKeyword: + return fmt.Sprintf("<%s>", i.val) + case len(i.val) > 10: return fmt.Sprintf("%.10q...", i.val) } return fmt.Sprintf("%q", i.val) } -// itemType identifies the type of lex item. +// itemType identifies the type of lex items. type itemType int const ( - itemError itemType = iota // error occurred; value is text of error - itemDot // the cursor, spelled '.'. + itemError itemType = iota // error occurred; value is text of error + itemBool // boolean constant + itemComplex // complex constant (1+2i); imaginary is just a number itemEOF - itemElse // else keyword - itemEnd // end keyword - itemField // alphanumeric identifier, starting with '.'. + itemField // alphanumeric identifier, starting with '.', possibly chained ('.x.y') itemIdentifier // alphanumeric identifier - itemIf // if keyword - itemLeftMeta // left meta-string - itemNumber // number + itemLeftDelim // left action delimiter + itemNumber // simple number, including imaginary itemPipe // pipe symbol - itemRange // range keyword itemRawString // raw quoted string (includes quotes) - itemRightMeta // right meta-string + itemRightDelim // right action delimiter itemString // quoted string (includes quotes) itemText // plain text + // Keywords appear after all the rest. + itemKeyword // used only to delimit the keywords + itemDot // the cursor, spelled '.'. + itemDefine // define keyword + itemElse // else keyword + itemEnd // end keyword + itemIf // if keyword + itemRange // range keyword + itemTemplate // template keyword + itemWith // with keyword ) // Make the types prettyprint. var itemName = map[itemType]string{ itemError: "error", - itemDot: ".", + itemBool: "bool", + itemComplex: "complex", itemEOF: "EOF", - itemElse: "else", - itemEnd: "end", itemField: "field", itemIdentifier: "identifier", - itemIf: "if", - itemLeftMeta: "left meta", + itemLeftDelim: "left delim", itemNumber: "number", itemPipe: "pipe", - itemRange: "range", itemRawString: "raw string", - itemRightMeta: "rightMeta", + itemRightDelim: "right delim", itemString: "string", - itemText: "text", + // keywords + itemDot: ".", + itemDefine: "define", + itemElse: "else", + itemIf: "if", + itemEnd: "end", + itemRange: "range", + itemTemplate: "template", + itemWith: "with", } func (i itemType) String() string { @@ -81,11 +94,14 @@ func (i itemType) String() string { } var key = map[string]itemType{ - ".": itemDot, - "else": itemElse, - "end": itemEnd, - "if": itemIf, - "range": itemRange, + ".": itemDot, + "define": itemDefine, + "else": itemElse, + "end": itemEnd, + "if": itemIf, + "range": itemRange, + "template": itemTemplate, + "with": itemWith, } const eof = -1 @@ -97,6 +113,7 @@ type stateFn func(*lexer) stateFn type lexer struct { name string // the name of the input; used only for error reports. input string // the string being scanned. + state stateFn // the next lexing function to enter pos int // current position in the input. start int // start position of this item. width int // width of last rune read from input. @@ -166,38 +183,47 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn { return nil } -// run lexes the input by executing state functions until nil. -func (l *lexer) run() { - for state := lexText; state != nil; { - state = state(l) +// nextItem returns the next item from the input. +func (l *lexer) nextItem() item { + for { + select { + case item := <-l.items: + return item + default: + l.state = l.state(l) + } } - close(l.items) + panic("not reached") } -// lex launches a new scanner and returns the channel of items. -func lex(name, input string) (*lexer, chan item) { +// lex creates a new scanner for the input string. +func lex(name, input string) *lexer { l := &lexer{ name: name, input: input, - items: make(chan item), + state: lexText, + items: make(chan item, 2), // Two items of buffering is sufficient for all state functions } - go l.run() - return l, l.items + return l } // state functions -const leftMeta = "{{" -const rightMeta = "}}" +const ( + leftDelim = "{{" + rightDelim = "}}" + leftComment = "{{/*" + rightComment = "*/}}" +) -// lexText scans until a metacharacter +// lexText scans until an opening action delimiter, "{{". func lexText(l *lexer) stateFn { for { - if strings.HasPrefix(l.input[l.pos:], leftMeta) { + if strings.HasPrefix(l.input[l.pos:], leftDelim) { if l.pos > l.start { l.emit(itemText) } - return lexLeftMeta + return lexLeftDelim } if l.next() == eof { break @@ -211,28 +237,42 @@ func lexText(l *lexer) stateFn { return nil } -// leftMeta scans the left "metacharacter", which is known to be present. -func lexLeftMeta(l *lexer) stateFn { - l.pos += len(leftMeta) - l.emit(itemLeftMeta) +// lexLeftDelim scans the left delimiter, which is known to be present. +func lexLeftDelim(l *lexer) stateFn { + if strings.HasPrefix(l.input[l.pos:], leftComment) { + return lexComment + } + l.pos += len(leftDelim) + l.emit(itemLeftDelim) return lexInsideAction } -// rightMeta scans the right "metacharacter", which is known to be present. -func lexRightMeta(l *lexer) stateFn { - l.pos += len(rightMeta) - l.emit(itemRightMeta) +// lexComment scans a comment. The left comment marker is known to be present. +func lexComment(l *lexer) stateFn { + i := strings.Index(l.input[l.pos:], rightComment) + if i < 0 { + return l.errorf("unclosed comment") + } + l.pos += i + len(rightComment) + l.ignore() + return lexText +} + +// lexRightDelim scans the right delimiter, which is known to be present. +func lexRightDelim(l *lexer) stateFn { + l.pos += len(rightDelim) + l.emit(itemRightDelim) return lexText } -// lexInsideAction scans the elements inside "metacharacters". +// lexInsideAction scans the elements inside action delimiters. func lexInsideAction(l *lexer) stateFn { // Either number, quoted string, or identifier. // Spaces separate and are ignored. // Pipe symbols separate and are emitted. for { - if strings.HasPrefix(l.input[l.pos:], rightMeta) { - return lexRightMeta + if strings.HasPrefix(l.input[l.pos:], rightDelim) { + return lexRightDelim } switch r := l.next(); { case r == eof || r == '\n': @@ -273,15 +313,19 @@ Loop: for { switch r := l.next(); { case isAlphaNumeric(r): - // absorb + // absorb. + case r == '.' && l.input[l.start] == '.': + // field chaining; absorb into one token. default: l.backup() word := l.input[l.start:l.pos] switch { - case key[word] != itemError: + case key[word] > itemKeyword: l.emit(key[word]) case word[0] == '.': l.emit(itemField) + case word == "true", word == "false": + l.emit(itemBool) default: l.emit(itemIdentifier) } @@ -295,8 +339,23 @@ Loop: // isn't a perfect number scanner - for instance it accepts "." and "0x0.2" // and "089" - but when it's wrong the input is invalid and the parser (via // strconv) will notice. -// TODO: without expressions you can do imaginary but not complex. func lexNumber(l *lexer) stateFn { + if !l.scanNumber() { + return l.errorf("bad number syntax: %q", l.input[l.start:l.pos]) + } + if sign := l.peek(); sign == '+' || sign == '-' { + // Complex: 1+2i. No spaces, must end in 'i'. + if !l.scanNumber() || l.input[l.pos-1] != 'i' { + return l.errorf("bad number syntax: %q", l.input[l.start:l.pos]) + } + l.emit(itemComplex) + } else { + l.emit(itemNumber) + } + return lexInsideAction +} + +func (l *lexer) scanNumber() bool { // Optional leading sign. l.accept("+-") // Is it hex? @@ -317,10 +376,9 @@ func lexNumber(l *lexer) stateFn { // Next thing mustn't be alphanumeric. if isAlphaNumeric(l.peek()) { l.next() - return l.errorf("bad number syntax: %q", l.input[l.start:l.pos]) + return false } - l.emit(itemNumber) - return lexInsideAction + return true } // lexQuote scans a quoted string. diff --git a/src/pkg/exp/template/lex_test.go b/src/pkg/exp/template/lex_test.go index 184e833ef..256ec04d8 100644 --- a/src/pkg/exp/template/lex_test.go +++ b/src/pkg/exp/template/lex_test.go @@ -17,8 +17,8 @@ type lexTest struct { var ( tEOF = item{itemEOF, ""} - tLeft = item{itemLeftMeta, "{{"} - tRight = item{itemRightMeta, "}}"} + tLeft = item{itemLeftDelim, "{{"} + tRight = item{itemRightDelim, "}}"} tRange = item{itemRange, "range"} tPipe = item{itemPipe, "|"} tFor = item{itemIdentifier, "for"} @@ -31,11 +31,16 @@ var lexTests = []lexTest{ {"empty", "", []item{tEOF}}, {"spaces", " \t\n", []item{{itemText, " \t\n"}, tEOF}}, {"text", `now is the time`, []item{{itemText, "now is the time"}, tEOF}}, + {"text with comment", "hello-{{/* this is a comment */}}-world", []item{ + {itemText, "hello-"}, + {itemText, "-world"}, + tEOF, + }}, {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}}, {"for", `{{for }}`, []item{tLeft, tFor, tRight, tEOF}}, {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}}, {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}}, - {"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4}}", []item{ + {"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{ tLeft, {itemNumber, "1"}, {itemNumber, "02"}, @@ -43,25 +48,40 @@ var lexTests = []lexTest{ {itemNumber, "-7.2i"}, {itemNumber, "1e3"}, {itemNumber, "+1.2e-4"}, + {itemNumber, "4.2i"}, + {itemComplex, "1+2i"}, tRight, tEOF, }}, - {"dots", "{{.x . .2 .x.y }}", []item{ + {"bools", "{{true false}}", []item{ + tLeft, + {itemBool, "true"}, + {itemBool, "false"}, + tRight, + tEOF, + }}, + {"dot", "{{.}}", []item{ + tLeft, + {itemDot, "."}, + tRight, + tEOF, + }}, + {"dots", "{{.x . .2 .x.y}}", []item{ tLeft, {itemField, ".x"}, {itemDot, "."}, {itemNumber, ".2"}, - {itemField, ".x"}, - {itemField, ".y"}, + {itemField, ".x.y"}, tRight, tEOF, }}, - {"keywords", "{{range if else end}}", []item{ + {"keywords", "{{range if else end with}}", []item{ tLeft, {itemRange, "range"}, {itemIf, "if"}, {itemElse, "else"}, {itemEnd, "end"}, + {itemWith, "with"}, tRight, tEOF, }}, @@ -112,9 +132,13 @@ var lexTests = []lexTest{ // collect gathers the emitted items into a slice. func collect(t *lexTest) (items []item) { - _, tokens := lex(t.name, t.input) - for i := range tokens { - items = append(items, i) + l := lex(t.name, t.input) + for { + item := l.nextItem() + items = append(items, item) + if item.typ == itemEOF || item.typ == itemError { + break + } } return } diff --git a/src/pkg/exp/template/parse.go b/src/pkg/exp/template/parse.go index 57ddb0084..8b2d60207 100644 --- a/src/pkg/exp/template/parse.go +++ b/src/pkg/exp/template/parse.go @@ -8,17 +8,21 @@ import ( "bytes" "fmt" "os" + "reflect" "runtime" "strconv" + "strings" + "unicode" ) // Template is the representation of a parsed template. type Template struct { - // TODO: At the moment, these are all internal to parsing. - name string - root *listNode + name string + root *listNode + funcs map[string]reflect.Value + // Parsing only; cleared after parse. + set *Set lex *lexer - tokens chan item token item // token lookahead for parser havePeek bool } @@ -28,7 +32,7 @@ func (t *Template) next() item { if t.havePeek { t.havePeek = false } else { - t.token = <-t.tokens + t.token = t.lex.nextItem() } return t.token } @@ -43,7 +47,7 @@ func (t *Template) peek() item { if t.havePeek { return t.token } - t.token = <-t.tokens + t.token = t.lex.nextItem() t.havePeek = true return t.token } @@ -64,14 +68,18 @@ const ( nodeText nodeType = iota nodeAction nodeCommand + nodeDot nodeElse nodeEnd nodeField nodeIdentifier + nodeIf nodeList nodeNumber nodeRange nodeString + nodeTemplate + nodeWith ) // Nodes. @@ -103,25 +111,26 @@ func (l *listNode) String() string { // textNode holds plain text. type textNode struct { nodeType - text string + text []byte } func newText(text string) *textNode { - return &textNode{nodeType: nodeText, text: text} + return &textNode{nodeType: nodeText, text: []byte(text)} } func (t *textNode) String() string { return fmt.Sprintf("(text: %q)", t.text) } -// actionNode holds an action (something bounded by metacharacters). +// actionNode holds an action (something bounded by delimiters). type actionNode struct { nodeType + line int pipeline []*commandNode } -func newAction() *actionNode { - return &actionNode{nodeType: nodeAction} +func newAction(line int, pipeline []*commandNode) *actionNode { + return &actionNode{nodeType: nodeAction, line: line, pipeline: pipeline} } func (a *actionNode) append(command *commandNode) { @@ -164,45 +173,88 @@ func (i *identifierNode) String() string { return fmt.Sprintf("I=%s", i.ident) } -// fieldNode holds a field (identifier starting with '.'). The period is dropped from the ident. +// dotNode holds the special identifier '.'. It is represented by a nil pointer. +type dotNode bool + +func newDot() *dotNode { + return nil +} + +func (d *dotNode) typ() nodeType { + return nodeDot +} + +func (d *dotNode) String() string { + return "{{<.>}}" +} + +// fieldNode holds a field (identifier starting with '.'). +// The names may be chained ('.x.y'). +// The period is dropped from each ident. type fieldNode struct { nodeType - ident string + ident []string } func newField(ident string) *fieldNode { - return &fieldNode{nodeType: nodeField, ident: ident[1:]} //drop period + return &fieldNode{nodeType: nodeField, ident: strings.Split(ident[1:], ".")} // [1:] to drop leading period } func (f *fieldNode) String() string { - return fmt.Sprintf("F=.%s", f.ident) + return fmt.Sprintf("F=%s", f.ident) +} + +// boolNode holds a boolean constant. +type boolNode struct { + nodeType + true bool +} + +func newBool(true bool) *boolNode { + return &boolNode{nodeType: nodeString, true: true} +} + +func (b *boolNode) String() string { + if b.true { + return fmt.Sprintf("B=true") + } + return fmt.Sprintf("B=false") } -// numberNode holds a number, signed or unsigned, integer, floating, or imaginary. +// numberNode holds a number, signed or unsigned integer, floating, or complex. // The value is parsed and stored under all the types that can represent the value. // This simulates in a small amount of code the behavior of Go's ideal constants. -// TODO: booleans, complex numbers. type numberNode struct { nodeType - isInt bool // number has an integral value - isUint bool // number has an unsigned integral value - isFloat bool // number has a floating-point value - imaginary bool // number is imaginary - int64 // the signed integer value - uint64 // the unsigned integer value - float64 // the positive floating-point value - text string -} - -func newNumber(text string) (*numberNode, os.Error) { + isInt bool // number has an integral value + isUint bool // number has an unsigned integral value + isFloat bool // number has a floating-point value + isComplex bool // number is complex + int64 // the signed integer value + uint64 // the unsigned integer value + float64 // the floating-point value + complex128 // the complex value + text string +} + +func newNumber(text string, isComplex bool) (*numberNode, os.Error) { n := &numberNode{nodeType: nodeNumber, text: text} - // Imaginary constants can only be floating-point. + if isComplex { + // fmt.Sscan can parse the pair, so let it do the work. + if _, err := fmt.Sscan(text, &n.complex128); err != nil { + return nil, err + } + n.isComplex = true + n.simplifyComplex() + return n, nil + } + // Imaginary constants can only be complex unless they are zero. if len(text) > 0 && text[len(text)-1] == 'i' { f, err := strconv.Atof64(text[:len(text)-1]) if err == nil { - n.imaginary = true - n.isFloat = true - n.float64 = f + n.isComplex = true + n.complex128 = complex(0, f) + n.simplifyComplex() return n, nil } } @@ -250,6 +302,23 @@ func newNumber(text string) (*numberNode, os.Error) { return n, nil } +// simplifyComplex pulls out any other types that are represented by the complex number. +// These all require that the imaginary part be zero. +func (n *numberNode) simplifyComplex() { + n.isFloat = imag(n.complex128) == 0 + if n.isFloat { + n.float64 = real(n.complex128) + n.isInt = float64(int64(n.float64)) == n.float64 + if n.isInt { + n.int64 = int64(n.float64) + } + n.isUint = float64(uint64(n.float64)) == n.float64 + if n.isUint { + n.uint64 = uint64(n.float64) + } + } +} + func (n *numberNode) String() string { return fmt.Sprintf("N=%s", n.text) } @@ -283,11 +352,14 @@ func (e *endNode) String() string { return "{{end}}" } -// elseNode represents an {{else}} action. It is represented by a nil pointer. -type elseNode bool +// elseNode represents an {{else}} action. +type elseNode struct { + nodeType + line int +} -func newElse() *elseNode { - return nil +func newElse(line int) *elseNode { + return &elseNode{nodeType: nodeElse, line: line} } func (e *elseNode) typ() nodeType { @@ -297,37 +369,106 @@ func (e *elseNode) typ() nodeType { func (e *elseNode) String() string { return "{{else}}" } +// ifNode represents an {{if}} action and its commands. +// TODO: what should evaluation look like? is a pipeline enough? +type ifNode struct { + nodeType + line int + pipeline []*commandNode + list *listNode + elseList *listNode +} + +func newIf(line int, pipeline []*commandNode, list, elseList *listNode) *ifNode { + return &ifNode{nodeType: nodeIf, line: line, pipeline: pipeline, list: list, elseList: elseList} +} + +func (i *ifNode) String() string { + if i.elseList != nil { + return fmt.Sprintf("({{if %s}} %s {{else}} %s)", i.pipeline, i.list, i.elseList) + } + return fmt.Sprintf("({{if %s}} %s)", i.pipeline, i.list) +} -// rangeNode represents an {{range}} action and its commands. +// rangeNode represents a {{range}} action and its commands. type rangeNode struct { nodeType - field node + line int + pipeline []*commandNode list *listNode elseList *listNode } -func newRange(field node, list *listNode) *rangeNode { - return &rangeNode{nodeType: nodeRange, field: field, list: list} +func newRange(line int, pipeline []*commandNode, list, elseList *listNode) *rangeNode { + return &rangeNode{nodeType: nodeRange, line: line, pipeline: pipeline, list: list, elseList: elseList} } func (r *rangeNode) String() string { if r.elseList != nil { - return fmt.Sprintf("({{range %s}} %s {{else}} %s)", r.field, r.list, r.elseList) + return fmt.Sprintf("({{range %s}} %s {{else}} %s)", r.pipeline, r.list, r.elseList) } - return fmt.Sprintf("({{range %s}} %s)", r.field, r.list) + return fmt.Sprintf("({{range %s}} %s)", r.pipeline, r.list) +} + +// templateNode represents a {{template}} action. +type templateNode struct { + nodeType + line int + name node + pipeline []*commandNode } +func newTemplate(line int, name node, pipeline []*commandNode) *templateNode { + return &templateNode{nodeType: nodeTemplate, line: line, name: name, pipeline: pipeline} +} + +func (t *templateNode) String() string { + return fmt.Sprintf("{{template %s %s}}", t.name, t.pipeline) +} + +// withNode represents a {{with}} action and its commands. +type withNode struct { + nodeType + line int + pipeline []*commandNode + list *listNode + elseList *listNode +} + +func newWith(line int, pipeline []*commandNode, list, elseList *listNode) *withNode { + return &withNode{nodeType: nodeWith, line: line, pipeline: pipeline, list: list, elseList: elseList} +} + +func (w *withNode) String() string { + if w.elseList != nil { + return fmt.Sprintf("({{with %s}} %s {{else}} %s)", w.pipeline, w.list, w.elseList) + } + return fmt.Sprintf("({{with %s}} %s)", w.pipeline, w.list) +} + + // Parsing. // New allocates a new template with the given name. func New(name string) *Template { return &Template{ - name: name, + name: name, + funcs: make(map[string]reflect.Value), } } +// Funcs adds to the template's function map the elements of the +// argument map. It panics if a value in the map is not a function +// with appropriate return type. +// The return value is the template, so calls can be chained. +func (t *Template) Funcs(funcMap FuncMap) *Template { + addFuncs(t.funcs, funcMap) + return t +} + // errorf formats the error and terminates processing. func (t *Template) errorf(format string, args ...interface{}) { + t.root = nil format = fmt.Sprintf("template: %s:%d: %s", t.name, t.lex.lineNumber(), format) panic(fmt.Errorf(format, args...)) } @@ -351,25 +492,80 @@ func (t *Template) unexpected(token item, context string) { t.errorf("unexpected %s in %s", token, context) } -// Parse parses the template definition string and constructs an efficient representation of the template. -func (t *Template) Parse(s string) (err os.Error) { - t.lex, t.tokens = lex(t.name, s) - defer func() { - e := recover() - if e != nil { - if _, ok := e.(runtime.Error); ok { - panic(e) +// recover is the handler that turns panics into returns from the top +// level of Parse or Execute. +func (t *Template) recover(errp *os.Error) { + e := recover() + if e != nil { + if _, ok := e.(runtime.Error); ok { + panic(e) + } + t.stopParse() + *errp = e.(os.Error) + } + return +} + +// startParse starts the template parsing from the lexer. +func (t *Template) startParse(set *Set, lex *lexer) { + t.root = nil + t.set = set + t.lex = lex +} + +// stopParse terminates parsing. +func (t *Template) stopParse() { + t.set, t.lex = nil, nil +} + +// atEOF returns true if, possibly after spaces, we're at EOF. +func (t *Template) atEOF() bool { + for { + token := t.peek() + switch token.typ { + case itemEOF: + return true + case itemText: + for _, r := range token.val { + if !unicode.IsSpace(r) { + return false + } } - err = e.(os.Error) + t.next() // skip spaces. + continue } - return - }() - var next node + break + } + return false +} + +// Parse parses the template definition string to construct an internal representation +// of the template for execution. +func (t *Template) Parse(s string) (err os.Error) { + t.startParse(nil, lex(t.name, s)) + defer t.recover(&err) + t.parse(true) + t.stopParse() + return +} + +// ParseInSet parses the template definition string to construct an internal representation +// of the template for execution. Function bindings are checked against those in the set. +func (t *Template) ParseInSet(s string, set *Set) (err os.Error) { + t.startParse(set, lex(t.name, s)) + defer t.recover(&err) + t.parse(true) + t.stopParse() + return +} + +// parse is the helper for Parse. It triggers an error if we expect EOF but don't reach it. +func (t *Template) parse(toEOF bool) (next node) { t.root, next = t.itemList(true) - if next != nil { + if toEOF && next != nil { t.errorf("unexpected %s", next) } - return nil + return next } // itemList: @@ -398,7 +594,7 @@ func (t *Template) textOrAction() node { switch token := t.next(); token.typ { case itemText: return newText(token.val) - case itemLeftMeta: + case itemLeftDelim: return t.action() default: t.unexpected(token, "input") @@ -409,63 +605,95 @@ func (t *Template) textOrAction() node { // Action: // control // command ("|" command)* -// Left meta is past. Now get actions. +// Left delim is past. Now get actions. +// First word could be a keyword such as range. func (t *Template) action() (n node) { - action := newAction() switch token := t.next(); token.typ { - case itemRange: - return t.rangeControl() case itemElse: return t.elseControl() case itemEnd: return t.endControl() + case itemIf: + return t.ifControl() + case itemRange: + return t.rangeControl() + case itemTemplate: + return t.templateControl() + case itemWith: + return t.withControl() } t.backup() -Loop: + return newAction(t.lex.lineNumber(), t.pipeline("command")) +} + +// Pipeline: +// field or command +// pipeline "|" pipeline +func (t *Template) pipeline(context string) (pipe []*commandNode) { for { switch token := t.next(); token.typ { - case itemRightMeta: - break Loop - case itemIdentifier, itemField: - t.backup() - cmd, err := t.command() - if err != nil { - t.error(err) + case itemRightDelim: + if len(pipe) == 0 { + t.errorf("missing value for %s", context) } - action.append(cmd) + return + case itemBool, itemComplex, itemDot, itemField, itemIdentifier, itemNumber, itemRawString, itemString: + t.backup() + pipe = append(pipe, t.command()) default: - t.unexpected(token, "command") + t.unexpected(token, context) } } - return action + return } -// Range: -// {{range field}} itemList {{end}} -// {{range field}} itemList {{else}} itemList {{end}} -// Range keyword is past. -func (t *Template) rangeControl() node { - field := t.expect(itemField, "range") - t.expect(itemRightMeta, "range") - list, next := t.itemList(false) - r := newRange(newField(field.val), list) +func (t *Template) parseControl(context string) (lineNum int, pipe []*commandNode, list, elseList *listNode) { + lineNum = t.lex.lineNumber() + pipe = t.pipeline(context) + var next node + list, next = t.itemList(false) switch next.typ() { case nodeEnd: //done case nodeElse: - elseList, next := t.itemList(false) + elseList, next = t.itemList(false) if next.typ() != nodeEnd { t.errorf("expected end; found %s", next) } - r.elseList = elseList + elseList = elseList } - return r + return lineNum, pipe, list, elseList +} + +// If: +// {{if pipeline}} itemList {{end}} +// {{if pipeline}} itemList {{else}} itemList {{end}} +// If keyword is past. +func (t *Template) ifControl() node { + return newIf(t.parseControl("if")) +} + +// Range: +// {{range pipeline}} itemList {{end}} +// {{range pipeline}} itemList {{else}} itemList {{end}} +// Range keyword is past. +func (t *Template) rangeControl() node { + return newRange(t.parseControl("range")) } +// With: +// {{with pipeline}} itemList {{end}} +// {{with pipeline}} itemList {{else}} itemList {{end}} +// If keyword is past. +func (t *Template) withControl() node { + return newWith(t.parseControl("with")) +} + + // End: // {{end}} // End keyword is past. func (t *Template) endControl() node { - t.expect(itemRightMeta, "end") + t.expect(itemRightDelim, "end") return newEnd() } @@ -473,50 +701,83 @@ func (t *Template) endControl() node { // {{else}} // Else keyword is past. func (t *Template) elseControl() node { - t.expect(itemRightMeta, "else") - return newElse() + t.expect(itemRightDelim, "else") + return newElse(t.lex.lineNumber()) +} + +// Template: +// {{template stringValue pipeline}} +// Template keyword is past. The name must be something that can evaluate +// to a string. +func (t *Template) templateControl() node { + var name node + switch token := t.next(); token.typ { + case itemIdentifier: + if _, ok := findFunction(token.val, t, t.set); !ok { + t.errorf("function %q not defined", token.val) + } + name = newIdentifier(token.val) + case itemDot: + name = newDot() + case itemField: + name = newField(token.val) + case itemString, itemRawString: + s, err := strconv.Unquote(token.val) + if err != nil { + t.error(err) + } + name = newString(s) + default: + t.unexpected(token, "template invocation") + } + pipeline := t.pipeline("template") + return newTemplate(t.lex.lineNumber(), name, pipeline) } // command: -// space-separated arguments up to a pipeline character or right metacharacter. -// we consume the pipe character but leave the right meta to terminate the action. -func (t *Template) command() (*commandNode, os.Error) { +// space-separated arguments up to a pipeline character or right delimiter. +// we consume the pipe character but leave the right delim to terminate the action. +func (t *Template) command() *commandNode { cmd := newCommand() Loop: for { switch token := t.next(); token.typ { - case itemRightMeta: + case itemRightDelim: t.backup() break Loop case itemPipe: break Loop case itemError: - return nil, os.NewError(token.val) + t.errorf("%s", token.val) case itemIdentifier: + if _, ok := findFunction(token.val, t, t.set); !ok { + t.errorf("function %q not defined", token.val) + } cmd.append(newIdentifier(token.val)) + case itemDot: + cmd.append(newDot()) case itemField: cmd.append(newField(token.val)) - case itemNumber: - if len(cmd.args) == 0 { - t.errorf("command cannot be %q", token.val) - } - number, err := newNumber(token.val) + case itemBool: + cmd.append(newBool(token.val == "true")) + case itemComplex, itemNumber: + number, err := newNumber(token.val, token.typ == itemComplex) if err != nil { t.error(err) } cmd.append(number) case itemString, itemRawString: - if len(cmd.args) == 0 { - t.errorf("command cannot be %q", token.val) - } s, err := strconv.Unquote(token.val) if err != nil { - return nil, err + t.error(err) } cmd.append(newString(s)) default: t.unexpected(token, "command") } } - return cmd, nil + if len(cmd.args) == 0 { + t.errorf("empty command") + } + return cmd } diff --git a/src/pkg/exp/template/parse_test.go b/src/pkg/exp/template/parse_test.go index f89eaa6ce..71580f8b6 100644 --- a/src/pkg/exp/template/parse_test.go +++ b/src/pkg/exp/template/parse_test.go @@ -5,52 +5,65 @@ package template import ( + "flag" "fmt" "testing" ) -const dumpErrors = true +var debug = flag.Bool("debug", false, "show the errors produced by the tests") type numberTest struct { text string isInt bool isUint bool isFloat bool - imaginary bool + isComplex bool int64 uint64 float64 + complex128 } var numberTests = []numberTest{ // basics - {"0", true, true, true, false, 0, 0, 0}, - {"-0", true, true, true, false, 0, 0, 0}, // check that -0 is a uint. - {"73", true, true, true, false, 73, 73, 73}, - {"-73", true, false, true, false, -73, 0, -73}, - {"+73", true, false, true, false, 73, 0, 73}, - {"100", true, true, true, false, 100, 100, 100}, - {"1e9", true, true, true, false, 1e9, 1e9, 1e9}, - {"-1e9", true, false, true, false, -1e9, 0, -1e9}, - {"-1.2", false, false, true, false, 0, 0, -1.2}, - {"1e19", false, true, true, false, 0, 1e19, 1e19}, - {"-1e19", false, false, true, false, 0, 0, -1e19}, - {"4i", false, false, true, true, 0, 0, 4}, + {"0", true, true, true, false, 0, 0, 0, 0}, + {"-0", true, true, true, false, 0, 0, 0, 0}, // check that -0 is a uint. + {"73", true, true, true, false, 73, 73, 73, 0}, + {"-73", true, false, true, false, -73, 0, -73, 0}, + {"+73", true, false, true, false, 73, 0, 73, 0}, + {"100", true, true, true, false, 100, 100, 100, 0}, + {"1e9", true, true, true, false, 1e9, 1e9, 1e9, 0}, + {"-1e9", true, false, true, false, -1e9, 0, -1e9, 0}, + {"-1.2", false, false, true, false, 0, 0, -1.2, 0}, + {"1e19", false, true, true, false, 0, 1e19, 1e19, 0}, + {"-1e19", false, false, true, false, 0, 0, -1e19, 0}, + {"4i", false, false, false, true, 0, 0, 0, 4i}, + {"-1.2+4.2i", false, false, false, true, 0, 0, 0, -1.2 + 4.2i}, + // complex with 0 imaginary are float (and maybe integer) + {"0i", true, true, true, true, 0, 0, 0, 0}, + {"-1.2+0i", false, false, true, true, 0, 0, -1.2, -1.2}, + {"-12+0i", true, false, true, true, -12, 0, -12, -12}, + {"13+0i", true, true, true, true, 13, 13, 13, 13}, // funny bases - {"0123", true, true, true, false, 0123, 0123, 0123}, - {"-0x0", true, true, true, false, 0, 0, 0}, - {"0xdeadbeef", true, true, true, false, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef}, + {"0123", true, true, true, false, 0123, 0123, 0123, 0}, + {"-0x0", true, true, true, false, 0, 0, 0, 0}, + {"0xdeadbeef", true, true, true, false, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef, 0}, // some broken syntax {text: "+-2"}, {text: "0x123."}, {text: "1e."}, {text: "0xi."}, + {text: "1+2."}, } func TestNumberParse(t *testing.T) { for _, test := range numberTests { - n, err := newNumber(test.text) - ok := test.isInt || test.isUint || test.isFloat + // If fmt.Sscan thinks it's complex, it's complex. We can't trust the output + // because imaginary comes out as a number. + var c complex128 + _, err := fmt.Sscan(test.text, &c) + n, err := newNumber(test.text, err == nil) + ok := test.isInt || test.isUint || test.isFloat || test.isComplex if ok && err != nil { t.Errorf("unexpected error for %q", test.text) continue @@ -62,8 +75,8 @@ func TestNumberParse(t *testing.T) { if !ok { continue } - if n.imaginary != test.imaginary { - t.Errorf("imaginary incorrect for %q; should be %t", test.text, test.imaginary) + if n.isComplex != test.isComplex { + t.Errorf("complex incorrect for %q; should be %t", test.text, test.isComplex) } if test.isInt { if !n.isInt { @@ -95,17 +108,19 @@ func TestNumberParse(t *testing.T) { } else if n.isFloat { t.Errorf("did not expect float for %q", test.text) } + if test.isComplex { + if !n.isComplex { + t.Errorf("expected complex for %q", test.text) + } + if n.complex128 != test.complex128 { + t.Errorf("complex128 for %q should be %g is %g", test.text, test.complex128, n.complex128) + } + } else if n.isComplex { + t.Errorf("did not expect complex for %q", test.text) + } } } -func num(s string) *numberNode { - n, err := newNumber(s) - if err != nil { - panic(err) - } - return n -} - type parseTest struct { name string input string @@ -125,29 +140,45 @@ var parseTests = []parseTest{ `[(text: " \t\n")]`}, {"text", "some text", noError, `[(text: "some text")]`}, - {"emptyMeta", "{{}}", noError, + {"emptyAction", "{{}}", hasError, `[(action: [])]`}, - {"simple command", "{{hello}}", noError, - `[(action: [(command: [I=hello])])]`}, - {"multi-word command", "{{hello world}}", noError, - `[(action: [(command: [I=hello I=world])])]`}, - {"multi-word command with number", "{{hello 80}}", noError, - `[(action: [(command: [I=hello N=80])])]`}, - {"multi-word command with string", "{{hello `quoted text`}}", noError, - "[(action: [(command: [I=hello S=`quoted text`])])]"}, - {"pipeline", "{{hello|world}}", noError, - `[(action: [(command: [I=hello]) (command: [I=world])])]`}, - {"simple range", "{{range .x}}hello{{end}}", noError, - `[({{range F=.x}} [(text: "hello")])]`}, - {"nested range", "{{range .x}}hello{{range .y}}goodbye{{end}}{{end}}", noError, - `[({{range F=.x}} [(text: "hello")({{range F=.y}} [(text: "goodbye")])])]`}, - {"range with else", "{{range .x}}true{{else}}false{{end}}", noError, - `[({{range F=.x}} [(text: "true")] {{else}} [(text: "false")])]`}, + {"field", "{{.X}}", noError, + `[(action: [(command: [F=[X]])])]`}, + {"simple command", "{{printf}}", noError, + `[(action: [(command: [I=printf])])]`}, + {"multi-word command", "{{printf `%d` 23}}", noError, + "[(action: [(command: [I=printf S=`%d` N=23])])]"}, + {"pipeline", "{{.X|.Y}}", noError, + `[(action: [(command: [F=[X]]) (command: [F=[Y]])])]`}, + {"simple if", "{{if .X}}hello{{end}}", noError, + `[({{if [(command: [F=[X]])]}} [(text: "hello")])]`}, + {"if with else", "{{if .X}}true{{else}}false{{end}}", noError, + `[({{if [(command: [F=[X]])]}} [(text: "true")] {{else}} [(text: "false")])]`}, + {"simple range", "{{range .X}}hello{{end}}", noError, + `[({{range [(command: [F=[X]])]}} [(text: "hello")])]`}, + {"chained field range", "{{range .X.Y.Z}}hello{{end}}", noError, + `[({{range [(command: [F=[X Y Z]])]}} [(text: "hello")])]`}, + {"nested range", "{{range .X}}hello{{range .Y}}goodbye{{end}}{{end}}", noError, + `[({{range [(command: [F=[X]])]}} [(text: "hello")({{range [(command: [F=[Y]])]}} [(text: "goodbye")])])]`}, + {"range with else", "{{range .X}}true{{else}}false{{end}}", noError, + `[({{range [(command: [F=[X]])]}} [(text: "true")] {{else}} [(text: "false")])]`}, + {"range over pipeline", "{{range .X|.M}}true{{else}}false{{end}}", noError, + `[({{range [(command: [F=[X]]) (command: [F=[M]])]}} [(text: "true")] {{else}} [(text: "false")])]`}, + {"range []int", "{{range .SI}}{{.}}{{end}}", noError, + `[({{range [(command: [F=[SI]])]}} [(action: [(command: [{{<.>}}])])])]`}, + {"constants", "{{range .SI 1 -3.2i true false }}{{end}}", noError, + `[({{range [(command: [F=[SI] N=1 N=-3.2i B=true B=false])]}} [])]`}, + {"template", "{{template `x` .Y}}", noError, + "[{{template S=`x` [(command: [F=[Y]])]}}]"}, + {"with", "{{with .X}}hello{{end}}", noError, + `[({{with [(command: [F=[X]])]}} [(text: "hello")])]`}, + {"with with else", "{{with .X}}hello{{else}}goodbye{{end}}", noError, + `[({{with [(command: [F=[X]])]}} [(text: "hello")] {{else}} [(text: "goodbye")])]`}, // Errors. {"unclosed action", "hello{{range", hasError, ""}, - {"not a field", "hello{{range x}}{{end}}", hasError, ""}, {"missing end", "hello{{range .x}}", hasError, ""}, {"missing end after else", "hello{{range .x}}{{else}}", hasError, ""}, + {"undefined function", "hello{{undefined}}", hasError, ""}, } func TestParse(t *testing.T) { @@ -163,7 +194,7 @@ func TestParse(t *testing.T) { continue case err != nil && !test.ok: // expected error, got one - if dumpErrors { + if *debug { fmt.Printf("%s: %s\n\t%s\n", test.name, test.input, err) } continue diff --git a/src/pkg/exp/template/set.go b/src/pkg/exp/template/set.go new file mode 100644 index 000000000..492e270e1 --- /dev/null +++ b/src/pkg/exp/template/set.go @@ -0,0 +1,115 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package template + +import ( + "fmt" + "io" + "os" + "reflect" + "runtime" + "strconv" +) + +// Set holds a set of related templates that can refer to one another by name. +// A template may be a member of multiple sets. +type Set struct { + tmpl map[string]*Template + funcs map[string]reflect.Value +} + +// NewSet allocates a new, empty template set. +func NewSet() *Set { + return &Set{ + tmpl: make(map[string]*Template), + funcs: make(map[string]reflect.Value), + } +} + +// Funcs adds to the set's function map the elements of the +// argument map. It panics if a value in the map is not a function +// with appropriate return type. +// The return value is the set, so calls can be chained. +func (s *Set) Funcs(funcMap FuncMap) *Set { + addFuncs(s.funcs, funcMap) + return s +} + +// Add adds the argument templates to the set. It panics if the call +// attempts to reuse a name defined in the template. +// The return value is the set, so calls can be chained. +func (s *Set) Add(templates ...*Template) *Set { + for _, t := range templates { + if _, ok := s.tmpl[t.name]; ok { + panic(fmt.Errorf("template: %q already defined in set", t.name)) + } + s.tmpl[t.name] = t + } + return s +} + +// Template returns the template with the given name in the set, +// or nil if there is no such template. +func (s *Set) Template(name string) *Template { + return s.tmpl[name] +} + +// Execute looks for the named template in the set and then applies that +// template to the specified data object, writing the output to wr. Nested +// template invocations will be resolved from the set. +func (s *Set) Execute(name string, wr io.Writer, data interface{}) os.Error { + tmpl := s.tmpl[name] + if tmpl == nil { + return fmt.Errorf("template: no template %q in set", name) + } + return tmpl.ExecuteInSet(wr, data, s) +} + +// recover is the handler that turns panics into returns from the top +// level of Parse. +func (s *Set) recover(errp *os.Error) { + e := recover() + if e != nil { + if _, ok := e.(runtime.Error); ok { + panic(e) + } + s.tmpl = nil + *errp = e.(os.Error) + } + return +} + +// Parse parses the file into a set of named templates. +func (s *Set) Parse(text string) (err os.Error) { + defer s.recover(&err) + lex := lex("set", text) + const context = "define clause" + for { + t := New("set") // name will be updated once we know it. + t.startParse(s, lex) + // Expect EOF or "{{ define name }}". + if t.atEOF() { + return + } + t.expect(itemLeftDelim, context) + t.expect(itemDefine, context) + name := t.expect(itemString, context) + t.name, err = strconv.Unquote(name.val) + if err != nil { + t.error(err) + } + t.expect(itemRightDelim, context) + end := t.parse(false) + if end == nil { + t.errorf("unexpected EOF in %s", context) + } + if end.typ() != nodeEnd { + t.errorf("unexpected %s in %s", end, context) + } + t.stopParse() + s.tmpl[t.name] = t + } + return nil +} diff --git a/src/pkg/exp/template/set_test.go b/src/pkg/exp/template/set_test.go new file mode 100644 index 000000000..c0115ec0a --- /dev/null +++ b/src/pkg/exp/template/set_test.go @@ -0,0 +1,101 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package template + +import ( + "fmt" + "testing" +) + +type setParseTest struct { + name string + input string + ok bool + names []string + results []string +} + +var setParseTests = []setParseTest{ + {"empty", "", noError, + nil, + nil}, + {"one", `{{define "foo"}} FOO {{end}}`, noError, + []string{"foo"}, + []string{`[(text: " FOO ")]`}}, + {"two", `{{define "foo"}} FOO {{end}}{{define "bar"}} BAR {{end}}`, noError, + []string{"foo", "bar"}, + []string{`[(text: " FOO ")]`, `[(text: " BAR ")]`}}, + // errors + {"missing end", `{{define "foo"}} FOO `, hasError, + nil, + nil}, + {"malformed name", `{{define "foo}} FOO `, hasError, + nil, + nil}, +} + +func TestSetParse(t *testing.T) { + for _, test := range setParseTests { + set := NewSet() + err := set.Parse(test.input) + switch { + case err == nil && !test.ok: + t.Errorf("%q: expected error; got none", test.name) + continue + case err != nil && test.ok: + t.Errorf("%q: unexpected error: %v", test.name, err) + continue + case err != nil && !test.ok: + // expected error, got one + if *debug { + fmt.Printf("%s: %s\n\t%s\n", test.name, test.input, err) + } + continue + } + if len(set.tmpl) != len(test.names) { + t.Errorf("%s: wrong number of templates; wanted %d got %d", test.name, len(test.names), len(set.tmpl)) + continue + } + for i, name := range test.names { + tmpl, ok := set.tmpl[name] + if !ok { + t.Errorf("%s: can't find template %q", test.name, name) + continue + } + result := tmpl.root.String() + if result != test.results[i] { + t.Errorf("%s=(%q): got\n\t%v\nexpected\n\t%v", test.name, test.input, result, test.results[i]) + } + } + } +} + + +var setExecTests = []execTest{ + {"empty", "", "", nil, true}, + {"text", "some text", "some text", nil, true}, + {"invoke text", `{{template "text" .SI}}`, "TEXT", tVal, true}, + {"invoke dot int", `{{template "dot" .I}}`, "17", tVal, true}, + {"invoke dot []int", `{{template "dot" .SI}}`, "[3 4 5]", tVal, true}, + {"invoke dotV", `{{template "dotV" .U}}`, "v", tVal, true}, + {"invoke nested int", `{{template "nested" .I}}`, "17", tVal, true}, +} + +const setText = ` + {{define "text"}}TEXT{{end}} + {{define "dotV"}}{{.V}}{{end}} + {{define "dot"}}{{.}}{{end}} + {{define "nested"}}{{template "dot" .}}{{end}} +` + +func TestSetExecute(t *testing.T) { + // Declare a set with a couple of templates first. + set := NewSet() + err := set.Parse(setText) + if err != nil { + t.Fatalf("error parsing set: %s", err) + } + testExecute(setExecTests, set, t) +} diff --git a/src/pkg/fmt/print.go b/src/pkg/fmt/print.go index 438e0ae26..5c083e5e9 100644 --- a/src/pkg/fmt/print.go +++ b/src/pkg/fmt/print.go @@ -162,19 +162,18 @@ func (p *pp) Write(b []byte) (ret int, err os.Error) { // Fprintf formats according to a format specifier and writes to w. // It returns the number of bytes written and any write error encountered. -func Fprintf(w io.Writer, format string, a ...interface{}) (n int, error os.Error) { +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err os.Error) { p := newPrinter() p.doPrintf(format, a) - n64, error := p.buf.WriteTo(w) + n64, err := p.buf.WriteTo(w) p.free() - return int(n64), error + return int(n64), err } // Printf formats according to a format specifier and writes to standard output. // It returns the number of bytes written and any write error encountered. -func Printf(format string, a ...interface{}) (n int, errno os.Error) { - n, errno = Fprintf(os.Stdout, format, a...) - return n, errno +func Printf(format string, a ...interface{}) (n int, err os.Error) { + return Fprintf(os.Stdout, format, a...) } // Sprintf formats according to a format specifier and returns the resulting string. @@ -197,20 +196,19 @@ func Errorf(format string, a ...interface{}) os.Error { // Fprint formats using the default formats for its operands and writes to w. // Spaces are added between operands when neither is a string. // It returns the number of bytes written and any write error encountered. -func Fprint(w io.Writer, a ...interface{}) (n int, error os.Error) { +func Fprint(w io.Writer, a ...interface{}) (n int, err os.Error) { p := newPrinter() p.doPrint(a, false, false) - n64, error := p.buf.WriteTo(w) + n64, err := p.buf.WriteTo(w) p.free() - return int(n64), error + return int(n64), err } // Print formats using the default formats for its operands and writes to standard output. // Spaces are added between operands when neither is a string. // It returns the number of bytes written and any write error encountered. -func Print(a ...interface{}) (n int, errno os.Error) { - n, errno = Fprint(os.Stdout, a...) - return n, errno +func Print(a ...interface{}) (n int, err os.Error) { + return Fprint(os.Stdout, a...) } // Sprint formats using the default formats for its operands and returns the resulting string. @@ -230,20 +228,19 @@ func Sprint(a ...interface{}) string { // Fprintln formats using the default formats for its operands and writes to w. // Spaces are always added between operands and a newline is appended. // It returns the number of bytes written and any write error encountered. -func Fprintln(w io.Writer, a ...interface{}) (n int, error os.Error) { +func Fprintln(w io.Writer, a ...interface{}) (n int, err os.Error) { p := newPrinter() p.doPrint(a, true, true) - n64, error := p.buf.WriteTo(w) + n64, err := p.buf.WriteTo(w) p.free() - return int(n64), error + return int(n64), err } // Println formats using the default formats for its operands and writes to standard output. // Spaces are always added between operands and a newline is appended. // It returns the number of bytes written and any write error encountered. -func Println(a ...interface{}) (n int, errno os.Error) { - n, errno = Fprintln(os.Stdout, a...) - return n, errno +func Println(a ...interface{}) (n int, err os.Error) { + return Fprintln(os.Stdout, a...) } // Sprintln formats using the default formats for its operands and returns the resulting string. diff --git a/src/pkg/fmt/scan.go b/src/pkg/fmt/scan.go index f48fcbb44..d93a8c1da 100644 --- a/src/pkg/fmt/scan.go +++ b/src/pkg/fmt/scan.go @@ -35,6 +35,10 @@ type ScanState interface { ReadRune() (rune int, size int, err os.Error) // UnreadRune causes the next call to ReadRune to return the same rune. UnreadRune() os.Error + // SkipSpace skips space in the input. Newlines are treated as space + // unless the scan operation is Scanln, Fscanln or Sscanln, in which case + // a newline is treated as EOF. + SkipSpace() // Token skips space in the input if skipSpace is true, then returns the // run of Unicode code points c satisfying f(c). If f is nil, // !unicode.IsSpace(c) is used; that is, the token will hold non-space @@ -267,6 +271,14 @@ func notSpace(r int) bool { return !unicode.IsSpace(r) } + +// skipSpace provides Scan() methods the ability to skip space and newline characters +// in keeping with the current scanning mode set by format strings and Scan()/Scanln(). +func (s *ss) SkipSpace() { + s.skipSpace(false) +} + + // readRune is a structure to enable reading UTF-8 encoded code points // from an io.Reader. It is used if the Reader given to the scanner does // not already implement io.RuneReader. diff --git a/src/pkg/go/ast/print_test.go b/src/pkg/go/ast/print_test.go index 0820dcfce..30b396fcf 100644 --- a/src/pkg/go/ast/print_test.go +++ b/src/pkg/go/ast/print_test.go @@ -53,7 +53,7 @@ var tests = []struct { // Split s into lines, trim whitespace from all lines, and return // the concatenated non-empty lines. func trim(s string) string { - lines := strings.Split(s, "\n", -1) + lines := strings.Split(s, "\n") i := 0 for _, line := range lines { line = strings.TrimSpace(line) diff --git a/src/pkg/go/build/dir.go b/src/pkg/go/build/dir.go index 20f8f2913..e0000b534 100644 --- a/src/pkg/go/build/dir.go +++ b/src/pkg/go/build/dir.go @@ -139,7 +139,7 @@ func goodOSArch(filename string) bool { if dot := strings.Index(filename, "."); dot != -1 { filename = filename[:dot] } - l := strings.Split(filename, "_", -1) + l := strings.Split(filename, "_") n := len(l) if n == 0 { return true diff --git a/src/pkg/go/build/path.go b/src/pkg/go/build/path.go index 8ad39fb0f..ea588abbd 100644 --- a/src/pkg/go/build/path.go +++ b/src/pkg/go/build/path.go @@ -88,6 +88,9 @@ func FindTree(path string) (tree *Tree, pkg string, err os.Error) { if path, err = filepath.Abs(path); err != nil { return } + if path, err = filepath.EvalSymlinks(path); err != nil { + return + } for _, t := range Path { tpath := t.SrcDir() + string(filepath.Separator) if !strings.HasPrefix(path, tpath) { diff --git a/src/pkg/go/doc/comment.go b/src/pkg/go/doc/comment.go index f1ebfa97b..85640af79 100644 --- a/src/pkg/go/doc/comment.go +++ b/src/pkg/go/doc/comment.go @@ -58,7 +58,7 @@ func CommentText(comment *ast.CommentGroup) string { } // Split on newlines. - cl := strings.Split(c, "\n", -1) + cl := strings.Split(c, "\n") // Walk lines, stripping trailing white space and adding to list. for _, l := range cl { diff --git a/src/pkg/go/doc/doc.go b/src/pkg/go/doc/doc.go index a7a7e0a32..b26cd2bed 100644 --- a/src/pkg/go/doc/doc.go +++ b/src/pkg/go/doc/doc.go @@ -551,7 +551,7 @@ func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc p := new(PackageDoc) p.PackageName = doc.pkgName p.ImportPath = importpath - sort.SortStrings(filenames) + sort.Strings(filenames) p.Filenames = filenames p.Doc = CommentText(doc.doc) // makeTypeDocs may extend the list of doc.values and diff --git a/src/pkg/go/types/testdata/exports.go b/src/pkg/go/types/testdata/exports.go index 1de2e00ad..035a13fb7 100644 --- a/src/pkg/go/types/testdata/exports.go +++ b/src/pkg/go/types/testdata/exports.go @@ -38,7 +38,7 @@ type ( T9 struct { a int b, c float32 - d []string "tag" + d []string `go:"tag"` } T10 struct { T8 diff --git a/src/pkg/gob/decode.go b/src/pkg/gob/decode.go index 415b30825..bf7cb95f2 100644 --- a/src/pkg/gob/decode.go +++ b/src/pkg/gob/decode.go @@ -741,7 +741,7 @@ func (dec *Decoder) ignoreInterface(state *decoderState) { // decodeGobDecoder decodes something implementing the GobDecoder interface. // The data is encoded as a byte slice. -func (dec *Decoder) decodeGobDecoder(state *decoderState, v reflect.Value, index int) { +func (dec *Decoder) decodeGobDecoder(state *decoderState, v reflect.Value) { // Read the bytes for the value. b := make([]byte, state.decodeUint()) _, err := state.b.Read(b) @@ -969,7 +969,7 @@ func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) (*decOp, int) { } else { v = reflect.ValueOf(unsafe.Unreflect(rcvrType, p)) } - state.dec.decodeGobDecoder(state, v, methodIndex(rcvrType, gobDecodeMethodName)) + state.dec.decodeGobDecoder(state, v) } return &op, int(ut.indir) diff --git a/src/pkg/gob/doc.go b/src/pkg/gob/doc.go index 850759bbd..aaf429c43 100644 --- a/src/pkg/gob/doc.go +++ b/src/pkg/gob/doc.go @@ -29,29 +29,29 @@ receiver and transmitter will do all necessary indirection and dereferencing to convert between gobs and actual Go values. For instance, a gob type that is schematically, - struct { a, b int } + struct { A, B int } can be sent from or received into any of these Go types: - struct { a, b int } // the same - *struct { a, b int } // extra indirection of the struct - struct { *a, **b int } // extra indirection of the fields - struct { a, b int64 } // different concrete value type; see below + struct { A, B int } // the same + *struct { A, B int } // extra indirection of the struct + struct { *A, **B int } // extra indirection of the fields + struct { A, B int64 } // different concrete value type; see below It may also be received into any of these: - struct { a, b int } // the same - struct { b, a int } // ordering doesn't matter; matching is by name - struct { a, b, c int } // extra field (c) ignored - struct { b int } // missing field (a) ignored; data will be dropped - struct { b, c int } // missing field (a) ignored; extra field (c) ignored. + struct { A, B int } // the same + struct { B, A int } // ordering doesn't matter; matching is by name + struct { A, B, C int } // extra field (C) ignored + struct { B int } // missing field (A) ignored; data will be dropped + struct { B, C int } // missing field (A) ignored; extra field (C) ignored. Attempting to receive into these types will draw a decode error: - struct { a int; b uint } // change of signedness for b - struct { a int; b float } // change of type for b + struct { A int; B uint } // change of signedness for B + struct { A int; B float } // change of type for B struct { } // no field names in common - struct { c, d int } // no field names in common + struct { C, D int } // no field names in common Integers are transmitted two ways: arbitrary precision signed integers or arbitrary precision unsigned integers. There is no int8, int16 etc. @@ -269,12 +269,12 @@ StructValue: /* For implementers and the curious, here is an encoded example. Given - type Point struct {x, y int} + type Point struct {X, Y int} and the value p := Point{22, 33} the bytes transmitted that encode p will be: 1f ff 81 03 01 01 05 50 6f 69 6e 74 01 ff 82 00 - 01 02 01 01 78 01 04 00 01 01 79 01 04 00 00 00 + 01 02 01 01 58 01 04 00 01 01 59 01 04 00 00 00 07 ff 82 01 2c 01 42 00 They are determined as follows. @@ -310,13 +310,13 @@ reserved). 02 // There are two fields in the type (len(structType.field)) 01 // Start of first field structure; add 1 to get field number 0: field[0].name 01 // 1 byte - 78 // structType.field[0].name = "x" + 58 // structType.field[0].name = "X" 01 // Add 1 to get field number 1: field[0].id 04 // structType.field[0].typeId is 2 (signed int). 00 // End of structType.field[0]; start structType.field[1]; set field number to -1. 01 // Add 1 to get field number 0: field[1].name 01 // 1 byte - 79 // structType.field[1].name = "y" + 59 // structType.field[1].name = "Y" 01 // Add 1 to get field number 1: field[0].id 04 // struct.Type.field[1].typeId is 2 (signed int). 00 // End of structType.field[1]; end of structType.field. diff --git a/src/pkg/gob/encode.go b/src/pkg/gob/encode.go index 743e853e9..941e26052 100644 --- a/src/pkg/gob/encode.go +++ b/src/pkg/gob/encode.go @@ -468,7 +468,7 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { // encGobEncoder encodes a value that implements the GobEncoder interface. // The data is sent as a byte array. -func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, v reflect.Value, index int) { +func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, v reflect.Value) { // TODO: should we catch panics from the called method? // We know it's a GobEncoder, so just call the method directly. data, err := v.Interface().(GobEncoder).GobEncode() @@ -592,17 +592,6 @@ func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp return &op, indir } -// methodIndex returns which method of rt implements the method. -func methodIndex(rt reflect.Type, method string) int { - for i := 0; i < rt.NumMethod(); i++ { - if rt.Method(i).Name == method { - return i - } - } - errorf("internal error: can't find method %s", method) - return 0 -} - // gobEncodeOpFor returns the op for a type that is known to implement // GobEncoder. func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { @@ -624,7 +613,7 @@ func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { v = reflect.ValueOf(unsafe.Unreflect(rt, p)) } state.update(i) - state.enc.encodeGobEncoder(state.b, v, methodIndex(rt, gobEncodeMethodName)) + state.enc.encodeGobEncoder(state.b, v) } return &op, int(ut.encIndir) // encIndir: op will get called with p == address of receiver. } diff --git a/src/pkg/gob/type.go b/src/pkg/gob/type.go index f8e3843a7..552faa4d6 100644 --- a/src/pkg/gob/type.go +++ b/src/pkg/gob/type.go @@ -80,11 +80,6 @@ func validUserType(rt reflect.Type) (ut *userTypeInfo, err os.Error) { return } -const ( - gobEncodeMethodName = "GobEncode" - gobDecodeMethodName = "GobDecode" -) - var ( gobEncoderInterfaceType = reflect.TypeOf(new(GobEncoder)).Elem() gobDecoderInterfaceType = reflect.TypeOf(new(GobDecoder)).Elem() diff --git a/src/pkg/html/parse.go b/src/pkg/html/parse.go index 2ef90a873..6a2bc1ea6 100644 --- a/src/pkg/html/parse.go +++ b/src/pkg/html/parse.go @@ -400,6 +400,7 @@ func inBodyIM(p *parser) (insertionMode, bool) { p.framesetOK = false default: // TODO. + p.addElement(p.tok.Data, p.tok.Attr) } case EndTagToken: switch p.tok.Data { @@ -413,7 +414,10 @@ func inBodyIM(p *parser) (insertionMode, bool) { p.pop() } default: - // TODO. + // TODO: any other end tag + if p.tok.Data == p.top().Data { + p.pop() + } } } if endP { diff --git a/src/pkg/html/token_test.go b/src/pkg/html/token_test.go index c17b436aa..c794612ab 100644 --- a/src/pkg/html/token_test.go +++ b/src/pkg/html/token_test.go @@ -161,7 +161,7 @@ func TestTokenizer(t *testing.T) { loop: for _, tt := range tokenTests { z := NewTokenizer(bytes.NewBuffer([]byte(tt.html))) - for i, s := range strings.Split(tt.golden, "$", -1) { + for i, s := range strings.Split(tt.golden, "$") { if z.Next() == ErrorToken { t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error()) continue loop diff --git a/src/pkg/http/cgi/host.go b/src/pkg/http/cgi/host.go index 2be3ede77..059fc758e 100644 --- a/src/pkg/http/cgi/host.go +++ b/src/pkg/http/cgi/host.go @@ -46,6 +46,12 @@ type Handler struct { Path string // path to the CGI executable Root string // root URI prefix of handler or empty for "/" + // Dir specifies the CGI executable's working directory. + // If Dir is empty, the base directory of Path is used. + // If Path has no base directory, the current working + // directory is used. + Dir string + Env []string // extra environment variables to set, if any, as "key=value" InheritEnv []string // environment variables to inherit from host, as "key" Logger *log.Logger // optional log for errors or nil to use log.Print @@ -125,11 +131,11 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { env = append(env, h.Env...) } - path := os.Getenv("PATH") - if path == "" { - path = "/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin" + envPath := os.Getenv("PATH") + if envPath == "" { + envPath = "/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin" } - env = append(env, "PATH="+path) + env = append(env, "PATH="+envPath) for _, e := range h.InheritEnv { if v := os.Getenv(e); v != "" { @@ -143,7 +149,13 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } } - cwd, pathBase := filepath.Split(h.Path) + var cwd, path string + if h.Dir != "" { + path = h.Path + cwd = h.Dir + } else { + cwd, path = filepath.Split(h.Path) + } if cwd == "" { cwd = "." } @@ -154,7 +166,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } cmd := &exec.Cmd{ - Path: pathBase, + Path: path, Args: append([]string{h.Path}, h.Args...), Dir: cwd, Env: env, @@ -197,7 +209,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if len(line) == 0 { break } - parts := strings.Split(string(line), ":", 2) + parts := strings.SplitN(string(line), ":", 2) if len(parts) < 2 { h.printf("cgi: bogus header line: %s", string(line)) continue diff --git a/src/pkg/http/cgi/host_test.go b/src/pkg/http/cgi/host_test.go index bbdb715cf..b08d8bbf6 100644 --- a/src/pkg/http/cgi/host_test.go +++ b/src/pkg/http/cgi/host_test.go @@ -13,8 +13,10 @@ import ( "http" "http/httptest" "os" + "path/filepath" "strings" "testing" + "runtime" ) func newRequest(httpreq string) *http.Request { @@ -46,7 +48,7 @@ readlines: } linesRead++ trimmedLine := strings.TrimRight(line, "\r\n") - split := strings.Split(trimmedLine, "=", 2) + split := strings.SplitN(trimmedLine, "=", 2) if len(split) != 2 { t.Fatalf("Unexpected %d parts from invalid line number %v: %q; existing map=%v", len(split), linesRead, line, m) @@ -301,3 +303,77 @@ func TestInternalRedirect(t *testing.T) { } runCgiTest(t, h, "GET /test.cgi?loc=/foo HTTP/1.0\nHost: example.com\n\n", expectedMap) } + +func TestDirUnix(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + + cwd, _ := os.Getwd() + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + Dir: cwd, + } + expectedMap := map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) + + cwd, _ = os.Getwd() + cwd = filepath.Join(cwd, "testdata") + h = &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap = map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestDirWindows(t *testing.T) { + if runtime.GOOS != "windows" { + return + } + + cgifile, _ := filepath.Abs("testdata/test.cgi") + + var perl string + var err os.Error + perl, err = exec.LookPath("perl") + if err != nil { + return + } + perl, _ = filepath.Abs(perl) + + cwd, _ := os.Getwd() + h := &Handler{ + Path: perl, + Root: "/test.cgi", + Dir: cwd, + Args: []string{cgifile}, + Env: []string{"SCRIPT_FILENAME=" + cgifile}, + } + expectedMap := map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) + + // If not specify Dir on windows, working directory should be + // base directory of perl. + cwd, _ = filepath.Split(perl) + if cwd != "" && cwd[len(cwd)-1] == filepath.Separator { + cwd = cwd[:len(cwd)-1] + } + h = &Handler{ + Path: perl, + Root: "/test.cgi", + Args: []string{cgifile}, + Env: []string{"SCRIPT_FILENAME=" + cgifile}, + } + expectedMap = map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) +} diff --git a/src/pkg/http/cgi/testdata/test.cgi b/src/pkg/http/cgi/testdata/test.cgi index a1b2ff893..36c107f76 100755 --- a/src/pkg/http/cgi/testdata/test.cgi +++ b/src/pkg/http/cgi/testdata/test.cgi @@ -6,9 +6,9 @@ # Test script run as a child process under cgi_test.go use strict; -use CGI; +use Cwd; -my $q = CGI->new; +my $q = MiniCGI->new; my $params = $q->Vars; if ($params->{"loc"}) { @@ -39,3 +39,50 @@ foreach my $k (sort keys %ENV) { $clean_env =~ s/[\n\r]//g; print "env-$k=$clean_env\n"; } + +# NOTE: don't call getcwd() for windows. +# msys return /c/go/src/... not C:\go\... +my $dir; +if ($^O eq 'MSWin32' || $^O eq 'msys') { + my $cmd = $ENV{'COMSPEC'} || 'c:\\windows\\system32\\cmd.exe'; + $cmd =~ s!\\!/!g; + $dir = `$cmd /c cd`; + chomp $dir; +} else { + $dir = getcwd(); +} +print "cwd=$dir\n"; + + +# A minimal version of CGI.pm, for people without the perl-modules +# package installed. (CGI.pm used to be part of the Perl core, but +# some distros now bundle perl-base and perl-modules separately...) +package MiniCGI; + +sub new { + my $class = shift; + return bless {}, $class; +} + +sub Vars { + my $self = shift; + my $pairs; + if ($ENV{CONTENT_LENGTH}) { + $pairs = do { local $/; }; + } else { + $pairs = $ENV{QUERY_STRING}; + } + my $vars = {}; + foreach my $kv (split(/&/, $pairs)) { + my ($k, $v) = split(/=/, $kv, 2); + $vars->{_urldecode($k)} = _urldecode($v); + } + return $vars; +} + +sub _urldecode { + my $v = shift; + $v =~ tr/+/ /; + $v =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; + return $v; +} diff --git a/src/pkg/http/chunked.go b/src/pkg/http/chunked.go index 59121c5a2..6c23e691f 100644 --- a/src/pkg/http/chunked.go +++ b/src/pkg/http/chunked.go @@ -9,6 +9,7 @@ import ( "log" "os" "strconv" + "bufio" ) // NewChunkedWriter returns a new writer that translates writes into HTTP @@ -64,3 +65,13 @@ func (cw *chunkedWriter) Close() os.Error { _, err := io.WriteString(cw.Wire, "0\r\n") return err } + +// NewChunkedReader returns a new reader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The reader returns os.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func NewChunkedReader(r *bufio.Reader) io.Reader { + return &chunkedReader{r: r} +} diff --git a/src/pkg/http/cookie.go b/src/pkg/http/cookie.go index 79c239b46..fe70431bb 100644 --- a/src/pkg/http/cookie.go +++ b/src/pkg/http/cookie.go @@ -41,7 +41,7 @@ type Cookie struct { func readSetCookies(h Header) []*Cookie { cookies := []*Cookie{} for _, line := range h["Set-Cookie"] { - parts := strings.Split(strings.TrimSpace(line), ";", -1) + parts := strings.Split(strings.TrimSpace(line), ";") if len(parts) == 1 && parts[0] == "" { continue } @@ -175,7 +175,7 @@ func readCookies(h Header, filter string) []*Cookie { } for _, line := range lines { - parts := strings.Split(strings.TrimSpace(line), ";", -1) + parts := strings.Split(strings.TrimSpace(line), ";") if len(parts) == 1 && parts[0] == "" { continue } diff --git a/src/pkg/http/fs.go b/src/pkg/http/fs.go index 56512980c..0b830053a 100644 --- a/src/pkg/http/fs.go +++ b/src/pkg/http/fs.go @@ -11,6 +11,7 @@ import ( "io" "mime" "os" + "path" "path/filepath" "strconv" "strings" @@ -18,6 +19,38 @@ import ( "utf8" ) +// A Dir implements http.FileSystem using the native file +// system restricted to a specific directory tree. +type Dir string + +func (d Dir) Open(name string) (File, os.Error) { + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 { + return nil, os.NewError("http: invalid character in file path") + } + f, err := os.Open(filepath.Join(string(d), filepath.FromSlash(path.Clean("/"+name)))) + if err != nil { + return nil, err + } + return f, nil +} + +// A FileSystem implements access to a collection of named files. +// The elements in a file path are separated by slash ('/', U+002F) +// characters, regardless of host operating system convention. +type FileSystem interface { + Open(name string) (File, os.Error) +} + +// A File is returned by a FileSystem's Open method and can be +// served by the FileServer implementation. +type File interface { + Close() os.Error + Stat() (*os.FileInfo, os.Error) + Readdir(count int) ([]os.FileInfo, os.Error) + Read([]byte) (int, os.Error) + Seek(offset int64, whence int) (int64, os.Error) +} + // Heuristic: b is text if it is valid UTF-8 and doesn't // contain any unprintable ASCII or Unicode characters. func isText(b []byte) bool { @@ -44,7 +77,7 @@ func isText(b []byte) bool { return true } -func dirList(w ResponseWriter, f *os.File) { +func dirList(w ResponseWriter, f File) { fmt.Fprintf(w, "
    \n")
     	for {
     		dirs, err := f.Readdir(100)
    @@ -63,7 +96,8 @@ func dirList(w ResponseWriter, f *os.File) {
     	fmt.Fprintf(w, "
    \n") } -func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { +// name is '/'-separated, not filepath.Separator. +func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirect bool) { const indexPage = "/index.html" // redirect .../index.html to .../ @@ -72,7 +106,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { return } - f, err := os.Open(name) + f, err := fs.Open(name) if err != nil { // TODO expose actual error? NotFound(w, r) @@ -113,7 +147,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { // use contents of index.html for directory, if present if d.IsDirectory() { index := name + filepath.FromSlash(indexPage) - ff, err := os.Open(index) + ff, err := fs.Open(index) if err == nil { defer ff.Close() dd, err := ff.Stat() @@ -188,28 +222,26 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { // ServeFile replies to the request with the contents of the named file or directory. func ServeFile(w ResponseWriter, r *Request, name string) { - serveFile(w, r, name, false) + serveFile(w, r, Dir(name), "", false) } type fileHandler struct { - root string - prefix string + root FileSystem } // FileServer returns a handler that serves HTTP requests // with the contents of the file system rooted at root. -// It strips prefix from the incoming requests before -// looking up the file name in the file system. -func FileServer(root, prefix string) Handler { return &fileHandler{root, prefix} } +// +// To use the operating system's file system implementation, +// use http.Dir: +// +// http.Handle("/", http.FileServer(http.Dir("/tmp"))) +func FileServer(root FileSystem) Handler { + return &fileHandler{root} +} func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { - path := r.URL.Path - if !strings.HasPrefix(path, f.prefix) { - NotFound(w, r) - return - } - path = path[len(f.prefix):] - serveFile(w, r, filepath.Join(f.root, filepath.FromSlash(path)), true) + serveFile(w, r, f.root, path.Clean(r.URL.Path), true) } // httpRange specifies the byte range to be sent to the client. @@ -227,7 +259,7 @@ func parseRange(s string, size int64) ([]httpRange, os.Error) { return nil, os.NewError("invalid range") } var ranges []httpRange - for _, ra := range strings.Split(s[len(b):], ",", -1) { + for _, ra := range strings.Split(s[len(b):], ",") { i := strings.Index(ra, "-") if i < 0 { return nil, os.NewError("invalid range") diff --git a/src/pkg/http/fs_test.go b/src/pkg/http/fs_test.go index 554053449..dbbdf05bd 100644 --- a/src/pkg/http/fs_test.go +++ b/src/pkg/http/fs_test.go @@ -85,6 +85,72 @@ func TestServeFile(t *testing.T) { } } +type testFileSystem struct { + open func(name string) (File, os.Error) +} + +func (fs *testFileSystem) Open(name string) (File, os.Error) { + return fs.open(name) +} + +func TestFileServerCleans(t *testing.T) { + ch := make(chan string, 1) + fs := FileServer(&testFileSystem{func(name string) (File, os.Error) { + ch <- name + return nil, os.ENOENT + }}) + tests := []struct { + reqPath, openArg string + }{ + {"/foo.txt", "/foo.txt"}, + {"//foo.txt", "/foo.txt"}, + {"/../foo.txt", "/foo.txt"}, + } + req, _ := NewRequest("GET", "http://example.com", nil) + for n, test := range tests { + rec := httptest.NewRecorder() + req.URL.Path = test.reqPath + fs.ServeHTTP(rec, req) + if got := <-ch; got != test.openArg { + t.Errorf("test %d: got %q, want %q", n, got, test.openArg) + } + } +} + +func TestDirJoin(t *testing.T) { + wfi, err := os.Stat("/etc/hosts") + if err != nil { + t.Logf("skipping test; no /etc/hosts file") + return + } + test := func(d Dir, name string) { + f, err := d.Open(name) + if err != nil { + t.Fatalf("open of %s: %v", name, err) + } + defer f.Close() + gfi, err := f.Stat() + if err != nil { + t.Fatalf("stat of %s: %v", err) + } + if gfi.Ino != wfi.Ino { + t.Errorf("%s got different inode") + } + } + test(Dir("/etc/"), "/hosts") + test(Dir("/etc/"), "hosts") + test(Dir("/etc/"), "../../../../hosts") + test(Dir("/etc"), "/hosts") + test(Dir("/etc"), "hosts") + test(Dir("/etc"), "../../../../hosts") + + // Not really directories, but since we use this trick in + // ServeFile, test it: + test(Dir("/etc/hosts"), "") + test(Dir("/etc/hosts"), "/") + test(Dir("/etc/hosts"), "../") +} + func TestServeFileContentType(t *testing.T) { const ctype = "icecream/chocolate" override := false diff --git a/src/pkg/http/header.go b/src/pkg/http/header.go index 95a25a814..08b077130 100644 --- a/src/pkg/http/header.go +++ b/src/pkg/http/header.go @@ -56,7 +56,7 @@ func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) os.Error { keys = append(keys, k) } } - sort.SortStrings(keys) + sort.Strings(keys) for _, k := range keys { for _, v := range h[k] { v = strings.Replace(v, "\n", " ", -1) diff --git a/src/pkg/http/persist.go b/src/pkg/http/persist.go index 62f9ff1b5..78bf9058f 100644 --- a/src/pkg/http/persist.go +++ b/src/pkg/http/persist.go @@ -24,6 +24,9 @@ var ( // to regain control over the connection. ServerConn supports pipe-lining, // i.e. requests can be read out of sync (but in the same order) while the // respective responses are sent. +// +// ServerConn is low-level and should not be needed by most applications. +// See Server. type ServerConn struct { lk sync.Mutex // read-write protects the following fields c net.Conn @@ -211,6 +214,9 @@ func (sc *ServerConn) Write(req *Request, resp *Response) os.Error { // connection, while respecting the HTTP keepalive logic. ClientConn // supports hijacking the connection calling Hijack to // regain control of the underlying net.Conn and deal with it as desired. +// +// ClientConn is low-level and should not be needed by most applications. +// See Client. type ClientConn struct { lk sync.Mutex // read-write protects the following fields c net.Conn diff --git a/src/pkg/http/readrequest_test.go b/src/pkg/http/readrequest_test.go index 0b92b7942..79f8de70d 100644 --- a/src/pkg/http/readrequest_test.go +++ b/src/pkg/http/readrequest_test.go @@ -13,11 +13,15 @@ import ( ) type reqTest struct { - Raw string - Req Request - Body string + Raw string + Req *Request + Body string + Error string } +var noError = "" +var noBody = "" + var reqTests = []reqTest{ // Baseline test; All Request fields included for template use { @@ -33,7 +37,7 @@ var reqTests = []reqTest{ "Proxy-Connection: keep-alive\r\n\r\n" + "abcdef\n???", - Request{ + &Request{ Method: "GET", RawURL: "http://www.techcrunch.com/", URL: &URL{ @@ -67,6 +71,34 @@ var reqTests = []reqTest{ }, "abcdef\n", + + noError, + }, + + // GET request with no body (the normal case) + { + "GET / HTTP/1.1\r\n" + + "Host: foo.com\r\n\r\n", + + &Request{ + Method: "GET", + RawURL: "/", + URL: &URL{ + Raw: "/", + Path: "/", + RawPath: "/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Close: false, + ContentLength: 0, + Host: "foo.com", + Form: Values{}, + }, + + noBody, + noError, }, // Tests that we don't parse a path that looks like a @@ -75,7 +107,7 @@ var reqTests = []reqTest{ "GET //user@host/is/actually/a/path/ HTTP/1.1\r\n" + "Host: test\r\n\r\n", - Request{ + &Request{ Method: "GET", RawURL: "//user@host/is/actually/a/path/", URL: &URL{ @@ -94,12 +126,31 @@ var reqTests = []reqTest{ ProtoMinor: 1, Header: Header{}, Close: false, - ContentLength: -1, + ContentLength: 0, Host: "test", Form: Values{}, }, - "", + noBody, + noError, + }, + + // Tests a bogus abs_path on the Request-Line (RFC 2616 section 5.1.2) + { + "GET ../../../../etc/passwd HTTP/1.1\r\n" + + "Host: test\r\n\r\n", + nil, + noBody, + "parse ../../../../etc/passwd: invalid URI for request", + }, + + // Tests missing URL: + { + "GET HTTP/1.1\r\n" + + "Host: test\r\n\r\n", + nil, + noBody, + "parse : empty url", }, } @@ -110,12 +161,14 @@ func TestReadRequest(t *testing.T) { braw.WriteString(tt.Raw) req, err := ReadRequest(bufio.NewReader(&braw)) if err != nil { - t.Errorf("#%d: %s", i, err) + if err.String() != tt.Error { + t.Errorf("#%d: error %q, want error %q", i, err.String(), tt.Error) + } continue } rbody := req.Body req.Body = nil - diff(t, fmt.Sprintf("#%d Request", i), req, &tt.Req) + diff(t, fmt.Sprintf("#%d Request", i), req, tt.Req) var bout bytes.Buffer if rbody != nil { io.Copy(&bout, rbody) diff --git a/src/pkg/http/request.go b/src/pkg/http/request.go index 183a35c71..2917cc1e6 100644 --- a/src/pkg/http/request.go +++ b/src/pkg/http/request.go @@ -428,10 +428,6 @@ type chunkedReader struct { err os.Error } -func newChunkedReader(r *bufio.Reader) *chunkedReader { - return &chunkedReader{r: r} -} - func (cr *chunkedReader) beginChunk() { // chunk-size CRLF var line string @@ -511,13 +507,6 @@ func NewRequest(method, url string, body io.Reader) (*Request, os.Error) { req.ContentLength = int64(v.Len()) case *bytes.Buffer: req.ContentLength = int64(v.Len()) - default: - req.ContentLength = -1 // chunked - } - if req.ContentLength == 0 { - // To prevent chunking and disambiguate this - // from the default ContentLength zero value. - req.TransferEncoding = []string{"identity"} } } @@ -550,7 +539,7 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { } var f []string - if f = strings.Split(s, " ", 3); len(f) < 3 { + if f = strings.SplitN(s, " ", 3); len(f) < 3 { return nil, &badStringError{"malformed HTTP request", s} } req.Method, req.RawURL, req.Proto = f[0], f[1], f[2] @@ -669,11 +658,11 @@ func ParseQuery(query string) (m Values, err os.Error) { } func parseQuery(m Values, query string) (err os.Error) { - for _, kv := range strings.Split(query, "&", -1) { + for _, kv := range strings.Split(query, "&") { if len(kv) == 0 { continue } - kvPair := strings.Split(kv, "=", 2) + kvPair := strings.SplitN(kv, "=", 2) var key, value string var e os.Error @@ -710,7 +699,7 @@ func (r *Request) ParseForm() (err os.Error) { return os.NewError("missing form body") } ct := r.Header.Get("Content-Type") - switch strings.Split(ct, ";", 2)[0] { + switch strings.SplitN(ct, ";", 2)[0] { case "text/plain", "application/x-www-form-urlencoded", "": const maxFormSize = int64(10 << 20) // 10 MB is a lot of text. b, e := ioutil.ReadAll(io.LimitReader(r.Body, maxFormSize+1)) diff --git a/src/pkg/http/requestwrite_test.go b/src/pkg/http/requestwrite_test.go index 43ad5252d..0052c0cfc 100644 --- a/src/pkg/http/requestwrite_test.go +++ b/src/pkg/http/requestwrite_test.go @@ -6,6 +6,7 @@ package http import ( "bytes" + "fmt" "io" "io/ioutil" "os" @@ -15,7 +16,7 @@ import ( type reqWriteTest struct { Req Request - Body []byte + Body interface{} // optional []byte or func() io.ReadCloser to populate Req.Body Raw string RawProxy string } @@ -98,13 +99,13 @@ var reqWriteTests = []reqWriteTest{ "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + - "6\r\nabcdef\r\n0\r\n\r\n", + chunk("abcdef") + chunk(""), "GET http://www.google.com/search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + - "6\r\nabcdef\r\n0\r\n\r\n", + chunk("abcdef") + chunk(""), }, // HTTP/1.1 POST => chunked coding; body; empty trailer { @@ -129,14 +130,14 @@ var reqWriteTests = []reqWriteTest{ "User-Agent: Go http package\r\n" + "Connection: close\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + - "6\r\nabcdef\r\n0\r\n\r\n", + chunk("abcdef") + chunk(""), "POST http://www.google.com/search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + - "6\r\nabcdef\r\n0\r\n\r\n", + chunk("abcdef") + chunk(""), }, // HTTP/1.1 POST with Content-Length, no chunking @@ -224,13 +225,72 @@ var reqWriteTests = []reqWriteTest{ "User-Agent: Go http package\r\n" + "\r\n", }, + + // Request with a 0 ContentLength and a 0 byte body. + { + Request{ + Method: "POST", + RawURL: "/", + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 0)) }, + + "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go http package\r\n" + + "\r\n", + + "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go http package\r\n" + + "\r\n", + }, + + // Request with a 0 ContentLength and a 1 byte body. + { + Request{ + Method: "POST", + RawURL: "/", + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 1)) }, + + "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go http package\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("x") + chunk(""), + + "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go http package\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("x") + chunk(""), + }, } func TestRequestWrite(t *testing.T) { for i := range reqWriteTests { tt := &reqWriteTests[i] + + setBody := func() { + switch b := tt.Body.(type) { + case []byte: + tt.Req.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + case func() io.ReadCloser: + tt.Req.Body = b() + } + } if tt.Body != nil { - tt.Req.Body = ioutil.NopCloser(bytes.NewBuffer(tt.Body)) + setBody() } if tt.Req.Header == nil { tt.Req.Header = make(Header) @@ -248,7 +308,7 @@ func TestRequestWrite(t *testing.T) { } if tt.Body != nil { - tt.Req.Body = ioutil.NopCloser(bytes.NewBuffer(tt.Body)) + setBody() } var praw bytes.Buffer err = tt.Req.WriteProxy(&praw) @@ -280,41 +340,30 @@ func (rc *closeChecker) Close() os.Error { func TestRequestWriteClosesBody(t *testing.T) { rc := &closeChecker{Reader: strings.NewReader("my body")} req, _ := NewRequest("POST", "http://foo.com/", rc) - if g, e := req.ContentLength, int64(-1); g != e { - t.Errorf("got req.ContentLength %d, want %d", g, e) + if req.ContentLength != 0 { + t.Errorf("got req.ContentLength %d, want 0", req.ContentLength) } buf := new(bytes.Buffer) req.Write(buf) if !rc.closed { t.Error("body not closed after write") } - if g, e := buf.String(), "POST / HTTP/1.1\r\nHost: foo.com\r\nUser-Agent: Go http package\r\nTransfer-Encoding: chunked\r\n\r\n7\r\nmy body\r\n0\r\n\r\n"; g != e { - t.Errorf("write:\n got: %s\nwant: %s", g, e) + expected := "POST / HTTP/1.1\r\n" + + "Host: foo.com\r\n" + + "User-Agent: Go http package\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + // TODO: currently we don't buffer before chunking, so we get a + // single "m" chunk before the other chunks, as this was the 1-byte + // read from our MultiReader where we stiched the Body back together + // after sniffing whether the Body was 0 bytes or not. + chunk("m") + + chunk("y body") + + chunk("") + if buf.String() != expected { + t.Errorf("write:\n got: %s\nwant: %s", buf.String(), expected) } } -func TestZeroLengthNewRequest(t *testing.T) { - var buf bytes.Buffer - - // Writing with default identity encoding - req, _ := NewRequest("PUT", "http://foo.com/", strings.NewReader("")) - if len(req.TransferEncoding) == 0 || req.TransferEncoding[0] != "identity" { - t.Fatalf("got req.TransferEncoding of %v, want %v", req.TransferEncoding, []string{"identity"}) - } - if g, e := req.ContentLength, int64(0); g != e { - t.Errorf("got req.ContentLength %d, want %d", g, e) - } - req.Write(&buf) - if g, e := buf.String(), "PUT / HTTP/1.1\r\nHost: foo.com\r\nUser-Agent: Go http package\r\nContent-Length: 0\r\n\r\n"; g != e { - t.Errorf("identity write:\n got: %s\nwant: %s", g, e) - } - - // Overriding identity encoding and forcing chunked. - req, _ = NewRequest("PUT", "http://foo.com/", strings.NewReader("")) - req.TransferEncoding = nil - buf.Reset() - req.Write(&buf) - if g, e := buf.String(), "PUT / HTTP/1.1\r\nHost: foo.com\r\nUser-Agent: Go http package\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n"; g != e { - t.Errorf("chunked write:\n got: %s\nwant: %s", g, e) - } +func chunk(s string) string { + return fmt.Sprintf("%x\r\n%s\r\n", len(s), s) } diff --git a/src/pkg/http/response.go b/src/pkg/http/response.go index 6c0c441a9..915327a69 100644 --- a/src/pkg/http/response.go +++ b/src/pkg/http/response.go @@ -95,7 +95,7 @@ func ReadResponse(r *bufio.Reader, req *Request) (resp *Response, err os.Error) } return nil, err } - f := strings.Split(line, " ", 3) + f := strings.SplitN(line, " ", 3) if len(f) < 2 { return nil, &badStringError{"malformed HTTP response", line} } diff --git a/src/pkg/http/reverseproxy_test.go b/src/pkg/http/reverseproxy_test.go index bc0861481..b2dd24633 100644 --- a/src/pkg/http/reverseproxy_test.go +++ b/src/pkg/http/reverseproxy_test.go @@ -17,6 +17,9 @@ func TestReverseProxy(t *testing.T) { const backendResponse = "I am the backend" const backendStatus = 404 backend := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if len(r.TransferEncoding) > 0 { + t.Errorf("backend got unexpected TransferEncoding: %v", r.TransferEncoding) + } if r.Header.Get("X-Forwarded-For") == "" { t.Errorf("didn't get X-Forwarded-For header") } diff --git a/src/pkg/http/serve_test.go b/src/pkg/http/serve_test.go index 40de54747..55a9cbf70 100644 --- a/src/pkg/http/serve_test.go +++ b/src/pkg/http/serve_test.go @@ -373,11 +373,8 @@ func TestIdentityResponse(t *testing.T) { } } -// TestServeHTTP10Close verifies that HTTP/1.0 requests won't be kept alive. -func TestServeHTTP10Close(t *testing.T) { - s := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { - ServeFile(w, r, "testdata/file") - })) +func testTcpConnectionCloses(t *testing.T, req string, h Handler) { + s := httptest.NewServer(h) defer s.Close() conn, err := net.Dial("tcp", s.Listener.Addr().String()) @@ -386,7 +383,7 @@ func TestServeHTTP10Close(t *testing.T) { } defer conn.Close() - _, err = fmt.Fprint(conn, "GET / HTTP/1.0\r\n\r\n") + _, err = fmt.Fprint(conn, req) if err != nil { t.Fatal("print error:", err) } @@ -414,6 +411,27 @@ func TestServeHTTP10Close(t *testing.T) { success <- true } +// TestServeHTTP10Close verifies that HTTP/1.0 requests won't be kept alive. +func TestServeHTTP10Close(t *testing.T) { + testTcpConnectionCloses(t, "GET / HTTP/1.0\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, "testdata/file") + })) +} + +// TestHandlersCanSetConnectionClose verifies that handlers can force a connection to close, +// even for HTTP/1.1 requests. +func TestHandlersCanSetConnectionClose11(t *testing.T) { + testTcpConnectionCloses(t, "GET / HTTP/1.1\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + })) +} + +func TestHandlersCanSetConnectionClose10(t *testing.T) { + testTcpConnectionCloses(t, "GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + })) +} + func TestSetsRemoteAddr(t *testing.T) { ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "%s", r.RemoteAddr) @@ -522,7 +540,12 @@ func TestHeadResponses(t *testing.T) { func TestTLSServer(t *testing.T) { ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { - fmt.Fprintf(w, "tls=%v", r.TLS != nil) + if r.TLS != nil { + w.Header().Set("X-TLS-Set", "true") + if r.TLS.HandshakeComplete { + w.Header().Set("X-TLS-HandshakeComplete", "true") + } + } })) defer ts.Close() if !strings.HasPrefix(ts.URL, "https://") { @@ -530,20 +553,17 @@ func TestTLSServer(t *testing.T) { } res, err := Get(ts.URL) if err != nil { - t.Error(err) + t.Fatal(err) } if res == nil { t.Fatalf("got nil Response") } - if res.Body == nil { - t.Fatalf("got nil Response.Body") - } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Error(err) + defer res.Body.Close() + if res.Header.Get("X-TLS-Set") != "true" { + t.Errorf("expected X-TLS-Set response header") } - if e, g := "tls=true", string(body); e != g { - t.Errorf("expected body %q; got %q", e, g) + if res.Header.Get("X-TLS-HandshakeComplete") != "true" { + t.Errorf("expected X-TLS-HandshakeComplete header") } } @@ -796,6 +816,30 @@ func TestNoDate(t *testing.T) { } } +func TestStripPrefix(t *testing.T) { + h := HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Path", r.URL.Path) + }) + ts := httptest.NewServer(StripPrefix("/foo", h)) + defer ts.Close() + + res, err := Get(ts.URL + "/foo/bar") + if err != nil { + t.Fatal(err) + } + if g, e := res.Header.Get("X-Path"), "/bar"; g != e { + t.Errorf("test 1: got %s, want %s", g, e) + } + + res, err = Get(ts.URL + "/bar") + if err != nil { + t.Fatal(err) + } + if g, e := res.StatusCode, 404; g != e { + t.Errorf("test 2: got status %v, want %v", g, e) + } +} + type errorListener struct { errs []os.Error } diff --git a/src/pkg/http/server.go b/src/pkg/http/server.go index 7f1b8a2bc..ab960f4f0 100644 --- a/src/pkg/http/server.go +++ b/src/pkg/http/server.go @@ -152,6 +152,7 @@ func newConn(rwc net.Conn, handler Handler) (c *conn, err os.Error) { c.buf = bufio.NewReadWriter(br, bw) if tlsConn, ok := rwc.(*tls.Conn); ok { + tlsConn.Handshake() c.tlsState = new(tls.ConnectionState) *c.tlsState = tlsConn.ConnectionState() } @@ -314,6 +315,10 @@ func (w *response) WriteHeader(code int) { w.closeAfterReply = true } + if w.header.Get("Connection") == "close" { + w.closeAfterReply = true + } + // Cannot use Content-Length with non-identity Transfer-Encoding. if w.chunking { w.header.Del("Content-Length") @@ -416,7 +421,7 @@ func errorKludge(w *response) { msg += " would ignore this error page if this text weren't here.\n" // Is it text? ("Content-Type" is always in the map) - baseType := strings.Split(w.header.Get("Content-Type"), ";", 2)[0] + baseType := strings.SplitN(w.header.Get("Content-Type"), ";", 2)[0] switch baseType { case "text/html": io.WriteString(w, "