diff options
| author | Ondřej Surý <ondrej@sury.org> | 2011-04-20 15:44:41 +0200 |
|---|---|---|
| committer | Ondřej Surý <ondrej@sury.org> | 2011-04-20 15:44:41 +0200 |
| commit | 50104cc32a498f7517a51c8dc93106c51c7a54b4 (patch) | |
| tree | 47af80be259cc7c45d0eaec7d42e61fa38c8e4fb | |
| parent | c072558b90f1bbedc2022b0f30c8b1ac4712538e (diff) | |
| download | golang-50104cc32a498f7517a51c8dc93106c51c7a54b4.tar.gz | |
Imported Upstream version 2011.03.07.1upstream/2011.03.07.1
367 files changed, 13901 insertions, 4288 deletions
@@ -27,6 +27,7 @@ Ben Olive <sionide21@gmail.com> Benny Siegert <bsiegert@gmail.com> Caine Tighe <arctanofyourface@gmail.com> Charles L. Dorian <cldorian@gmail.com> +Chris Dollin <ehog.hedge@gmail.com> Chris Jones <chris@cjones.org> Chris Lennert <calennert@gmail.com> Christian Himpel <chressie@googlemail.com> @@ -87,13 +88,17 @@ Micah Stetson <micah.stetson@gmail.com> Michael Elkins <michael.elkins@gmail.com> Michael Hoisie <hoisie@gmail.com> Miek Gieben <miek@miek.nl> +Mikael Tillenius <mikti42@gmail.com> Mikio Hara <mikioh.mikioh@gmail.com> Mikkel Krautz <mikkel@krautz.dk> Moriyoshi Koizumi <mozo@mozo.jp> Môshe van der Sterre <moshevds@gmail.com> Nicholas Waples <nwaples@gmail.com> Nigel Kerr <nigel.kerr@gmail.com> +Olivier Antoine <olivier.antoine@gmail.com> +Padraig Kitterick <padraigkitterick@gmail.com> Paolo Giarrusso <p.giarrusso@gmail.com> +Pascal S. de Kloe <pascal@quies.net> Patrick Gavlin <pgavlin@gmail.com> Petar Maymounkov <petarm@gmail.com> Peter Froehlich <peter.hans.froehlich@gmail.com> @@ -102,6 +107,7 @@ Peter Williams <pwil3058@gmail.com> Pieter Droogendijk <pieter@binky.org.uk> Raif S. Naffah <go@naffah-raif.name> Risto Jaakko Saarelma <rsaarelm@gmail.com> +Robert Hencke <robert.hencke@gmail.com> Roger Peppe <rogpeppe@gmail.com> Ross Light <rlight2@gmail.com> Ryan Hitchman <hitchmanr@gmail.com> @@ -121,7 +127,7 @@ Vincent Ambo <tazjin@googlemail.com> Vinu Rajashekhar <vinutheraj@gmail.com> Wei Guangjing <vcc.163@gmail.com> William Josephson <wjosephson@gmail.com> -Yongjian Xu <i3dmaster@gmail.com> Yasuhiro Matsumoto <mattn.jp@gmail.com> +Yongjian Xu <i3dmaster@gmail.com> Yuusei Kuwana <kuwana@kumama.org> Yuval Pavel Zholkover <paulzhol@gmail.com> diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 1f19a0d3f..913116e1e 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -54,12 +54,14 @@ Ben Eitzen <eitzenb@golang.org> Ben Lynn <benlynn@gmail.com> Ben Olive <sionide21@gmail.com> Benny Siegert <bsiegert@gmail.com> +Berengar Lehr <Berengar.Lehr@gmx.de> Bill Neubauer <wcn@golang.org> <wcn@google.com> Brad Fitzpatrick <bradfitz@golang.org> <bradfitz@gmail.com> Brendan O'Dea <bod@golang.org> Caine Tighe <arctanofyourface@gmail.com> Cary Hull <chull@google.com> Charles L. Dorian <cldorian@gmail.com> +Chris Dollin <ehog.hedge@gmail.com> Chris Jones <chris@cjones.org> <chris.jones.yar@gmail.com> Chris Lennert <calennert@gmail.com> Christian Himpel <chressie@googlemail.com> <chressie@gmail.com> @@ -71,7 +73,6 @@ Corey Thomasson <cthom.lists@gmail.com> Dan Sinclair <dan.sinclair@gmail.com> Daniel Fleischman <danielfleischman@gmail.com> Daniel Nadasi <dnadasi@google.com> -Berengar Lehr <Berengar.Lehr@gmx.de> Daniel Theophanes <kardianos@gmail.com> Dave Cheney <dave@cheney.net> David Anderson <danderson@google.com> @@ -136,6 +137,7 @@ Micah Stetson <micah.stetson@gmail.com> Michael Elkins <michael.elkins@gmail.com> Michael Hoisie <hoisie@gmail.com> Miek Gieben <miek@miek.nl> <remigius.gieben@gmail.com> +Mikael Tillenius <mikti42@gmail.com> Mikio Hara <mikioh.mikioh@gmail.com> Mikkel Krautz <mikkel@krautz.dk> <krautz@gmail.com> Moriyoshi Koizumi <mozo@mozo.jp> @@ -143,7 +145,10 @@ Môshe van der Sterre <moshevds@gmail.com> Nicholas Waples <nwaples@gmail.com> Nigel Kerr <nigel.kerr@gmail.com> Nigel Tao <nigeltao@golang.org> +Olivier Antoine <olivier.antoine@gmail.com> +Padraig Kitterick <padraigkitterick@gmail.com> Paolo Giarrusso <p.giarrusso@gmail.com> +Pascal S. de Kloe <pascal@quies.net> Patrick Gavlin <pgavlin@gmail.com> Petar Maymounkov <petarm@gmail.com> Peter Froehlich <peter.hans.froehlich@gmail.com> @@ -157,13 +162,14 @@ Raif S. Naffah <go@naffah-raif.name> Risto Jaakko Saarelma <rsaarelm@gmail.com> Rob Pike <r@golang.org> Robert Griesemer <gri@golang.org> +Robert Hencke <robert.hencke@gmail.com> Roger Peppe <rogpeppe@gmail.com> Ross Light <rlight2@gmail.com> Russ Cox <rsc@golang.org> Ryan Hitchman <hitchmanr@gmail.com> Sam Thorogood <thorogood@google.com> <sam.thorogood@gmail.com> -Scott Schwartz <scotts@golang.org> Scott Lawrence <bytbox@gmail.com> +Scott Schwartz <scotts@golang.org> Sebastien Binet <seb.binet@gmail.com> Sergei Skorobogatov <skorobo@rambler.ru> Sergey 'SnakE' Gromov <snake.scaly@gmail.com> @@ -174,8 +180,8 @@ Stephen Ma <stephenm@golang.org> Stephen Weinberg <stephen@q5comm.com> Sven Almgren <sven@tras.se> Tarmigan Casebolt <tarmigan@gmail.com> -Tom Szymanski <tgs@google.com> Timo Savola <timo.savola@gmail.com> +Tom Szymanski <tgs@google.com> Tor Andersson <tor.andersson@gmail.com> Trevor Strohman <trevor.strohman@gmail.com> Vincent Ambo <tazjin@googlemail.com> @@ -183,8 +189,8 @@ Vinu Rajashekhar <vinutheraj@gmail.com> Vish Subramanian <vish@google.com> Wei Guangjing <vcc.163@gmail.com> William Josephson <wjosephson@gmail.com> -Yongjian Xu <i3dmaster@gmail.com> Yasuhiro Matsumoto <mattn.jp@gmail.com> +Yongjian Xu <i3dmaster@gmail.com> Yuusei Kuwana <kuwana@kumama.org> Yuval Pavel Zholkover <paulzhol@gmail.com> Yves Junqueira <yves.junqueira@gmail.com> diff --git a/doc/code.html b/doc/code.html index 9236cf263..cdc60b071 100644 --- a/doc/code.html +++ b/doc/code.html @@ -229,7 +229,7 @@ run <code>gotest one_test.go</code>. <p> If your change affects performance, add a <code>Benchmark</code> function (see the <a href="/cmd/gotest/">gotest command documentation</a>) -and run it using <code>gotest -benchmarks=.</code>. +and run it using <code>gotest -test.bench=.</code>. </p> <p> @@ -322,3 +322,47 @@ reported. See the <a href="/cmd/gotest/">gotest documentation</a> and the <a href="/pkg/testing/">testing package</a> for more detail. </p> + +<h2 id="arch_os_specific">Architecture- and operating system-specific code</h2> + +<p>First, a disclaimer: very few Go packages should need to know about the +hardware and operating system they run on. In the vast majority of cases the +language and standard library handle most portability issues. This section is +a guide for experienced systems programmers who have a good reason to write +platform-specific code, such as assembly-language support for fast +trigonometric functions or code that implements a common interface above +different operating systems.</p> + +<p>To compile such code, use the <code>$GOOS</code> and <code>$GOARCH</code> +<a href="/doc/install.html#environment">environment variables</a> in your +source file names and <code>Makefile</code>.</p> + +<p>For example, this <code>Makefile</code> describes a package that builds on +different operating systems by parameterizing the file name with +<code>$GOOS</code>.</p> + +<pre> +include $(GOROOT)/src/Make.inc + +TARG=mypackage +GOFILES=\ + my.go\ + my_$(GOOS).go\ + +include $(GOROOT)/src/Make.pkg +</pre> + +<p>The OS-specific code goes in <code>my_linux.go</code>, +<code>my_darwin.go</code>, and so on.</p> + +<p>If you follow these conventional parameterizations, tools such as +<a href="/cmd/goinstall/">goinstall</a> will work seamlessly with your package: +</p> + +<pre> +my_$(GOOS).go +my_$(GOARCH).go +my_$(GOOS)_$(GOARCH).go +</pre> + +<p>The same holds for <code>.s</code> (assembly) files.</p> diff --git a/doc/codelab/wiki/Makefile b/doc/codelab/wiki/Makefile index 0d948ed4b..43f05b21d 100644 --- a/doc/codelab/wiki/Makefile +++ b/doc/codelab/wiki/Makefile @@ -11,7 +11,7 @@ include ../../../src/Make.common CLEANFILES+=index.html srcextract.bin htmlify.bin index.html: srcextract.bin htmlify.bin - awk '/^!/{system(substr($$0,2)); next} {print}' "$$@" < wiki.html > index.html + PATH=.:$$PATH awk '/^!/{system(substr($$0,2)); next} {print}' < wiki.html | tr -d '\r' > index.html test: get.bin bash ./test.sh diff --git a/doc/codelab/wiki/test.sh b/doc/codelab/wiki/test.sh index 95ff145b9..ed63ff20f 100755 --- a/doc/codelab/wiki/test.sh +++ b/doc/codelab/wiki/test.sh @@ -12,7 +12,7 @@ gomake get.bin addr=$(./get.bin -addr) sed s/:8080/$addr/ < final.go > final-test.go gomake final-test.bin -./final-test.bin & +(./final-test.bin) & wiki_pid=$! sleep 1 diff --git a/doc/codelab/wiki/wiki.html b/doc/codelab/wiki/wiki.html index 7ef97b45b..3628eeb56 100644 --- a/doc/codelab/wiki/wiki.html +++ b/doc/codelab/wiki/wiki.html @@ -76,7 +76,7 @@ the title and body. </p> <pre> -!./srcextract.bin -src=part1.go -name=Page +!srcextract.bin -src=part1.go -name=Page </pre> <p> @@ -95,7 +95,7 @@ But what about persistent storage? We can address that by creating a </p> <pre> -!./srcextract.bin -src=part1.go -name=save +!srcextract.bin -src=part1.go -name=save </pre> <p> @@ -131,7 +131,7 @@ We will want to load pages, too: </p> <pre> -!./srcextract.bin -src=part1-noerror.go -name=loadPage +!srcextract.bin -src=part1-noerror.go -name=loadPage </pre> <p> @@ -155,7 +155,7 @@ function to return <code>*Page</code> and <code>os.Error</code>. </p> <pre> -!./srcextract.bin -src=part1.go -name=loadPage +!srcextract.bin -src=part1.go -name=loadPage </pre> <p> @@ -173,7 +173,7 @@ written: </p> <pre> -!./srcextract.bin -src=part1.go -name=main +!srcextract.bin -src=part1.go -name=main </pre> <p> @@ -211,7 +211,7 @@ Here's a full working example of a simple web server: </p> <pre> -!./htmlify.bin < http-sample.go +!htmlify.bin < http-sample.go </pre> <p> @@ -276,9 +276,9 @@ Let's create a handler to view a wiki page: </p> <pre> -!./srcextract.bin -src=part2.go -name=lenPath +!srcextract.bin -src=part2.go -name=lenPath -!./srcextract.bin -src=part2.go -name=viewHandler +!srcextract.bin -src=part2.go -name=viewHandler </pre> <p> @@ -309,7 +309,7 @@ any requests under the path <code>/view/</code>. </p> <pre> -!./srcextract.bin -src=part2.go -name=main +!srcextract.bin -src=part2.go -name=main </pre> <p> @@ -348,7 +348,7 @@ First, we add them to <code>main()</code>: </p> <pre> -!./srcextract.bin -src=final-noclosure.go -name=main +!srcextract.bin -src=final-noclosure.go -name=main </pre> <p> @@ -358,7 +358,7 @@ and displays an HTML form. </p> <pre> -!./srcextract.bin -src=notemplate.go -name=editHandler +!srcextract.bin -src=notemplate.go -name=editHandler </pre> <p> @@ -394,7 +394,7 @@ Open a new file named <code>edit.html</code>, and add the following lines: </p> <pre> -!./htmlify.bin < edit.html +!htmlify.bin < edit.html </pre> <p> @@ -403,7 +403,7 @@ HTML: </p> <pre> -!./srcextract.bin -src=final-noerror.go -name=editHandler +!srcextract.bin -src=final-noerror.go -name=editHandler </pre> <p> @@ -438,7 +438,7 @@ While we're working with templates, let's create a template for our </p> <pre> -!./htmlify.bin < view.html +!htmlify.bin < view.html </pre> <p> @@ -446,7 +446,7 @@ Modify <code>viewHandler</code> accordingly: </p> <pre> -!./srcextract.bin -src=final-noerror.go -name=viewHandler +!srcextract.bin -src=final-noerror.go -name=viewHandler </pre> <p> @@ -456,11 +456,11 @@ to its own function: </p> <pre> -!./srcextract.bin -src=final-template.go -name=viewHandler +!srcextract.bin -src=final-template.go -name=viewHandler -!./srcextract.bin -src=final-template.go -name=editHandler +!srcextract.bin -src=final-template.go -name=editHandler -!./srcextract.bin -src=final-template.go -name=renderTemplate +!srcextract.bin -src=final-template.go -name=renderTemplate </pre> <p> @@ -477,7 +477,7 @@ redirect the client to the edit Page so the content may be created: </p> <pre> -!./srcextract.bin -src=final-noclosure.go -name=viewHandler +!srcextract.bin -src=final-noclosure.go -name=viewHandler </pre> <p> @@ -493,7 +493,7 @@ The function <code>saveHandler</code> will handle the form submission. </p> <pre> -!./srcextract.bin -src=final-template.go -name=saveHandler +!srcextract.bin -src=final-template.go -name=saveHandler </pre> <p> @@ -525,7 +525,7 @@ First, let's handle the errors in <code>renderTemplate</code>: </p> <pre> -!./srcextract.bin -src=final-parsetemplate.go -name=renderTemplate +!srcextract.bin -src=final-parsetemplate.go -name=renderTemplate </pre> <p> @@ -539,7 +539,7 @@ Now let's fix up <code>saveHandler</code>: </p> <pre> -!./srcextract.bin -src=final-noclosure.go -name=saveHandler +!srcextract.bin -src=final-noclosure.go -name=saveHandler </pre> <p> @@ -564,7 +564,7 @@ our <code>*Template</code> values, keyed by <code>string</code> </p> <pre> -!./srcextract.bin -src=final.go -name=templates +!srcextract.bin -src=final.go -name=templates </pre> <p> @@ -577,7 +577,7 @@ be loaded the only sensible thing to do is exit the program. </p> <pre> -!./srcextract.bin -src=final.go -name=init +!srcextract.bin -src=final.go -name=init </pre> <p> @@ -593,7 +593,7 @@ the <code>Execute</code> method on the appropriate <code>Template</code> from <code>templates</code>: <pre> -!./srcextract.bin -src=final.go -name=renderTemplate +!srcextract.bin -src=final.go -name=renderTemplate </pre> <h2>Validation</h2> @@ -610,7 +610,7 @@ Then we can create a global variable to store our validation regexp: </p> <pre> -!./srcextract.bin -src=final-noclosure.go -name=titleValidator +!srcextract.bin -src=final-noclosure.go -name=titleValidator </pre> <p> @@ -628,7 +628,7 @@ URL, and tests it against our <code>TitleValidator</code> expression: </p> <pre> -!./srcextract.bin -src=final-noclosure.go -name=getTitle +!srcextract.bin -src=final-noclosure.go -name=getTitle </pre> <p> @@ -643,11 +643,11 @@ Let's put a call to <code>getTitle</code> in each of the handlers: </p> <pre> -!./srcextract.bin -src=final-noclosure.go -name=viewHandler +!srcextract.bin -src=final-noclosure.go -name=viewHandler -!./srcextract.bin -src=final-noclosure.go -name=editHandler +!srcextract.bin -src=final-noclosure.go -name=editHandler -!./srcextract.bin -src=final-noclosure.go -name=saveHandler +!srcextract.bin -src=final-noclosure.go -name=saveHandler </pre> <h2>Introducing Function Literals and Closures</h2> @@ -700,7 +700,7 @@ Now we can take the code from <code>getTitle</code> and use it here </p> <pre> -!./srcextract.bin -src=final.go -name=makeHandler +!srcextract.bin -src=final.go -name=makeHandler </pre> <p> @@ -723,7 +723,7 @@ package: </p> <pre> -!./srcextract.bin -src=final.go -name=main +!srcextract.bin -src=final.go -name=main </pre> <p> @@ -732,11 +732,11 @@ making them much simpler: </p> <pre> -!./srcextract.bin -src=final.go -name=viewHandler +!srcextract.bin -src=final.go -name=viewHandler -!./srcextract.bin -src=final.go -name=editHandler +!srcextract.bin -src=final.go -name=editHandler -!./srcextract.bin -src=final.go -name=saveHandler +!srcextract.bin -src=final.go -name=saveHandler </pre> <h2>Try it out!</h2> diff --git a/doc/devel/release.html b/doc/devel/release.html index 57da6ca60..c7691c766 100644 --- a/doc/devel/release.html +++ b/doc/devel/release.html @@ -5,15 +5,164 @@ <p>This page summarizes the changes between tagged releases of Go. For full details, see the <a href="http://code.google.com/p/go/source/list">Mercurial change log</a>.</p> -<h3 id="2011-02-01">2011-02-15</h3> +<h3 id="2011-03-07">2011-03-07</h3> <pre> -This release includes changes to the io and template packages. +This release includes changes to the reflect and path packages. +Code that uses reflect or path may need to be updated. + +The reflect package's Value.Addr method has been renamed to Value.UnsafeAddr. +Code that uses the Addr method will have to call UnsafeAddr instead. + +The path package has been split into two packages: path and path/filepath. +Package path manipulates slash-separated paths, regardless of operating system. +Package filepath implements the local operating system's native file paths. +OS-specific functioanlity in pacakge path, such as Walk, moved to filepath. + +Other changes: +* build: fixes and simplifications (thanks Dave Cheney), + move $GOBIN ahead of /bin, /usr/bin in build $PATH. +* bzip2: speed up decompression. +* cgo: fix dwarf type parsing (thanks Gustavo Niemeyer), + put temporary source files in _obj (thanks Roger Peppe), + fix bug involving 0-argument callbacks. +* compress/lzw: optimizations. +* doc: add FAQ about "implements", + add FAQ about large binaries , + add FAQ about stack vs heap allocation, + add internationalization to roadmap, + describe platform-specific conventions in code.html. +* fmt: allow recursive calls to Fscan etc (thanks Roger Peppe), + make %#p suppress leading 0x. +* gc, gopack: add some missing flags to the docs. +* gc: fix init of packages named main (thanks Gustavo Niemeyer), +* gob: make recursive map and slice types work, and other fixes. + tentative support for GobEncoder/GobDecoder interfaces. +* gobuilder: add -package flag to build external packages and -v for verbose. +* gofmt: exclude test file that is not legal Go. +* goinstall: protect against malicious filenames (thanks Roger Peppe). +* goyacc: provide -p flag to set prefix for names, documentation update. +* http: add cookie support (thanks Petar Maymounkov), + allow handlers to send non-chunked responses, + export ParseHTTPVersion, + expose Client's Transport, + use WriteProxy, + rename ClientTransport to Transport. +* http/cgi: new package. +* http/httptest: new package. +* image: add a decoding test for common file formats. +* io/ioutil: add TempDir. +* mime/multipart: Header changed from map to MIMEHeader +* path/filepath: new OS-specific path support (thanks Gustavo Niemeyer). +* reflect: add PtrTo, add Value.Addr (old Addr is now UnsafeAddr). +* runtime: use kernel-supplied compare-and-swap on linux/arm. +* spec: minor clarification of scope rule for functions. +* sync/atomic: new package to expose atomic operations. +* syscall: regenerate zerrors_freebsd_amd64.go (thanks Mikio Hara), + work around FreeBSD execve kernel bug (thanks Devon H. O'Dell). +* template: document the delimiters. +* testing: run GC before each benchmark run (thanks Roger Peppe). +* unsafe: fix the documentation. +* websocket: use httptest.Server for tests (thanks Robert Hencke). +* xml: permit nested directives (thanks Chris Dollin). +</pre> + +<h3 id="2011-02-24">2011-02-24</h3> + +<pre> +This release includes changes to the http package and a small language change. +Your code will require changes if it manipulates http Headers or omits the +condition in if statements. + +The new http.Header type replaces map[string]string in the Header and Trailer +fields of http.Request and http.Response. +A Header value can be manipulated via its Get, Set, Add, and Del methods. +See http://golang.org/pkg/http/#Header + +The condition is now mandatory in if statements. +Previously it would default to true, as in switch and for statements. +This code is now illegal: + if x := foo(); { + // code that is always executed + } +The same effect can be achieved like this: + if x := foo(); true { + // code + } +Or, in a simpler form: + { + x := foo() + // code + } + +Other changes: +* 6l: new -Hwindowsgui flag allows to build windows gui pe (thanks Alex Brainman), + pe fixes (thanks Wei Guangjing). +* 8l, 6l: allow for more os threads to be created on Windows (thanks Alex Brainman), +* build: reduce the use of subshells in recursive make, and + remove unused NaCl conditional from make.bash (thanks Dave Cheney). +* codereview: fix clpatch with empty diffs (thanks Gustavo Niemeyer). +* compress/bzip2: add package. +* compress/lzw: implement a decoder. +* crypto/openpgp: add package. +* crypto/rand: add read buffer to speed up small requests (thanks Albert Strasheim). +* crypto/rsa: left-pad OAEP results when needed. +* crypto/tls: make protocol negotiation failure fatal. +* fmt: stop giving characters to the Scan method of Scanner when we hit a newline in Scanln. +* gc: interface error message fixes, + make string const comparison unsigned (thanks Jeff R. Allen). +* go spec: minor clarification on channel types. +* go/ast, parser: condition in if statement is mandatory. +* gob: compute information about a user's type once. + protect against pure recursive types. +* godoc: accept symbolic links as path names provided to -path, + add robots.txt, log errors when reading filter files. +* html: tokenize HTML comments. +* http: add proxy support (thanks Yasuhiro Matsumoto), + implement with net/textproto (thanks Petar Maymounkov), + send full URL in proxy requests, + introduce start of Client and ClientTransport. +* image/png: support for more formats (thanks Mikael Tillenius). +* json: only use alphanumeric tags, + use base64 to encode []byte (thanks Roger Peppe). +* ld: detect stack overflow due to NOSPLIT, drop rpath, support weak symbols. +* misc/dashboard/builder: talk to hg with utf-8 encoding. +* misc/dashboard: notify golang-dev on build failure. +* net: *netFD.Read to return os.EOF on eof under windows (thanks Alex Brainman), + add IPv4 multicast to UDPConn (thanks Dave Cheney), + more accurate IPv4-in-IPv6 API test (thanks Mikio Hara), + reject invalid net:proto network names (thanks Olivier Antoine). +* netchan: allow use of arbitrary connections (thanks Roger Peppe). +* os: add ENODATA and ENOTCONN (thanks Albert Strasheim). +* reflect: add a couple of sentences explaining how Methods operate, + add a secret method to ArrayOrSliceType to ensure it’s only implemented by arrays and slices, + add pointer word to CommonType (placeholder for future work). +* runtime-gdb.py: gdb pretty printer for go strings properly handles length. +* runtime: various bug fixes, more complete stack traces, + record $GOROOT_FINAL for runtime.GOROOT. +* spec: delete incorrect mention of selector working on pointer to interface type. +* sync: add Cond (thanks Gustavo Niemeyer). +* syscall: add MCL_* flags for mlockall (thanks Albert Strasheim), + implement chmod() for win32 (thanks Yasuhiro Matsumoto). +* test/bench: update timings for new GC. +* testing: rename cmdline flags to avoid conflicts (thanks Gustavo Niemeyer). +* textproto: introduce Header type (thanks Petar Maymounkov). +* websocket: use new interface to access Header. +</pre> + +<h3 id="2011-02-15">2011-02-15</h3> + +<pre> +This release includes changes to the io, os, and template packages. You may need to update your code. The io.ReadByter and io.ReadRuner interface types have been renamed to io.ByteReader and io.RuneReader respectively. +The os package's ForkExec function has been superseded by the new StartProcess +function and an API built around the Process type: + http://golang.org/pkg/os/#Process + The order of arguments to template.Execute has been reversed to be consistent the notion of "destination first", as with io.Copy, fmt.Fprint, and others. diff --git a/doc/devel/roadmap.html b/doc/devel/roadmap.html index 9a3c4eaba..97d8a08b8 100644 --- a/doc/devel/roadmap.html +++ b/doc/devel/roadmap.html @@ -47,6 +47,8 @@ App Engine support. Improved CGO including some mechanism for calling back from C to Go. <li> Improved implementation documentation. +<li> +Comprehensive support for internationalization. </ul> <h4 id="Gc_roadmap"> diff --git a/doc/effective_go.html b/doc/effective_go.html index 8f94f467b..a32179298 100644 --- a/doc/effective_go.html +++ b/doc/effective_go.html @@ -194,9 +194,13 @@ Comments do not need extra formatting such as banners of stars. The generated output may not even be presented in a fixed-width font, so don't depend on spacing for alignment—<code>godoc</code>, like <code>gofmt</code>, takes care of that. -Finally, the comments are uninterpreted plain text, so HTML and other +The comments are uninterpreted plain text, so HTML and other annotations such as <code>_this_</code> will reproduce <i>verbatim</i> and should not be used. +Depending on the context, <code>godoc</code> might not even +reformat comments, so make sure they look good straight up: +use correct spelling, punctuation, and sentence structure, +fold long lines, and so on. </p> <p> diff --git a/doc/gccgo_contribute.html b/doc/gccgo_contribute.html index cab6967f3..8eeb3a5c5 100644 --- a/doc/gccgo_contribute.html +++ b/doc/gccgo_contribute.html @@ -45,7 +45,7 @@ a <code>gcc-interface</code> subdirectory. </p> <p> -The runtime library for <code>gccgo</code> is mostly the same as the +The run-time library for <code>gccgo</code> is mostly the same as the library in <a href="http://code.google.com/p/go">the main Go repository</a>. The library code in the Go repository is periodically copied into the <code>gofrontend</code> and the <code>gcc</code> diff --git a/doc/gccgo_install.html b/doc/gccgo_install.html index 2ab6dcdae..159fab7bb 100644 --- a/doc/gccgo_install.html +++ b/doc/gccgo_install.html @@ -116,7 +116,7 @@ gccgo -o file file.o <p> To run the resulting file, you will need to tell the program where to -find the Go runtime library. This can be done either by setting +find the compiled Go packages. This can be done either by setting <code>LD_LIBRARY_PATH</code> in your environment: <pre> diff --git a/doc/go_faq.html b/doc/go_faq.html index 3f9c1d246..4be811068 100644 --- a/doc/go_faq.html +++ b/doc/go_faq.html @@ -508,6 +508,71 @@ Regarding operator overloading, it seems more a convenience than an absolute requirement. Again, things are simpler without it. </p> +<h3 id="implements_interface"> +Why doesn't Go have "implements" declarations?</h3> + +<p> +A Go type satisfies an interface by implementing the methods of that interface, +nothing more. This property allows interfaces to be defined and used without +having to modify existing code. It enables a kind of "duck typing" that +promotes separation of concerns and improves code re-use, and makes it easier +to build on patterns that emerge as the code develops. +The semantics of interfaces is one of the main reasons for Go's nimble, +lightweight feel. +</p> + +<p> +See the <a href="#inheritance">question on type inheritance</a> for more detail. +</p> + +<h3 id="guarantee_satisfies_interface"> +How can I guarantee my type satisfies an interface?</h3> + +<p> +You can ask the compiler to check that the type <code>T</code> implements the +interface <code>I</code> by attempting an assignment: +</p> + +<pre> +type T struct{} +var _ I = T{} +</pre> + +<p> +If <code>T</code> doesn't implement <code>I</code>, the mistake will be caught +at compile time. +</p> + +<p> +If you wish the users of an interface to explicitly declare that they implement +it, you can add a method with a descriptive name to the interface's method set. +For example: +</p> + +<pre> +type Fooer interface { + Foo() + ImplementsFooer() +} +</pre> + +<p> +A type must then implement the <code>ImplementsFooer</code> method to be a +<code>Fooer</code>, clearly documenting the fact. +</p> + +<pre> +type Bar struct{} +func (b Bar) ImplementsFooer() {} +func (b Bar) Foo() {} +</pre> + +<p> +Most code doesn't make use of such constraints, since they limit the utility of +the interface idea. Sometimes, though, they're necessary to resolve ambiguities +among similar interfaces. +</p> + <h2 id="values">Values</h2> @@ -677,6 +742,28 @@ floating-point numbers. The default size of a floating-point constant is <code>float64</code>. </p> +<h3 id="stack_or_heap"> +How do I know whether a variable is allocated on the heap or the stack?</h3> + +<p> +From a correctness standpoint, you don't need to know. +Each variable in Go exists as long as there are references to it. +The storage location chosen by the implementation is irrelevant to the +semantics of the language. + +<p> +The storage location does have an effect on writing efficient programs. +When possible, the Go compilers will allocate variables that are +local to a function in that function's stack frame. However, if the +compiler cannot prove that the variable is not referenced after the +function returns, then the compiler must allocate the variable on the +garbage-collected heap to avoid dangling pointer errors. + +<p> +In the current compilers, the analysis is crude: if a variable has its address +taken, that variable is allocated on the heap. We are working to improve this +analysis so that more data is kept on the stack. + <h2 id="Concurrency">Concurrency</h2> <h3 id="What_operations_are_atomic_What_about_mutexes"> @@ -708,7 +795,7 @@ Why doesn't my multi-goroutine program use multiple CPUs?</h3> <p> Under the gc compilers you must set <code>GOMAXPROCS</code> to allow the -runtime to utilise more than one OS thread. Under <code>gccgo</code> an OS +run-time support to utilise more than one OS thread. Under <code>gccgo</code> an OS thread will be created for each goroutine, and <code>GOMAXPROCS</code> is effectively equal to the number of running goroutines. </p> @@ -716,7 +803,7 @@ effectively equal to the number of running goroutines. <p> Programs that perform concurrent computation should benefit from an increase in <code>GOMAXPROCS</code>. (See the <a -href="http://golang.org/pkg/runtime/#GOMAXPROCS">runtime package +href="http://golang.org/pkg/runtime/#GOMAXPROCS"><code>runtime</code> package's documentation</a>.) </p> @@ -737,8 +824,8 @@ penalty involved in sending data between threads. </p> <p> -The Go runtime's scheduler is not as good as it needs to be. In future, it -should recognise such cases and optimize its use of OS threads. For now, +Go's goroutine scheduler is not as good as it needs to be. In future, it +should recognize such cases and optimize its use of OS threads. For now, <code>GOMAXPROCS</code> should be set on a per-application basis. </p> @@ -920,13 +1007,13 @@ parser are already available in <a href="/pkg/go/"><code>/pkg/go</code></a>.) We also considered using LLVM for <code>6g</code> but we felt it was too large and slow to meet our performance goals. -<h3 id="How_is_the_runtime_implemented"> -How is the runtime implemented?</h3> +<h3 id="How_is_the_run_time_support_implemented"> +How is the run-time support implemented?</h3> <p> -Again due to bootstrapping issues, the runtime is mostly in C (with a +Again due to bootstrapping issues, the run-time code is mostly in C (with a tiny bit of assembler) although Go is capable of implementing most of -it now. <code>Gccgo</code>'s runtime uses <code>glibc</code>. +it now. <code>Gccgo</code>'s run-time support uses <code>glibc</code>. <code>Gc</code> uses a custom library, to keep the footprint under control; it is compiled with a version of the Plan 9 C compiler that supports @@ -934,6 +1021,21 @@ segmented stacks for goroutines. Work is underway to provide the same stack management in <code>gccgo</code>. +<h3 id="Why_is_my_trivial_program_such_a_large_binary"> +Why is my trivial program such a large binary?</h3> + +<p> +The gc tool chain (<code>5l</code>, <code>6l</code>, and <code>8l</code>) only +generate statically linked binaries. All Go binaries therefore include the Go +run-time, along with the run-time type information necessary to support dynamic +type checks, reflection, and even panic-time stack traces. + +<p> +A trivial C "hello, world" program compiled and linked statically using gcc +on Linux is around 750 kB. An equivalent Go program is around 1.8 MB, but +that includes more powerful run-time support. We believe that with some effort +the size of Go binaries can be reduced. + <h2 id="Performance">Performance</h2> <h3 id="Why_does_Go_perform_badly_on_benchmark_x"> diff --git a/doc/go_for_cpp_programmers.html b/doc/go_for_cpp_programmers.html index 608ab147b..7168f1d05 100644 --- a/doc/go_for_cpp_programmers.html +++ b/doc/go_for_cpp_programmers.html @@ -555,7 +555,7 @@ When you want the equivalent of a virtual function, use an interface. A variable which has an interface type may be converted to have a different interface type using a special construct called a type assertion. This is implemented dynamically -at runtime, like C++ <code>dynamic_cast</code>. Unlike +at run time, like C++ <code>dynamic_cast</code>. Unlike <code>dynamic_cast</code>, there does not need to be any declared relationship between the two interfaces. @@ -589,7 +589,7 @@ must unbox using a type assertion to recover values of the contained type. As the typing is dynamic rather than static, there is no equivalent of the way that a C++ template may inline the relevant operations. The operations are fully type-checked -at runtime, but all operations will involve a function call. +at run time, but all operations will involve a function call. <pre> type iterator interface { diff --git a/doc/go_spec.html b/doc/go_spec.html index a95ed704a..85dfc44bd 100644 --- a/doc/go_spec.html +++ b/doc/go_spec.html @@ -1,5 +1,5 @@ <!-- title The Go Programming Language Specification --> -<!-- subtitle Version of February 8, 2011 --> +<!-- subtitle Version of March 3, 2011 --> <!-- TODO @@ -1227,9 +1227,11 @@ make(chan int, 100) <p> The capacity, in number of elements, sets the size of the buffer in the channel. If the -capacity is greater than zero, the channel is asynchronous: provided the -buffer is not full, sends can succeed without blocking. If the capacity is zero -or absent, the communication succeeds only when both a sender and receiver are ready. +capacity is greater than zero, the channel is asynchronous: communication operations +succeed without blocking if the buffer is not full (sends) or not empty (receives), +and elements are received in the order they are sent. +If the capacity is zero or absent, the communication succeeds only when both a sender and +receiver are ready. </p> <p> @@ -1429,8 +1431,8 @@ Go is lexically scoped using blocks: <li>The scope of a predeclared identifier is the universe block.</li> <li>The scope of an identifier denoting a constant, type, variable, - or function declared at top level (outside any function) is the - package block.</li> + or function (but not method) declared at top level (outside any + function) is the package block.</li> <li>The scope of an imported package identifier is the file block of the file containing the import declaration.</li> @@ -2265,7 +2267,7 @@ If there is not exactly one <code>f</code> with shallowest depth, the selector expression is illegal. </li> <li> -For a variable <code>x</code> of type <code>I</code> or <code>*I</code> +For a variable <code>x</code> of type <code>I</code> where <code>I</code> is an interface type, <code>x.f</code> denotes the actual method with name <code>f</code> of the value assigned to <code>x</code> if there is such a method. @@ -2742,7 +2744,7 @@ and finally <code>||</code> (logical or): Precedence Operator 5 * / % << >> & &^ 4 + - | ^ - 3 == != < <= > >= + 3 == != < <= > >= 2 && 1 || </pre> @@ -2758,7 +2760,7 @@ For instance, <code>x / y * z</code> is the same as <code>(x / y) * z</code>. x <= f() ^a >> b f() || g() -x == y+1 && <-chan_ptr > 0 +x == y+1 && <-chan_ptr > 0 </pre> @@ -3684,17 +3686,16 @@ complex, or string constant. "If" statements specify the conditional execution of two branches according to the value of a boolean expression. If the expression evaluates to true, the "if" branch is executed, otherwise, if -present, the "else" branch is executed. A missing condition -is equivalent to <code>true</code>. +present, the "else" branch is executed. </p> <pre class="ebnf"> -IfStmt = "if" [ SimpleStmt ";" ] [ Expression ] Block [ "else" Statement ] . +IfStmt = "if" [ SimpleStmt ";" ] Expression Block [ "else" Statement ] . </pre> <pre> -if x > 0 { - return true; +if x > max { + x = max } </pre> @@ -3706,7 +3707,7 @@ executes before the expression is evaluated. <pre> if x := f(); x < y { return x -} else if x > z { +} else if x > z { return z } else { return y @@ -4697,7 +4698,7 @@ func protect(g func()) { defer func() { log.Println("done") // Println executes normally even in there is a panic if x := recover(); x != nil { - log.Printf("runtime panic: %v", x) + log.Printf("run time panic: %v", x) } } log.Println("start") @@ -4794,7 +4795,7 @@ The PackageName is used in <a href="#Qualified_identifiers">qualified identifier to access the exported identifiers of the package within the importing source file. It is declared in the <a href="#Blocks">file block</a>. If the PackageName is omitted, it defaults to the identifier specified in the -<a href="#Package_clauses">package clause</a> of the imported package. +<a href="#Package_clause">package clause</a> of the imported package. If an explicit period (<code>.</code>) appears instead of a name, all the package's exported identifiers will be declared in the current file's file block and can be accessed without a qualifier. @@ -5151,7 +5152,6 @@ The following minimal alignment properties are guaranteed: <h2 id="Implementation_differences"><span class="alert">Implementation differences - TODO</span></h2> <ul> <li><span class="alert">Implementation does not honor the restriction on goto statements and targets (no intervening declarations).</span></li> - <li><span class="alert">Gccgo: The <code>append</code> built-in function is not yet implemented.</span></li> <li><span class="alert">Gccgo: Method expressions are partially implemented.</span></li> <li><span class="alert">Gccgo: allows only one init() function per source file.</span></li> </ul> diff --git a/doc/install.html b/doc/install.html index d8fa8b468..816e6e654 100644 --- a/doc/install.html +++ b/doc/install.html @@ -54,7 +54,7 @@ architectures. </dl> <p> -Except for things like low-level operating system interface code, the runtime +Except for things like low-level operating system interface code, the run-time support is the same in all ports and includes a mark-and-sweep garbage collector (a fancier one is in the works), efficient array and string slicing, support for segmented stacks, and a strong goroutine implementation. @@ -163,8 +163,7 @@ The compiler is 6g. </pre> <p> -where <var>N</var> is a number that varies from release to release -and the details on the last few lines will reflect the operating system, +where the details on the last few lines reflect the operating system, architecture, and root directory used during the install. </p> @@ -419,9 +418,9 @@ to override the defaults. <code>$GOARM</code> (arm, default=6) </dt> <dd> - The ARM architecture version the runtime libraries should target. + The ARM architecture version the run-time libraries should target. ARMv6 cores have more efficient synchronization primitives. Setting - <code>$GOARM</code> to 5 will compile the runtime libraries using + <code>$GOARM</code> to 5 will compile the run-time libraries using just SWP instructions that work on older architectures as well. Running v6 code on an older core will cause an illegal instruction trap. </dd> diff --git a/doc/talks/go_talk-20091030.pdf b/doc/talks/go_talk-20091030.pdf Binary files differdeleted file mode 100644 index 5139ff2bd..000000000 --- a/doc/talks/go_talk-20091030.pdf +++ /dev/null diff --git a/lib/codereview/codereview.py b/lib/codereview/codereview.py index 96efc855b..fa703c711 100644 --- a/lib/codereview/codereview.py +++ b/lib/codereview/codereview.py @@ -1136,11 +1136,14 @@ def clpatch(ui, repo, clname, **opts): return missing_codereview cl, patch, err = DownloadCL(ui, repo, clname) + if err != "": + return err + if patch == emptydiff: + return "codereview issue %s has no diff" % clname + argv = ["hgpatch"] if opts["no_incoming"]: argv += ["--checksync=false"] - if err != "": - return err try: cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32") except: @@ -1151,6 +1154,8 @@ def clpatch(ui, repo, clname, **opts): return "hgpatch failed" cl.local = True cl.files = out.strip().split() + if not cl.files: + return "codereview issue %s has no diff" % clname files = ChangedFiles(ui, repo, [], opts) extra = Sub(cl.files, files) if extra: diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile index 7270a3f42..cff47942a 100644 --- a/misc/dashboard/builder/Makefile +++ b/misc/dashboard/builder/Makefile @@ -10,5 +10,6 @@ GOFILES=\ hg.go\ http.go\ main.go\ + package.go\ include ../../../src/Make.cmd diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go index 54a9adfc0..a28658a95 100644 --- a/misc/dashboard/builder/doc.go +++ b/misc/dashboard/builder/doc.go @@ -38,6 +38,15 @@ Optional flags: -release: Build and deliver binary release archive + -rev=N: Build revision N and exit + + -cmd="./all.bash": Build command (specify absolute or relative to go/src) + + -v: Verbose logging + + -external: External package builder mode (will not report Go build + state to dashboard, issue releases, or run benchmarks) + The key file should be located at $HOME/.gobuilder or, for a builder-specific key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64). diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go index 6236c915a..53ea93ac5 100644 --- a/misc/dashboard/builder/exec.go +++ b/misc/dashboard/builder/exec.go @@ -1,15 +1,23 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package main import ( "bytes" "exec" "io" + "log" "os" "strings" ) // run is a simple wrapper for exec.Run/Close func run(envv []string, dir string, argv ...string) os.Error { + if *verbose { + log.Println("run", argv) + } bin, err := pathLookup(argv[0]) if err != nil { return err @@ -25,6 +33,9 @@ func run(envv []string, dir string, argv ...string) os.Error { // runLog runs a process and returns the combined stdout/stderr, // as well as writing it to logfile (if specified). func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) { + if *verbose { + log.Println("runLog", argv) + } bin, err := pathLookup(argv[0]) if err != nil { return diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go index 5d2f63a17..d4310845d 100644 --- a/misc/dashboard/builder/hg.go +++ b/misc/dashboard/builder/hg.go @@ -1,8 +1,13 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package main import ( "fmt" "os" + "regexp" "strconv" "strings" ) @@ -46,9 +51,36 @@ func getCommit(rev string) (c Commit, err os.Error) { func getCommitParts(rev string) (parts []string, err os.Error) { const format = "{rev}>{node}>{author|escape}>{date}>{desc}" s, _, err := runLog(nil, "", goroot, - "hg", "log", "-r", rev, "-l", "1", "--template", format) + "hg", "log", + "--encoding", "utf-8", + "--rev", rev, + "--limit", "1", + "--template", format, + ) if err != nil { return } return strings.Split(s, ">", 5), nil } + +var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`) + +// getTag fetches a Commit by finding the first hg tag that matches re. +func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n", -1) { + tag = re.FindString(l) + if tag == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + c, err = getCommit(s[1]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index 02f281061..dba19ba8f 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package main import ( @@ -6,8 +10,11 @@ import ( "encoding/binary" "fmt" "http" + "json" + "log" "os" "regexp" + "strconv" ) // getHighWater returns the current highwater revision hash for this builder @@ -63,7 +70,46 @@ func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error { }) } +// getPackages fetches a list of package paths from the dashboard +func getPackages() (pkgs []string, err os.Error) { + r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard)) + if err != nil { + return + } + defer r.Body.Close() + d := json.NewDecoder(r.Body) + var resp struct { + Packages []struct { + Path string + } + } + if err = d.Decode(&resp); err != nil { + return + } + for _, p := range resp.Packages { + pkgs = append(pkgs, p.Path) + } + return +} + +// updatePackage sends package build results and info to the dashboard +func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error { + args := map[string]string{ + "builder": b.name, + "key": b.key, + "path": pkg, + "state": strconv.Btoa(state), + "log": buildLog, + "info": info, + "go_rev": strconv.Itoa(c.num), + } + return httpCommand("package", args) +} + func httpCommand(cmd string, args map[string]string) os.Error { + if *verbose { + log.Println("httpCommand", cmd, args) + } url := fmt.Sprintf("http://%v/%v", *dashboard, cmd) _, err := http.PostForm(url, args) return err diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index 7e80934e1..fc11d365e 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package main import ( @@ -15,13 +19,24 @@ import ( ) const ( - codeProject = "go" - codePyScript = "misc/dashboard/googlecode_upload.py" - hgUrl = "https://go.googlecode.com/hg/" - waitInterval = 10e9 // time to wait before checking for new revs - mkdirPerm = 0750 + codeProject = "go" + codePyScript = "misc/dashboard/googlecode_upload.py" + hgUrl = "https://go.googlecode.com/hg/" + waitInterval = 10e9 // time to wait before checking for new revs + mkdirPerm = 0750 + pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours ) +// These variables are copied from the gobuilder's environment +// to the envv of its subprocesses. +var extraEnv = []string{ + "GOHOSTOS", + "GOHOSTARCH", + "PATH", + "DISABLE_NET_TESTS", + "GOARM", +} + type Builder struct { name string goos, goarch string @@ -43,6 +58,8 @@ var ( buildRelease = flag.Bool("release", false, "Build and upload binary release archives") buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") + external = flag.Bool("external", false, "Build external packages") + verbose = flag.Bool("v", false, "verbose") ) var ( @@ -70,6 +87,8 @@ func main() { } builders[i] = b } + + // set up work environment if err := os.RemoveAll(*buildroot); err != nil { log.Fatalf("Error removing build root (%s): %s", *buildroot, err) } @@ -79,6 +98,7 @@ func main() { if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil { log.Fatal("Error cloning repository:", err) } + // if specified, build revision and return if *buildRevision != "" { c, err := getCommit(*buildRevision) @@ -93,6 +113,16 @@ func main() { } return } + + // external package build mode + if *external { + if len(builders) != 1 { + log.Fatal("only one goos-goarch should be specified with -external") + } + builders[0].buildExternal() + } + + // go continuous build mode (default) // check for new commits and build them for { err := run(nil, goroot, "hg", "pull", "-u") @@ -179,6 +209,44 @@ func NewBuilder(builder string) (*Builder, os.Error) { return b, nil } +// buildExternal downloads and builds external packages, and +// reports their build status to the dashboard. +// It will re-build all packages after pkgBuildInterval nanoseconds or +// a new release tag is found. +func (b *Builder) buildExternal() { + var prevTag string + var nextBuild int64 + for { + time.Sleep(waitInterval) + err := run(nil, goroot, "hg", "pull", "-u") + if err != nil { + log.Println("hg pull failed:", err) + continue + } + c, tag, err := getTag(releaseRegexp) + if err != nil { + log.Println(err) + continue + } + if *verbose { + log.Println("latest release:", tag) + } + // don't rebuild if there's no new release + // and it's been less than pkgBuildInterval + // nanoseconds since the last build. + if tag == prevTag && time.Nanoseconds() < nextBuild { + continue + } + // buildCommit will also build the packages + if err := b.buildCommit(c); err != nil { + log.Println(err) + continue + } + prevTag = tag + nextBuild = time.Nanoseconds() + pkgBuildInterval + } +} + // build checks for a new commit for this builder // and builds it if one is found. // It returns true if a build was attempted. @@ -262,23 +330,23 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { return } - // set up environment for build/bench execution - env := []string{ - "GOOS=" + b.goos, - "GOARCH=" + b.goarch, - "GOHOSTOS=" + os.Getenv("GOHOSTOS"), - "GOHOSTARCH=" + os.Getenv("GOHOSTARCH"), - "GOROOT_FINAL=/usr/local/go", - "PATH=" + os.Getenv("PATH"), - } srcDir := path.Join(workpath, "go", "src") // build logfile := path.Join(workpath, "build.log") - buildLog, status, err := runLog(env, logfile, srcDir, *buildCmd) + buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) if err != nil { return fmt.Errorf("all.bash: %s", err) } + + // if we're in external mode, build all packages and return + if *external { + if status != 0 { + return os.NewError("go build failed") + } + return b.buildPackages(workpath, c) + } + if status != 0 { // record failure return b.recordResult(buildLog, c) @@ -307,7 +375,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // if this is a release, create tgz and upload to google code if release := releaseRegexp.FindString(c.desc); release != "" { // clean out build state - err = run(env, srcDir, "./clean.bash", "--nopkg") + err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") if err != nil { return fmt.Errorf("clean.bash: %s", err) } @@ -329,6 +397,19 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { return } +// envv returns an environment for build/bench execution +func (b *Builder) envv() []string { + e := []string{ + "GOOS=" + b.goos, + "GOARCH=" + b.goarch, + "GOROOT_FINAL=/usr/local/go", + } + for _, k := range extraEnv { + e = append(e, k+"="+os.Getenv(k)) + } + return e +} + func isDirectory(name string) bool { s, err := os.Stat(name) return err == nil && s.IsDirectory() diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go new file mode 100644 index 000000000..6e9f9ff39 --- /dev/null +++ b/misc/dashboard/builder/package.go @@ -0,0 +1,66 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/doc" + "go/parser" + "go/token" + "log" + "os" + "path" +) + +func (b *Builder) buildPackages(workpath string, c Commit) os.Error { + pkgs, err := getPackages() + if err != nil { + return err + } + for _, p := range pkgs { + goroot := path.Join(workpath, "go") + goinstall := path.Join(goroot, "bin", "goinstall") + envv := append(b.envv(), "GOROOT="+goroot) + + // goinstall + buildLog, code, err := runLog(envv, "", goroot, goinstall, p) + if err != nil { + log.Printf("goinstall %v: %v", p, err) + continue + } + built := code != 0 + + // get doc comment from package source + info, err := getPackageComment(p, path.Join(goroot, "pkg", p)) + if err != nil { + log.Printf("goinstall %v: %v", p, err) + } + + // update dashboard with build state + info + err = b.updatePackage(p, built, buildLog, info, c) + if err != nil { + log.Printf("updatePackage %v: %v", p, err) + } + } + return nil +} + +func getPackageComment(pkg, pkgpath string) (info string, err os.Error) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments) + if err != nil { + return + } + for name := range pkgs { + if name == "main" { + continue + } + if info != "" { + return "", os.NewError("multiple non-main package docs") + } + pdoc := doc.NewPackageDoc(pkgs[name], pkg) + info = pdoc.Doc + } + return +} diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py new file mode 100644 index 000000000..b0110c635 --- /dev/null +++ b/misc/dashboard/godashboard/const.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +mail_from = "Go Dashboard <builder@golang.org>" + +mail_submit_to = "adg@golang.org" +mail_submit_subject = "New Project Submitted" + +mail_fail_to = "golang-dev@googlegroups.com" +mail_fail_reply_to = "golang-dev@googlegroups.com" +mail_fail_subject = "%s broken by %s" + diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt new file mode 100644 index 000000000..a699005ea --- /dev/null +++ b/misc/dashboard/godashboard/fail-notify.txt @@ -0,0 +1,6 @@ +Change {{node}} broke the {{builder}} build: +http://godashboard.appspot.com/log/{{loghash}} + +{{desc}} + +http://code.google.com/p/go/source/detail?r={{node}} diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 46aeef9f9..08d70ec64 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -5,6 +5,7 @@ # This is the server part of the continuous build system for Go. It must be run # by AppEngine. +from google.appengine.api import mail from google.appengine.api import memcache from google.appengine.runtime import DeadlineExceededError from google.appengine.ext import db @@ -24,6 +25,7 @@ import bz2 # local imports import key +import const # The majority of our state are commit objects. One of these exists for each of # the commits known to the build system. Their key names are of the form @@ -47,6 +49,8 @@ class Commit(db.Model): # successful. builds = db.StringListProperty() + fail_notification_sent = db.BooleanProperty() + class Benchmark(db.Model): name = db.StringProperty() version = db.IntegerProperty() @@ -259,7 +263,7 @@ class Init(webapp.RequestHandler): commit.num = 0 commit.node = node commit.parentnode = '' - commit.user = self.request.get('user') + commit.user = self.request.get('user').encode('utf8') commit.date = date commit.desc = self.request.get('desc').encode('utf8') @@ -285,34 +289,37 @@ class Build(webapp.RequestHandler): l.put() date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') node = self.request.get('node') - parent = self.request.get('parent') - if not validNode(node) or not validNode(parent) or date is None: + parenthash = self.request.get('parent') + if not validNode(node) or not validNode(parenthash) or date is None: logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) self.response.set_status(500) return q = Commit.all() - q.filter('node =', parent) - p = q.get() - if p is None: - logging.error('Cannot find parent %s of node %s' % (parent, node)) + q.filter('node =', parenthash) + parent = q.get() + if parent is None: + logging.error('Cannot find parent %s of node %s' % (parenthash, node)) self.response.set_status(404) return - parentnum, _ = p.key().name().split('-', 1) + parentnum, _ = parent.key().name().split('-', 1) nodenum = int(parentnum, 16) + 1 + key_name = '%08x-%s' % (nodenum, node) + def add_build(): - key_name = '%08x-%s' % (nodenum, node) n = Commit.get_by_key_name(key_name) if n is None: n = Commit(key_name = key_name) n.num = nodenum n.node = node - n.parentnode = parent - n.user = self.request.get('user') + n.parentnode = parenthash + n.user = user n.date = date - n.desc = self.request.get('desc').encode('utf8') + n.desc = desc s = '%s`%s' % (builder, loghash) for i, b in enumerate(n.builds): if b.split('`', 1)[0] == builder: @@ -333,8 +340,40 @@ class Build(webapp.RequestHandler): memcache.delete(key) memcache.delete('hw') + def mark_sent(): + n = Commit.get_by_key_name(key_name) + n.fail_notification_sent = True + n.put() + + n = Commit.get_by_key_name(key_name) + if loghash and not failed(parent, builder) and not n.fail_notification_sent: + subject = const.mail_fail_subject % (builder, desc.split("\n")[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": node[:12], + "user": user, + "desc": desc, + "loghash": loghash + }) + mail.send_mail( + sender=const.mail_from, + reply_to=const.mail_fail_reply_to, + to=const.mail_fail_to, + subject=subject, + body=body + ) + db.run_in_transaction(mark_sent) + self.response.set_status(200) +def failed(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return len(p[1]) > 0 + return False + class Benchmarks(webapp.RequestHandler): def json(self): q = Benchmark.all() diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py index cf59bf3e8..7570d2218 100644 --- a/misc/dashboard/godashboard/package.py +++ b/misc/dashboard/godashboard/package.py @@ -5,10 +5,6 @@ # This is the server part of the package dashboard. # It must be run by App Engine. -mail_to = "adg@golang.org" -mail_from = "Go Dashboard <adg@golang.org>" -mail_subject = "New Project Submitted" - from google.appengine.api import memcache from google.appengine.runtime import DeadlineExceededError from google.appengine.ext import db @@ -32,6 +28,7 @@ import sets # local imports import toutf8 +import const template.register_template_library('toutf8') @@ -241,7 +238,9 @@ class ProjectPage(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') mail.send_mail( - sender=mail_from, to=mail_to, subject=mail_subject, + sender=const.mail_from, + to=const.mail_submit_to, + subject=const.mail_submit_subject, body=template.render(path, {'project': p})) self.list({"submitMsg": "Your project has been submitted."}) diff --git a/robots.txt b/robots.txt new file mode 100644 index 000000000..1f53798bb --- /dev/null +++ b/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/src/Make.common b/src/Make.common index e3f415a1f..af6d04adc 100644 --- a/src/Make.common +++ b/src/Make.common @@ -6,7 +6,7 @@ clean: rm -rf *.o *.a *.[$(OS)] [$(OS)].out $(CLEANFILES) %.make: - (cd $* && gomake install) + $(MAKE) -C $* install .PHONY: all clean nuke install coverage test bench testpackage-clean\ importpath dir diff --git a/src/Make.pkg b/src/Make.pkg index ca0fa9ee2..3d616ca99 100644 --- a/src/Make.pkg +++ b/src/Make.pkg @@ -36,7 +36,7 @@ INSTALLFILES+=$(pkgdir)/$(TARG).a # The rest of the cgo rules are below, but these variable updates # must be done here so they apply to the main rules. ifdef CGOFILES -GOFILES+=$(patsubst %.go,%.cgo1.go,$(CGOFILES)) _cgo_gotypes.go +GOFILES+=$(patsubst %.go,_obj/%.cgo1.go,$(CGOFILES)) _obj/_cgo_gotypes.go CGO_OFILES+=$(patsubst %.go,%.cgo2.o,$(CGOFILES)) _cgo_export.o OFILES+=_cgo_defun.$O _cgo_import.$O $(CGO_OFILES) endif @@ -47,21 +47,19 @@ coverage: gotest 6cov -g $(shell pwd) $O.out | grep -v '_test\.go:' -CLEANFILES+=*.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* -CLEANFILES+=_cgo_.c _cgo_import.c _cgo_main.c _cgo_flags _cgo_run -CLEANFILES+=*.so _obj _test _testmain.go *.exe +CLEANFILES+=*.so _obj _test _testmain.go *.exe _cgo* *.cgo[12].* test: gotest bench: - gotest -benchmarks=. -match="Do not run tests" + gotest -test.bench=. -test.run="Do not run tests" nuke: clean rm -f $(pkgdir)/$(TARG).a testpackage-clean: - rm -f _test/$(TARG).a _gotest_.$O + rm -f _test/$(TARG).a install: $(INSTALLFILES) @@ -103,33 +101,34 @@ dir: # x.go and y.go. # Cgo translates each x.go file listed in $(CGOFILES) into a basic -# translation of x.go, called x.cgo1.go. Additionally, three other +# translation of x.go, called _obj/x.cgo1.go. Additionally, three other # files are created: # -# _cgo_gotypes.go - declarations needed for all .go files in the package; imports "unsafe" -# _cgo_defun.c - C trampoline code to be compiled with 6c and linked into the package -# x.cgo2.c - C implementations compiled with gcc to create a dynamic library +# _obj/_cgo_gotypes.go - declarations needed for all .go files in the package; imports "unsafe" +# _obj/_cgo_defun.c - C trampoline code to be compiled with 6c and linked into the package +# _obj/x.cgo2.c - C implementations compiled with gcc to create a dynamic library # ifdef CGOFILES -_cgo_run: $(CGOFILES) +_obj/_cgo_run: $(CGOFILES) + @mkdir -p _obj CGOPKGPATH=$(dir) cgo -- $(CGO_CFLAGS) $(CGOFILES) - touch _cgo_run + touch _obj/_cgo_run # _CGO_CFLAGS and _CGO_LDFLAGS are defined via the evaluation of _cgo_flags. # The include happens before the commands in the recipe run, # so it cannot be done in the same recipe that runs cgo. -_load_cgo_flags: _cgo_run - $(eval include _cgo_flags) +_obj/_load_cgo_flags: _obj/_cgo_run + $(eval include _obj/_cgo_flags) # Include any previous flags in case cgo files are up to date. --include _cgo_flags +-include _obj/_cgo_flags # Ugly but necessary - cgo writes these files too. -_cgo_gotypes.go _cgo_export.c _cgo_export.h _cgo_main.c _cgo_defun.c: _load_cgo_flags +_obj/_cgo_gotypes.go _obj/_cgo_export.c _obj/_cgo_export.h _obj/_cgo_main.c _obj/_cgo_defun.c: _obj/_load_cgo_flags @true -%.cgo1.go %.cgo2.c: _cgo_defun.c +_obj/%.cgo1.go _obj/%.cgo2.c: _obj/_cgo_defun.c @true endif @@ -137,6 +136,9 @@ endif %.o: %.c $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -g -fPIC -O2 -o $@ -c $(CGO_CFLAGS) $(_CGO_CFLAGS) $*.c +%.o: _obj/%.c + $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -I . -g -fPIC -O2 -o $@ -c $(CGO_CFLAGS) $(_CGO_CFLAGS) $^ + # To find out which symbols are needed from external libraries # and which libraries are needed, we build a simple a.out that # links all the objects we just created and then use cgo -dynimport @@ -145,14 +147,12 @@ endif # After main we have to define all the symbols that will be provided # by Go code. That's crosscall2 and any exported symbols. -_cgo_main.o: _cgo_main.c - $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -g -fPIC -O2 -o $@ -c $(CGO_CFLAGS) $(_CGO_CFLAGS) _cgo_main.c - _cgo1_.o: _cgo_main.o $(CGO_OFILES) $(HOST_CC) $(_CGO_CFLAGS_$(GOARCH)) -g -fPIC -O2 -o $@ $^ $(CGO_LDFLAGS) $(_CGO_LDFLAGS) -_cgo_import.c: _cgo1_.o - cgo -dynimport _cgo1_.o >_$@ && mv -f _$@ $@ +_obj/_cgo_import.c: _cgo1_.o + @mkdir -p _obj + cgo -dynimport _cgo1_.o >$@_ && mv -f $@_ $@ # The rules above added x.cgo1.go and _cgo_gotypes.go to $(GOFILES), # added _cgo_defun.$O to $OFILES, and added the installed copy of @@ -170,17 +170,17 @@ _CGO_LDFLAGS_windows=-shared -lm -mthreads RUNTIME_CFLAGS=-I$(pkgdir) # Compile _cgo_defun.c with 6c; needs access to the runtime headers. -_cgo_defun.$O: _cgo_defun.c - $(CC) $(CFLAGS) $(RUNTIME_CFLAGS) _cgo_defun.c +_cgo_defun.$O: _obj/_cgo_defun.c + $(CC) $(CFLAGS) $(RUNTIME_CFLAGS) -I . -o "$@" _obj/_cgo_defun.c # Generic build rules. # These come last so that the rules above can override them # for more specific file names. -%.$O: %.c - $(CC) $(CFLAGS) $*.c +%.$O: %.c $(HFILES) + $(CC) $(CFLAGS) -o "$@" $*.c + +%.$O: _obj/%.c $(HFILES) + $(CC) $(CFLAGS) -I . -o "$@" _obj/$*.c %.$O: %.s $(AS) $*.s - -%.$O: $(HFILES) - diff --git a/src/clean.bash b/src/clean.bash index 5c1dded56..7969e2cd0 100755 --- a/src/clean.bash +++ b/src/clean.bash @@ -22,11 +22,6 @@ rm -f "$GOROOT"/lib/*.a for i in lib9 libbio libmach cmd pkg \ ../misc/cgo/gmp ../misc/cgo/stdio \ ../test/bench ../test/garbage -do( - cd "$GOROOT"/src/$i || exit 1 - if test -f clean.bash; then - bash clean.bash --gomake $MAKE - else - $MAKE clean - fi -)done +do + gomake -C "$GOROOT/src/$i" clean +done diff --git a/src/cmd/5a/lex.c b/src/cmd/5a/lex.c index e762f5646..dbee3657f 100644 --- a/src/cmd/5a/lex.c +++ b/src/cmd/5a/lex.c @@ -364,7 +364,9 @@ struct "MOVWF", LTYPE3, AMOVWF, "LDREX", LTYPE3, ALDREX, + "LDREXD", LTYPE3, ALDREXD, "STREX", LTYPE9, ASTREX, + "STREXD", LTYPE9, ASTREXD, /* "ABSF", LTYPEI, AABSF, diff --git a/src/cmd/5c/txt.c b/src/cmd/5c/txt.c index 0f17cea89..f5619f800 100644 --- a/src/cmd/5c/txt.c +++ b/src/cmd/5c/txt.c @@ -1194,8 +1194,10 @@ gpseudo(int a, Sym *s, Node *n) p->from.type = D_OREG; p->from.sym = s; p->from.name = D_EXTERN; - if(a == ATEXT) + if(a == ATEXT) { p->reg = textflag; + textflag = 0; + } if(s->class == CSTATIC) p->from.name = D_STATIC; naddr(n, &p->to); diff --git a/src/cmd/5g/gg.h b/src/cmd/5g/gg.h index 603c09fc8..ce4575be9 100644 --- a/src/cmd/5g/gg.h +++ b/src/cmd/5g/gg.h @@ -27,6 +27,7 @@ struct Addr uchar type; char name; char reg; + char pun; uchar etype; }; #define A ((Addr*)0) diff --git a/src/cmd/5g/gsubr.c b/src/cmd/5g/gsubr.c index 133a21b3e..83a9949d6 100644 --- a/src/cmd/5g/gsubr.c +++ b/src/cmd/5g/gsubr.c @@ -1168,6 +1168,7 @@ naddr(Node *n, Addr *a, int canemitcode) a->etype = simtype[n->type->etype]; a->width = n->type->width; } + a->pun = n->pun; a->offset = n->xoffset; a->sym = n->sym; if(a->sym == S) diff --git a/src/cmd/5g/reg.c b/src/cmd/5g/reg.c index f31f70535..1cbeb3e3d 100644 --- a/src/cmd/5g/reg.c +++ b/src/cmd/5g/reg.c @@ -697,8 +697,8 @@ mkvar(Reg *r, Adr *a) n = D_NONE; flag = 0; -// if(a->pun) -// flag = 1; + if(a->pun) + flag = 1; switch(t) { default: diff --git a/src/cmd/5l/5.out.h b/src/cmd/5l/5.out.h index a25c0f71d..002b46d45 100644 --- a/src/cmd/5l/5.out.h +++ b/src/cmd/5l/5.out.h @@ -179,6 +179,9 @@ enum as ALDREX, ASTREX, + + ALDREXD, + ASTREXD, ALAST, }; diff --git a/src/cmd/5l/asm.c b/src/cmd/5l/asm.c index 7163997c0..af6d1dfda 100644 --- a/src/cmd/5l/asm.c +++ b/src/cmd/5l/asm.c @@ -331,21 +331,21 @@ asmb(void) Bprint(&bso, "%5.2f sym\n", cputime()); Bflush(&bso); switch(HEADTYPE) { - case 0: - case 1: - case 4: - case 5: + case Hnoheader: + case Hrisc: + case Hixp1200: + case Hipaq: debug['s'] = 1; break; - case 2: + case Hplan9x32: OFFSET = HEADR+textsize+segdata.filelen; seek(cout, OFFSET, 0); break; - case 3: + case Hnetbsd: OFFSET += rnd(segdata.filelen, 4096); seek(cout, OFFSET, 0); break; - case 6: + case Hlinux: OFFSET += segdata.filelen; seek(cout, rnd(OFFSET, INITRND), 0); break; @@ -362,9 +362,9 @@ asmb(void) OFFSET = 0; seek(cout, OFFSET, 0); switch(HEADTYPE) { - case 0: /* no header */ + case Hnoheader: /* no header */ break; - case 1: /* aif for risc os */ + case Hrisc: /* aif for risc os */ lputl(0xe1a00000); /* NOP - decompress code */ lputl(0xe1a00000); /* NOP - relocation code */ lputl(0xeb000000 + 12); /* BL - zero init code */ @@ -394,7 +394,7 @@ asmb(void) lputl(0xe1a00000); /* NOP - zero init code */ lputl(0xe1a0f00e); /* B (R14) - zero init return */ break; - case 2: /* plan 9 */ + case Hplan9x32: /* plan 9 */ lput(0x647); /* magic */ lput(textsize); /* sizes */ lput(segdata.filelen); @@ -404,7 +404,7 @@ asmb(void) lput(0L); lput(lcsize); break; - case 3: /* boot for NetBSD */ + case Hnetbsd: /* boot for NetBSD */ lput((143<<16)|0413); /* magic */ lputl(rnd(HEADR+textsize, 4096)); lputl(rnd(segdata.filelen, 4096)); @@ -414,15 +414,15 @@ asmb(void) lputl(0L); lputl(0L); break; - case 4: /* boot for IXP1200 */ + case Hixp1200: /* boot for IXP1200 */ break; - case 5: /* boot for ipaq */ + case Hipaq: /* boot for ipaq */ lputl(0xe3300000); /* nop */ lputl(0xe3300000); /* nop */ lputl(0xe3300000); /* nop */ lputl(0xe3300000); /* nop */ break; - case 6: + case Hlinux: /* elf arm */ eh = getElfEhdr(); fo = HEADR; @@ -1463,7 +1463,7 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na aclass(&p->from); if(instoffset != 0) diag("offset must be zero in STREX"); - o1 = (0x3<<23) | (0xf9<<4); + o1 = (0x18<<20) | (0xf90); o1 |= p->from.reg << 16; o1 |= p->reg << 0; o1 |= p->to.reg << 12; @@ -1553,6 +1553,25 @@ if(debug['G']) print("%ux: %s: arm %d %d %d\n", (uint32)(p->pc), p->from.sym->na o1 = oprrr(ACMP+AEND, p->scond); o1 |= p->from.reg<<16; break; + case 91: /* ldrexd oreg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in LDREX"); + o1 = (0x1b<<20) | (0xf9f); + o1 |= p->from.reg << 16; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; + case 92: /* strexd reg,oreg,reg */ + aclass(&p->from); + if(instoffset != 0) + diag("offset must be zero in STREX"); + o1 = (0x1a<<20) | (0xf90); + o1 |= p->from.reg << 16; + o1 |= p->reg << 0; + o1 |= p->to.reg << 12; + o1 |= (p->scond & C_SCOND) << 28; + break; } out[0] = o1; diff --git a/src/cmd/5l/doc.go b/src/cmd/5l/doc.go index d266b9233..aa7ccebfc 100644 --- a/src/cmd/5l/doc.go +++ b/src/cmd/5l/doc.go @@ -23,6 +23,8 @@ Options new in this version: -F Force use of software floating point. Also implied by setting GOARM=5 in the environment. +-Hlinux + Write Linux ELF binaries (default when $GOOS is linux) -I interpreter Set the ELF dynamic linker to use. -L dir1 -L dir2 diff --git a/src/cmd/5l/l.h b/src/cmd/5l/l.h index e42be4e98..2e887dad7 100644 --- a/src/cmd/5l/l.h +++ b/src/cmd/5l/l.h @@ -35,6 +35,7 @@ enum { + thechar = '5', PtrSize = 4 }; @@ -109,6 +110,7 @@ struct Prog Prog* dlink; int32 pc; int32 line; + int32 spadj; uchar mark; uchar optab; uchar as; @@ -122,6 +124,8 @@ struct Prog #define datasize reg #define textflag reg +#define iscall(p) ((p)->as == ABL) + struct Sym { char* name; @@ -131,6 +135,7 @@ struct Sym uchar reachable; uchar dynexport; uchar leaf; + uchar stkcheck; int32 dynid; int32 plt; int32 got; diff --git a/src/cmd/5l/list.c b/src/cmd/5l/list.c index b4df89587..2ae25d491 100644 --- a/src/cmd/5l/list.c +++ b/src/cmd/5l/list.c @@ -98,6 +98,10 @@ Pconv(Fmt *fp) fmtprint(fp, "(%d) DWORD %D %D", p->line, &p->from, &p->to); break; } + + if(p->spadj) + fmtprint(fp, " (spadj%+d)", p->spadj); + return 0; } diff --git a/src/cmd/5l/noop.c b/src/cmd/5l/noop.c index a9439c27a..a5e66f038 100644 --- a/src/cmd/5l/noop.c +++ b/src/cmd/5l/noop.c @@ -227,7 +227,7 @@ noops(void) #ifdef CALLEEBX if(p->from.sym->foreign){ if(thumb) - // don't allow literal pool to seperate these + // don't allow literal pool to separate these p = adword(0xe28f7001, 0xe12fff17, p); // arm add 1, pc, r7 and bx r7 // p = aword(0xe12fff17, aword(0xe28f7001, p)); // arm add 1, pc, r7 and bx r7 else @@ -282,6 +282,7 @@ noops(void) q1->to.type = D_OREG; q1->to.offset = -autosize; q1->to.reg = REGSP; + q1->spadj = autosize; q1->link = p->link; p->link = q1; } else if (autosize < StackBig) { @@ -376,6 +377,7 @@ noops(void) p->to.type = D_OREG; p->to.offset = -autosize; p->to.reg = REGSP; + p->spadj = autosize; } else { // > StackBig // MOVW $autosize, R1 // MOVW $args, R2 @@ -424,6 +426,7 @@ noops(void) p->to.type = D_OREG; p->to.offset = -autosize; p->to.reg = REGSP; + p->spadj = autosize; } break; @@ -527,9 +530,20 @@ noops(void) p->from.reg = REGSP; p->to.type = D_REG; p->to.reg = REGPC; + // no spadj because it doesn't fall through } break; + case AADD: + if(p->from.type == D_CONST && p->from.reg == NREG && p->to.type == D_REG && p->to.reg == REGSP) + p->spadj = -p->from.offset; + break; + + case ASUB: + if(p->from.type == D_CONST && p->from.reg == NREG && p->to.type == D_REG && p->to.reg == REGSP) + p->spadj = p->from.offset; + break; + case ADIV: case ADIVU: case AMOD: @@ -635,6 +649,7 @@ noops(void) p->reg = NREG; p->to.type = D_REG; p->to.reg = REGSP; + p->spadj = -8; /* SUB $8,SP */ q1->as = ASUB; @@ -644,6 +659,7 @@ noops(void) q1->reg = NREG; q1->to.type = D_REG; q1->to.reg = REGSP; + q1->spadj = 8; break; case AMOVW: @@ -653,6 +669,12 @@ noops(void) if(a->type == D_CONST && ((a->name == D_NONE && a->reg == REGSP) || a->name == D_AUTO || a->name == D_PARAM) && (a->offset & 3)) diag("SP offset not multiple of 4"); } + if((p->scond & C_WBIT) && p->to.type == D_OREG && p->to.reg == REGSP) + p->spadj = -p->to.offset; + if((p->scond & C_PBIT) && p->from.type == D_OREG && p->from.reg == REGSP && p->to.reg != REGPC) + p->spadj = -p->from.offset; + if(p->from.type == D_CONST && p->from.reg == REGSP && p->to.type == D_REG && p->to.reg == REGSP) + p->spadj = -p->from.offset; break; case AMOVB: case AMOVBU: diff --git a/src/cmd/5l/obj.c b/src/cmd/5l/obj.c index 5b778d777..f252f9fc5 100644 --- a/src/cmd/5l/obj.c +++ b/src/cmd/5l/obj.c @@ -41,15 +41,26 @@ #endif char *noname = "<none>"; -char thechar = '5'; char *thestring = "arm"; +Header headers[] = { + "noheader", Hnoheader, + "risc", Hrisc, + "plan9", Hplan9x32, + "netbsd", Hnetbsd, + "ixp1200", Hixp1200, + "ipaq", Hipaq, + "linux", Hlinux, + 0, 0 +}; + /* - * -H1 -T0x10005000 -R4 is aif for risc os - * -H2 -T4128 -R4096 is plan9 format - * -H3 -T0xF0000020 -R4 is NetBSD format - * -H4 is IXP1200 (raw) - * -H5 -T0xC0008010 -R1024 is ipaq + * -Hrisc -T0x10005000 -R4 is aif for risc os + * -Hplan9 -T4128 -R4096 is plan9 format + * -Hnetbsd -T0xF0000020 -R4 is NetBSD format + * -Hixp1200 is IXP1200 (raw) + * -Hipaq -T0xC0008010 -R1024 is ipaq + * -Hlinux -Tx -Rx is linux elf */ static char* @@ -119,7 +130,7 @@ main(int argc, char *argv[]) rpath = EARGF(usage()); break; case 'H': - HEADTYPE = atolwhex(EARGF(usage())); + HEADTYPE = headtype(EARGF(usage())); /* do something about setting INITTEXT */ break; case 'V': @@ -133,25 +144,23 @@ main(int argc, char *argv[]) usage(); libinit(); - if(rpath == nil) - rpath = smprint("%s/pkg/%s_%s", goroot, goos, goarch); if(!debug['9'] && !debug['U'] && !debug['B']) debug[DEFAULT] = 1; if(HEADTYPE == -1) { if(debug['U']) - HEADTYPE = 0; + HEADTYPE = Hnoheader; if(debug['B']) - HEADTYPE = 1; + HEADTYPE = Hrisc; if(debug['9']) - HEADTYPE = 2; - HEADTYPE = 6; + HEADTYPE = Hplan9x32; + HEADTYPE = Hlinux; } switch(HEADTYPE) { default: diag("unknown -H option"); errorexit(); - case 0: /* no header */ + case Hnoheader: /* no header */ HEADR = 0L; if(INITTEXT == -1) INITTEXT = 0; @@ -160,7 +169,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4; break; - case 1: /* aif for risc os */ + case Hrisc: /* aif for risc os */ HEADR = 128L; if(INITTEXT == -1) INITTEXT = 0x10005000 + HEADR; @@ -169,7 +178,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4; break; - case 2: /* plan 9 */ + case Hplan9x32: /* plan 9 */ HEADR = 32L; if(INITTEXT == -1) INITTEXT = 4128; @@ -178,7 +187,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 3: /* boot for NetBSD */ + case Hnetbsd: /* boot for NetBSD */ HEADR = 32L; if(INITTEXT == -1) INITTEXT = 0xF0000020L; @@ -187,7 +196,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 4: /* boot for IXP1200 */ + case Hixp1200: /* boot for IXP1200 */ HEADR = 0L; if(INITTEXT == -1) INITTEXT = 0x0; @@ -196,7 +205,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4; break; - case 5: /* boot for ipaq */ + case Hipaq: /* boot for ipaq */ HEADR = 16L; if(INITTEXT == -1) INITTEXT = 0xC0008010; @@ -205,7 +214,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 1024; break; - case 6: /* arm elf */ + case Hlinux: /* arm elf */ debug['d'] = 1; // no dynamic linking elfinit(); HEADR = ELFRESERVE; @@ -265,11 +274,13 @@ main(int argc, char *argv[]) follow(); softfloat(); noops(); + dostkcheck(); span(); pclntab(); symtab(); dodata(); address(); + doweak(); reloc(); asmb(); undef(); diff --git a/src/cmd/5l/optab.c b/src/cmd/5l/optab.c index 8b3135e06..625b66812 100644 --- a/src/cmd/5l/optab.c +++ b/src/cmd/5l/optab.c @@ -253,5 +253,8 @@ Optab optab[] = { ATST, C_REG, C_NONE, C_NONE, 90, 4, 0 }, + { ALDREXD, C_SOREG,C_NONE, C_REG, 91, 4, 0 }, + { ASTREXD, C_SOREG,C_REG, C_REG, 92, 4, 0 }, + { AXXX, C_NONE, C_NONE, C_NONE, 0, 4, 0 }, }; diff --git a/src/cmd/5l/pass.c b/src/cmd/5l/pass.c index e16b34171..7e1ba6a09 100644 --- a/src/cmd/5l/pass.c +++ b/src/cmd/5l/pass.c @@ -35,18 +35,6 @@ static void xfol(Prog*, Prog**); -void -undef(void) -{ - int i; - Sym *s; - - for(i=0; i<NHASH; i++) - for(s = hash[i]; s != S; s = s->hash) - if(s->type == SXREF) - diag("%s: not defined", s->name); -} - Prog* brchain(Prog *p) { diff --git a/src/cmd/5l/span.c b/src/cmd/5l/span.c index 220140f43..482d3e90a 100644 --- a/src/cmd/5l/span.c +++ b/src/cmd/5l/span.c @@ -1054,6 +1054,8 @@ buildop(void) case ALDREX: case ASTREX: + case ALDREXD: + case ASTREXD: case ATST: break; } diff --git a/src/cmd/6l/asm.c b/src/cmd/6l/asm.c index d179e77b1..fb041d83a 100644 --- a/src/cmd/6l/asm.c +++ b/src/cmd/6l/asm.c @@ -262,7 +262,7 @@ adddynrel(Sym *s, Reloc *r) r->type = 256; // ignore during relocsym return; } - if(HEADTYPE == 6 && s->size == PtrSize && r->off == 0) { + if(HEADTYPE == Hdarwin && s->size == PtrSize && r->off == 0) { // Mach-O relocations are a royal pain to lay out. // They use a compact stateful bytecode representation // that is too much bother to deal with. @@ -365,7 +365,7 @@ addpltsym(Sym *s) adduint64(rela, 0); s->plt = plt->size - 16; - } else if(HEADTYPE == 6) { // Mach-O + } else if(HEADTYPE == Hdarwin) { // To do lazy symbol lookup right, we're supposed // to tell the dynamic loader which library each // symbol comes from and format the link info @@ -412,7 +412,7 @@ addgotsym(Sym *s) addaddrplus(rela, got, s->got); adduint64(rela, ELF64_R_INFO(s->dynid, R_X86_64_GLOB_DAT)); adduint64(rela, 0); - } else if(HEADTYPE == 6) { // Mach-O + } else if(HEADTYPE == Hdarwin) { adduint32(lookup(".linkedit.got", 0), s->dynid); } else { diag("addgotsym: unsupported binary format"); @@ -486,7 +486,7 @@ adddynsym(Sym *s) elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, addstring(lookup(".dynstr", 0), s->dynimplib)); } - } else if(HEADTYPE == 6) { + } else if(HEADTYPE == Hdarwin) { // Mach-o symbol nlist64 d = lookup(".dynsym", 0); name = s->dynimpname; @@ -539,7 +539,7 @@ adddynlib(char *lib) if(s->size == 0) addstring(s, ""); elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, addstring(s, lib)); - } else if(HEADTYPE == 6) { // Mach-O + } else if(HEADTYPE == Hdarwin) { machoadddynlib(lib); } else { diag("adddynlib: unsupported binary format"); @@ -551,7 +551,7 @@ doelf(void) { Sym *s, *shstrtab, *dynstr; - if(HEADTYPE != 7 && HEADTYPE != 9) + if(HEADTYPE != Hlinux && HEADTYPE != Hfreebsd) return; /* predefine strings we need for section headers */ @@ -717,20 +717,20 @@ asmb(void) datblk(segdata.vaddr, segdata.filelen); machlink = 0; - if(HEADTYPE == 6) + if(HEADTYPE == Hdarwin) machlink = domacholink(); switch(HEADTYPE) { default: diag("unknown header type %d", HEADTYPE); - case 2: - case 5: + case Hplan9x32: + case Helf: break; - case 6: + case Hdarwin: debug['8'] = 1; /* 64-bit addresses */ break; - case 7: - case 9: + case Hlinux: + case Hfreebsd: debug['8'] = 1; /* 64-bit addresses */ /* index of elf text section; needed by asmelfsym, double-checked below */ /* !debug['d'] causes extra sections before the .text section */ @@ -738,7 +738,7 @@ asmb(void) if(!debug['d']) elftextsh += 10; break; - case 10: + case Hwindows: break; } @@ -752,20 +752,20 @@ asmb(void) Bflush(&bso); switch(HEADTYPE) { default: - case 2: - case 5: + case Hplan9x32: + case Helf: debug['s'] = 1; symo = HEADR+segtext.len+segdata.filelen; break; - case 6: + case Hdarwin: symo = rnd(HEADR+segtext.len, INITRND)+rnd(segdata.filelen, INITRND)+machlink; break; - case 7: - case 9: + case Hlinux: + case Hfreebsd: symo = rnd(HEADR+segtext.len, INITRND)+segdata.filelen; symo = rnd(symo, INITRND); break; - case 10: + case Hwindows: symo = rnd(HEADR+segtext.filelen, PEFILEALIGN)+segdata.filelen; symo = rnd(symo, PEFILEALIGN); break; @@ -791,7 +791,7 @@ asmb(void) lputl(symsize); lputl(lcsize); cflush(); - if(HEADTYPE != 10 && !debug['s']) { + if(HEADTYPE != Hwindows && !debug['s']) { elfsymo = symo+8+symsize+lcsize; seek(cout, elfsymo, 0); asmelfsym64(); @@ -813,7 +813,7 @@ asmb(void) seek(cout, 0L, 0); switch(HEADTYPE) { default: - case 2: /* plan9 */ + case Hplan9x32: /* plan9 */ magic = 4*26*26+7; magic |= 0x00008000; /* fat header */ lputb(magic); /* magic */ @@ -827,7 +827,7 @@ asmb(void) lputb(lcsize); /* line offsets */ vputb(vl); /* va of entry */ break; - case 3: /* plan9 */ + case Hplan9x64: /* plan9 */ magic = 4*26*26+7; lputb(magic); /* magic */ lputb(segtext.filelen); /* sizes */ @@ -838,11 +838,11 @@ asmb(void) lputb(spsize); /* sp offsets */ lputb(lcsize); /* line offsets */ break; - case 6: + case Hdarwin: asmbmacho(); break; - case 7: - case 9: + case Hlinux: + case Hfreebsd: /* elf amd-64 */ eh = getElfEhdr(); @@ -871,10 +871,10 @@ asmb(void) sh->addralign = 1; if(interpreter == nil) { switch(HEADTYPE) { - case 7: + case Hlinux: interpreter = linuxdynld; break; - case 9: + case Hfreebsd: interpreter = freebsddynld; break; } @@ -1032,7 +1032,7 @@ asmb(void) eh->ident[EI_MAG1] = 'E'; eh->ident[EI_MAG2] = 'L'; eh->ident[EI_MAG3] = 'F'; - if(HEADTYPE == 9) + if(HEADTYPE == Hfreebsd) eh->ident[EI_OSABI] = 9; eh->ident[EI_CLASS] = ELFCLASS64; eh->ident[EI_DATA] = ELFDATA2LSB; @@ -1055,7 +1055,7 @@ asmb(void) if(a+elfwriteinterp() > ELFRESERVE) diag("ELFRESERVE too small: %d > %d", a, ELFRESERVE); break; - case 10: + case Hwindows: asmbpe(); break; } diff --git a/src/cmd/6l/doc.go b/src/cmd/6l/doc.go index 97fa2cc5a..cc7782cfe 100644 --- a/src/cmd/6l/doc.go +++ b/src/cmd/6l/doc.go @@ -28,10 +28,14 @@ Options new in this version: -e Emit an extra ELF-compatible symbol table useful with tools such as nm, gdb, and oprofile. This option makes the binary file considerably larger. --H6 +-Hdarwin Write Apple Mach-O binaries (default when $GOOS is darwin) --H7 +-Hlinux Write Linux ELF binaries (default when $GOOS is linux) +-Hfreebsd + Write FreeBSD ELF binaries (default when $GOOS is freebsd) +-Hwindows + Write Windows PE32+ binaries (default when $GOOS is windows) -I interpreter Set the ELF dynamic linker to use. -L dir1 -L dir2 diff --git a/src/cmd/6l/l.h b/src/cmd/6l/l.h index 70473ecd2..6933d8eb1 100644 --- a/src/cmd/6l/l.h +++ b/src/cmd/6l/l.h @@ -39,6 +39,7 @@ enum { + thechar = '6', PtrSize = 8 }; @@ -111,6 +112,7 @@ struct Prog }; #define datasize from.scale #define textflag from.scale +#define iscall(p) ((p)->as == ACALL) struct Auto { @@ -129,6 +131,7 @@ struct Sym uchar reachable; uchar dynexport; uchar special; + uchar stkcheck; int32 dynid; int32 sig; int32 plt; @@ -367,7 +370,6 @@ EXTERN Sym* fromgotype; // type symbol on last p->from read EXTERN vlong textstksiz; EXTERN vlong textarg; -extern char thechar; EXTERN int elfstrsize; EXTERN char* elfstrdat; EXTERN int elftextsh; diff --git a/src/cmd/6l/obj.c b/src/cmd/6l/obj.c index f9e257842..f113e3ec1 100644 --- a/src/cmd/6l/obj.c +++ b/src/cmd/6l/obj.c @@ -40,17 +40,29 @@ #include <ar.h> char *noname = "<none>"; -char thechar = '6'; char* thestring = "amd64"; char* paramspace = "FP"; +Header headers[] = { + "plan9x32", Hplan9x32, + "plan9", Hplan9x64, + "elf", Helf, + "darwin", Hdarwin, + "linux", Hlinux, + "freebsd", Hfreebsd, + "windows", Hwindows, + "windowsgui", Hwindows, + 0, 0 +}; + /* - * -H2 -T4136 -R4096 is plan9 64-bit format - * -H3 -T4128 -R4096 is plan9 32-bit format - * -H5 -T0x80110000 -R4096 is ELF32 - * -H6 -Tx -Rx is apple MH-exec - * -H7 -Tx -Rx is linux elf-exec - * -H9 -Tx -Rx is FreeBSD elf-exec + * -Hplan9x32 -T4136 -R4096 is plan9 64-bit format + * -Hplan9 -T4128 -R4096 is plan9 32-bit format + * -Helf -T0x80110000 -R4096 is ELF32 + * -Hdarwin -Tx -Rx is apple MH-exec + * -Hlinux -Tx -Rx is linux elf-exec + * -Hfreebsd -Tx -Rx is FreeBSD elf-exec + * -Hwindows -Tx -Rx is MS Windows PE32+ * * options used: 189BLQSWabcjlnpsvz */ @@ -94,7 +106,7 @@ main(int argc, char *argv[]) INITENTRY = EARGF(usage()); break; case 'H': - HEADTYPE = atolwhex(EARGF(usage())); + HEADTYPE = headtype(EARGF(usage())); break; case 'I': interpreter = EARGF(usage()); @@ -123,31 +135,15 @@ main(int argc, char *argv[]) usage(); libinit(); - if(rpath == nil) - rpath = smprint("%s/pkg/%s_%s", goroot, goos, goarch); - if(HEADTYPE == -1) { - HEADTYPE = 2; - if(strcmp(goos, "linux") == 0) - HEADTYPE = 7; - else - if(strcmp(goos, "darwin") == 0) - HEADTYPE = 6; - else - if(strcmp(goos, "freebsd") == 0) - HEADTYPE = 9; - else - if(strcmp(goos, "windows") == 0) - HEADTYPE = 10; - else - print("goos is not known: %s\n", goos); - } + if(HEADTYPE == -1) + HEADTYPE = headtype(goos); switch(HEADTYPE) { default: diag("unknown -H option"); errorexit(); - case 2: /* plan 9 */ + case Hplan9x32: /* plan 9 */ HEADR = 32L+8L; if(INITTEXT == -1) INITTEXT = 4096+HEADR; @@ -156,7 +152,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 3: /* plan 9 */ + case Hplan9x64: /* plan 9 */ HEADR = 32L; if(INITTEXT == -1) INITTEXT = 4096+32; @@ -165,7 +161,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 5: /* elf32 executable */ + case Helf: /* elf32 executable */ HEADR = rnd(52L+3*32L, 16); if(INITTEXT == -1) INITTEXT = 0x80110000L; @@ -174,7 +170,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 6: /* apple MACH */ + case Hdarwin: /* apple MACH */ /* * OS X system constant - offset from 0(GS) to our TLS. * Explained in ../../libcgo/darwin_amd64.c. @@ -189,8 +185,8 @@ main(int argc, char *argv[]) if(INITDAT == -1) INITDAT = 0; break; - case 7: /* elf64 executable */ - case 9: /* freebsd */ + case Hlinux: /* elf64 executable */ + case Hfreebsd: /* freebsd */ /* * ELF uses TLS offset negative from FS. * Translate 0(FS) and 8(FS) into -16(FS) and -8(FS). @@ -207,7 +203,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 10: /* PE executable */ + case Hwindows: /* PE executable */ peinit(); HEADR = PEFILEHEADR; if(INITTEXT == -1) @@ -252,9 +248,10 @@ main(int argc, char *argv[]) patch(); follow(); doelf(); - if(HEADTYPE == 6) + if(HEADTYPE == Hdarwin) domacho(); dostkoff(); + dostkcheck(); paramspace = "SP"; /* (FP) now (SP) on output */ if(debug['p']) if(debug['1']) @@ -262,7 +259,7 @@ main(int argc, char *argv[]) else doprof2(); span(); - if(HEADTYPE == 10) + if(HEADTYPE == Hwindows) dope(); addexport(); textaddress(); @@ -270,6 +267,7 @@ main(int argc, char *argv[]) symtab(); dodata(); address(); + doweak(); reloc(); asmb(); undef(); diff --git a/src/cmd/6l/pass.c b/src/cmd/6l/pass.c index 5eb221a35..8fda94392 100644 --- a/src/cmd/6l/pass.c +++ b/src/cmd/6l/pass.c @@ -32,16 +32,10 @@ #include "l.h" #include "../ld/lib.h" +#include "../../pkg/runtime/stack.h" static void xfol(Prog*, Prog**); -// see ../../runtime/proc.c:/StackGuard -enum -{ - StackSmall = 128, - StackBig = 4096, -}; - Prog* brchain(Prog *p) { @@ -277,7 +271,7 @@ patch(void) vexit = s->value; for(cursym = textp; cursym != nil; cursym = cursym->next) for(p = cursym->text; p != P; p = p->link) { - if(HEADTYPE == 10) { + if(HEADTYPE == Hwindows) { // Windows // Convert // op n(GS), reg @@ -289,7 +283,7 @@ patch(void) // a different method is used to access them. if(p->from.type == D_INDIR+D_GS && p->to.type >= D_AX && p->to.type <= D_DI - && p->from.offset != 0x58) { + && p->from.offset <= 8) { q = appendp(p); q->from = p->from; q->from.type = D_INDIR + p->to.type; @@ -300,7 +294,7 @@ patch(void) p->from.offset = 0x58; } } - if(HEADTYPE == 7 || HEADTYPE == 9) { + if(HEADTYPE == Hlinux || HEADTYPE == Hfreebsd) { // ELF uses FS instead of GS. if(p->from.type == D_INDIR+D_GS) p->from.type = D_INDIR+D_FS; @@ -428,13 +422,13 @@ dostkoff(void) if(!(p->from.scale & NOSPLIT)) { p = appendp(p); // load g into CX p->as = AMOVQ; - if(HEADTYPE == 7 || HEADTYPE == 9) // ELF uses FS + if(HEADTYPE == Hlinux || HEADTYPE == Hfreebsd) // ELF uses FS p->from.type = D_INDIR+D_FS; else p->from.type = D_INDIR+D_GS; p->from.offset = tlsoffset+0; p->to.type = D_CX; - if(HEADTYPE == 10) { // Windows + if(HEADTYPE == Hwindows) { // movq %gs:0x58, %rcx // movq (%rcx), %rcx p->as = AMOVQ; @@ -724,15 +718,3 @@ atolwhex(char *s) n = -n; return n; } - -void -undef(void) -{ - int i; - Sym *s; - - for(i=0; i<NHASH; i++) - for(s = hash[i]; s != S; s = s->hash) - if(s->type == SXREF) - diag("%s: not defined", s->name); -} diff --git a/src/cmd/8a/lex.c b/src/cmd/8a/lex.c index d5fa959aa..ca18b69ce 100644 --- a/src/cmd/8a/lex.c +++ b/src/cmd/8a/lex.c @@ -332,6 +332,7 @@ struct "CMPSB", LTYPE0, ACMPSB, "CMPSL", LTYPE0, ACMPSL, "CMPSW", LTYPE0, ACMPSW, + "CMPXCHG8B", LTYPE1, ACMPXCHG8B, "CMPXCHGB", LTYPE3, ACMPXCHGB, "CMPXCHGL", LTYPE3, ACMPXCHGL, "CMPXCHGW", LTYPE3, ACMPXCHGW, @@ -546,6 +547,9 @@ struct "VERW", LTYPE2, AVERW, "WAIT", LTYPE0, AWAIT, "WORD", LTYPE2, AWORD, + "XADDB", LTYPE3, AXADDB, + "XADDL", LTYPE3, AXADDL, + "XADDW", LTYPE3, AXADDW, "XCHGB", LTYPE3, AXCHGB, "XCHGL", LTYPE3, AXCHGL, "XCHGW", LTYPE3, AXCHGW, diff --git a/src/cmd/8l/8.out.h b/src/cmd/8l/8.out.h index 0866f05f0..03db0016b 100644 --- a/src/cmd/8l/8.out.h +++ b/src/cmd/8l/8.out.h @@ -392,6 +392,11 @@ enum as ACMPXCHGB, ACMPXCHGL, ACMPXCHGW, + ACMPXCHG8B, + + AXADDB, + AXADDL, + AXADDW, /* conditional move */ ACMOVLCC, diff --git a/src/cmd/8l/asm.c b/src/cmd/8l/asm.c index d90eab7e7..1e760d89e 100644 --- a/src/cmd/8l/asm.c +++ b/src/cmd/8l/asm.c @@ -246,7 +246,7 @@ adddynrel(Sym *s, Reloc *r) r->sym = S; return; } - if(HEADTYPE == 6 && s->size == PtrSize && r->off == 0) { + if(HEADTYPE == Hdarwin && s->size == PtrSize && r->off == 0) { // Mach-O relocations are a royal pain to lay out. // They use a compact stateful bytecode representation // that is too much bother to deal with. @@ -356,7 +356,7 @@ addpltsym(Sym *s) adduint32(rel, ELF32_R_INFO(s->dynid, R_386_JMP_SLOT)); s->plt = plt->size - 16; - } else if(HEADTYPE == 6) { // Mach-O + } else if(HEADTYPE == Hdarwin) { // Same laziness as in 6l. Sym *plt; @@ -395,7 +395,7 @@ addgotsym(Sym *s) rel = lookup(".rel", 0); addaddrplus(rel, got, s->got); adduint32(rel, ELF32_R_INFO(s->dynid, R_386_GLOB_DAT)); - } else if(HEADTYPE == 6) { // Mach-O + } else if(HEADTYPE == Hdarwin) { adduint32(lookup(".linkedit.got", 0), s->dynid); } else { diag("addgotsym: unsupported binary format"); @@ -465,7 +465,7 @@ adddynsym(Sym *s) } adduint16(d, t); } - } else if(HEADTYPE == 6) { + } else if(HEADTYPE == Hdarwin) { // Mach-O symbol nlist32 d = lookup(".dynsym", 0); name = s->dynimpname; @@ -481,7 +481,7 @@ adddynsym(Sym *s) adduint8(d, 0); // section adduint16(d, 0); // desc adduint32(d, 0); // value - } else if(HEADTYPE != 10) { + } else if(HEADTYPE != Hwindows) { diag("adddynsym: unsupported binary format"); } } @@ -499,9 +499,9 @@ adddynlib(char *lib) if(s->size == 0) addstring(s, ""); elfwritedynent(lookup(".dynamic", 0), DT_NEEDED, addstring(s, lib)); - } else if(HEADTYPE == 6) { // Mach-O + } else if(HEADTYPE == Hdarwin) { machoadddynlib(lib); - } else if(HEADTYPE != 10) { + } else if(HEADTYPE != Hwindows) { diag("adddynlib: unsupported binary format"); } } @@ -673,7 +673,7 @@ asmb(void) datblk(segdata.vaddr, segdata.filelen); machlink = 0; - if(HEADTYPE == 6) + if(HEADTYPE == Hdarwin) machlink = domacholink(); if(iself) { @@ -697,28 +697,28 @@ asmb(void) default: if(iself) goto Elfsym; - case 0: + case Hgarbunix: seek(cout, rnd(HEADR+segtext.filelen, 8192)+segdata.filelen, 0); break; - case 1: + case Hunixcoff: seek(cout, rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen, 0); break; - case 2: + case Hplan9x32: symo = HEADR+segtext.filelen+segdata.filelen; break; - case 3: - case 4: + case Hmsdoscom: + case Hmsdosexe: debug['s'] = 1; symo = HEADR+segtext.filelen+segdata.filelen; break; - case 6: + case Hdarwin: symo = rnd(HEADR+segtext.filelen, INITRND)+rnd(segdata.filelen, INITRND)+machlink; break; Elfsym: symo = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; symo = rnd(symo, INITRND); break; - case 10: + case Hwindows: // TODO(brainman): not sure what symo meant to be, but it is not used for Windows PE for now anyway symo = rnd(HEADR+segtext.filelen, PEFILEALIGN)+segdata.filelen; symo = rnd(symo, PEFILEALIGN); @@ -727,7 +727,7 @@ asmb(void) if(!debug['s']) { seek(cout, symo, 0); - if(HEADTYPE == 2) { + if(HEADTYPE == Hplan9x32) { asmplan9sym(); cflush(); @@ -740,7 +740,7 @@ asmb(void) cflush(); } - } else if(HEADTYPE != 10) { + } else if(HEADTYPE != Hwindows) { if(debug['v']) Bprint(&bso, "%5.2f dwarf\n", cputime()); dwarfemitdebugsections(); @@ -755,7 +755,7 @@ asmb(void) default: if(iself) goto Elfput; - case 0: /* garbage */ + case Hgarbunix: /* garbage */ lputb(0x160L<<16); /* magic and sections */ lputb(0L); /* time and date */ lputb(rnd(HEADR+segtext.filelen, 4096)+segdata.filelen); @@ -777,7 +777,7 @@ asmb(void) lputb(~0L); /* gp value ?? */ break; lputl(0); /* x */ - case 1: /* unix coff */ + case Hunixcoff: /* unix coff */ /* * file header */ @@ -845,7 +845,7 @@ asmb(void) lputl(0); /* relocation, line numbers */ lputl(0x200); /* flags comment only */ break; - case 2: /* plan9 */ + case Hplan9x32: /* plan9 */ magic = 4*11*11+7; lputb(magic); /* magic */ lputb(segtext.filelen); /* sizes */ @@ -856,10 +856,10 @@ asmb(void) lputb(spsize); /* sp offsets */ lputb(lcsize); /* line offsets */ break; - case 3: + case Hmsdoscom: /* MS-DOS .COM */ break; - case 4: + case Hmsdosexe: /* fake MS-DOS .EXE */ v = rnd(HEADR+segtext.filelen, INITRND)+segdata.filelen; wputl(0x5A4D); /* 'MZ' */ @@ -882,13 +882,13 @@ asmb(void) wputl(0x0000); /* overlay number */ break; - case 6: + case Hdarwin: asmbmacho(); break; Elfput: /* elf 386 */ - if(HEADTYPE == 11) + if(HEADTYPE == Htiny) debug['d'] = 1; eh = getElfEhdr(); @@ -917,10 +917,10 @@ asmb(void) sh->addralign = 1; if(interpreter == nil) { switch(HEADTYPE) { - case 7: + case Hlinux: interpreter = linuxdynld; break; - case 9: + case Hfreebsd: interpreter = freebsddynld; break; } @@ -1068,7 +1068,7 @@ asmb(void) eh->ident[EI_DATA] = ELFDATA2LSB; eh->ident[EI_VERSION] = EV_CURRENT; switch(HEADTYPE) { - case 9: + case Hfreebsd: eh->ident[EI_OSABI] = 9; break; } @@ -1093,7 +1093,7 @@ asmb(void) diag("ELFRESERVE too small: %d > %d", a, ELFRESERVE); break; - case 10: + case Hwindows: asmbpe(); break; } diff --git a/src/cmd/8l/doc.go b/src/cmd/8l/doc.go index ef5ebc31d..b70888907 100644 --- a/src/cmd/8l/doc.go +++ b/src/cmd/8l/doc.go @@ -25,10 +25,16 @@ Options new in this version: Elide the dynamic linking header. With this option, the binary is statically linked and does not refer to dynld. Without this option (the default), the binary's contents are identical but it is loaded with dynld. --H6 +-Hplan9 + Write Plan 9 32-bit format binaries (default when $GOOS is plan9) +-Hdarwin Write Apple Mach-O binaries (default when $GOOS is darwin) --H7 +-Hlinux Write Linux ELF binaries (default when $GOOS is linux) +-Hfreebsd + Write FreeBSD ELF binaries (default when $GOOS is freebsd) +-Hwindows + Write Windows PE32 binaries (default when $GOOS is windows) -I interpreter Set the ELF dynamic linker to use. -L dir1 -L dir2 diff --git a/src/cmd/8l/l.h b/src/cmd/8l/l.h index f2546cf20..e4650ee58 100644 --- a/src/cmd/8l/l.h +++ b/src/cmd/8l/l.h @@ -39,6 +39,7 @@ enum { + thechar = '8', PtrSize = 4 }; @@ -110,6 +111,7 @@ struct Prog }; #define datasize from.scale #define textflag from.scale +#define iscall(p) ((p)->as == ACALL) struct Auto { @@ -128,6 +130,7 @@ struct Sym uchar reachable; uchar dynexport; uchar special; + uchar stkcheck; int32 value; int32 size; int32 sig; diff --git a/src/cmd/8l/obj.c b/src/cmd/8l/obj.c index 9c687f2fc..d505dc10e 100644 --- a/src/cmd/8l/obj.c +++ b/src/cmd/8l/obj.c @@ -44,21 +44,36 @@ #endif char *noname = "<none>"; -char thechar = '8'; char *thestring = "386"; +Header headers[] = { + "garbunix", Hgarbunix, + "unixcoff", Hunixcoff, + "plan9", Hplan9x32, + "msdoscom", Hmsdoscom, + "msdosexe", Hmsdosexe, + "darwin", Hdarwin, + "linux", Hlinux, + "nacl", Hnacl, + "freebsd", Hfreebsd, + "windows", Hwindows, + "windowsgui", Hwindows, + "tiny", Htiny, + 0, 0 +}; + /* - * -H0 -T0x40004C -D0x10000000 is garbage unix - * -H1 -T0xd0 -R4 is unix coff - * -H2 -T4128 -R4096 is plan9 format - * -H3 -Tx -Rx is MS-DOS .COM - * -H4 -Tx -Rx is fake MS-DOS .EXE - * -H6 -Tx -Rx is Apple Mach-O - * -H7 -Tx -Rx is Linux ELF32 - * -H8 -Tx -Rx was Google Native Client - * -H9 -Tx -Rx is FreeBSD ELF32 - * -H10 -Tx -Rx is MS Windows PE - * -H11 -Tx -Rx is tiny (os image) + * -Hgarbunix -T0x40004C -D0x10000000 is garbage unix + * -Hunixcoff -T0xd0 -R4 is unix coff + * -Hplan9 -T4128 -R4096 is plan9 format + * -Hmsdoscom -Tx -Rx is MS-DOS .COM + * -Hmsdosexe -Tx -Rx is fake MS-DOS .EXE + * -Hdarwin -Tx -Rx is Apple Mach-O + * -Hlinux -Tx -Rx is Linux ELF32 + * -Hnacl -Tx -Rx was Google Native Client + * -Hfreebsd -Tx -Rx is FreeBSD ELF32 + * -Hwindows -Tx -Rx is MS Windows PE32 + * -Htiny -Tx -Rx is tiny (os image) */ void @@ -100,7 +115,7 @@ main(int argc, char *argv[]) INITENTRY = EARGF(usage()); break; case 'H': - HEADTYPE = atolwhex(EARGF(usage())); + HEADTYPE = headtype(EARGF(usage())); break; case 'I': interpreter = EARGF(usage()); @@ -130,46 +145,24 @@ main(int argc, char *argv[]) mywhatsys(); // get goos - if(HEADTYPE == -1) { - HEADTYPE = 2; - if(strcmp(goos, "linux") == 0) - HEADTYPE = 7; - else - if(strcmp(goos, "darwin") == 0) - HEADTYPE = 6; - else - if(strcmp(goos, "freebsd") == 0) - HEADTYPE = 9; - else - if(strcmp(goos, "windows") == 0) - HEADTYPE = 10; - else - if(strcmp(goos, "tiny") == 0) - HEADTYPE = 11; - else - if(strcmp(goos, "plan9") == 0) - HEADTYPE = 2; - else - print("goos is not known: %s\n", goos); - } + if(HEADTYPE == -1) + HEADTYPE = headtype(goos); if(outfile == nil) { - if(HEADTYPE == 10) + if(HEADTYPE == Hwindows) outfile = "8.out.exe"; else outfile = "8.out"; } libinit(); - if(rpath == nil) - rpath = smprint("%s/pkg/%s_%s", goroot, goos, goarch); switch(HEADTYPE) { default: diag("unknown -H option"); errorexit(); - case 0: /* this is garbage */ + case Hgarbunix: /* this is garbage */ HEADR = 20L+56L; if(INITTEXT == -1) INITTEXT = 0x40004CL; @@ -178,7 +171,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 0; break; - case 1: /* is unix coff */ + case Hunixcoff: /* is unix coff */ HEADR = 0xd0L; if(INITTEXT == -1) INITTEXT = 0xd0; @@ -187,7 +180,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 0; break; - case 2: /* plan 9 */ + case Hplan9x32: /* plan 9 */ tlsoffset = -8; HEADR = 32L; if(INITTEXT == -1) @@ -197,7 +190,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 1; break; - case 3: /* MS-DOS .COM */ + case Hmsdoscom: /* MS-DOS .COM */ HEADR = 0; if(INITTEXT == -1) INITTEXT = 0x0100; @@ -206,7 +199,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4; break; - case 4: /* fake MS-DOS .EXE */ + case Hmsdosexe: /* fake MS-DOS .EXE */ HEADR = 0x200; if(INITTEXT == -1) INITTEXT = 0x0100; @@ -218,7 +211,7 @@ main(int argc, char *argv[]) if(debug['v']) Bprint(&bso, "HEADR = 0x%d\n", HEADR); break; - case 6: /* apple MACH */ + case Hdarwin: /* apple MACH */ /* * OS X system constant - offset from %gs to our TLS. * Explained in ../../libcgo/darwin_386.c. @@ -233,8 +226,8 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 7: /* elf32 executable */ - case 9: + case Hlinux: /* elf32 executable */ + case Hfreebsd: /* * ELF uses TLS offsets negative from %gs. * Translate 0(GS) and 4(GS) into -8(GS) and -4(GS). @@ -251,7 +244,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = 4096; break; - case 10: /* PE executable */ + case Hwindows: /* PE executable */ peinit(); HEADR = PEFILEHEADR; if(INITTEXT == -1) @@ -261,7 +254,7 @@ main(int argc, char *argv[]) if(INITRND == -1) INITRND = PESECTALIGN; break; - case 11: + case Htiny: tlsoffset = 0; elfinit(); HEADR = ELFRESERVE; @@ -306,9 +299,9 @@ main(int argc, char *argv[]) patch(); follow(); doelf(); - if(HEADTYPE == 6) + if(HEADTYPE == Hdarwin) domacho(); - if(HEADTYPE == 10) + if(HEADTYPE == Hwindows) dope(); dostkoff(); if(debug['p']) @@ -323,6 +316,7 @@ main(int argc, char *argv[]) symtab(); dodata(); address(); + doweak(); reloc(); asmb(); undef(); diff --git a/src/cmd/8l/optab.c b/src/cmd/8l/optab.c index fceab785d..1e89a2105 100644 --- a/src/cmd/8l/optab.c +++ b/src/cmd/8l/optab.c @@ -702,6 +702,11 @@ Optab optab[] = { ACMPXCHGB, yrb_mb, Pm, 0xb0 }, { ACMPXCHGL, yrl_ml, Pm, 0xb1 }, { ACMPXCHGW, yrl_ml, Pm, 0xb1 }, + { ACMPXCHG8B, yscond, Pm, 0xc7,(01) }, + + { AXADDB, yrb_mb, Pb, 0x0f,0xc0 }, + { AXADDL, yrl_ml, Pm, 0xc1 }, + { AXADDW, yrl_ml, Pe, 0x0f,0xc1 }, { ACMOVLCC, yml_rl, Pm, 0x43 }, { ACMOVLCS, yml_rl, Pm, 0x42 }, diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c index 67acfa167..294926f29 100644 --- a/src/cmd/8l/pass.c +++ b/src/cmd/8l/pass.c @@ -262,12 +262,13 @@ patch(void) s = lookup("exit", 0); vexit = s->value; - if(HEADTYPE == 2) + plan9_tos = S; + if(HEADTYPE == Hplan9x32) plan9_tos = lookup("_tos", 0); for(cursym = textp; cursym != nil; cursym = cursym->next) { for(p = cursym->text; p != P; p = p->link) { - if(HEADTYPE == 10) { // Windows + if(HEADTYPE == Hwindows) { // Convert // op n(GS), reg // to @@ -288,7 +289,7 @@ patch(void) p->from.offset = 0x2C; } } - if(HEADTYPE == 7) { // Linux + if(HEADTYPE == Hlinux) { // Running binaries under Xen requires using // MOVL 0(GS), reg // and then off(reg) instead of saying off(GS) directly @@ -305,7 +306,7 @@ patch(void) p->from.offset = 0; } } - if(HEADTYPE == 2) { // Plan 9 + if(HEADTYPE == Hplan9x32) { if(p->from.type == D_INDIR+D_GS && p->to.type >= D_AX && p->to.type <= D_DI) { q = appendp(p); @@ -412,7 +413,8 @@ dostkoff(void) symmorestack->text->from.scale |= NOSPLIT; } - if(HEADTYPE == 2) + plan9_tos = S; + if(HEADTYPE == Hplan9x32) plan9_tos = lookup("_tos", 0); for(cursym = textp; cursym != nil; cursym = cursym->next) { @@ -430,7 +432,7 @@ dostkoff(void) if(!(p->from.scale & NOSPLIT)) { p = appendp(p); // load g into CX switch(HEADTYPE) { - case 10: // Windows + case Hwindows: p->as = AMOVL; p->from.type = D_INDIR+D_FS; p->from.offset = 0x2c; @@ -443,7 +445,7 @@ dostkoff(void) p->to.type = D_CX; break; - case 7: // Linux + case Hlinux: p->as = AMOVL; p->from.type = D_INDIR+D_GS; p->from.offset = 0; @@ -456,7 +458,7 @@ dostkoff(void) p->to.type = D_CX; break; - case 2: // Plan 9 + case Hplan9x32: p->as = AMOVL; p->from.type = D_EXTERN; p->from.sym = plan9_tos; @@ -664,15 +666,3 @@ atolwhex(char *s) n = -n; return n; } - -void -undef(void) -{ - int i; - Sym *s; - - for(i=0; i<NHASH; i++) - for(s = hash[i]; s != S; s = s->hash) - if(s->type == SXREF) - diag("%s(%d): not defined", s->name, s->version); -} diff --git a/src/cmd/Makefile b/src/cmd/Makefile new file mode 100644 index 000000000..104e9f5df --- /dev/null +++ b/src/cmd/Makefile @@ -0,0 +1,66 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../Make.inc + +all: install + +# Only build tools for current architecture, and only tools written in C. +# The tools written in Go are managed by ../pkg/Makefile. +DIRS=\ + $(O)a\ + $(O)c\ + $(O)g\ + $(O)l\ + cc\ + cov\ + gc\ + godefs\ + gopack\ + gotest\ + nm\ + prof\ + +# Clean applies to all directories, even for other architectures or +# written in Go. +CLEANDIRS=\ + $(DIRS)\ + 5a\ + 5c\ + 5g\ + 5l\ + 6a\ + 6c\ + 6g\ + 6l\ + 8a\ + 8c\ + 8g\ + 8l\ + cgo\ + ebnflint\ + godoc\ + gofmt\ + goinstall\ + goyacc\ + hgpatch\ + +install: $(patsubst %,%.install,$(DIRS)) +clean: $(patsubst %,%.clean,$(CLEANDIRS)) + +%.install: + @echo + @echo %%%% making $* %%%% + @echo + $(MAKE) -C $* install + +gc.install $(O)c.install: cc.install +$(O)g.install: gc.install +$(O)a.install $(O)c.install $(O)g.install: $(O)l.install + +%.clean: + $(MAKE) -C $* clean + +echo-dirs: + @echo $(DIRS) diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index e6ce21ed3..f7ecc9e14 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -599,7 +599,7 @@ func (p *Package) gccMachine() string { return "-m32" } -const gccTmp = "_cgo_.o" +const gccTmp = "_obj/_cgo_.o" // gccCmd returns the gcc command line to use for compiling // the input. @@ -776,6 +776,32 @@ var dwarfToName = map[string]string{ const signedDelta = 64 +// String returns the current type representation. Format arguments +// are assembled within this method so that any changes in mutable +// values are taken into account. +func (tr *TypeRepr) String() string { + if len(tr.Repr) == 0 { + return "" + } + if len(tr.FormatArgs) == 0 { + return tr.Repr + } + return fmt.Sprintf(tr.Repr, tr.FormatArgs...) +} + +// Empty returns true if the result of String would be "". +func (tr *TypeRepr) Empty() bool { + return len(tr.Repr) == 0 +} + +// Set modifies the type representation. +// If fargs are provided, repr is used as a format for fmt.Sprintf. +// Otherwise, repr is used unprocessed as the type representation. +func (tr *TypeRepr) Set(repr string, fargs ...interface{}) { + tr.Repr = repr + tr.FormatArgs = fargs +} + // Type returns a *Type with the same memory layout as // dtype when used as the type of a variable or a struct field. func (c *typeConv) Type(dtype dwarf.Type) *Type { @@ -789,16 +815,15 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { t := new(Type) t.Size = dtype.Size() t.Align = -1 - t.C = dtype.Common().Name - t.EnumValues = nil + t.C = &TypeRepr{Repr: dtype.Common().Name} c.m[dtype] = t if t.Size < 0 { // Unsized types are [0]byte t.Size = 0 t.Go = c.Opaque(0) - if t.C == "" { - t.C = "void" + if t.C.Empty() { + t.C.Set("void") } return t } @@ -827,7 +852,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { sub := c.Type(dt.Type) t.Align = sub.Align gt.Elt = sub.Go - t.C = fmt.Sprintf("typeof(%s[%d])", sub.C, dt.Count) + t.C.Set("typeof(%s[%d])", sub.C, dt.Count) case *dwarf.BoolType: t.Go = c.bool @@ -844,7 +869,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { if t.Align = t.Size; t.Align >= c.ptrSize { t.Align = c.ptrSize } - t.C = "enum " + dt.EnumName + t.C.Set("enum " + dt.EnumName) signed := 0 t.EnumValues = make(map[string]int64) for _, ev := range dt.Val { @@ -932,7 +957,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { // Translate void* as unsafe.Pointer if _, ok := base(dt.Type).(*dwarf.VoidType); ok { t.Go = c.unsafePointer - t.C = "void*" + t.C.Set("void*") break } @@ -940,7 +965,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { t.Go = gt // publish before recursive call sub := c.Type(dt.Type) gt.X = sub.Go - t.C = sub.C + "*" + t.C.Set("%s*", sub.C) case *dwarf.QualType: // Ignore qualifier. @@ -955,21 +980,21 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { if tag == "" { tag = "__" + strconv.Itoa(tagGen) tagGen++ - } else if t.C == "" { - t.C = dt.Kind + " " + tag + } else if t.C.Empty() { + t.C.Set(dt.Kind + " " + tag) } name := c.Ident("_Ctype_" + dt.Kind + "_" + tag) t.Go = name // publish before recursive calls switch dt.Kind { case "union", "class": typedef[name.Name] = c.Opaque(t.Size) - if t.C == "" { - t.C = fmt.Sprintf("typeof(unsigned char[%d])", t.Size) + if t.C.Empty() { + t.C.Set("typeof(unsigned char[%d])", t.Size) } case "struct": g, csyntax, align := c.Struct(dt) - if t.C == "" { - t.C = csyntax + if t.C.Empty() { + t.C.Set(csyntax) } t.Align = align typedef[name.Name] = g @@ -1024,7 +1049,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { case *dwarf.VoidType: t.Go = c.void - t.C = "void" + t.C.Set("void") } switch dtype.(type) { @@ -1041,7 +1066,7 @@ func (c *typeConv) Type(dtype dwarf.Type) *Type { } } - if t.C == "" { + if t.C.Empty() { fatal("internal error: did not create C name for %s", dtype) } @@ -1056,11 +1081,13 @@ func (c *typeConv) FuncArg(dtype dwarf.Type) *Type { case *dwarf.ArrayType: // Arrays are passed implicitly as pointers in C. // In Go, we must be explicit. + tr := &TypeRepr{} + tr.Set("%s*", t.C) return &Type{ Size: c.ptrSize, Align: c.ptrSize, Go: &ast.StarExpr{X: t.Go}, - C: t.C + "*", + C: tr, } case *dwarf.TypedefType: // C has much more relaxed rules than Go for @@ -1189,7 +1216,7 @@ func (c *typeConv) Struct(dt *dwarf.StructType) (expr *ast.StructType, csyntax s fld[n] = &ast.Field{Names: []*ast.Ident{c.Ident(ident[f.Name])}, Type: t.Go} off += t.Size - buf.WriteString(t.C) + buf.WriteString(t.C.String()) buf.WriteString(" ") buf.WriteString(f.Name) buf.WriteString("; ") diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go index b15d34527..2dc662de5 100644 --- a/src/cmd/cgo/main.go +++ b/src/cmd/cgo/main.go @@ -82,11 +82,17 @@ type ExpFunc struct { ExpName string // name to use from C } +// A TypeRepr contains the string representation of a type. +type TypeRepr struct { + Repr string + FormatArgs []interface{} +} + // A Type collects information about a type in both the C and Go worlds. type Type struct { Size int64 Align int64 - C string + C *TypeRepr Go ast.Expr EnumValues map[string]int64 } @@ -215,6 +221,10 @@ func main() { fs[i] = f } + // make sure that _obj directory exists, so that we can write + // all the output files there. + os.Mkdir("_obj", 0777) + for i, input := range goFiles { f := fs[i] p.Translate(f) diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index ede8f57d8..4a5fa6a73 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -20,20 +20,11 @@ import ( // writeDefs creates output files to be compiled by 6g, 6c, and gcc. // (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.) func (p *Package) writeDefs() { - // The path for the shared object is slash-free so that ELF loaders - // will treat it as a relative path. We rewrite slashes to underscores. - sopath := "cgo_" + strings.Map(slashToUnderscore, p.PackagePath) - soprefix := "" - if os.Getenv("GOOS") == "darwin" { - // OS X requires its own prefix for a relative path - soprefix = "@rpath/" - } - - fgo2 := creat("_cgo_gotypes.go") - fc := creat("_cgo_defun.c") - fm := creat("_cgo_main.c") + fgo2 := creat("_obj/_cgo_gotypes.go") + fc := creat("_obj/_cgo_defun.c") + fm := creat("_obj/_cgo_main.c") - fflg := creat("_cgo_flags") + fflg := creat("_obj/_cgo_flags") for k, v := range p.CgoFlags { fmt.Fprintf(fflg, "_CGO_%s=%s\n", k, v) } @@ -94,7 +85,7 @@ func (p *Package) writeDefs() { for _, n := range p.Name { if n.FuncType != nil { - p.writeDefsFunc(fc, fgo2, n, soprefix, sopath) + p.writeDefsFunc(fc, fgo2, n) } } @@ -172,7 +163,7 @@ func (p *Package) structType(n *Name) (string, int64) { off += pad } qual := "" - if t.C[len(t.C)-1] == '*' { + if c := t.C.String(); c[len(c)-1] == '*' { qual = "const " } fmt.Fprintf(&buf, "\t\t%s%s r;\n", qual, t.C) @@ -189,13 +180,12 @@ func (p *Package) structType(n *Name) (string, int64) { } if off == 0 { fmt.Fprintf(&buf, "\t\tchar unused;\n") // avoid empty struct - off++ } fmt.Fprintf(&buf, "\t}") return buf.String(), off } -func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name, soprefix, sopath string) { +func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { name := n.Go gtype := n.FuncType.Go if n.AddError { @@ -234,6 +224,9 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name, soprefix, sopath str fmt.Fprintf(fc, "void _cgo%s%s(void*);\n", cPrefix, n.Mangle) fmt.Fprintf(fc, "\n") fmt.Fprintf(fc, "void\n") + if argSize == 0 { + argSize++ + } fmt.Fprintf(fc, "·%s(struct{uint8 x[%d];}p)\n", n.Mangle, argSize) fmt.Fprintf(fc, "{\n") fmt.Fprintf(fc, "\truntime·cgocall(_cgo%s%s, &p);\n", cPrefix, n.Mangle) @@ -271,8 +264,8 @@ func (p *Package) writeOutput(f *File, srcfile string) { base = base[0 : len(base)-3] } base = strings.Map(slashToUnderscore, base) - fgo1 := creat(base + ".cgo1.go") - fgcc := creat(base + ".cgo2.c") + fgo1 := creat("_obj/" + base + ".cgo1.go") + fgcc := creat("_obj/" + base + ".cgo2.c") p.GoFiles = append(p.GoFiles, base+".cgo1.go") p.GccFiles = append(p.GccFiles, base+".cgo2.c") @@ -340,7 +333,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) { // Write out the various stubs we need to support functions exported // from Go so that they are callable from C. func (p *Package) writeExports(fgo2, fc, fm *os.File) { - fgcc := creat("_cgo_export.c") + fgcc := creat("_obj/_cgo_export.c") fgcch := creat("_cgo_export.h") fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n") @@ -401,7 +394,6 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { } if ctype == "struct {\n" { ctype += "\t\tchar unused;\n" // avoid empty struct - off++ } ctype += "\t}" @@ -411,7 +403,7 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { if fntype.Results == nil || len(fntype.Results.List) == 0 { gccResult = "void" } else if len(fntype.Results.List) == 1 && len(fntype.Results.List[0].Names) <= 1 { - gccResult = p.cgoType(fntype.Results.List[0].Type).C + gccResult = p.cgoType(fntype.Results.List[0].Type).C.String() } else { fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName) fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName) @@ -426,7 +418,7 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { // Build the wrapper function compiled by gcc. s := fmt.Sprintf("%s %s(", gccResult, exp.ExpName) if fn.Recv != nil { - s += p.cgoType(fn.Recv.List[0].Type).C + s += p.cgoType(fn.Recv.List[0].Type).C.String() s += " recv" } forFieldList(fntype.Params, @@ -453,7 +445,7 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { func(i int, atype ast.Expr) { fmt.Fprintf(fgcc, "\ta.p%d = p%d;\n", i, i) }) - fmt.Fprintf(fgcc, "\tcrosscall2(_cgoexp%s_%s, &a, (int) sizeof a);\n", cPrefix, exp.ExpName) + fmt.Fprintf(fgcc, "\tcrosscall2(_cgoexp%s_%s, &a, %d);\n", cPrefix, exp.ExpName, off) if gccResult != "void" { if len(fntype.Results.List) == 1 && len(fntype.Results.List[0].Names) <= 1 { fmt.Fprintf(fgcc, "\treturn a.r0;\n") @@ -542,24 +534,28 @@ func forFieldList(fl *ast.FieldList, fn func(int, ast.Expr)) { } } +func c(repr string, args ...interface{}) *TypeRepr { + return &TypeRepr{repr, args} +} + // Map predeclared Go types to Type. var goTypes = map[string]*Type{ - "int": &Type{Size: 4, Align: 4, C: "int"}, - "uint": &Type{Size: 4, Align: 4, C: "uint"}, - "int8": &Type{Size: 1, Align: 1, C: "schar"}, - "uint8": &Type{Size: 1, Align: 1, C: "uchar"}, - "int16": &Type{Size: 2, Align: 2, C: "short"}, - "uint16": &Type{Size: 2, Align: 2, C: "ushort"}, - "int32": &Type{Size: 4, Align: 4, C: "int"}, - "uint32": &Type{Size: 4, Align: 4, C: "uint"}, - "int64": &Type{Size: 8, Align: 8, C: "int64"}, - "uint64": &Type{Size: 8, Align: 8, C: "uint64"}, - "float": &Type{Size: 4, Align: 4, C: "float"}, - "float32": &Type{Size: 4, Align: 4, C: "float"}, - "float64": &Type{Size: 8, Align: 8, C: "double"}, - "complex": &Type{Size: 8, Align: 8, C: "__complex float"}, - "complex64": &Type{Size: 8, Align: 8, C: "__complex float"}, - "complex128": &Type{Size: 16, Align: 16, C: "__complex double"}, + "int": &Type{Size: 4, Align: 4, C: c("int")}, + "uint": &Type{Size: 4, Align: 4, C: c("uint")}, + "int8": &Type{Size: 1, Align: 1, C: c("schar")}, + "uint8": &Type{Size: 1, Align: 1, C: c("uchar")}, + "int16": &Type{Size: 2, Align: 2, C: c("short")}, + "uint16": &Type{Size: 2, Align: 2, C: c("ushort")}, + "int32": &Type{Size: 4, Align: 4, C: c("int")}, + "uint32": &Type{Size: 4, Align: 4, C: c("uint")}, + "int64": &Type{Size: 8, Align: 8, C: c("int64")}, + "uint64": &Type{Size: 8, Align: 8, C: c("uint64")}, + "float": &Type{Size: 4, Align: 4, C: c("float")}, + "float32": &Type{Size: 4, Align: 4, C: c("float")}, + "float64": &Type{Size: 8, Align: 8, C: c("double")}, + "complex": &Type{Size: 8, Align: 8, C: c("__complex float")}, + "complex64": &Type{Size: 8, Align: 8, C: c("__complex float")}, + "complex128": &Type{Size: 16, Align: 16, C: c("__complex double")}, } // Map an ast type to a Type. @@ -567,21 +563,21 @@ func (p *Package) cgoType(e ast.Expr) *Type { switch t := e.(type) { case *ast.StarExpr: x := p.cgoType(t.X) - return &Type{Size: p.PtrSize, Align: p.PtrSize, C: x.C + "*"} + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("%s*", x.C)} case *ast.ArrayType: if t.Len == nil { - return &Type{Size: p.PtrSize + 8, Align: p.PtrSize, C: "GoSlice"} + return &Type{Size: p.PtrSize + 8, Align: p.PtrSize, C: c("GoSlice")} } case *ast.StructType: // TODO case *ast.FuncType: - return &Type{Size: p.PtrSize, Align: p.PtrSize, C: "void*"} + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")} case *ast.InterfaceType: - return &Type{Size: 3 * p.PtrSize, Align: p.PtrSize, C: "GoInterface"} + return &Type{Size: 3 * p.PtrSize, Align: p.PtrSize, C: c("GoInterface")} case *ast.MapType: - return &Type{Size: p.PtrSize, Align: p.PtrSize, C: "GoMap"} + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("GoMap")} case *ast.ChanType: - return &Type{Size: p.PtrSize, Align: p.PtrSize, C: "GoChan"} + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("GoChan")} case *ast.Ident: // Look up the type in the top level declarations. // TODO: Handle types defined within a function. @@ -606,10 +602,10 @@ func (p *Package) cgoType(e ast.Expr) *Type { } } if t.Name == "uintptr" { - return &Type{Size: p.PtrSize, Align: p.PtrSize, C: "uintptr"} + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("uintptr")} } if t.Name == "string" { - return &Type{Size: p.PtrSize + 4, Align: p.PtrSize, C: "GoString"} + return &Type{Size: p.PtrSize + 4, Align: p.PtrSize, C: c("GoString")} } if r, ok := goTypes[t.Name]; ok { if r.Align > p.PtrSize { @@ -620,11 +616,11 @@ func (p *Package) cgoType(e ast.Expr) *Type { case *ast.SelectorExpr: id, ok := t.X.(*ast.Ident) if ok && id.Name == "unsafe" && t.Sel.Name == "Pointer" { - return &Type{Size: p.PtrSize, Align: p.PtrSize, C: "void*"} + return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")} } } error(e.Pos(), "unrecognized Go type %T", e) - return &Type{Size: 4, Align: 4, C: "int"} + return &Type{Size: 4, Align: 4, C: c("int")} } const gccProlog = ` diff --git a/src/cmd/clean.bash b/src/cmd/clean.bash deleted file mode 100644 index 92d8cc5c9..000000000 --- a/src/cmd/clean.bash +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -gomake=gomake -if [ "$1" == "--gomake" -a "$2" != "" ]; then - gomake=$2 -fi - -for i in cc 6l 6a 6c 8l 8a 8c 8g 5l 5a 5c 5g gc 6g gopack nm cgo cov ebnflint godefs godoc gofmt goinstall gotest goyacc hgpatch prof -do - cd $i - $gomake clean - cd .. -done diff --git a/src/cmd/ebnflint/ebnflint.go b/src/cmd/ebnflint/ebnflint.go index 5eb398735..cac39179f 100644 --- a/src/cmd/ebnflint/ebnflint.go +++ b/src/cmd/ebnflint/ebnflint.go @@ -13,7 +13,7 @@ import ( "go/token" "io/ioutil" "os" - "path" + "path/filepath" ) @@ -91,7 +91,7 @@ func main() { os.Exit(1) } - if path.Ext(filename) == ".html" { + if filepath.Ext(filename) == ".html" { src = extractEBNF(src) } diff --git a/src/cmd/gc/const.c b/src/cmd/gc/const.c index 0ee693c02..a54c40f6c 100644 --- a/src/cmd/gc/const.c +++ b/src/cmd/gc/const.c @@ -1051,12 +1051,12 @@ int cmpslit(Node *l, Node *r) { int32 l1, l2, i, m; - char *s1, *s2; + uchar *s1, *s2; l1 = l->val.u.sval->len; l2 = r->val.u.sval->len; - s1 = l->val.u.sval->s; - s2 = r->val.u.sval->s; + s1 = (uchar*)l->val.u.sval->s; + s2 = (uchar*)r->val.u.sval->s; m = l1; if(l2 < m) diff --git a/src/cmd/gc/doc.go b/src/cmd/gc/doc.go index 21e1b103b..3fe7fafdd 100644 --- a/src/cmd/gc/doc.go +++ b/src/cmd/gc/doc.go @@ -43,6 +43,8 @@ Flags: disable optimization -S write assembly language text to standard output + -u + disallow importing packages not marked as safe -V print the compiler version diff --git a/src/cmd/gc/export.c b/src/cmd/gc/export.c index 594509915..09b963f27 100644 --- a/src/cmd/gc/export.c +++ b/src/cmd/gc/export.c @@ -51,6 +51,12 @@ exportname(char *s) return isupperrune(r); } +static int +initname(char *s) +{ + return strcmp(s, "init") == 0; +} + void autoexport(Node *n, int ctxt) { @@ -60,7 +66,7 @@ autoexport(Node *n, int ctxt) return; if(n->ntype && n->ntype->op == OTFUNC && n->ntype->left) // method return; - if(exportname(n->sym->name) || strcmp(n->sym->name, "init") == 0) + if(exportname(n->sym->name) || initname(n->sym->name)) exportsym(n); else packagesym(n); @@ -304,7 +310,7 @@ importsym(Sym *s, int op) // mark the symbol so it is not reexported if(s->def == N) { - if(exportname(s->name)) + if(exportname(s->name) || initname(s->name)) s->flags |= SymExport; else s->flags |= SymPackage; // package scope @@ -374,7 +380,7 @@ importvar(Sym *s, Type *t, int ctxt) { Node *n; - if(!exportname(s->name) && !mypackage(s)) + if(!exportname(s->name) && !initname(s->name) && !mypackage(s)) return; importsym(s, ONAME); diff --git a/src/cmd/gc/go.y b/src/cmd/gc/go.y index 86e3cae33..4b838a491 100644 --- a/src/cmd/gc/go.y +++ b/src/cmd/gc/go.y @@ -640,10 +640,15 @@ if_stmt: { markdcl(); } - if_header loop_body + if_header + { + if($3->ntest == N) + yyerror("missing condition in if statement"); + } + loop_body { $$ = $3; - $$->nbody = $4; + $$->nbody = $5; // no popdcl; maybe there's an LELSE } diff --git a/src/cmd/gc/init.c b/src/cmd/gc/init.c index dc073443e..af4eb0336 100644 --- a/src/cmd/gc/init.c +++ b/src/cmd/gc/init.c @@ -30,19 +30,19 @@ renameinit(Node *n) /* * hand-craft the following initialization code - * var initdone·<file> uint8 (1) - * func Init·<file>() (2) - * if initdone·<file> != 0 { (3) - * if initdone·<file> == 2 (4) + * var initdone· uint8 (1) + * func init() (2) + * if initdone· != 0 { (3) + * if initdone· == 2 (4) * return * throw(); (5) * } - * initdone.<file> = 1; (6) + * initdone· = 1; (6) * // over all matching imported symbols - * <pkg>.init·<file>() (7) + * <pkg>.init() (7) * { <init stmts> } (8) - * init·<file>() // if any (9) - * initdone.<file> = 2; (10) + * init·<n>() // if any (9) + * initdone· = 2; (10) * return (11) * } */ @@ -79,7 +79,7 @@ anyinit(NodeList *n) // are there any imported init functions for(h=0; h<NHASH; h++) for(s = hash[h]; s != S; s = s->link) { - if(s->name[0] != 'I' || strncmp(s->name, "Init·", 6) != 0) + if(s->name[0] != 'i' || strcmp(s->name, "init") != 0) continue; if(s->def == N) continue; @@ -118,12 +118,7 @@ fninit(NodeList *n) // (2) maxarg = 0; - snprint(namebuf, sizeof(namebuf), "Init·"); - - // this is a botch since we need a known name to - // call the top level init function out of rt0 - if(strcmp(localpkg->name, "main") == 0) - snprint(namebuf, sizeof(namebuf), "init"); + snprint(namebuf, sizeof(namebuf), "init"); fn = nod(ODCLFUNC, N, N); initsym = lookup(namebuf); @@ -154,7 +149,7 @@ fninit(NodeList *n) // (7) for(h=0; h<NHASH; h++) for(s = hash[h]; s != S; s = s->link) { - if(s->name[0] != 'I' || strncmp(s->name, "Init·", 6) != 0) + if(s->name[0] != 'i' || strcmp(s->name, "init") != 0) continue; if(s->def == N) continue; diff --git a/src/cmd/gc/reflect.c b/src/cmd/gc/reflect.c index 36c245d47..8129bf1ce 100644 --- a/src/cmd/gc/reflect.c +++ b/src/cmd/gc/reflect.c @@ -10,6 +10,7 @@ static NodeList* signatlist; static Sym* dtypesym(Type*); +static Sym* weaktypesym(Type*); static int sigcmp(Sig *a, Sig *b) @@ -570,9 +571,17 @@ dcommontype(Sym *s, int ot, Type *t) { int i; Sym *s1; + Sym *sptr; char *p; dowidth(t); + + sptr = nil; + if(t->sym != nil && !isptr[t->etype]) + sptr = dtypesym(ptrto(t)); + else + sptr = weaktypesym(ptrto(t)); + s1 = dextratype(t); // empty interface pointing at this type. @@ -592,7 +601,8 @@ dcommontype(Sym *s, int ot, Type *t) // fieldAlign uint8; // kind uint8; // string *string; - // *nameInfo; + // *extraType; + // ptrToThis *Type // } ot = duintptr(s, ot, t->width); ot = duint32(s, ot, typehash(t)); @@ -616,7 +626,7 @@ dcommontype(Sym *s, int ot, Type *t) ot = dsymptr(s, ot, s1, 0); // extraType else ot = duintptr(s, ot, 0); - + ot = dsymptr(s, ot, sptr, 0); // ptr to type return ot; } @@ -662,6 +672,25 @@ typename(Type *t) } static Sym* +weaktypesym(Type *t) +{ + char *p; + Sym *s; + static Pkg *weak; + + if(weak == nil) { + weak = mkpkg(strlit("weak.type")); + weak->name = "weak.type"; + weak->prefix = "weak.type"; // not weak%2etype + } + + p = smprint("%#-T", t); + s = pkglookup(p, weak); + free(p); + return s; +} + +static Sym* dtypesym(Type *t) { int ot, n, isddd, dupok; diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c index 0755ca3cd..142e5ba41 100644 --- a/src/cmd/gc/subr.c +++ b/src/cmd/gc/subr.c @@ -203,7 +203,6 @@ fatal(char *fmt, ...) flusherrors(); -*(int*)0=0; print("%L: internal compiler error: ", lineno); va_start(arg, fmt); vfprint(1, fmt, arg); @@ -1909,8 +1908,12 @@ assignop(Type *src, Type *dst, char **why) return 0; } if(src->etype == TINTER && dst->etype != TBLANK) { - if(why != nil) - *why = ": need type assertion"; + if(why != nil) { + if(isptrto(dst, TINTER)) + *why = smprint(":\n\t%T is interface, not pointer to interface", src); + else + *why = ": need type assertion"; + } return 0; } @@ -2265,7 +2268,7 @@ syslook(char *name, int copy) s = pkglookup(name, runtimepkg); if(s == S || s->def == N) - fatal("looksys: cant find runtime.%s", name); + fatal("syslook: can't find runtime.%s", name); if(!copy) return s->def; diff --git a/src/cmd/gc/typecheck.c b/src/cmd/gc/typecheck.c index 5edca964a..3e8f35877 100644 --- a/src/cmd/gc/typecheck.c +++ b/src/cmd/gc/typecheck.c @@ -318,7 +318,7 @@ reswitch: n->left = N; goto ret; } - if(!isptr[t->etype]) { + if(!isptr[t->etype] || (t->type != T && t->type->etype == TANY) /* unsafe.Pointer */) { yyerror("invalid indirect of %+N", n->left); goto error; } @@ -1613,7 +1613,7 @@ typecheckaste(int op, Node *call, int isddd, Type *tstruct, NodeList *nl, char * exportassignok(tn->type, desc); if(assignop(tn->type, tl->type->type, &why) == 0) { if(call != N) - yyerror("cannot use %T as type %T in argument to %#N%s", tn->type, tl->type->type, desc, call, why); + yyerror("cannot use %T as type %T in argument to %#N%s", tn->type, tl->type->type, call, why); else yyerror("cannot use %T as type %T in %s%s", tn->type, tl->type->type, desc, why); } @@ -1625,7 +1625,7 @@ typecheckaste(int op, Node *call, int isddd, Type *tstruct, NodeList *nl, char * exportassignok(tn->type, desc); if(assignop(tn->type, tl->type, &why) == 0) { if(call != N) - yyerror("cannot use %T as type %T in argument to %#N%s", tn->type, tl->type, desc, call, why); + yyerror("cannot use %T as type %T in argument to %#N%s", tn->type, tl->type, call, why); else yyerror("cannot use %T as type %T in %s%s", tn->type, tl->type, desc, why); } diff --git a/src/cmd/godoc/dirtrees.go b/src/cmd/godoc/dirtrees.go index edb4a169d..3ad7c8cfc 100644 --- a/src/cmd/godoc/dirtrees.go +++ b/src/cmd/godoc/dirtrees.go @@ -12,8 +12,9 @@ import ( "go/parser" "go/token" "io/ioutil" + "log" "os" - pathutil "path" + "path/filepath" "strings" "unicode" ) @@ -31,7 +32,7 @@ type Directory struct { func isGoFile(f *os.FileInfo) bool { return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && // ignore .files - pathutil.Ext(f.Name) == ".go" + filepath.Ext(f.Name) == ".go" } @@ -100,7 +101,13 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i return &Directory{depth, path, name, "", nil} } - list, _ := ioutil.ReadDir(path) // ignore errors + list, err := ioutil.ReadDir(path) + if err != nil { + // newDirTree is called with a path that should be a package + // directory; errors here should not happen, but if they do, + // we want to know about them + log.Printf("ioutil.ReadDir(%s): %s", path, err) + } // determine number of subdirectories and if there are package files ndirs := 0 @@ -116,7 +123,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i // though the directory doesn't contain any real package files - was bug) if synopses[0] == "" { // no "optimal" package synopsis yet; continue to collect synopses - file, err := parser.ParseFile(fset, pathutil.Join(path, d.Name), nil, + file, err := parser.ParseFile(fset, filepath.Join(path, d.Name), nil, parser.ParseComments|parser.PackageClauseOnly) if err == nil { hasPkgFiles = true @@ -149,7 +156,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i i := 0 for _, d := range list { if isPkgDir(d) { - dd := b.newDirTree(fset, pathutil.Join(path, d.Name), d.Name, depth+1) + dd := b.newDirTree(fset, filepath.Join(path, d.Name), d.Name, depth+1) if dd != nil { dirs[i] = dd i++ @@ -188,8 +195,16 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i // (i.e., in this case the tree may contain directories w/o any package files). // func newDirectory(root string, pathFilter func(string) bool, maxDepth int) *Directory { - d, err := os.Lstat(root) - if err != nil || !isPkgDir(d) { + // The root could be a symbolic link so use os.Stat not os.Lstat. + d, err := os.Stat(root) + // If we fail here, report detailed error messages; otherwise + // is is hard to see why a directory tree was not built. + switch { + case err != nil: + log.Printf("newDirectory(%s): %s", root, err) + return nil + case !isPkgDir(d): + log.Printf("newDirectory(%s): not a package directory", root) return nil } if maxDepth < 0 { diff --git a/src/cmd/godoc/godoc.go b/src/cmd/godoc/godoc.go index c91dc33db..9dce5edf9 100644 --- a/src/cmd/godoc/godoc.go +++ b/src/cmd/godoc/godoc.go @@ -18,7 +18,8 @@ import ( "io/ioutil" "log" "os" - pathutil "path" + "path" + "path/filepath" "regexp" "runtime" "sort" @@ -81,8 +82,8 @@ var ( func initHandlers() { fsMap.Init(*pkgPath) fileServer = http.FileServer(*goroot, "") - cmdHandler = httpHandler{"/cmd/", pathutil.Join(*goroot, "src/cmd"), false} - pkgHandler = httpHandler{"/pkg/", pathutil.Join(*goroot, "src/pkg"), true} + cmdHandler = httpHandler{"/cmd/", filepath.Join(*goroot, "src", "cmd"), false} + pkgHandler = httpHandler{"/pkg/", filepath.Join(*goroot, "src", "pkg"), true} } @@ -91,12 +92,13 @@ func registerPublicHandlers(mux *http.ServeMux) { mux.Handle(pkgHandler.pattern, &pkgHandler) mux.HandleFunc("/doc/codewalk/", codewalk) mux.HandleFunc("/search", search) + mux.Handle("/robots.txt", fileServer) mux.HandleFunc("/", serveFile) } func initFSTree() { - fsTree.set(newDirectory(pathutil.Join(*goroot, *testDir), nil, -1)) + fsTree.set(newDirectory(filepath.Join(*goroot, *testDir), nil, -1)) invalidateIndex() } @@ -147,8 +149,13 @@ func readDirList(filename string) ([]string, os.Error) { } // create a sorted list of valid directory names filter := func(path string) bool { - d, err := os.Lstat(path) - return err == nil && isPkgDir(d) + d, e := os.Lstat(path) + if e != nil && err == nil { + // remember first error and return it from readDirList + // so we have at least some information if things go bad + err = e + } + return e == nil && isPkgDir(d) } list := canonicalizePaths(strings.Split(string(contents), "\n", -1), filter) // for each parent path, remove all it's children q @@ -160,7 +167,7 @@ func readDirList(filename string) ([]string, os.Error) { i++ } } - return list[0:i], nil + return list[0:i], err } @@ -207,9 +214,10 @@ func initDirTrees() { if *filter != "" { list, err := readDirList(*filter) if err != nil { - log.Printf("%s", err) - } else if len(list) == 0 { - log.Printf("no directory paths in file %s", *filter) + log.Printf("readDirList(%s): %s", *filter, err) + } + if *verbose || len(list) == 0 { + log.Printf("found %d directory paths in file %s", len(list), *filter) } setPathFilter(list) } @@ -239,27 +247,30 @@ func initDirTrees() { // ---------------------------------------------------------------------------- // Path mapping -func absolutePath(path, defaultRoot string) string { - abspath := fsMap.ToAbsolute(path) +// Absolute paths are file system paths (backslash-separated on Windows), +// but relative paths are always slash-separated. + +func absolutePath(relpath, defaultRoot string) string { + abspath := fsMap.ToAbsolute(relpath) if abspath == "" { // no user-defined mapping found; use default mapping - abspath = pathutil.Join(defaultRoot, path) + abspath = filepath.Join(defaultRoot, filepath.FromSlash(relpath)) } return abspath } -func relativePath(path string) string { - relpath := fsMap.ToRelative(path) +func relativeURL(abspath string) string { + relpath := fsMap.ToRelative(abspath) if relpath == "" { - // prefix must end in '/' + // prefix must end in a path separator prefix := *goroot - if len(prefix) > 0 && prefix[len(prefix)-1] != '/' { - prefix += "/" + if len(prefix) > 0 && prefix[len(prefix)-1] != filepath.Separator { + prefix += string(filepath.Separator) } - if strings.HasPrefix(path, prefix) { + if strings.HasPrefix(abspath, prefix) { // no user-defined mapping found; use default mapping - relpath = path[len(prefix):] + relpath = filepath.ToSlash(abspath[len(prefix):]) } } // Only if path is an invalid absolute path is relpath == "" @@ -474,7 +485,7 @@ func urlFmt(w io.Writer, format string, x ...interface{}) { } // map path - relpath := relativePath(path) + relpath := relativeURL(path) // convert to relative URLs so that they can also // be used as relative file names in .txt templates @@ -591,7 +602,7 @@ func dirslashFmt(w io.Writer, format string, x ...interface{}) { // Template formatter for "localname" format. func localnameFmt(w io.Writer, format string, x ...interface{}) { - _, localname := pathutil.Split(x[0].(string)) + _, localname := filepath.Split(x[0].(string)) template.HTMLEscape(w, []byte(localname)) } @@ -623,7 +634,7 @@ var fmap = template.FormatterMap{ func readTemplate(name string) *template.Template { - path := pathutil.Join(*goroot, "lib/godoc/"+name) + path := filepath.Join(*goroot, "lib", "godoc", name) data, err := ioutil.ReadFile(path) if err != nil { log.Fatalf("ReadFile %s: %v", path, err) @@ -760,14 +771,13 @@ func applyTemplate(t *template.Template, name string, data interface{}) []byte { func redirect(w http.ResponseWriter, r *http.Request) (redirected bool) { - if canonical := pathutil.Clean(r.URL.Path) + "/"; r.URL.Path != canonical { + if canonical := path.Clean(r.URL.Path) + "/"; r.URL.Path != canonical { http.Redirect(w, r, canonical, http.StatusMovedPermanently) redirected = true } return } - func serveTextFile(w http.ResponseWriter, r *http.Request, abspath, relpath, title string) { src, err := ioutil.ReadFile(abspath) if err != nil { @@ -778,7 +788,7 @@ func serveTextFile(w http.ResponseWriter, r *http.Request, abspath, relpath, tit var buf bytes.Buffer buf.WriteString("<pre>") - FormatText(&buf, src, 1, pathutil.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s"))) + FormatText(&buf, src, 1, filepath.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s"))) buf.WriteString("</pre>") servePage(w, title+" "+relpath, "", "", buf.Bytes()) @@ -815,7 +825,7 @@ func serveFile(w http.ResponseWriter, r *http.Request) { // pick off special cases and hand the rest to the standard file server switch r.URL.Path { case "/": - serveHTMLDoc(w, r, pathutil.Join(*goroot, "doc/root.html"), "doc/root.html") + serveHTMLDoc(w, r, filepath.Join(*goroot, "doc", "root.html"), "doc/root.html") return case "/doc/root.html": @@ -824,9 +834,9 @@ func serveFile(w http.ResponseWriter, r *http.Request) { return } - switch pathutil.Ext(abspath) { + switch path.Ext(relpath) { case ".html": - if strings.HasSuffix(abspath, "/index.html") { + if strings.HasSuffix(relpath, "/index.html") { // We'll show index.html for the directory. // Use the dir/ version as canonical instead of dir/index.html. http.Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len("index.html")], http.StatusMovedPermanently) @@ -851,8 +861,8 @@ func serveFile(w http.ResponseWriter, r *http.Request) { if redirect(w, r) { return } - if index := abspath + "/index.html"; isTextFile(index) { - serveHTMLDoc(w, r, index, relativePath(index)) + if index := filepath.Join(abspath, "index.html"); isTextFile(index) { + serveHTMLDoc(w, r, index, relativeURL(index)) return } serveDirectory(w, r, abspath, relpath) @@ -948,13 +958,13 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf // the package with dirname, and the 3rd choice is a package // that is not called "main" if there is exactly one such // package. Otherwise, don't select a package. - dirpath, dirname := pathutil.Split(abspath) + dirpath, dirname := filepath.Split(abspath) // If the dirname is "go" we might be in a sub-directory for // .go files - use the outer directory name instead for better // results. if dirname == "go" { - _, dirname = pathutil.Split(pathutil.Clean(dirpath)) + _, dirname = filepath.Split(filepath.Clean(dirpath)) } var choice3 *ast.Package @@ -995,7 +1005,7 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf ast.PackageExports(pkg) } if mode&genDoc != 0 { - pdoc = doc.NewPackageDoc(pkg, pathutil.Clean(relpath)) // no trailing '/' in importpath + pdoc = doc.NewPackageDoc(pkg, path.Clean(relpath)) // no trailing '/' in importpath } else { past = ast.MergePackageFiles(pkg, ast.FilterUnassociatedComments) } @@ -1081,13 +1091,13 @@ func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { title = "Package " + info.PDoc.PackageName case info.PDoc.PackageName == fakePkgName: // assume that the directory name is the command name - _, pkgname := pathutil.Split(pathutil.Clean(relpath)) + _, pkgname := path.Split(path.Clean(relpath)) title = "Command " + pkgname default: title = "Command " + info.PDoc.PackageName } default: - title = "Directory " + relativePath(info.Dirname) + title = "Directory " + relativeURL(info.Dirname) if *showTimestamps { subtitle = "Last update: " + time.SecondsToLocalTime(info.DirTime).String() } diff --git a/src/cmd/godoc/index.go b/src/cmd/godoc/index.go index 56f31f5cf..5af4d15cb 100644 --- a/src/cmd/godoc/index.go +++ b/src/cmd/godoc/index.go @@ -47,7 +47,7 @@ import ( "index/suffixarray" "io/ioutil" "os" - "path" + "path/filepath" "regexp" "sort" "strings" @@ -718,7 +718,7 @@ var whitelisted = map[string]bool{ // of "permitted" files for indexing. The filename must // be the directory-local name of the file. func isWhitelisted(filename string) bool { - key := path.Ext(filename) + key := filepath.Ext(filename) if key == "" { // file has no extension - use entire filename key = filename @@ -732,7 +732,7 @@ func (x *Indexer) visitFile(dirname string, f *os.FileInfo, fulltextIndex bool) return } - filename := path.Join(dirname, f.Name) + filename := filepath.Join(dirname, f.Name) goFile := false switch { @@ -757,7 +757,7 @@ func (x *Indexer) visitFile(dirname string, f *os.FileInfo, fulltextIndex bool) if fast != nil { // we've got a Go file to index x.current = file - dir, _ := path.Split(filename) + dir, _ := filepath.Split(filename) pak := Pak{dir, fast.Name.Name} x.file = &File{filename, pak} ast.Walk(x, fast) diff --git a/src/cmd/godoc/main.go b/src/cmd/godoc/main.go index ea1e3c42e..1ebb80279 100644 --- a/src/cmd/godoc/main.go +++ b/src/cmd/godoc/main.go @@ -36,7 +36,7 @@ import ( "io" "log" "os" - pathutil "path" + "path/filepath" "regexp" "runtime" "strings" @@ -314,14 +314,14 @@ func main() { if len(path) > 0 && path[0] == '.' { // assume cwd; don't assume -goroot cwd, _ := os.Getwd() // ignore errors - path = pathutil.Join(cwd, path) + path = filepath.Join(cwd, path) } relpath := path abspath := path - if !pathutil.IsAbs(path) { + if !filepath.IsAbs(path) { abspath = absolutePath(path, pkgHandler.fsRoot) } else { - relpath = relativePath(path) + relpath = relativeURL(path) } var mode PageInfoMode @@ -339,7 +339,7 @@ func main() { if info.IsEmpty() { // try again, this time assume it's a command - if !pathutil.IsAbs(path) { + if !filepath.IsAbs(path) { abspath = absolutePath(path, cmdHandler.fsRoot) } cmdInfo := cmdHandler.getPageInfo(abspath, relpath, "", mode) diff --git a/src/cmd/godoc/mapping.go b/src/cmd/godoc/mapping.go index 1d87bbc76..6ae9032e4 100644 --- a/src/cmd/godoc/mapping.go +++ b/src/cmd/godoc/mapping.go @@ -10,7 +10,8 @@ import ( "fmt" "io" "os" - pathutil "path" + "path" + "path/filepath" "sort" "strings" ) @@ -59,10 +60,10 @@ type mapping struct { } -// Init initializes the Mapping from a list of ':'-separated -// paths. Empty paths are ignored; relative paths are assumed -// to be relative to the current working directory and converted -// to absolute paths. For each path of the form: +// Init initializes the Mapping from a list of paths separated by +// filepath.ListSeparator. Empty paths are ignored; relative paths +// are assumed to be relative to the current working directory and +// converted to absolute paths. For each path of the form: // // dirname/localname // @@ -71,7 +72,7 @@ type mapping struct { // localname -> path // // is added to the Mapping object, in the order of occurrence. -// For instance, the argument: +// For instance, under Unix, the argument: // // /home/user:/home/build/public // @@ -81,12 +82,12 @@ type mapping struct { // public -> /home/build/public // func (m *Mapping) Init(paths string) { - pathlist := canonicalizePaths(strings.Split(paths, ":", -1), nil) + pathlist := canonicalizePaths(filepath.SplitList(paths), nil) list := make([]mapping, len(pathlist)) // create mapping list for i, path := range pathlist { - _, prefix := pathutil.Split(path) + _, prefix := filepath.Split(path) list[i] = mapping{prefix, path, new(RWValue)} } @@ -147,7 +148,7 @@ func (m *Mapping) Fprint(w io.Writer) { func splitFirst(path string) (head, tail string) { - i := strings.Index(path, "/") + i := strings.Index(path, string(filepath.Separator)) if i > 0 { // 0 < i < len(path) return path[0:i], path[i+1:] @@ -156,22 +157,23 @@ func splitFirst(path string) (head, tail string) { } -// ToAbsolute maps a relative path to an absolute path using the Mapping -// specified by the receiver. If the path cannot be mapped, the empty -// string is returned. +// ToAbsolute maps a slash-separated relative path to an absolute filesystem +// path using the Mapping specified by the receiver. If the path cannot +// be mapped, the empty string is returned. // -func (m *Mapping) ToAbsolute(path string) string { - prefix, tail := splitFirst(path) +func (m *Mapping) ToAbsolute(spath string) string { + fpath := filepath.FromSlash(spath) + prefix, tail := splitFirst(fpath) for _, e := range m.list { switch { case e.prefix == prefix: // use tail case e.prefix == "": - tail = path + tail = fpath default: continue // no match } - abspath := pathutil.Join(e.path, tail) + abspath := filepath.Join(e.path, tail) if _, err := os.Stat(abspath); err == nil { return abspath } @@ -181,15 +183,16 @@ func (m *Mapping) ToAbsolute(path string) string { } -// ToRelative maps an absolute path to a relative path using the Mapping -// specified by the receiver. If the path cannot be mapped, the empty -// string is returned. +// ToRelative maps an absolute filesystem path to a relative slash-separated +// path using the Mapping specified by the receiver. If the path cannot +// be mapped, the empty string is returned. // -func (m *Mapping) ToRelative(path string) string { +func (m *Mapping) ToRelative(fpath string) string { for _, e := range m.list { - if strings.HasPrefix(path, e.path) { + if strings.HasPrefix(fpath, e.path) { + spath := filepath.ToSlash(fpath) // /absolute/prefix/foo -> prefix/foo - return pathutil.Join(e.prefix, path[len(e.path):]) // Join will remove a trailing '/' + return path.Join(e.prefix, spath[len(e.path):]) // Join will remove a trailing '/' } } return "" // no match diff --git a/src/cmd/godoc/utils.go b/src/cmd/godoc/utils.go index cc028cc4d..9517aee7a 100644 --- a/src/cmd/godoc/utils.go +++ b/src/cmd/godoc/utils.go @@ -10,7 +10,7 @@ import ( "io" "io/ioutil" "os" - pathutil "path" + "path/filepath" "sort" "strings" "sync" @@ -60,10 +60,10 @@ func canonicalizePaths(list []string, filter func(path string) bool) []string { continue // ignore empty paths (don't assume ".") } // len(path) > 0: normalize path - if pathutil.IsAbs(path) { - path = pathutil.Clean(path) + if filepath.IsAbs(path) { + path = filepath.Clean(path) } else { - path = pathutil.Join(cwd, path) + path = filepath.Join(cwd, path) } // we have a non-empty absolute path if filter != nil && !filter(path) { @@ -95,7 +95,7 @@ func canonicalizePaths(list []string, filter func(path string) bool) []string { // atomically renames that file to the file named by filename. // func writeFileAtomically(filename string, data []byte) os.Error { - f, err := ioutil.TempFile(cwd, filename) + f, err := ioutil.TempFile(filepath.Split(filename)) if err != nil { return err } @@ -149,7 +149,7 @@ var textExt = map[string]bool{ // func isTextFile(filename string) bool { // if the extension is known, use it for decision making - if isText, found := textExt[pathutil.Ext(filename)]; found { + if isText, found := textExt[filepath.Ext(filename)]; found { return isText } diff --git a/src/cmd/gofmt/gofmt.go b/src/cmd/gofmt/gofmt.go index 41c12b88d..224aee717 100644 --- a/src/cmd/gofmt/gofmt.go +++ b/src/cmd/gofmt/gofmt.go @@ -15,7 +15,7 @@ import ( "go/token" "io/ioutil" "os" - pathutil "path" + "path/filepath" "strings" ) @@ -181,7 +181,7 @@ func walkDir(path string) { done <- true }() // walk the tree - pathutil.Walk(path, v, v) + filepath.Walk(path, v, v) close(v) // terminate error handler loop <-done // wait for all errors to be reported } diff --git a/src/cmd/gofmt/test.sh b/src/cmd/gofmt/test.sh index 2f60a3e7b..3340c48f0 100755 --- a/src/cmd/gofmt/test.sh +++ b/src/cmd/gofmt/test.sh @@ -42,7 +42,7 @@ apply1() { bug163.go | bug166.go | bug169.go | bug217.go | bug222.go | \ bug226.go | bug228.go | bug248.go | bug274.go | bug280.go | \ bug282.go | bug287.go | bug298.go | bug299.go | bug300.go | \ - bug302.go | bug306.go | bug322.go ) return ;; + bug302.go | bug306.go | bug322.go | bug324.go ) return ;; esac # the following directories are skipped because they contain test # cases for syntax errors and thus won't parse in the first place: diff --git a/src/cmd/goinstall/download.go b/src/cmd/goinstall/download.go index 889f9d857..88befc0dc 100644 --- a/src/cmd/goinstall/download.go +++ b/src/cmd/goinstall/download.go @@ -9,7 +9,7 @@ package main import ( "http" "os" - "path" + "path/filepath" "regexp" "strings" ) @@ -42,7 +42,7 @@ func download(pkg string) (string, os.Error) { return "", os.ErrorString("invalid path (contains ..)") } if m := bitbucket.FindStringSubmatch(pkg); m != nil { - if err := vcsCheckout(&hg, root+m[1], "http://"+m[1], m[1]); err != nil { + if err := vcsCheckout(&hg, m[1], "http://"+m[1], m[1]); err != nil { return "", err } return root + pkg, nil @@ -58,7 +58,7 @@ func download(pkg string) (string, os.Error) { // regexp only allows hg, svn to get through panic("missing case in download: " + pkg) } - if err := vcsCheckout(v, root+m[1], "https://"+m[1], m[1]); err != nil { + if err := vcsCheckout(v, m[1], "https://"+m[1], m[1]); err != nil { return "", err } return root + pkg, nil @@ -67,7 +67,7 @@ func download(pkg string) (string, os.Error) { if strings.HasSuffix(m[1], ".git") { return "", os.ErrorString("repository " + pkg + " should not have .git suffix") } - if err := vcsCheckout(&git, root+m[1], "http://"+m[1]+".git", m[1]); err != nil { + if err := vcsCheckout(&git, m[1], "http://"+m[1]+".git", m[1]); err != nil { return "", err } return root + pkg, nil @@ -75,7 +75,7 @@ func download(pkg string) (string, os.Error) { if m := launchpad.FindStringSubmatch(pkg); m != nil { // Either lp.net/<project>[/<series>[/<path>]] // or lp.net/~<user or team>/<project>/<branch>[/<path>] - if err := vcsCheckout(&bzr, root+m[1], "https://"+m[1], m[1]); err != nil { + if err := vcsCheckout(&bzr, m[1], "https://"+m[1], m[1]); err != nil { return "", err } return root + pkg, nil @@ -172,17 +172,18 @@ func (v *vcs) updateRepo(dst string) os.Error { // exists and -u was specified on the command line) // the repository at tag/branch "release". If there is no // such tag or branch, it falls back to the repository tip. -func vcsCheckout(vcs *vcs, dst, repo, dashpath string) os.Error { - dir, err := os.Stat(dst + "/" + vcs.metadir) +func vcsCheckout(vcs *vcs, pkgprefix, repo, dashpath string) os.Error { + dst := filepath.Join(root, filepath.FromSlash(pkgprefix)) + dir, err := os.Stat(filepath.Join(dst, vcs.metadir)) if err == nil && !dir.IsDirectory() { return os.ErrorString("not a directory: " + dst) } if err != nil { - parent, _ := path.Split(dst) + parent, _ := filepath.Split(dst) if err := os.MkdirAll(parent, 0777); err != nil { return err } - if err := run("/", nil, vcs.cmd, vcs.clone, repo, dst); err != nil { + if err := run(string(filepath.Separator), nil, vcs.cmd, vcs.clone, repo, dst); err != nil { return err } if err := vcs.updateRepo(dst); err != nil { diff --git a/src/cmd/goinstall/main.go b/src/cmd/goinstall/main.go index f13aeb3bc..34441be45 100644 --- a/src/cmd/goinstall/main.go +++ b/src/cmd/goinstall/main.go @@ -15,7 +15,7 @@ import ( "io" "io/ioutil" "os" - "path" + "path/filepath" "runtime" "strings" ) @@ -34,7 +34,7 @@ var ( parents = make(map[string]string) root = runtime.GOROOT() visit = make(map[string]status) - logfile = path.Join(root, "goinstall.log") + logfile = filepath.Join(root, "goinstall.log") installedPkgs = make(map[string]bool) allpkg = flag.Bool("a", false, "install all previously installed packages") @@ -59,7 +59,7 @@ func main() { fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0) os.Exit(1) } - root += "/src/pkg/" + root += filepath.FromSlash("/src/pkg/") // special case - "unsafe" is already installed visit["unsafe"] = done @@ -160,7 +160,7 @@ func install(pkg, parent string) { dir = pkg local = true } else if isStandardPath(pkg) { - dir = path.Join(root, pkg) + dir = filepath.Join(root, filepath.FromSlash(pkg)) local = true } else { var err os.Error @@ -216,7 +216,8 @@ func install(pkg, parent string) { // Is this a local path? /foo ./foo ../foo . .. func isLocalPath(s string) bool { - return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") || s == "." || s == ".." + const sep = string(filepath.Separator) + return strings.HasPrefix(s, sep) || strings.HasPrefix(s, "."+sep) || strings.HasPrefix(s, ".."+sep) || s == "." || s == ".." } // Is this a standard package path? strings container/vector etc. diff --git a/src/cmd/goinstall/make.go b/src/cmd/goinstall/make.go index 8d4d6c5d2..e2d99bb47 100644 --- a/src/cmd/goinstall/make.go +++ b/src/cmd/goinstall/make.go @@ -44,6 +44,9 @@ func domake(dir, pkg string, local bool) (err os.Error) { // installing as package pkg. It includes all *.go files in the directory // except those in package main and those ending in _test.go. func makeMakefile(dir, pkg string) ([]byte, os.Error) { + if !safeName(pkg) { + return nil, os.ErrorString("unsafe name: " + pkg) + } dirInfo, err := scanDir(dir, false) if err != nil { return nil, err @@ -58,16 +61,25 @@ func makeMakefile(dir, pkg string) ([]byte, os.Error) { cgoFiles := dirInfo.cgoFiles isCgo := make(map[string]bool, len(cgoFiles)) for _, file := range cgoFiles { + if !safeName(file) { + return nil, os.ErrorString("bad name: " + file) + } isCgo[file] = true } oFiles := make([]string, 0, len(dirInfo.cFiles)) for _, file := range dirInfo.cFiles { + if !safeName(file) { + return nil, os.ErrorString("unsafe name: " + file) + } oFiles = append(oFiles, file[:len(file)-2]+".o") } goFiles := make([]string, 0, len(dirInfo.goFiles)) for _, file := range dirInfo.goFiles { + if !safeName(file) { + return nil, os.ErrorString("unsafe name: " + file) + } if !isCgo[file] { goFiles = append(goFiles, file) } @@ -81,6 +93,17 @@ func makeMakefile(dir, pkg string) ([]byte, os.Error) { return buf.Bytes(), nil } +var safeBytes = []byte("+-./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +func safeName(s string) bool { + for i := 0; i < len(s); i++ { + if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 { + return false + } + } + return true +} + // makedata is the data type for the makefileTemplate. type makedata struct { Pkg string // package import path diff --git a/src/cmd/goinstall/parse.go b/src/cmd/goinstall/parse.go index 679edfabc..014b8fcb2 100644 --- a/src/cmd/goinstall/parse.go +++ b/src/cmd/goinstall/parse.go @@ -7,13 +7,13 @@ package main import ( - "path" - "os" - "log" - "strings" - "strconv" "go/ast" "go/parser" + "log" + "os" + "path/filepath" + "strconv" + "strings" ) @@ -64,7 +64,7 @@ func scanDir(dir string, allowMain bool) (info *dirInfo, err os.Error) { if !strings.HasSuffix(d.Name, ".go") || strings.HasSuffix(d.Name, "_test.go") { continue } - filename := path.Join(dir, d.Name) + filename := filepath.Join(dir, d.Name) pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly) if err != nil { return nil, err diff --git a/src/cmd/gopack/ar.c b/src/cmd/gopack/ar.c index 702f104a6..a7e2c41af 100644 --- a/src/cmd/gopack/ar.c +++ b/src/cmd/gopack/ar.c @@ -109,7 +109,7 @@ typedef struct Hashchain /* constants and flags */ char *man = "mrxtdpq"; -char *opt = "uvnbailoS"; +char *opt = "uvnbailogS"; char artemp[] = "/tmp/vXXXXX"; char movtemp[] = "/tmp/v1XXXXX"; char tailtemp[] = "/tmp/v2XXXXX"; diff --git a/src/cmd/gopack/doc.go b/src/cmd/gopack/doc.go index 74c272fd2..08711e72e 100644 --- a/src/cmd/gopack/doc.go +++ b/src/cmd/gopack/doc.go @@ -12,10 +12,12 @@ It adds a special Go-specific section __.PKGDEF that collects all the Go type information from the files in the archive; that section is used by the compiler when importing the package during compilation. -Usage: gopack [uvnbailo][mrxtdpq] archive files ... +Usage: gopack [uvnbailogS][mrxtdpq] archive files ... The new option 'g' causes gopack to maintain the __.PKGDEF section as files are added to the archive. +The new option 'S' forces gopack to mark the archive as safe. + */ package documentation diff --git a/src/cmd/gotest/doc.go b/src/cmd/gotest/doc.go index 40c40fc1f..581eaaab9 100644 --- a/src/cmd/gotest/doc.go +++ b/src/cmd/gotest/doc.go @@ -20,7 +20,7 @@ They should have signature func TestXXX(t *testing.T) { ... } Benchmark functions can be written as well; they will be run only -when the -benchmarks flag is provided. Benchmarks should have +when the -test.bench flag is provided. Benchmarks should have signature func BenchmarkXXX(b *testing.B) { ... } @@ -42,15 +42,15 @@ The resulting binary, called (for amd64) 6.out, has a couple of arguments. Usage: - 6.out [-v] [-match pattern] [-benchmarks pattern] - -The -v flag causes the tests to be logged as they run. The -match -flag causes only those tests whose names match the regular expression -pattern to be run. By default all tests are run silently. If all -the specified test pass, 6.out prints PASS and exits with a 0 exit -code. If any tests fail, it prints FAIL and exits with a non-zero -code. The -benchmarks flag is analogous to the -match flag, but -applies to benchmarks. No benchmarks run by default. + 6.out [-test.v] [-test.run pattern] [-test.bench pattern] + +The -test.v flag causes the tests to be logged as they run. The +-test.run flag causes only those tests whose names match the regular +expression pattern to be run. By default all tests are run silently. +If all the specified test pass, 6.out prints PASS and exits with a 0 +exit code. If any tests fail, it prints FAIL and exits with a +non-zero code. The -test.bench flag is analogous to the -test.run +flag, but applies to benchmarks. No benchmarks run by default. */ package documentation diff --git a/src/cmd/govet/govet.go b/src/cmd/govet/govet.go index 5619b12ba..ff6421de8 100644 --- a/src/cmd/govet/govet.go +++ b/src/cmd/govet/govet.go @@ -15,7 +15,7 @@ import ( "go/parser" "go/token" "os" - "path" + "path/filepath" "strconv" "strings" ) @@ -99,7 +99,7 @@ func doFile(name string, reader io.Reader) { file.checkFile(name, parsedFile) } -// Visitor for path.Walk - trivial. Just calls doFile on each file. +// Visitor for filepath.Walk - trivial. Just calls doFile on each file. // TODO: if govet becomes richer, might want to process // a directory (package) at a time. type V struct{} @@ -124,7 +124,7 @@ func walkDir(root string) { } done <- true }() - path.Walk(root, V{}, errors) + filepath.Walk(root, V{}, errors) close(errors) <-done } diff --git a/src/cmd/goyacc/Makefile b/src/cmd/goyacc/Makefile index 54b8f3360..ac0f427cc 100644 --- a/src/cmd/goyacc/Makefile +++ b/src/cmd/goyacc/Makefile @@ -11,7 +11,7 @@ GOFILES=\ include ../../Make.cmd units: goyacc units.y - ./goyacc units.y + ./goyacc -p units_ units.y $(GC) y.go $(LD) -o units y.$O diff --git a/src/cmd/goyacc/doc.go b/src/cmd/goyacc/doc.go index 686f75745..5dd6abe69 100644 --- a/src/cmd/goyacc/doc.go +++ b/src/cmd/goyacc/doc.go @@ -17,7 +17,8 @@ Yacc adepts will have no trouble adapting to this form of the tool. The file units.y in this directory is a yacc grammar for a version of the Unix tool units, also written in Go and largely transliterated -from the Plan 9 C version. +from the Plan 9 C version. It needs the flag "-p units_" (see +below). The generated parser is reentrant. Parse expects to be given an argument that conforms to the following interface: @@ -31,8 +32,15 @@ Lex should return the token identifier, and place other token information in lval (which replaces the usual yylval). Error is equivalent to yyerror in the original yacc. -Code inside the parser may refer to the variable yylex +Code inside the parser may refer to the variable yylex, which holds the yyLexer passed to Parse. +Multiple grammars compiled into a single program should be placed in +distinct packages. If that is impossible, the "-p prefix" flag to +goyacc sets the prefix, by default yy, that begins the names of +symbols, including types, the parser, and the lexer, generated and +referenced by goyacc's generated code. Setting it to distinct values +allows multiple grammars to be placed in a single package. + */ package documentation diff --git a/src/cmd/goyacc/goyacc.go b/src/cmd/goyacc/goyacc.go index c9fa6bfb9..32816b700 100644 --- a/src/cmd/goyacc/goyacc.go +++ b/src/cmd/goyacc/goyacc.go @@ -153,9 +153,17 @@ var ftable *bufio.Writer // y.go file var fcode = &bytes.Buffer{} // saved code var foutput *bufio.Writer // y.output file -var oflag string // -o [y.go] - y.go file -var vflag string // -v [y.output] - y.output file -var lflag bool // -l - disable line directives +var oflag string // -o [y.go] - y.go file +var vflag string // -v [y.output] - y.output file +var lflag bool // -l - disable line directives +var prefix string // name prefix for identifiers, default yy + +func init() { + flag.StringVar(&oflag, "o", "y.go", "parser output") + flag.StringVar(&prefix, "p", "yy", "name prefix to use in generated code") + flag.StringVar(&vflag, "v", "y.output", "create parsing tables") + flag.BoolVar(&lflag, "l", false, "disable line directives") +} var stacksize = 200 @@ -349,10 +357,6 @@ func setup() { stderr = bufio.NewWriter(os.NewFile(2, "stderr")) foutput = nil - flag.StringVar(&oflag, "o", "", "parser output") - flag.StringVar(&vflag, "v", "", "create parsing tables") - flag.BoolVar(&lflag, "l", false, "disable line directives") - flag.Parse() if flag.NArg() != 1 { usage() @@ -362,6 +366,7 @@ func setup() { fmt.Fprintf(stderr, "yacc: stack size too small\n") usage() } + yaccpar = strings.Replace(yaccpartext, "$$", prefix, -1) openup() defin(0, "$end") @@ -506,20 +511,20 @@ outer: } // put out names of token names - fmt.Fprintf(ftable, "var\tyyToknames\t =[]string {\n") + fmt.Fprintf(ftable, "var\t%sToknames\t =[]string {\n", prefix) for i := TOKSTART; i <= ntokens; i++ { fmt.Fprintf(ftable, "\t\"%v\",\n", tokset[i].name) } fmt.Fprintf(ftable, "}\n") // put out names of state names - fmt.Fprintf(ftable, "var\tyyStatenames\t =[]string {\n") + fmt.Fprintf(ftable, "var\t%sStatenames\t =[]string {\n", prefix) // for i:=TOKSTART; i<=ntokens; i++ { // fmt.Fprintf(ftable, "\t\"%v\",\n", tokset[i].name); // } fmt.Fprintf(ftable, "}\n") - fmt.Fprintf(fcode, "switch yynt {\n") + fmt.Fprintf(fcode, "switch %snt {\n", prefix) moreprod() prdptr[0] = []int{NTBASE, start, 1, 0} @@ -648,8 +653,8 @@ outer: error("default action causes potential type clash") } fmt.Fprintf(fcode, "\ncase %v:", nprod) - fmt.Fprintf(fcode, "\n\tYYVAL.%v = YYS[yypt-0].%v;", - typeset[tempty], typeset[tempty]) + fmt.Fprintf(fcode, "\n\t%sVAL.%v = %sS[%spt-0].%v;", + prefix, typeset[tempty], prefix, prefix, typeset[tempty]) } moreprod() prdptr[nprod] = make([]int, mem) @@ -666,9 +671,9 @@ outer: fmt.Fprintf(fcode, "\n\t}") - fmt.Fprintf(ftable, "const yyEofCode = 1\n") - fmt.Fprintf(ftable, "const yyErrCode = 2\n") - fmt.Fprintf(ftable, "const yyMaxDepth = %v\n", stacksize) + fmt.Fprintf(ftable, "const %sEofCode = 1\n", prefix) + fmt.Fprintf(ftable, "const %sErrCode = 2\n", prefix) + fmt.Fprintf(ftable, "const %sMaxDepth = %v\n", prefix, stacksize) // // copy any postfix code @@ -1034,7 +1039,7 @@ func cpyunion() { if !lflag { fmt.Fprintf(ftable, "\n//line %v:%v\n", infile, lineno) } - fmt.Fprintf(ftable, "type\tyySymType\tstruct") + fmt.Fprintf(ftable, "type\t%sSymType\tstruct", prefix) level := 0 @@ -1197,7 +1202,7 @@ loop: c = getrune(finput) } if c == '$' { - fmt.Fprintf(fcode, "YYVAL") + fmt.Fprintf(fcode, "%sVAL", prefix) // put out the proper tag... if ntypes != 0 { @@ -1258,7 +1263,7 @@ loop: ungetrune(finput, c) continue loop } - fmt.Fprintf(fcode, "YYS[yypt-%v]", max-j-1) + fmt.Fprintf(fcode, "%sS[%spt-%v]", prefix, prefix, max-j-1) // put out the proper tag if ntypes != 0 { @@ -2067,7 +2072,7 @@ func output() { var c, u, v int fmt.Fprintf(ftable, "\n//line yacctab:1\n") - fmt.Fprintf(ftable, "var\tyyExca = []int {\n") + fmt.Fprintf(ftable, "var\t%sExca = []int {\n", prefix) noset := mkset() @@ -2140,10 +2145,10 @@ func output() { } fmt.Fprintf(ftable, "}\n") - fmt.Fprintf(ftable, "const\tyyNprod\t= %v\n", nprod) - fmt.Fprintf(ftable, "const\tyyPrivate\t= %v\n", PRIVATE) - fmt.Fprintf(ftable, "var\tyyTokenNames []string\n") - fmt.Fprintf(ftable, "var\tyyStates []string\n") + fmt.Fprintf(ftable, "const\t%sNprod\t= %v\n", prefix, nprod) + fmt.Fprintf(ftable, "const\t%sPrivate\t= %v\n", prefix, PRIVATE) + fmt.Fprintf(ftable, "var\t%sTokenNames []string\n", prefix) + fmt.Fprintf(ftable, "var\t%sStates []string\n", prefix) } // @@ -2718,10 +2723,10 @@ nextn: // write out the optimized parser // func aoutput() { - fmt.Fprintf(ftable, "const\tyyLast\t= %v\n", maxa+1) - arout("yyAct", amem, maxa+1) - arout("yyPact", indgo, nstate) - arout("yyPgo", pgo, nnonter+1) + fmt.Fprintf(ftable, "const\t%sLast\t= %v\n", prefix, maxa+1) + arout("Act", amem, maxa+1) + arout("Pact", indgo, nstate) + arout("Pgo", pgo, nnonter+1) } // @@ -2730,7 +2735,7 @@ func aoutput() { func others() { var i, j int - arout("yyR1", levprd, nprod) + arout("R1", levprd, nprod) aryfil(temp1, nprod, 0) // @@ -2739,7 +2744,7 @@ func others() { for i = 1; i < nprod; i++ { temp1[i] = len(prdptr[i]) - 2 } - arout("yyR2", temp1, nprod) + arout("R2", temp1, nprod) aryfil(temp1, nstate, -1000) for i = 0; i <= ntokens; i++ { @@ -2752,8 +2757,8 @@ func others() { temp1[j] = -i } } - arout("yyChk", temp1, nstate) - arout("yyDef", defact, nstate) + arout("Chk", temp1, nstate) + arout("Def", defact, nstate) // put out token translation tables // table 1 has 0-256 @@ -2778,7 +2783,7 @@ func others() { temp1[i] = YYLEXUNK } } - arout("yyTok1", temp1, c+1) + arout("Tok1", temp1, c+1) // table 2 has PRIVATE-PRIVATE+256 aryfil(temp1, 256, 0) @@ -2797,10 +2802,10 @@ func others() { } } } - arout("yyTok2", temp1, c+1) + arout("Tok2", temp1, c+1) // table 3 has everything else - fmt.Fprintf(ftable, "var\tyyTok3\t= []int {\n") + fmt.Fprintf(ftable, "var\t%sTok3\t= []int {\n", prefix) c = 0 for i = 1; i <= ntokens; i++ { j = tokset[i].value @@ -2829,13 +2834,14 @@ func others() { // copy yaccpar fmt.Fprintf(ftable, "\n//line yaccpar:1\n") - parts := strings.Split(yaccpar, "yyrun()", 2) + parts := strings.Split(yaccpar, prefix+"run()", 2) fmt.Fprintf(ftable, "%v", parts[0]) ftable.Write(fcode.Bytes()) fmt.Fprintf(ftable, "%v", parts[1]) } func arout(s string, v []int, n int) { + s = prefix + s fmt.Fprintf(ftable, "var\t%v\t= []int {\n", s) for i := 0; i < n; i++ { if i%10 == 0 { @@ -3076,86 +3082,84 @@ func exit(status int) { os.Exit(status) } -var yaccpar = ` +var yaccpar string // will be processed version of yaccpartext: s/$$/prefix/g +var yaccpartext = ` /* parser for yacc output */ -var yyDebug = 0 +var $$Debug = 0 -type yyLexer interface { - Lex(lval *yySymType) int +type $$Lexer interface { + Lex(lval *$$SymType) int Error(s string) } -const yyFlag = -1000 +const $$Flag = -1000 -func yyTokname(yyc int) string { - if yyc > 0 && yyc <= len(yyToknames) { - if yyToknames[yyc-1] != "" { - return yyToknames[yyc-1] +func $$Tokname(c int) string { + if c > 0 && c <= len($$Toknames) { + if $$Toknames[c-1] != "" { + return $$Toknames[c-1] } } - return fmt.Sprintf("tok-%v", yyc) + return fmt.Sprintf("tok-%v", c) } -func yyStatname(yys int) string { - if yys >= 0 && yys < len(yyStatenames) { - if yyStatenames[yys] != "" { - return yyStatenames[yys] +func $$Statname(s int) string { + if s >= 0 && s < len($$Statenames) { + if $$Statenames[s] != "" { + return $$Statenames[s] } } - return fmt.Sprintf("state-%v", yys) + return fmt.Sprintf("state-%v", s) } -func yylex1(yylex yyLexer, lval *yySymType) int { - var yychar int - var c int - - yychar = yylex.Lex(lval) - if yychar <= 0 { - c = yyTok1[0] +func $$lex1(lex $$Lexer, lval *$$SymType) int { + c := 0 + char := lex.Lex(lval) + if char <= 0 { + c = $$Tok1[0] goto out } - if yychar < len(yyTok1) { - c = yyTok1[yychar] + if char < len($$Tok1) { + c = $$Tok1[char] goto out } - if yychar >= yyPrivate { - if yychar < yyPrivate+len(yyTok2) { - c = yyTok2[yychar-yyPrivate] + if char >= $$Private { + if char < $$Private+len($$Tok2) { + c = $$Tok2[char-$$Private] goto out } } - for i := 0; i < len(yyTok3); i += 2 { - c = yyTok3[i+0] - if c == yychar { - c = yyTok3[i+1] + for i := 0; i < len($$Tok3); i += 2 { + c = $$Tok3[i+0] + if c == char { + c = $$Tok3[i+1] goto out } } - c = 0 out: if c == 0 { - c = yyTok2[1] /* unknown char */ + c = $$Tok2[1] /* unknown char */ } - if yyDebug >= 3 { - fmt.Printf("lex %U %s\n", uint(yychar), yyTokname(c)) + if $$Debug >= 3 { + fmt.Printf("lex %U %s\n", uint(char), $$Tokname(c)) } return c } -func yyParse(yylex yyLexer) int { - var yyn int - var yylval yySymType - var YYVAL yySymType - YYS := make([]yySymType, yyMaxDepth) +func $$Parse($$lex $$Lexer) int { + var $$n int + var $$lval $$SymType + var $$VAL $$SymType + $$S := make([]$$SymType, $$MaxDepth) Nerrs := 0 /* number of errors */ Errflag := 0 /* error recovery flag */ - yystate := 0 - yychar := -1 - yyp := -1 - goto yystack + $$state := 0 + $$char := -1 + $$p := -1 + goto $$stack ret0: return 0 @@ -3163,80 +3167,80 @@ ret0: ret1: return 1 -yystack: +$$stack: /* put a state and value onto the stack */ - if yyDebug >= 4 { - fmt.Printf("char %v in %v\n", yyTokname(yychar), yyStatname(yystate)) + if $$Debug >= 4 { + fmt.Printf("char %v in %v\n", $$Tokname($$char), $$Statname($$state)) } - yyp++ - if yyp >= len(YYS) { - nyys := make([]yySymType, len(YYS)*2) - copy(nyys, YYS) - YYS = nyys + $$p++ + if $$p >= len($$S) { + nyys := make([]$$SymType, len($$S)*2) + copy(nyys, $$S) + $$S = nyys } - YYS[yyp] = YYVAL - YYS[yyp].yys = yystate + $$S[$$p] = $$VAL + $$S[$$p].yys = $$state -yynewstate: - yyn = yyPact[yystate] - if yyn <= yyFlag { - goto yydefault /* simple state */ +$$newstate: + $$n = $$Pact[$$state] + if $$n <= $$Flag { + goto $$default /* simple state */ } - if yychar < 0 { - yychar = yylex1(yylex, &yylval) + if $$char < 0 { + $$char = $$lex1($$lex, &$$lval) } - yyn += yychar - if yyn < 0 || yyn >= yyLast { - goto yydefault + $$n += $$char + if $$n < 0 || $$n >= $$Last { + goto $$default } - yyn = yyAct[yyn] - if yyChk[yyn] == yychar { /* valid shift */ - yychar = -1 - YYVAL = yylval - yystate = yyn + $$n = $$Act[$$n] + if $$Chk[$$n] == $$char { /* valid shift */ + $$char = -1 + $$VAL = $$lval + $$state = $$n if Errflag > 0 { Errflag-- } - goto yystack + goto $$stack } -yydefault: +$$default: /* default state action */ - yyn = yyDef[yystate] - if yyn == -2 { - if yychar < 0 { - yychar = yylex1(yylex, &yylval) + $$n = $$Def[$$state] + if $$n == -2 { + if $$char < 0 { + $$char = $$lex1($$lex, &$$lval) } /* look through exception table */ - yyxi := 0 + xi := 0 for { - if yyExca[yyxi+0] == -1 && yyExca[yyxi+1] == yystate { + if $$Exca[xi+0] == -1 && $$Exca[xi+1] == $$state { break } - yyxi += 2 + xi += 2 } - for yyxi += 2; ; yyxi += 2 { - yyn = yyExca[yyxi+0] - if yyn < 0 || yyn == yychar { + for xi += 2; ; xi += 2 { + $$n = $$Exca[xi+0] + if $$n < 0 || $$n == $$char { break } } - yyn = yyExca[yyxi+1] - if yyn < 0 { + $$n = $$Exca[xi+1] + if $$n < 0 { goto ret0 } } - if yyn == 0 { + if $$n == 0 { /* error ... attempt to resume parsing */ switch Errflag { case 0: /* brand new error */ - yylex.Error("syntax error") + $$lex.Error("syntax error") Nerrs++ - if yyDebug >= 1 { - fmt.Printf("%s", yyStatname(yystate)) - fmt.Printf("saw %s\n", yyTokname(yychar)) + if $$Debug >= 1 { + fmt.Printf("%s", $$Statname($$state)) + fmt.Printf("saw %s\n", $$Tokname($$char)) } fallthrough @@ -3244,64 +3248,64 @@ yydefault: Errflag = 3 /* find a state where "error" is a legal shift action */ - for yyp >= 0 { - yyn = yyPact[YYS[yyp].yys] + yyErrCode - if yyn >= 0 && yyn < yyLast { - yystate = yyAct[yyn] /* simulate a shift of "error" */ - if yyChk[yystate] == yyErrCode { - goto yystack + for $$p >= 0 { + $$n = $$Pact[$$S[$$p].yys] + $$ErrCode + if $$n >= 0 && $$n < $$Last { + $$state = $$Act[$$n] /* simulate a shift of "error" */ + if $$Chk[$$state] == $$ErrCode { + goto $$stack } } - /* the current yyp has no shift onn "error", pop stack */ - if yyDebug >= 2 { + /* the current p has no shift onn "error", pop stack */ + if $$Debug >= 2 { fmt.Printf("error recovery pops state %d, uncovers %d\n", - YYS[yyp].yys, YYS[yyp-1].yys) + $$S[$$p].yys, $$S[$$p-1].yys) } - yyp-- + $$p-- } /* there is no state on the stack with an error shift ... abort */ goto ret1 case 3: /* no shift yet; clobber input char */ - if yyDebug >= 2 { - fmt.Printf("error recovery discards %s\n", yyTokname(yychar)) + if $$Debug >= 2 { + fmt.Printf("error recovery discards %s\n", $$Tokname($$char)) } - if yychar == yyEofCode { + if $$char == $$EofCode { goto ret1 } - yychar = -1 - goto yynewstate /* try again in the same state */ + $$char = -1 + goto $$newstate /* try again in the same state */ } } - /* reduction by production yyn */ - if yyDebug >= 2 { - fmt.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) + /* reduction by production $$n */ + if $$Debug >= 2 { + fmt.Printf("reduce %v in:\n\t%v\n", $$n, $$Statname($$state)) } - yynt := yyn - yypt := yyp - _ = yypt // guard against "declared and not used" + $$nt := $$n + $$pt := $$p + _ = $$pt // guard against "declared and not used" - yyp -= yyR2[yyn] - YYVAL = YYS[yyp+1] + $$p -= $$R2[$$n] + $$VAL = $$S[$$p+1] /* consult goto table to find next state */ - yyn = yyR1[yyn] - yyg := yyPgo[yyn] - yyj := yyg + YYS[yyp].yys + 1 + $$n = $$R1[$$n] + $$g := $$Pgo[$$n] + $$j := $$g + $$S[$$p].yys + 1 - if yyj >= yyLast { - yystate = yyAct[yyg] + if $$j >= $$Last { + $$state = $$Act[$$g] } else { - yystate = yyAct[yyj] - if yyChk[yystate] != -yyn { - yystate = yyAct[yyg] + $$state = $$Act[$$j] + if $$Chk[$$state] != -$$n { + $$state = $$Act[$$g] } } // dummy call; replaced with literal code - yyrun() - goto yystack /* stack new state and value */ + $$run() + goto $$stack /* stack new state and value */ } ` diff --git a/src/cmd/goyacc/units.y b/src/cmd/goyacc/units.y index a7d472fc6..5d3f9aca2 100644 --- a/src/cmd/goyacc/units.y +++ b/src/cmd/goyacc/units.y @@ -6,6 +6,9 @@ // Distributed under the terms of the Lucent Public License Version 1.02 // See http://plan9.bell-labs.com/plan9/license.html +// Generate parser with prefix "units_": +// goyacc -p "units_" + %{ // units.y @@ -215,7 +218,7 @@ expr0: type UnitsLex int -func (UnitsLex) Lex(yylval *yySymType) int { +func (UnitsLex) Lex(yylval *units_SymType) int { var c, i int c = peekrune @@ -319,7 +322,7 @@ func main() { continue } peekrune = ':' - yyParse(UnitsLex(0)) + units_Parse(UnitsLex(0)) } /* @@ -340,7 +343,7 @@ func main() { } peekrune = '?' nerrors = 0 - yyParse(UnitsLex(0)) + units_Parse(UnitsLex(0)) if nerrors != 0 { continue } diff --git a/src/cmd/hgpatch/main.go b/src/cmd/hgpatch/main.go index bd4b563f9..2dcb5234c 100644 --- a/src/cmd/hgpatch/main.go +++ b/src/cmd/hgpatch/main.go @@ -14,7 +14,7 @@ import ( "io/ioutil" "os" "patch" - "path" + "path/filepath" "sort" "strings" ) @@ -186,7 +186,7 @@ func main() { // make parent directory for name, if necessary func makeParent(name string) { - parent, _ := path.Split(name) + parent, _ := filepath.Split(name) chk(mkdirAll(parent, 0755)) } diff --git a/src/cmd/ld/data.c b/src/cmd/ld/data.c index 0551232cf..a20b057ce 100644 --- a/src/cmd/ld/data.c +++ b/src/cmd/ld/data.c @@ -241,7 +241,7 @@ dynrelocsym(Sym *s) { Reloc *r; - if(thechar == '8' && HEADTYPE == 10) { // Windows PE + if(HEADTYPE == Hwindows) { Sym *rel, *targ; rel = lookup(".rel", 0); @@ -898,9 +898,9 @@ address(void) segdata.rwx = 06; segdata.vaddr = va; segdata.fileoff = va - segtext.vaddr + segtext.fileoff; - if((thechar == '6' || thechar == '8') && HEADTYPE == 10) // Windows PE + if(HEADTYPE == Hwindows) segdata.fileoff = segtext.fileoff + rnd(segtext.len, PEFILEALIGN); - if(thechar == '8' && HEADTYPE == 2) { // Plan 9 + if(HEADTYPE == Hplan9x32) { segdata.vaddr = va = rnd(va, 4096); segdata.fileoff = segtext.fileoff + segtext.filelen; } diff --git a/src/cmd/ld/dwarf.c b/src/cmd/ld/dwarf.c index 5df3515f5..5ba4b7c64 100644 --- a/src/cmd/ld/dwarf.c +++ b/src/cmd/ld/dwarf.c @@ -772,6 +772,9 @@ enum { KindUnsafePointer, KindNoPointers = 1<<7, + + // size of Type interface header + CommonType structure. + CommonSize = 2*PtrSize+ 4*PtrSize + 8, }; static Reloc* @@ -849,59 +852,59 @@ decodetype_size(Sym *s) static Sym* decodetype_arrayelem(Sym *s) { - return decode_reloc_sym(s, 5*PtrSize + 8); // 0x1c / 0x30 + return decode_reloc_sym(s, CommonSize); // 0x1c / 0x30 } static vlong decodetype_arraylen(Sym *s) { - return decode_inuxi(s->p + 6*PtrSize + 8, PtrSize); + return decode_inuxi(s->p + CommonSize+PtrSize, PtrSize); } // Type.PtrType.elem static Sym* decodetype_ptrelem(Sym *s) { - return decode_reloc_sym(s, 5*PtrSize + 8); // 0x1c / 0x30 + return decode_reloc_sym(s, CommonSize); // 0x1c / 0x30 } // Type.MapType.key, elem static Sym* decodetype_mapkey(Sym *s) { - return decode_reloc_sym(s, 5*PtrSize + 8); // 0x1c / 0x30 + return decode_reloc_sym(s, CommonSize); // 0x1c / 0x30 } static Sym* decodetype_mapvalue(Sym *s) { - return decode_reloc_sym(s, 6*PtrSize + 8); // 0x20 / 0x38 + return decode_reloc_sym(s, CommonSize+PtrSize); // 0x20 / 0x38 } // Type.ChanType.elem static Sym* decodetype_chanelem(Sym *s) { - return decode_reloc_sym(s, 5*PtrSize + 8); // 0x1c / 0x30 + return decode_reloc_sym(s, CommonSize); // 0x1c / 0x30 } // Type.FuncType.dotdotdot static int decodetype_funcdotdotdot(Sym *s) { - return s->p[5*PtrSize + 8]; + return s->p[CommonSize]; } // Type.FuncType.in.len static int decodetype_funcincount(Sym *s) { - return decode_inuxi(s->p + 7*PtrSize + 8, 4); + return decode_inuxi(s->p + CommonSize+2*PtrSize, 4); } static int decodetype_funcoutcount(Sym *s) { - return decode_inuxi(s->p + 8*PtrSize + 16, 4); + return decode_inuxi(s->p + CommonSize+3*PtrSize + 2*4, 4); } static Sym* @@ -909,7 +912,7 @@ decodetype_funcintype(Sym *s, int i) { Reloc *r; - r = decode_reloc(s, 6*PtrSize + 8); + r = decode_reloc(s, CommonSize + PtrSize); if (r == nil) return nil; return decode_reloc_sym(r->sym, r->add + i * PtrSize); @@ -920,7 +923,7 @@ decodetype_funcouttype(Sym *s, int i) { Reloc *r; - r = decode_reloc(s, 7*PtrSize + 16); + r = decode_reloc(s, CommonSize + 2*PtrSize + 2*4); if (r == nil) return nil; return decode_reloc_sym(r->sym, r->add + i * PtrSize); @@ -930,15 +933,18 @@ decodetype_funcouttype(Sym *s, int i) static int decodetype_structfieldcount(Sym *s) { - return decode_inuxi(s->p + 6*PtrSize + 8, 4); // 0x20 / 0x38 + return decode_inuxi(s->p + CommonSize + PtrSize, 4); } -// Type.StructType.fields[]-> name, typ and offset. sizeof(structField) = 5*PtrSize +enum { + StructFieldSize = 5*PtrSize +}; +// Type.StructType.fields[]-> name, typ and offset. static char* decodetype_structfieldname(Sym *s, int i) { // go.string."foo" 0x28 / 0x40 - s = decode_reloc_sym(s, 6*PtrSize + 0x10 + i*5*PtrSize); + s = decode_reloc_sym(s, CommonSize + PtrSize + 2*4 + i*StructFieldSize); if (s == nil) // embedded structs have a nil name. return nil; s = decode_reloc_sym(s, 0); // string."foo" @@ -950,20 +956,20 @@ decodetype_structfieldname(Sym *s, int i) static Sym* decodetype_structfieldtype(Sym *s, int i) { - return decode_reloc_sym(s, 8*PtrSize + 0x10 + i*5*PtrSize); // 0x30 / 0x50 + return decode_reloc_sym(s, CommonSize + PtrSize + 2*4 + i*StructFieldSize + 2*PtrSize); } static vlong decodetype_structfieldoffs(Sym *s, int i) { - return decode_inuxi(s->p + 10*PtrSize + 0x10 + i*5*PtrSize, 4); // 0x38 / 0x60 + return decode_inuxi(s->p + CommonSize + PtrSize + 2*4 + i*StructFieldSize + 4*PtrSize, 4); } // InterfaceTYpe.methods.len static vlong decodetype_ifacemethodcount(Sym *s) { - return decode_inuxi(s->p + 6*PtrSize + 8, 4); + return decode_inuxi(s->p + CommonSize + PtrSize, 4); } @@ -2302,7 +2308,7 @@ writegdbscript(void) static void align(vlong size) { - if((thechar == '6' || thechar == '8') && HEADTYPE == 10) // Only Windows PE need section align. + if(HEADTYPE == Hwindows) // Only Windows PE need section align. strnput("", rnd(size, PEFILEALIGN) - size); } diff --git a/src/cmd/ld/go.c b/src/cmd/ld/go.c index 2c6a6d084..3c1e230b4 100644 --- a/src/cmd/ld/go.c +++ b/src/cmd/ld/go.c @@ -550,6 +550,8 @@ mark(Sym *s) if(s == S || s->reachable) return; + if(strncmp(s->name, "weak.", 5) == 0) + return; s->reachable = 1; if(s->text) marktext(s); @@ -654,6 +656,37 @@ deadcode(void) textp = nil; else last->next = nil; + + for(i=0; i<NHASH; i++) + for(s = hash[i]; s != S; s = s->hash) + if(strncmp(s->name, "weak.", 5) == 0) { + s->special = 1; // do not lay out in data segment + s->reachable = 1; + } +} + +void +doweak(void) +{ + int i; + Sym *s, *t; + + // resolve weak references only if + // target symbol will be in binary anyway. + for(i=0; i<NHASH; i++) + for(s = hash[i]; s != S; s = s->hash) { + if(strncmp(s->name, "weak.", 5) == 0) { + t = lookup(s->name+5, s->version); + if(t->type != 0 && t->reachable) { + s->value = t->value; + s->type = t->type; + } else { + s->type = SCONST; + s->value = 0; + } + continue; + } + } } void diff --git a/src/cmd/ld/lib.c b/src/cmd/ld/lib.c index c144d4295..e645502b3 100644 --- a/src/cmd/ld/lib.c +++ b/src/cmd/ld/lib.c @@ -31,6 +31,8 @@ #include "l.h" #include "lib.h" +#include "../../pkg/runtime/stack.h" + #include <ar.h> int iconv(Fmt*); @@ -1084,3 +1086,208 @@ be64(uchar *b) Endian be = { be16, be32, be64 }; Endian le = { le16, le32, le64 }; + +typedef struct Chain Chain; +struct Chain +{ + Sym *sym; + Chain *up; + int limit; // limit on entry to sym +}; + +static int stkcheck(Chain*, int); +static void stkprint(Chain*, int); +static void stkbroke(Chain*, int); +static Sym *morestack; +static Sym *newstack; + +enum +{ + HasLinkRegister = (thechar == '5'), + CallSize = (!HasLinkRegister)*PtrSize, // bytes of stack required for a call +}; + +void +dostkcheck(void) +{ + Chain ch; + Sym *s; + + morestack = lookup("runtime.morestack", 0); + newstack = lookup("runtime.newstack", 0); + + // First the nosplits on their own. + for(s = textp; s != nil; s = s->next) { + if(s->text == nil || s->text->link == nil || (s->text->textflag & NOSPLIT) == 0) + continue; + cursym = s; + ch.up = nil; + ch.sym = s; + ch.limit = StackLimit - CallSize; + stkcheck(&ch, 0); + s->stkcheck = 1; + } + + // Check calling contexts. + // Some nosplits get called a little further down, + // like newproc and deferproc. We could hard-code + // that knowledge but it's more robust to look at + // the actual call sites. + for(s = textp; s != nil; s = s->next) { + if(s->text == nil || s->text->link == nil || (s->text->textflag & NOSPLIT) != 0) + continue; + cursym = s; + ch.up = nil; + ch.sym = s; + ch.limit = StackLimit - CallSize; + stkcheck(&ch, 0); + } +} + +static int +stkcheck(Chain *up, int depth) +{ + Chain ch, ch1; + Prog *p; + Sym *s; + int limit, prolog; + + limit = up->limit; + s = up->sym; + p = s->text; + + // Small optimization: don't repeat work at top. + if(s->stkcheck && limit == StackLimit-CallSize) + return 0; + + if(depth > 100) { + diag("nosplit stack check too deep"); + stkbroke(up, 0); + return -1; + } + + if(p == nil || p->link == nil) { + // external function. + // should never be called directly. + // only diagnose the direct caller. + if(depth == 1) + diag("call to external function %s", s->name); + return -1; + } + + if(limit < 0) { + stkbroke(up, limit); + return -1; + } + + // morestack looks like it calls functions, + // but it switches the stack pointer first. + if(s == morestack) + return 0; + + ch.up = up; + prolog = (s->text->textflag & NOSPLIT) == 0; + for(p = s->text; p != P; p = p->link) { + limit -= p->spadj; + if(prolog && p->spadj != 0) { + // The first stack adjustment in a function with a + // split-checking prologue marks the end of the + // prologue. Assuming the split check is correct, + // after the adjustment there should still be at least + // StackLimit bytes available below the stack pointer. + // If this is not the top call in the chain, no need + // to duplicate effort, so just stop. + if(depth > 0) + return 0; + prolog = 0; + limit = StackLimit; + } + if(limit < 0) { + stkbroke(up, limit); + return -1; + } + if(iscall(p)) { + limit -= CallSize; + ch.limit = limit; + if(p->to.type == D_BRANCH) { + // Direct call. + ch.sym = p->to.sym; + if(stkcheck(&ch, depth+1) < 0) + return -1; + } else { + // Indirect call. Assume it is a splitting function, + // so we have to make sure it can call morestack. + limit -= CallSize; + ch.sym = nil; + ch1.limit = limit; + ch1.up = &ch; + ch1.sym = morestack; + if(stkcheck(&ch1, depth+2) < 0) + return -1; + limit += CallSize; + } + limit += CallSize; + } + + } + return 0; +} + +static void +stkbroke(Chain *ch, int limit) +{ + diag("nosplit stack overflow"); + stkprint(ch, limit); +} + +static void +stkprint(Chain *ch, int limit) +{ + char *name; + + if(ch->sym) + name = ch->sym->name; + else + name = "function pointer"; + + if(ch->up == nil) { + // top of chain. ch->sym != nil. + if(ch->sym->text->textflag & NOSPLIT) + print("\t%d\tassumed on entry to %s\n", ch->limit, name); + else + print("\t%d\tguaranteed after split check in %s\n", ch->limit, name); + } else { + stkprint(ch->up, ch->limit + (!HasLinkRegister)*PtrSize); + if(!HasLinkRegister) + print("\t%d\ton entry to %s\n", ch->limit, name); + } + if(ch->limit != limit) + print("\t%d\tafter %s uses %d\n", limit, name, ch->limit - limit); +} + +int +headtype(char *name) +{ + int i; + + for(i=0; headers[i].name; i++) + if(strcmp(name, headers[i].name) == 0) { + headstring = headers[i].name; + return headers[i].val; + } + fprint(2, "unknown header type -H %s\n", name); + errorexit(); + return -1; // not reached +} + +void +undef(void) +{ + int i; + Sym *s; + + for(i=0; i<NHASH; i++) + for(s = hash[i]; s != S; s = s->hash) + if(s->type == SXREF) + diag("%s(%d): not defined", s->name, s->version); +} diff --git a/src/cmd/ld/lib.h b/src/cmd/ld/lib.h index 16dfb0dc3..adde2c9ff 100644 --- a/src/cmd/ld/lib.h +++ b/src/cmd/ld/lib.h @@ -74,7 +74,6 @@ extern int nlibdir; extern int cout; EXTERN char* INITENTRY; -EXTERN char thechar; EXTERN char* thestring; EXTERN Library* library; EXTERN int libraryp; @@ -167,6 +166,9 @@ void adddynlib(char*); int archreloc(Reloc*, Sym*, vlong*); void adddynsym(Sym*); void addexport(void); +void dostkcheck(void); +void undef(void); +void doweak(void); int pathchar(void); void* mal(uint32); @@ -208,3 +210,36 @@ enum { ArchiveObj, Pkgdef }; + +/* executable header types */ +enum { + Hgarbunix = 0, // garbage unix + Hnoheader, // no header + Hunixcoff, // unix coff + Hrisc, // aif for risc os + Hplan9x32, // plan 9 32-bit format + Hplan9x64, // plan 9 64-bit format + Hmsdoscom, // MS-DOS .COM + Hnetbsd, // NetBSD + Hmsdosexe, // fake MS-DOS .EXE + Hixp1200, // IXP1200 (raw) + Helf, // ELF32 + Hipaq, // ipaq + Hdarwin, // Apple Mach-O + Hlinux, // Linux ELF + Hnacl, // Google Native Client + Hfreebsd, // FreeBSD ELF + Hwindows, // MS Windows PE + Htiny // tiny (os image) +}; + +typedef struct Header Header; +struct Header { + char *name; + int val; +}; + +EXTERN char* headstring; +extern Header headers[]; + +int headtype(char*); diff --git a/src/cmd/ld/macho.c b/src/cmd/ld/macho.c index 402e0ec63..c8d7c4a6d 100644 --- a/src/cmd/ld/macho.c +++ b/src/cmd/ld/macho.c @@ -276,7 +276,6 @@ asmbmacho(void) vlong v, w; vlong va; int a, i; - char *pkgroot; MachoHdr *mh; MachoSect *msect; MachoSeg *ms; @@ -428,12 +427,6 @@ asmbmacho(void) ml->data[0] = 12; /* offset to string */ strcpy((char*)&ml->data[1], "/usr/lib/dyld"); - if(ndylib > 0) { /* add reference to where .so files are installed */ - pkgroot = smprint("%s/pkg/%s_%s", goroot, goos, goarch); - ml = newMachoLoad(0x80000000 | 0x1c, 1+(strlen(pkgroot)+1+7)/8*2); /* LC_RPATH */ - ml->data[0] = 12; /* offset of string from beginning of load */ - strcpy((char*)&ml->data[1], pkgroot); - } for(i=0; i<ndylib; i++) { ml = newMachoLoad(12, 4+(strlen(dylib[i])+1+7)/8*2); /* LC_LOAD_DYLIB */ ml->data[0] = 24; /* offset of string from beginning of load */ diff --git a/src/cmd/ld/pe.c b/src/cmd/ld/pe.c index 2c34daab4..e72b0b2a0 100644 --- a/src/cmd/ld/pe.c +++ b/src/cmd/ld/pe.c @@ -500,6 +500,7 @@ asmbpe(void) IMAGE_FILE_EXECUTABLE_IMAGE|IMAGE_FILE_DEBUG_STRIPPED; if (pe64) { fh.SizeOfOptionalHeader = sizeof(oh64); + fh.Characteristics |= IMAGE_FILE_LARGE_ADDRESS_AWARE; set(Magic, 0x20b); // PE32+ } else { fh.SizeOfOptionalHeader = sizeof(oh); @@ -525,8 +526,11 @@ asmbpe(void) set(MinorSubsystemVersion, 0); set(SizeOfImage, nextsectoff); set(SizeOfHeaders, PEFILEHEADR); - set(Subsystem, 3); // WINDOWS_CUI - set(SizeOfStackReserve, 0x00200000); + if(strcmp(headstring, "windowsgui") == 0) + set(Subsystem, IMAGE_SUBSYSTEM_WINDOWS_GUI); + else + set(Subsystem, IMAGE_SUBSYSTEM_WINDOWS_CUI); + set(SizeOfStackReserve, 0x0040000); set(SizeOfStackCommit, 0x00001000); set(SizeOfHeapReserve, 0x00100000); set(SizeOfHeapCommit, 0x00001000); diff --git a/src/cmd/ld/pe.h b/src/cmd/ld/pe.h index 6dbf6a5be..2180fb88c 100644 --- a/src/cmd/ld/pe.h +++ b/src/cmd/ld/pe.h @@ -131,6 +131,9 @@ enum { IMAGE_DIRECTORY_ENTRY_IAT = 12, IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT = 13, IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR = 14, + + IMAGE_SUBSYSTEM_WINDOWS_GUI = 2, + IMAGE_SUBSYSTEM_WINDOWS_CUI = 3, }; void peinit(void); diff --git a/src/cmd/make.bash b/src/cmd/make.bash deleted file mode 100755 index 63da74625..000000000 --- a/src/cmd/make.bash +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e - -bash clean.bash - -eval $(gomake --no-print-directory -f ../Make.inc go-env) -if [ -z "$O" ]; then - echo 'missing $O - maybe no Make.$GOARCH?' 1>&2 - exit 1 -fi - -cd ${O}l -bash mkenam -gomake enam.o -cd .. - -# Note: commands written in Go are not listed here. -# They are in ../pkg/Makefile so that they can be built -# after the Go libraries on which they depend. -for i in cc ${O}l ${O}a ${O}c gc ${O}g cov godefs gopack gotest nm prof -do - echo; echo; echo %%%% making $i %%%%; echo - cd $i - gomake install - cd .. -done diff --git a/src/env.bash b/src/env.bash index 4fc762821..c1055d561 100644 --- a/src/env.bash +++ b/src/env.bash @@ -29,7 +29,7 @@ if [ ! -d "$GOBIN" -a "$GOBIN" != "$GOROOT/bin" ]; then fi export OLDPATH=$PATH -export PATH=/bin:/usr/bin:"$GOBIN":$PATH +export PATH="$GOBIN":/bin:/usr/bin:$PATH MAKE=make if ! make --version 2>/dev/null | grep 'GNU Make' >/dev/null; then diff --git a/src/make.bash b/src/make.bash index 43c70a87b..d9ca40d42 100755 --- a/src/make.bash +++ b/src/make.bash @@ -61,29 +61,8 @@ bash "$GOROOT"/src/clean.bash # pkg builds libcgo and the Go programs in cmd. for i in lib9 libbio libmach cmd pkg do - case "$i-$GOOS-$GOARCH" in - cmd/*-nacl-*) - ;; - *) - # The ( ) here are to preserve the current directory - # for the next round despite the cd $i below. - # set -e does not apply to ( ) so we must explicitly - # test the exit status. - ( - echo; echo; echo %%%% making $i %%%%; echo - cd "$GOROOT"/src/$i - case $i in - cmd) - bash make.bash - ;; - pkg) - gomake install - ;; - *) - gomake install - esac - ) || exit 1 - esac + echo; echo; echo %%%% making $i %%%%; echo + gomake -C $i install done # Print post-install messages. diff --git a/src/pkg/Makefile b/src/pkg/Makefile index 619167ca4..6e70690d1 100644 --- a/src/pkg/Makefile +++ b/src/pkg/Makefile @@ -21,8 +21,10 @@ DIRS=\ bufio\ bytes\ cmath\ + compress/bzip2\ compress/flate\ compress/gzip\ + compress/lzw \ compress/zlib\ container/heap\ container/list\ @@ -40,6 +42,11 @@ DIRS=\ crypto/md4\ crypto/md5\ crypto/ocsp\ + crypto/openpgp\ + crypto/openpgp/armor\ + crypto/openpgp/error\ + crypto/openpgp/packet\ + crypto/openpgp/s2k\ crypto/rand\ crypto/rc4\ crypto/ripemd160\ @@ -89,7 +96,9 @@ DIRS=\ hash/crc64\ html\ http\ + http/cgi\ http/pprof\ + http/httptest\ image\ image/jpeg\ image/png\ @@ -109,6 +118,7 @@ DIRS=\ os/signal\ patch\ path\ + path/filepath\ rand\ reflect\ regexp\ @@ -124,6 +134,7 @@ DIRS=\ strconv\ strings\ sync\ + sync/atomic\ syscall\ syslog\ tabwriter\ @@ -156,6 +167,7 @@ endif NOTEST=\ crypto\ + crypto/openpgp/error\ debug/proc\ exp/draw/x11\ go/ast\ @@ -163,7 +175,7 @@ NOTEST=\ go/token\ hash\ http/pprof\ - image\ + http/httptest\ image/jpeg\ net/dict\ rand\ @@ -211,19 +223,19 @@ test.dirs: $(addsuffix .test, $(TEST)) bench.dirs: $(addsuffix .bench, $(BENCH)) %.clean: - +cd $* && $(MAKE) clean + +$(MAKE) -C $* clean %.install: - +cd $* && $(MAKE) install + +$(MAKE) -C $* install %.nuke: - +cd $* && $(MAKE) nuke + +$(MAKE) -C $* nuke %.test: - +cd $* && $(MAKE) test + +$(MAKE) -C $* test %.bench: - +cd $* && $(MAKE) bench + +$(MAKE) -C $* bench clean: clean.dirs diff --git a/src/pkg/compress/bzip2/Makefile b/src/pkg/compress/bzip2/Makefile new file mode 100644 index 000000000..a4bceef16 --- /dev/null +++ b/src/pkg/compress/bzip2/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/bzip2 +GOFILES=\ + bit_reader.go\ + bzip2.go\ + huffman.go\ + move_to_front.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/bzip2/bit_reader.go b/src/pkg/compress/bzip2/bit_reader.go new file mode 100644 index 000000000..50f0ec836 --- /dev/null +++ b/src/pkg/compress/bzip2/bit_reader.go @@ -0,0 +1,88 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bufio" + "io" + "os" +) + +// bitReader wraps an io.Reader and provides the ability to read values, +// bit-by-bit, from it. Its Read* methods don't return the usual os.Error +// because the error handling was verbose. Instead, any error is kept and can +// be checked afterwards. +type bitReader struct { + r byteReader + n uint64 + bits uint + err os.Error +} + +// bitReader needs to read bytes from an io.Reader. We attempt to cast the +// given io.Reader to this interface and, if it doesn't already fit, we wrap in +// a bufio.Reader. +type byteReader interface { + ReadByte() (byte, os.Error) +} + +func newBitReader(r io.Reader) bitReader { + byter, ok := r.(byteReader) + if !ok { + byter = bufio.NewReader(r) + } + return bitReader{r: byter} +} + +// ReadBits64 reads the given number of bits and returns them in the +// least-significant part of a uint64. In the event of an error, it returns 0 +// and the error can be obtained by calling Error(). +func (br *bitReader) ReadBits64(bits uint) (n uint64) { + for bits > br.bits { + b, err := br.r.ReadByte() + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + br.err = err + return 0 + } + br.n <<= 8 + br.n |= uint64(b) + br.bits += 8 + } + + // br.n looks like this (assuming that br.bits = 14 and bits = 6): + // Bit: 111111 + // 5432109876543210 + // + // (6 bits, the desired output) + // |-----| + // V V + // 0101101101001110 + // ^ ^ + // |------------| + // br.bits (num valid bits) + // + // This the next line right shifts the desired bits into the + // least-significant places and masks off anything above. + n = (br.n >> (br.bits - bits)) & ((1 << bits) - 1) + br.bits -= bits + return +} + +func (br *bitReader) ReadBits(bits uint) (n int) { + n64 := br.ReadBits64(bits) + return int(n64) +} + +func (br *bitReader) ReadBit() bool { + n := br.ReadBits(1) + return n != 0 +} + +func (br *bitReader) Error() os.Error { + return br.err +} diff --git a/src/pkg/compress/bzip2/bzip2.go b/src/pkg/compress/bzip2/bzip2.go new file mode 100644 index 000000000..9e97edec1 --- /dev/null +++ b/src/pkg/compress/bzip2/bzip2.go @@ -0,0 +1,390 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bzip2 implements bzip2 decompression. +package bzip2 + +import ( + "io" + "os" +) + +// There's no RFC for bzip2. I used the Wikipedia page for reference and a lot +// of guessing: http://en.wikipedia.org/wiki/Bzip2 +// The source code to pyflate was useful for debugging: +// http://www.paul.sladen.org/projects/pyflate + +// A StructuralError is returned when the bzip2 data is found to be +// syntactically invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "bzip2 data invalid: " + string(s) +} + +// A reader decompresses bzip2 compressed data. +type reader struct { + br bitReader + setupDone bool // true if we have parsed the bzip2 header. + blockSize int // blockSize in bytes, i.e. 900 * 1024. + eof bool + buf []byte // stores Burrows-Wheeler transformed data. + c [256]uint // the `C' array for the inverse BWT. + tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits. + tPos uint32 // Index of the next output byte in tt. + + preRLE []uint32 // contains the RLE data still to be processed. + preRLEUsed int // number of entries of preRLE used. + lastByte int // the last byte value seen. + byteRepeats uint // the number of repeats of lastByte seen. + repeats uint // the number of copies of lastByte to output. +} + +// NewReader returns an io.Reader which decompresses bzip2 data from r. +func NewReader(r io.Reader) io.Reader { + bz2 := new(reader) + bz2.br = newBitReader(r) + return bz2 +} + +const bzip2FileMagic = 0x425a // "BZ" +const bzip2BlockMagic = 0x314159265359 +const bzip2FinalMagic = 0x177245385090 + +// setup parses the bzip2 header. +func (bz2 *reader) setup() os.Error { + br := &bz2.br + + magic := br.ReadBits(16) + if magic != bzip2FileMagic { + return StructuralError("bad magic value") + } + + t := br.ReadBits(8) + if t != 'h' { + return StructuralError("non-Huffman entropy encoding") + } + + level := br.ReadBits(8) + if level < '1' || level > '9' { + return StructuralError("invalid compression level") + } + + bz2.blockSize = 100 * 1024 * (int(level) - '0') + bz2.tt = make([]uint32, bz2.blockSize) + return nil +} + +func (bz2 *reader) Read(buf []byte) (n int, err os.Error) { + if bz2.eof { + return 0, os.EOF + } + + if !bz2.setupDone { + err = bz2.setup() + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + if err != nil { + return 0, err + } + bz2.setupDone = true + } + + n, err = bz2.read(buf) + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + return +} + +func (bz2 *reader) read(buf []byte) (n int, err os.Error) { + // bzip2 is a block based compressor, except that it has a run-length + // preprocessing step. The block based nature means that we can + // preallocate fixed-size buffers and reuse them. However, the RLE + // preprocessing would require allocating huge buffers to store the + // maximum expansion. Thus we process blocks all at once, except for + // the RLE which we decompress as required. + + for (bz2.repeats > 0 || bz2.preRLEUsed < len(bz2.preRLE)) && n < len(buf) { + // We have RLE data pending. + + // The run-length encoding works like this: + // Any sequence of four equal bytes is followed by a length + // byte which contains the number of repeats of that byte to + // include. (The number of repeats can be zero.) Because we are + // decompressing on-demand our state is kept in the reader + // object. + + if bz2.repeats > 0 { + buf[n] = byte(bz2.lastByte) + n++ + bz2.repeats-- + if bz2.repeats == 0 { + bz2.lastByte = -1 + } + continue + } + + bz2.tPos = bz2.preRLE[bz2.tPos] + b := byte(bz2.tPos) + bz2.tPos >>= 8 + bz2.preRLEUsed++ + + if bz2.byteRepeats == 3 { + bz2.repeats = uint(b) + bz2.byteRepeats = 0 + continue + } + + if bz2.lastByte == int(b) { + bz2.byteRepeats++ + } else { + bz2.byteRepeats = 0 + } + bz2.lastByte = int(b) + + buf[n] = b + n++ + } + + if n > 0 { + return + } + + // No RLE data is pending so we need to read a block. + + br := &bz2.br + magic := br.ReadBits64(48) + if magic == bzip2FinalMagic { + br.ReadBits64(32) // ignored CRC + bz2.eof = true + return 0, os.EOF + } else if magic != bzip2BlockMagic { + return 0, StructuralError("bad magic value found") + } + + err = bz2.readBlock() + if err != nil { + return 0, err + } + + return bz2.read(buf) +} + +// readBlock reads a bzip2 block. The magic number should already have been consumed. +func (bz2 *reader) readBlock() (err os.Error) { + br := &bz2.br + br.ReadBits64(32) // skip checksum. TODO: check it if we can figure out what it is. + randomized := br.ReadBits(1) + if randomized != 0 { + return StructuralError("deprecated randomized files") + } + origPtr := uint(br.ReadBits(24)) + + // If not every byte value is used in the block (i.e., it's text) then + // the symbol set is reduced. The symbols used are stored as a + // two-level, 16x16 bitmap. + symbolRangeUsedBitmap := br.ReadBits(16) + symbolPresent := make([]bool, 256) + numSymbols := 0 + for symRange := uint(0); symRange < 16; symRange++ { + if symbolRangeUsedBitmap&(1<<(15-symRange)) != 0 { + bits := br.ReadBits(16) + for symbol := uint(0); symbol < 16; symbol++ { + if bits&(1<<(15-symbol)) != 0 { + symbolPresent[16*symRange+symbol] = true + numSymbols++ + } + } + } + } + + // A block uses between two and six different Huffman trees. + numHuffmanTrees := br.ReadBits(3) + if numHuffmanTrees < 2 || numHuffmanTrees > 6 { + return StructuralError("invalid number of Huffman trees") + } + + // The Huffman tree can switch every 50 symbols so there's a list of + // tree indexes telling us which tree to use for each 50 symbol block. + numSelectors := br.ReadBits(15) + treeIndexes := make([]uint8, numSelectors) + + // The tree indexes are move-to-front transformed and stored as unary + // numbers. + mtfTreeDecoder := newMTFDecoderWithRange(numHuffmanTrees) + for i := range treeIndexes { + c := 0 + for { + inc := br.ReadBits(1) + if inc == 0 { + break + } + c++ + } + if c >= numHuffmanTrees { + return StructuralError("tree index too large") + } + treeIndexes[i] = uint8(mtfTreeDecoder.Decode(c)) + } + + // The list of symbols for the move-to-front transform is taken from + // the previously decoded symbol bitmap. + symbols := make([]byte, numSymbols) + nextSymbol := 0 + for i := 0; i < 256; i++ { + if symbolPresent[i] { + symbols[nextSymbol] = byte(i) + nextSymbol++ + } + } + mtf := newMTFDecoder(symbols) + + numSymbols += 2 // to account for RUNA and RUNB symbols + huffmanTrees := make([]huffmanTree, numHuffmanTrees) + + // Now we decode the arrays of code-lengths for each tree. + lengths := make([]uint8, numSymbols) + for i := 0; i < numHuffmanTrees; i++ { + // The code lengths are delta encoded from a 5-bit base value. + length := br.ReadBits(5) + for j := 0; j < numSymbols; j++ { + for { + if !br.ReadBit() { + break + } + if br.ReadBit() { + length-- + } else { + length++ + } + } + if length < 0 || length > 20 { + return StructuralError("Huffman length out of range") + } + lengths[j] = uint8(length) + } + huffmanTrees[i], err = newHuffmanTree(lengths) + if err != nil { + return err + } + } + + selectorIndex := 1 // the next tree index to use + currentHuffmanTree := huffmanTrees[treeIndexes[0]] + bufIndex := 0 // indexes bz2.buf, the output buffer. + // The output of the move-to-front transform is run-length encoded and + // we merge the decoding into the Huffman parsing loop. These two + // variables accumulate the repeat count. See the Wikipedia page for + // details. + repeat := 0 + repeat_power := 0 + + // The `C' array (used by the inverse BWT) needs to be zero initialised. + for i := range bz2.c { + bz2.c[i] = 0 + } + + decoded := 0 // counts the number of symbols decoded by the current tree. + for { + if decoded == 50 { + currentHuffmanTree = huffmanTrees[treeIndexes[selectorIndex]] + selectorIndex++ + decoded = 0 + } + + v := currentHuffmanTree.Decode(br) + decoded++ + + if v < 2 { + // This is either the RUNA or RUNB symbol. + if repeat == 0 { + repeat_power = 1 + } + repeat += repeat_power << v + repeat_power <<= 1 + + // This limit of 2 million comes from the bzip2 source + // code. It prevents repeat from overflowing. + if repeat > 2*1024*1024 { + return StructuralError("repeat count too large") + } + continue + } + + if repeat > 0 { + // We have decoded a complete run-length so we need to + // replicate the last output symbol. + for i := 0; i < repeat; i++ { + b := byte(mtf.First()) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + repeat = 0 + } + + if int(v) == numSymbols-1 { + // This is the EOF symbol. Because it's always at the + // end of the move-to-front list, and nevers gets moved + // to the front, it has this unique value. + break + } + + // Since two metasymbols (RUNA and RUNB) have values 0 and 1, + // one would expect |v-2| to be passed to the MTF decoder. + // However, the front of the MTF list is never referenced as 0, + // it's always referenced with a run-length of 1. Thus 0 + // doesn't need to be encoded and we have |v-1| in the next + // line. + b := byte(mtf.Decode(int(v - 1))) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + + if origPtr >= uint(bufIndex) { + return StructuralError("origPtr out of bounds") + } + + // We have completed the entropy decoding. Now we can perform the + // inverse BWT and setup the RLE buffer. + bz2.preRLE = bz2.tt[:bufIndex] + bz2.preRLEUsed = 0 + bz2.tPos = inverseBWT(bz2.preRLE, origPtr, bz2.c[:]) + bz2.lastByte = -1 + bz2.byteRepeats = 0 + bz2.repeats = 0 + + return nil +} + +// inverseBWT implements the inverse Burrows-Wheeler transform as described in +// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf, section 4.2. +// In that document, origPtr is called `I' and c is the `C' array after the +// first pass over the data. It's an argument here because we merge the first +// pass with the Huffman decoding. +// +// This also implements the `single array' method from the bzip2 source code +// which leaves the output, still shuffled, in the bottom 8 bits of tt with the +// index of the next byte in the top 24-bits. The index of the first byte is +// returned. +func inverseBWT(tt []uint32, origPtr uint, c []uint) uint32 { + sum := uint(0) + for i := 0; i < 256; i++ { + sum += c[i] + c[i] = sum - c[i] + } + + for i := range tt { + b := tt[i] & 0xff + tt[c[b]] |= uint32(i) << 8 + c[b]++ + } + + return tt[origPtr] >> 8 +} diff --git a/src/pkg/compress/bzip2/bzip2_test.go b/src/pkg/compress/bzip2/bzip2_test.go new file mode 100644 index 000000000..156eea83f --- /dev/null +++ b/src/pkg/compress/bzip2/bzip2_test.go @@ -0,0 +1,158 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestBitReader(t *testing.T) { + buf := bytes.NewBuffer([]byte{0xaa}) + br := newBitReader(buf) + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 1 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 2 wrong") + } + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 3 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 4 wrong") + } +} + +func TestBitReaderLarge(t *testing.T) { + buf := bytes.NewBuffer([]byte{0x12, 0x34, 0x56, 0x78}) + br := newBitReader(buf) + if n := br.ReadBits(32); n != 0x12345678 { + t.Errorf("got: %x want: %x", n, 0x12345678) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func decompressHex(s string) (out []byte, err os.Error) { + r := NewReader(readerFromHex(s)) + return ioutil.ReadAll(r) +} + +func TestHelloWorldBZ2(t *testing.T) { + out, err := decompressHex(helloWorldBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + if !bytes.Equal(helloWorld, out) { + t.Errorf("got %x, want %x", out, helloWorld) + } +} + +func testZeros(t *testing.T, inHex string, n int) { + out, err := decompressHex(inHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, n) + + if !bytes.Equal(expected, out) { + allZeros := true + for _, b := range out { + if b != 0 { + allZeros = false + break + } + } + t.Errorf("incorrect result, got %d bytes (allZeros: %t)", len(out), allZeros) + } +} + +func Test32Zeros(t *testing.T) { + testZeros(t, thirtyTwoZerosBZ2Hex, 32) +} + +func Test1MBZeros(t *testing.T) { + testZeros(t, oneMBZerosBZ2Hex, 1024*1024) +} + +func testRandomData(t *testing.T, compressedHex, uncompressedHex string) { + out, err := decompressHex(compressedHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected, _ := hex.DecodeString(uncompressedHex) + + if !bytes.Equal(out, expected) { + t.Errorf("incorrect result\ngot: %x\nwant: %x", out, expected) + } +} + +func TestRandomData1(t *testing.T) { + testRandomData(t, randBZ2Hex, randHex) +} + +func TestRandomData2(t *testing.T) { + // This test involves several repeated bytes in the output, but they + // should trigger RLE decoding. + testRandomData(t, rand2BZ2Hex, rand2Hex) +} + +func TestRandomData3(t *testing.T) { + // This test uses the full range of symbols. + testRandomData(t, rand3BZ2Hex, rand3Hex) +} + +func Test1MBSawtooth(t *testing.T) { + out, err := decompressHex(oneMBSawtoothBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, 1024*1024) + + for i := range expected { + expected[i] = byte(i) + } + + if !bytes.Equal(out, expected) { + t.Error("incorrect result") + } +} + +const helloWorldBZ2Hex = "425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0" + +var helloWorld = []byte("hello world\n") + +const thirtyTwoZerosBZ2Hex = "425a6839314159265359b5aa5098000000600040000004200021008283177245385090b5aa5098" +const oneMBZerosBZ2Hex = "425a683931415926535938571ce50008084000c0040008200030cc0529a60806c4201e2ee48a70a12070ae39ca" + +const randBZ2Hex = "425a6839314159265359905d990d0001957fffffffffffafffffffffffffffffbfff6fffdfffffffffffffffffffffffffffffc002b6dd75676ed5b77720098320d11a64626981323d4da47a83131a13d09e8040f534cd4f4d27a464d193008cd09804601347a980026350c9886234d36864193d1351b44c136919e90340d26127a4cd264c32023009898981310c0344c340027a8303427a99a04c00003534c230d034f5006468d268cf54d36a3009a69a62626261311b40026013d34201a6934c9a604c98ca6c8460989fa9346234d30d3469a2604fd4131a7aa6d0046043d4c62098479269e89e835190d018d4c046001a11e801a0264792321932308c43a130688c260d46686804cd01a9e80981193684c6a68c00000004c4c20c04627a4c0000260003400d04c0681a01334026009a6f48041466132581ec5212b081d96b0effc16543e2228b052fcd30f2567ee8d970e0f10aabca68dd8270591c376cfc1baae0dba00aaff2d6caf6b211322c997cc18eaee5927f75185336bf907021324c71626c1dd20e22b9b0977f05d0f901eaa51db9fbaf7c603b4c87bc82890e6dd7e61d0079e27ec050dd788fd958152061cd01e222f9547cb9efc465d775b6fc98bac7d387bffd151ae09dadf19494f7a638e2eae58e550faba5fe6820ea520eb986096de4e527d80def3ba625e71fbefdcf7e7844e0a25d29b52dcd1344fca083737d42692aab38d230485f3c8ed54c2ed31f15cf0270c8143765b10b92157233fa1dfe0d7ce8ffe70b8b8f7250071701dfe9f1c94de362c9031455951c93eb098a6b50ee45c6131fefc3b6f9643e21f4adc59497138e246f5c57d834aa67c4f10d8bd8b3908d8130dd7388409c299a268eab3664fa4907c5c31574874bd8d388a4ab22b339660804e53e1b8d05867d40e3082560608d35d5d2c6054e8bab23da28f61f83efd41d25529ad6ea15fb50505cacfabb0902166427354ca3830a2c8415f21b19e592690fbe447020d685a4bcd16ecc4ff1a1c0e572627d0ef6265c008a43fc243240541061ed7840606be466d1c0dac2c53250ed567507d926c844154560d631960c65e15157829b2c7f16859f111a3a8cb72bf24ffa57a680c3be67b1be67c8dd8aea73ac2437a78df5b686d427080ebc01bd30b71a49f6ea31dc0f08e4849e38face96717690239538bc08b6cc5aa8d467cb9c36aa83d40ac7e58bddbfa185b22065e89a86c0145569d9e23726651aec49e31588d70f40fe9a4449dcf4f89eac220171e9c938e803dc195679651004b79ad33cc0c13aeeba5941b33ffeeb8fbe16e76c7811445c67b4269c90479433ddf9e8ed1d00c166b6c17217fb22c3ef1b0c1c7e28e185446a111c37f1ea6c07a59fbcc6546ecc6968d36ba58bc5489a5640647e426b0c39350cb6f07d5dc7a717648c4ec7f841467597ae1f65f408fd2d9940a4b1b860b3c9ae351dcae0b4425f7e8538710f2e40b7f70d13b51ac05ccc6ecda8264a88cad2d721d18132a9b9110a9e759c2483c77dcefc7e464ec88588174cb0c9abff93230ea0bed8decdd8ed8bfe2b5df0a253803678df04fab44c03b9ab7cc97d6e6d6fd0c4c840ce0efc498436f453bbb181603459471f2b588724592b222ec990614db530e10cadd84705621cfdd9261fa44a5f5806a2d74b575056b3c915255c65678f9c16e6dc00a99180fef1a840aff0e842ac02731080cc92782538360a60a727991013984da4fad95f79d5030677b7528d076b2483685fca4429edf804682fdc110dfc2f7c30e23e20a72e039108a0ad6fdee2f76985a4b4be4f5afc6101bf9d5042b657a05dc914e1424241766434" +const randHex = "c95138082bdf2b9bfa5b1072b23f729735d42c785eeb94320fb14c265b9c2ca421d01a3db986df1ac2acde5a0e6bf955d6f95e61261540905928e195f1a66644cc7f37281744fff4dc6df35566a494c41a8167151950eb74f5fc45f85ad0e5ed28b49adfe218aa7ec1707e8e1d55825f61f72beda3b4c006b8c9188d7336a5d875329b1b58c27cc4e89ecbae02c7712400c39dd131d2c6de82e2863da51d472bdfb21ecce62cc9cf769ed28aedc7583d755da45a0d90874bda269dd53283a9bdfd05f95fc8e9a304bb338ea1a2111894678c18134f17d31a15d9bfc1237894650f3e715e2548639ecbddb845cfe4a46a7b3a3c540f48629488e8c869f1e9f3f4c552243a8105b20eb8e264994214349dae83b165fd6c2a5b8e83fce09fc0a80d3281c8d53a9a08095bd19cbc1388df23975646ed259e003d39261ee68cbece8bcf32971f7fe7e588e8ba8f5e8597909abaea693836a79a1964050ed910a45a0f13a58cd2d3ae18992c5b23082407fd920d0bf01e33118a017bb5e39f44931346845af52128f7965206759433a346034ea481671f501280067567619f5ecef6cded077f92ed7f3b3ce8e308c80f34ba06939e9303f91b4318c8c1dd4cc223c1f057ac0c91211c629cd30e46ee9ec1d9fd493086b7bc2bc83e33f08749a5d430b0ed4f79d70f481940c9b0930b16321886a0df4fa5a1465d5208c7d3494a7987d9a5e42aa256f0c9523947f8318d0ef0af3d59a45cfc2418d0785c9a548b32b81e7de18be7d55a69a4c156bbb3d7579c0ac8e9c72b24646e54b0d0e8725f8f49fb44ae3c6b9d0287be118586255a90a4a83483ed0328518037e52aa959c5748ed83e13023e532306be98b8288da306bbb040bcf5d92176f84a9306dc6b274b040370b61d71fde58dd6d20e6fee348eae0c54bd0a5a487b2d005f329794f2a902c296af0a4c1f638f63292a1fa18e006c1b1838636f4de71c73635b25660d32e88a0917e1a5677f6a02ca65585b82cbd99fb4badbfa97a585da1e6cadf6737b4ec6ca33f245d66ee6a9fae6785d69b003c17b9fc6ec34fe5824ab8caae5e8e14dc6f9e116e7bf4a60c04388783c8ae929e1b46b3ef3bbe81b38f2fa6da771bf39dfba2374d3d2ed356b8e2c42081d885a91a3afb2f31986d2f9873354c48cf5448492c32e62385af423aa4f83db6d1b2669650379a1134b0a04cbca0862d6f9743c791cbb527d36cd5d1f0fc7f503831c8bd1b7a0ef8ae1a5ed1155dfdd9e32b6bb33138112d3d476b802179cb85a2a6c354ccfed2f31604fbd8d6ec4baf9f1c8454f72c6588c06a7df3178c43a6970bfa02dd6f74cb5ec3b63f9eddaa17db5cbf27fac6de8e57c384afd0954179f7b5690c3bee42abc4fa79b4b12101a9cf5f0b9aecdda945def0bd04163237247d3539850e123fe18139f316fa0256d5bd2faa8" + +const oneMBSawtoothBZ2Hex = "425a683931415926535971931ea00006ddffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe007de00000000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000009aaaaa0000000000000000000000000000000000000000000000000000000000000000498002600026000000000000000000000000000000000000000000000000000000007fc42271980d044c0a822607411304a08982d044c1a82260f411308a08984d044c2a82261741130ca08986d044c3a82261f411310a08988d044c4a822627411314a0898ad044c5a82262f411318a0898cd044c6a82263741131ca0898ed044c7a82263f411320a08990d044c8a822647411324a08992d044c9a82264f411328a08994d044caa82265741132ca08996d044cba82265f411330a08998d044cca822667411334a0899ad044cda82266f411338a0899cd044cea82267741133ca0899ed044cfa82267f411340a089a0d044d0a822687411344a089a2d044d1a82268f411348a089a4d044d2a82269741134ca089a6d044d3a82269f411350a089a8d044d4a8226a7411354a089aad044d5a8226af411358a089acd044d6a8226b741135ca089aed044d7a8226bf411360a089b0d044d8a8226c7411364a089b2d044d9a8226cf411368a089b4d044daa8226d741136ca089b6d044dba8226df411370a089b8d044dca8226e7411374a089bad044dda8226ef411378a089bcd044dea8226f741137ca089bed044dfa8226ff411380a089c0d044e0a822707411384a089c2d044e1a82270f411388a089c4d044e2a82271741138ca089c59089c69089c71089c79089c81089c89089c91089c99089ca1089ca9089cb1089cb9089cc1089cc9089cd1089cd9089ce1089ce9089cf1089cf9089d01089d09089d11089d19089d21089d29089d31089d39089d41089d49089d51089d59089d61089d69089d71089d79089d81089d89089d91089d99089da1089da9089db1089db9089dc1089dc9089dd1089dd9089de1089de9089df1089df9089e01089e09089e11089e19089e21089e29089e31089e39089e41089e49089e51089e59089e61089e69089e71089e79089e81089e89089e91089e99089ea1089ea9089eb1089eb9089ec1089ec9089ed1089ed9089ee1089ee9089ef1089ef9089f01089f09089f11089f19089f21089f29089f31089f39089f41089f49089f51089f59089f61089f69089f71089f79089f81089f89089f91089f99089fa1089fa9089fb1089fb9089fc1089fc9089fd1089fd9089fe1089fe9089ff1089ff98a0ac9329acf23ba884804fdd3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0034f800000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000024c0013000130000000000000000000000000000000000000000000000000000000002955540000000000000000000000000000000000000000000000000000000000000001ff108c00846024230221181908c108460a4230621183908c20846124230a21185908c308461a4230e21187908c40846224231221189908c508462a423162118b908c60846324231a2118d908c708463a4231e2118f908c80846424232221191908c908464a4232621193908ca0846524232a21195908cb08465a4232e21197908cc0846624233221199908cd08466a423362119b908ce0846724233a2119d908cf08467a4233e2119f908d008468242342211a1908d108468a42346211a3908d20846924234a211a5908d308469a4234e211a7908d40846a242352211a9908d50846aa42356211ab908d60846b24235a211ad908d70846ba4235e211af908d80846c242362211b1908d90846ca42366211b3908da0846d24236a211b5908db0846da4236e211b7908dc0846e242372211b9908dd0846ea42376211bb908de0846f24237a211bd908df0846fa4237e211bf908e008470242382211c1908e108470a42386211c3908e20847124238a211c5908e2f8c211c6c8471d211c7c84721211c8c84725211c9c84729211cac8472d211cbc84731211ccc84735211cdc84739211cec8473d211cfc84741211d0c84745211d1c84749211d2c8474d211d3c84751211d4c84755211d5c84759211d6c8475d211d7c84761211d8c84765211d9c84769211dac8476d211dbc84771211dcc84775211ddc84779211dec8477d211dfc84781211e0c84785211e1c84789211e2c8478d211e3c84791211e4c84795211e5c84799211e6c8479d211e7c847a1211e8c847a5211e9c847a9211eac847ad211ebc847b1211ecc847b5211edc847b9211eec847bd211efc847c1211f0c847c5211f1c847c9211f2c847cd211f3c847d1211f4c847d5211f5c847d9211f6c847dd211f7c847e1211f8c847e5211f9c847e9211fac847ed211fbc847f1211fcc847f5211fdc847f9211fec847fd211ff8bb9229c284803a8b6248" + +const rand2BZ2Hex = "425a6839314159265359d992d0f60000137dfe84020310091c1e280e100e042801099210094806c0110002e70806402000546034000034000000f2830000032000d3403264049270eb7a9280d308ca06ad28f6981bee1bf8160727c7364510d73a1e123083421b63f031f63993a0f40051fbf177245385090d992d0f60" +const rand2Hex = "92d5652616ac444a4a04af1a8a3964aca0450d43d6cf233bd03233f4ba92f8719e6c2a2bd4f5f88db07ecd0da3a33b263483db9b2c158786ad6363be35d17335ba" + +const rand3BZ2Hex = "425a68393141592653593be669d00000327ffffffffffffffffffffffffffffffffffff7ffffffffffffffffffffffffffffffc002b3b2b1b6e2bae400004c00132300004c0d268c004c08c0130026001a008683234c0684c34008c230261a04c0260064d07a8d00034000d27a1268c9931a8d327a3427a41faa69ea0da264c1a34219326869b51b49a6469a3268c689fa53269a62794687a9a68f5189994c9e487a8f534fd49a3d34043629e8c93d04da4f4648d30d4f44d3234c4d3023d0840680984d309934c234d3131a000640984f536a6132601300130130c8d00d04d1841ea7a8d31a02609b40023460010c01a34d4c1a0d04d3069306810034d0d0d4c0046130d034d0131a9a64d321804c68003400098344c13000991808c0001a00000000098004d3d4da4604c47a13012140aadf8d673c922c607ef6212a8c0403adea4b28aee578900e653b9cdeb8d11e6b838815f3ebaad5a01c5408d84a332170aff8734d4e06612d3c2889f31925fb89e33561f5100ae89b1f7047102e729373d3667e58d73aaa80fa7be368a1cc2dadd81d81ec8e1b504bd772ca31d03649269b01ceddaca07bf3d4eba24de141be3f86f93601e03714c0f64654671684f9f9528626fd4e1b76753dc0c54b842486b8d59d8ab314e86ca818e7a1f079463cbbd70d9b79b283c7edc419406311022e4be98c2c1374df9cdde2d008ce1d00e5f06ad1024baf555631f70831fc1023034e62be7c4bcb648caf276963ffa20e96bb50377fe1c113da0db4625b50741c35a058edb009c6ee5dbf93b8a6b060eec568180e8db791b82aab96cbf4326ca98361461379425ba8dcc347be670bdba7641883e5526ae3d833f6e9cb9bac9557747c79e206151072f7f0071dff3880411846f66bf4075c7462f302b53cb3400a74cf35652ad5641ed33572fd54e7ed7f85f58a0acba89327e7c6be5c58cb71528b99df2431f1d0358f8d28d81d95292da631fb06701decabb205fac59ff0fb1df536afc681eece6ea658c4d9eaa45f1342aa1ff70bdaff2ddaf25ec88c22f12829a0553db1ec2505554cb17d7b282e213a5a2aa30431ded2bce665bb199d023840832fedb2c0c350a27291407ff77440792872137df281592e82076a05c64c345ffb058c64f7f7c207ef78420b7010520610f17e302cc4dfcfaef72a0ed091aab4b541eb0531bbe941ca2f792bf7b31ca6162882b68054a8470115bc2c19f2df2023f7800432b39b04d3a304e8085ba3f1f0ca5b1ba4d38d339e6084de979cdea6d0e244c6c9fa0366bd890621e3d30846f5e8497e21597b8f29bbf52c961a485dfbea647600da0fc1f25ce4d203a8352ece310c39073525044e7ac46acf2ed9120bae1b4f6f02364abfe343f80b290983160c103557af1c68416480d024cc31b6c06cfec011456f1e95c420a12b48b1c3fe220c2879a982fb099948ac440db844b9a112a5188c7783fd3b19593290785f908d95c9db4b280bafe89c1313aeec24772046d9bc089645f0d182a21184e143823c5f52de50e5d7e98d3d7ab56f5413bbccd1415c9bcff707def475b643fb7f29842582104d4cc1dbaaca8f10a2f44273c339e0984f2b1e06ab2f0771db01fafa8142298345f3196f23e5847bda024034b6f59b11c29e981c881456e40d211929fd4f766200258aad8212016322bd5c605790dcfdf1bd2a93d99c9b8f498722d311d7eae7ff420496a31804c55f4759a7b13aaaf5f7ce006c3a8a998897d5e0a504398c2b627852545baf440798bcc5cc049357cf3f17d9771e4528a1af3d77dc794a11346e1bdf5efe37a405b127b4c43b616d61fbc5dc914e14240ef99a7400" +const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c593553f33bd786e3d0ce31626f511bc985f59d1a88aa38ba8ad6218d306abee60dd9172540232b95be1af146c69e72e5fde667a090dc3f93bdc5c5af0ab80acdbaa7a505f628c59dc0247b31a439cacf5010a94376d71521df08c178b02fb96fdb1809144ea38c68536187c53201fea8631fb0a880b4451ccdca7cc61f6aafca21cc7449d920599db61789ac3b1e164b3390124f95022aeea39ccca3ec1053f4fa10de2978e2861ea58e477085c2220021a0927aa94c5d0006b5055abba340e4f9eba22e969978dfd18e278a8b89d877328ae34268bc0174cfe211954c0036f078025217d1269fac1932a03b05a0b616012271bbe1fb554171c7a59b196d8a4479f45a77931b5d97aaf6c0c673cbe597b79b96e2a0c1eae2e66e46ccc8c85798e23ffe972ebdaa3f6caea243c004e60321eb47cd79137d78fd0613be606feacc5b3637bdc96a89c13746db8cad886f3ccf912b2178c823bcac395f06d28080269bdca2debf3419c66c690fd1adcfbd53e32e79443d7a42511a84cb22ca94fffad9149275a075b2f8ae0b021dcde9bf62b102db920733b897560518b06e1ad7f4b03458493ddaa7f4fa2c1609f7a1735aeeb1b3e2cea3ab45fc376323cc91873b7e9c90d07c192e38d3f5dfc9bfab1fd821c854da9e607ea596c391c7ec4161c6c4493929a8176badaa5a5af7211c623f29643a937677d3df0da9266181b7c4da5dd40376db677fe8f4a1dc456adf6f33c1e37cec471dd318c2647644fe52f93707a77da7d1702380a80e14cc0fdce7bf2eed48a529090bae0388ee277ce6c7018c5fb00b88362554362205c641f0d0fab94fd5b8357b5ff08b207fee023709bc126ec90cfb17c006754638f8186aaeb1265e80be0c1189ec07d01d5f6f96cb9ce82744147d18490de7dc72862f42f024a16968891a356f5e7e0e695d8c933ba5b5e43ad4c4ade5399bc2cae9bb6189b7870d7f22956194d277f28b10e01c10c6ffe3e065f7e2d6d056aa790db5649ca84dc64c35566c0af1b68c32b5b7874aaa66467afa44f40e9a0846a07ae75360a641dd2acc69d93219b2891f190621511e62a27f5e4fbe641ece1fa234fc7e9a74f48d2a760d82160d9540f649256b169d1fed6fbefdc491126530f3cbad7913e19fbd7aa53b1e243fbf28d5f38c10ebd77c8b986775975cc1d619efb27cdcd733fa1ca36cffe9c0a33cc9f02463c91a886601fd349efee85ef1462065ef9bd2c8f533220ad93138b8382d5938103ab25b2d9af8ae106e1211eb9b18793fba033900c809c02cd6d17e2f3e6fc84dae873411f8e87c3f0a8f1765b7825d185ce3730f299c3028d4a62da9ee95c2b870fb70c79370d485f9d5d9acb78926d20444033d960524d2776dc31988ec7c0dbf23b9905d" diff --git a/src/pkg/compress/bzip2/huffman.go b/src/pkg/compress/bzip2/huffman.go new file mode 100644 index 000000000..732bc4a21 --- /dev/null +++ b/src/pkg/compress/bzip2/huffman.go @@ -0,0 +1,223 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "os" + "sort" +) + +// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a +// symbol. +type huffmanTree struct { + // nodes contains all the non-leaf nodes in the tree. nodes[0] is the + // root of the tree and nextNode contains the index of the next element + // of nodes to use when the tree is being constructed. + nodes []huffmanNode + nextNode int +} + +// A huffmanNode is a node in the tree. left and right contain indexes into the +// nodes slice of the tree. If left or right is invalidNodeValue then the child +// is a left node and its value is in leftValue/rightValue. +// +// The symbols are uint16s because bzip2 encodes not only MTF indexes in the +// tree, but also two magic values for run-length encoding and an EOF symbol. +// Thus there are more than 256 possible symbols. +type huffmanNode struct { + left, right uint16 + leftValue, rightValue uint16 +} + +// invalidNodeValue is an invalid index which marks a leaf node in the tree. +const invalidNodeValue = 0xffff + +// Decode reads bits from the given bitReader and navigates the tree until a +// symbol is found. +func (t huffmanTree) Decode(br *bitReader) (v uint16) { + nodeIndex := uint16(0) // node 0 is the root of the tree. + + for { + node := &t.nodes[nodeIndex] + bit := br.ReadBit() + // bzip2 encodes left as a true bit. + if bit { + // left + if node.left == invalidNodeValue { + return node.leftValue + } + nodeIndex = node.left + } else { + // right + if node.right == invalidNodeValue { + return node.rightValue + } + nodeIndex = node.right + } + } + + panic("unreachable") +} + +// newHuffmanTree builds a Huffman tree from a slice containing the code +// lengths of each symbol. The maximum code length is 32 bits. +func newHuffmanTree(lengths []uint8) (huffmanTree, os.Error) { + // There are many possible trees that assign the same code length to + // each symbol (consider reflecting a tree down the middle, for + // example). Since the code length assignments determine the + // efficiency of the tree, each of these trees is equally good. In + // order to minimise the amount of information needed to build a tree + // bzip2 uses a canonical tree so that it can be reconstructed given + // only the code length assignments. + + if len(lengths) < 2 { + panic("newHuffmanTree: too few symbols") + } + + var t huffmanTree + + // First we sort the code length assignments by ascending code length, + // using the symbol value to break ties. + pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths))) + for i, length := range lengths { + pairs[i].value = uint16(i) + pairs[i].length = length + } + + sort.Sort(pairs) + + // Now we assign codes to the symbols, starting with the longest code. + // We keep the codes packed into a uint32, at the most-significant end. + // So branches are taken from the MSB downwards. This makes it easy to + // sort them later. + code := uint32(0) + length := uint8(32) + + codes := huffmanCodes(make([]huffmanCode, len(lengths))) + for i := len(pairs) - 1; i >= 0; i-- { + if length > pairs[i].length { + // If the code length decreases we shift in order to + // zero any bits beyond the end of the code. + length >>= 32 - pairs[i].length + length <<= 32 - pairs[i].length + length = pairs[i].length + } + codes[i].code = code + codes[i].codeLen = length + codes[i].value = pairs[i].value + // We need to 'increment' the code, which means treating |code| + // like a |length| bit number. + code += 1 << (32 - length) + } + + // Now we can sort by the code so that the left half of each branch are + // grouped together, recursively. + sort.Sort(codes) + + t.nodes = make([]huffmanNode, len(codes)) + _, err := buildHuffmanNode(&t, codes, 0) + return t, err +} + +// huffmanSymbolLengthPair contains a symbol and its code length. +type huffmanSymbolLengthPair struct { + value uint16 + length uint8 +} + +// huffmanSymbolLengthPair is used to provide an interface for sorting. +type huffmanSymbolLengthPairs []huffmanSymbolLengthPair + +func (h huffmanSymbolLengthPairs) Len() int { + return len(h) +} + +func (h huffmanSymbolLengthPairs) Less(i, j int) bool { + if h[i].length < h[j].length { + return true + } + if h[i].length > h[j].length { + return false + } + if h[i].value < h[j].value { + return true + } + return false +} + +func (h huffmanSymbolLengthPairs) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +// huffmanCode contains a symbol, its code and code length. +type huffmanCode struct { + code uint32 + codeLen uint8 + value uint16 +} + +// huffmanCodes is used to provide an interface for sorting. +type huffmanCodes []huffmanCode + +func (n huffmanCodes) Len() int { + return len(n) +} + +func (n huffmanCodes) Less(i, j int) bool { + return n[i].code < n[j].code +} + +func (n huffmanCodes) Swap(i, j int) { + n[i], n[j] = n[j], n[i] +} + +// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in +// the Huffman tree at the given level. It returns the index of the newly +// constructed node. +func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err os.Error) { + test := uint32(1) << (31 - level) + + // We have to search the list of codes to find the divide between the left and right sides. + firstRightIndex := len(codes) + for i, code := range codes { + if code.code&test != 0 { + firstRightIndex = i + break + } + } + + left := codes[:firstRightIndex] + right := codes[firstRightIndex:] + + if len(left) == 0 || len(right) == 0 { + return 0, StructuralError("superfluous level in Huffman tree") + } + + nodeIndex = uint16(t.nextNode) + node := &t.nodes[t.nextNode] + t.nextNode++ + + if len(left) == 1 { + // leaf node + node.left = invalidNodeValue + node.leftValue = left[0].value + } else { + node.left, err = buildHuffmanNode(t, left, level+1) + } + + if err != nil { + return + } + + if len(right) == 1 { + // leaf node + node.right = invalidNodeValue + node.rightValue = right[0].value + } else { + node.right, err = buildHuffmanNode(t, right, level+1) + } + + return +} diff --git a/src/pkg/compress/bzip2/move_to_front.go b/src/pkg/compress/bzip2/move_to_front.go new file mode 100644 index 000000000..0ed19dec3 --- /dev/null +++ b/src/pkg/compress/bzip2/move_to_front.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +// moveToFrontDecoder implements a move-to-front list. Such a list is an +// efficient way to transform a string with repeating elements into one with +// many small valued numbers, which is suitable for entropy encoding. It works +// by starting with an initial list of symbols and references symbols by their +// index into that list. When a symbol is referenced, it's moved to the front +// of the list. Thus, a repeated symbol ends up being encoded with many zeros, +// as the symbol will be at the front of the list after the first access. +type moveToFrontDecoder struct { + // Rather than actually keep the list in memory, the symbols are stored + // as a circular, double linked list with the symbol indexed by head + // at the front of the list. + symbols []byte + next []uint8 + prev []uint8 + head uint8 +} + +// newMTFDecoder creates a move-to-front decoder with an explicit initial list +// of symbols. +func newMTFDecoder(symbols []byte) *moveToFrontDecoder { + if len(symbols) > 256 { + panic("too many symbols") + } + + m := &moveToFrontDecoder{ + symbols: symbols, + next: make([]uint8, len(symbols)), + prev: make([]uint8, len(symbols)), + } + + m.threadLinkedList() + return m +} + +// newMTFDecoderWithRange creates a move-to-front decoder with an initial +// symbol list of 0...n-1. +func newMTFDecoderWithRange(n int) *moveToFrontDecoder { + if n > 256 { + panic("newMTFDecoderWithRange: cannot have > 256 symbols") + } + + m := &moveToFrontDecoder{ + symbols: make([]uint8, n), + next: make([]uint8, n), + prev: make([]uint8, n), + } + + for i := 0; i < n; i++ { + m.symbols[i] = byte(i) + } + + m.threadLinkedList() + return m +} + +// threadLinkedList creates the initial linked-list pointers. +func (m *moveToFrontDecoder) threadLinkedList() { + if len(m.symbols) == 0 { + return + } + + m.prev[0] = uint8(len(m.symbols) - 1) + + for i := 0; i < len(m.symbols)-1; i++ { + m.next[i] = uint8(i + 1) + m.prev[i+1] = uint8(i) + } + + m.next[len(m.symbols)-1] = 0 +} + +func (m *moveToFrontDecoder) Decode(n int) (b byte) { + // Most of the time, n will be zero so it's worth dealing with this + // simple case. + if n == 0 { + return m.symbols[m.head] + } + + i := m.head + for j := 0; j < n; j++ { + i = m.next[i] + } + b = m.symbols[i] + + m.next[m.prev[i]] = m.next[i] + m.prev[m.next[i]] = m.prev[i] + m.next[i] = m.head + m.prev[i] = m.prev[m.head] + m.next[m.prev[m.head]] = i + m.prev[m.head] = i + m.head = i + + return +} + +// First returns the symbol at the front of the list. +func (m *moveToFrontDecoder) First() byte { + return m.symbols[m.head] +} diff --git a/src/pkg/compress/flate/deflate_test.go b/src/pkg/compress/flate/deflate_test.go index 68dcd7bcc..ed5884a4b 100644 --- a/src/pkg/compress/flate/deflate_test.go +++ b/src/pkg/compress/flate/deflate_test.go @@ -191,9 +191,16 @@ func testSync(t *testing.T, level int, input []byte, name string) { t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) return } - if i == 0 && buf.buf.Len() != 0 { - t.Errorf("testSync/%d (%d, %d, %s): extra data after %d", i, level, len(input), name, hi-lo) - } + // This test originally checked that after reading + // the first half of the input, there was nothing left + // in the read buffer (buf.buf.Len() != 0) but that is + // not necessarily the case: the write Flush may emit + // some extra framing bits that are not necessary + // to process to obtain the first half of the uncompressed + // data. The test ran correctly most of the time, because + // the background goroutine had usually read even + // those extra bits by now, but it's not a useful thing to + // check. buf.WriteMode() } buf.ReadMode() @@ -262,135 +269,9 @@ func TestReverseBits(t *testing.T) { } func TestDeflateInflateString(t *testing.T) { - gold := bytes.NewBufferString(getEdata()).Bytes() + gold, err := ioutil.ReadFile("../testdata/e.txt") + if err != nil { + t.Error(err) + } testToFromWithLevel(t, 1, gold, "2.718281828...") } - -func getEdata() string { - return "2.718281828459045235360287471352662497757247093699959574966967627724076630353547" + - "59457138217852516642742746639193200305992181741359662904357290033429526059563073" + - "81323286279434907632338298807531952510190115738341879307021540891499348841675092" + - "44761460668082264800168477411853742345442437107539077744992069551702761838606261" + - "33138458300075204493382656029760673711320070932870912744374704723069697720931014" + - "16928368190255151086574637721112523897844250569536967707854499699679468644549059" + - "87931636889230098793127736178215424999229576351482208269895193668033182528869398" + - "49646510582093923982948879332036250944311730123819706841614039701983767932068328" + - "23764648042953118023287825098194558153017567173613320698112509961818815930416903" + - "51598888519345807273866738589422879228499892086805825749279610484198444363463244" + - "96848756023362482704197862320900216099023530436994184914631409343173814364054625" + - "31520961836908887070167683964243781405927145635490613031072085103837505101157477" + - "04171898610687396965521267154688957035035402123407849819334321068170121005627880" + - "23519303322474501585390473041995777709350366041699732972508868769664035557071622" + - "68447162560798826517871341951246652010305921236677194325278675398558944896970964" + - "09754591856956380236370162112047742722836489613422516445078182442352948636372141" + - "74023889344124796357437026375529444833799801612549227850925778256209262264832627" + - "79333865664816277251640191059004916449982893150566047258027786318641551956532442" + - "58698294695930801915298721172556347546396447910145904090586298496791287406870504" + - "89585867174798546677575732056812884592054133405392200011378630094556068816674001" + - "69842055804033637953764520304024322566135278369511778838638744396625322498506549" + - "95886234281899707733276171783928034946501434558897071942586398772754710962953741" + - "52111513683506275260232648472870392076431005958411661205452970302364725492966693" + - "81151373227536450988890313602057248176585118063036442812314965507047510254465011" + - "72721155519486685080036853228183152196003735625279449515828418829478761085263981" + - "39559900673764829224437528718462457803619298197139914756448826260390338144182326" + - "25150974827987779964373089970388867782271383605772978824125611907176639465070633" + - "04527954661855096666185664709711344474016070462621568071748187784437143698821855" + - "96709591025968620023537185887485696522000503117343920732113908032936344797273559" + - "55277349071783793421637012050054513263835440001863239914907054797780566978533580" + - "48966906295119432473099587655236812859041383241160722602998330535370876138939639" + - "17795745401613722361878936526053815584158718692553860616477983402543512843961294" + - "60352913325942794904337299085731580290958631382683291477116396337092400316894586" + - "36060645845925126994655724839186564209752685082307544254599376917041977780085362" + - "73094171016343490769642372229435236612557250881477922315197477806056967253801718" + - "07763603462459278778465850656050780844211529697521890874019660906651803516501792" + - "50461950136658543663271254963990854914420001457476081930221206602433009641270489" + - "43903971771951806990869986066365832322787093765022601492910115171776359446020232" + - "49300280401867723910288097866605651183260043688508817157238669842242201024950551" + - "88169480322100251542649463981287367765892768816359831247788652014117411091360116" + - "49950766290779436460058519419985601626479076153210387275571269925182756879893027" + - "61761146162549356495903798045838182323368612016243736569846703785853305275833337" + - "93990752166069238053369887956513728559388349989470741618155012539706464817194670" + - "83481972144888987906765037959036696724949925452790337296361626589760394985767413" + - "97359441023744329709355477982629614591442936451428617158587339746791897571211956" + - "18738578364475844842355558105002561149239151889309946342841393608038309166281881" + - "15037152849670597416256282360921680751501777253874025642534708790891372917228286" + - "11515915683725241630772254406337875931059826760944203261924285317018781772960235" + - "41306067213604600038966109364709514141718577701418060644363681546444005331608778" + - "31431744408119494229755993140118886833148328027065538330046932901157441475631399" + - "97221703804617092894579096271662260740718749975359212756084414737823303270330168" + - "23719364800217328573493594756433412994302485023573221459784328264142168487872167" + - "33670106150942434569844018733128101079451272237378861260581656680537143961278887" + - "32527373890392890506865324138062796025930387727697783792868409325365880733988457" + - "21874602100531148335132385004782716937621800490479559795929059165547050577751430" + - "81751126989851884087185640260353055837378324229241856256442550226721559802740126" + - "17971928047139600689163828665277009752767069777036439260224372841840883251848770" + - "47263844037953016690546593746161932384036389313136432713768884102681121989127522" + - "30562567562547017250863497653672886059667527408686274079128565769963137897530346" + - "60616669804218267724560530660773899624218340859882071864682623215080288286359746" + - "83965435885668550377313129658797581050121491620765676995065971534476347032085321" + - "56036748286083786568030730626576334697742956346437167093971930608769634953288468" + - "33613038829431040800296873869117066666146800015121143442256023874474325250769387" + - "07777519329994213727721125884360871583483562696166198057252661220679754062106208" + - "06498829184543953015299820925030054982570433905535701686531205264956148572492573" + - "86206917403695213533732531666345466588597286659451136441370331393672118569553952" + - "10845840724432383558606310680696492485123263269951460359603729725319836842336390" + - "46321367101161928217111502828016044880588023820319814930963695967358327420249882" + - "45684941273860566491352526706046234450549227581151709314921879592718001940968866" + - "98683703730220047531433818109270803001720593553052070070607223399946399057131158" + - "70996357773590271962850611465148375262095653467132900259943976631145459026858989" + - "79115837093419370441155121920117164880566945938131183843765620627846310490346293" + - "95002945834116482411496975832601180073169943739350696629571241027323913874175492" + - "30718624545432220395527352952402459038057445028922468862853365422138157221311632" + - "88112052146489805180092024719391710555390113943316681515828843687606961102505171" + - "00739276238555338627255353883096067164466237092264680967125406186950214317621166" + - "81400975952814939072226011126811531083873176173232352636058381731510345957365382" + - "23534992935822836851007810884634349983518404451704270189381994243410090575376257" + - "76757111809008816418331920196262341628816652137471732547772778348877436651882875" + - "21566857195063719365653903894493664217640031215278702223664636357555035655769488" + - "86549500270853923617105502131147413744106134445544192101336172996285694899193369" + - "18472947858072915608851039678195942983318648075608367955149663644896559294818785" + - "17840387733262470519450504198477420141839477312028158868457072905440575106012852" + - "58056594703046836344592652552137008068752009593453607316226118728173928074623094" + - "68536782310609792159936001994623799343421068781349734695924646975250624695861690" + - "91785739765951993929939955675427146549104568607020990126068187049841780791739240" + - "71945996323060254707901774527513186809982284730860766536866855516467702911336827" + - "56310722334672611370549079536583453863719623585631261838715677411873852772292259" + - "47433737856955384562468010139057278710165129666367644518724656537304024436841408" + - "14488732957847348490003019477888020460324660842875351848364959195082888323206522" + - "12810419044804724794929134228495197002260131043006241071797150279343326340799596" + - "05314460532304885289729176598760166678119379323724538572096075822771784833616135" + - "82612896226118129455927462767137794487586753657544861407611931125958512655759734" + - "57301533364263076798544338576171533346232527057200530398828949903425956623297578" + - "24887350292591668258944568946559926584547626945287805165017206747854178879822768" + - "06536650641910973434528878338621726156269582654478205672987756426325321594294418" + - "03994321700009054265076309558846589517170914760743713689331946909098190450129030" + - "70995662266203031826493657336984195557769637876249188528656866076005660256054457" + - "11337286840205574416030837052312242587223438854123179481388550075689381124935386" + - "31863528708379984569261998179452336408742959118074745341955142035172618420084550" + - "91708456823682008977394558426792142734775608796442792027083121501564063413416171" + - "66448069815483764491573900121217041547872591998943825364950514771379399147205219" + - "52907939613762110723849429061635760459623125350606853765142311534966568371511660" + - "42207963944666211632551577290709784731562782775987881364919512574833287937715714" + - "59091064841642678309949723674420175862269402159407924480541255360431317992696739" + - "15754241929660731239376354213923061787675395871143610408940996608947141834069836" + - "29936753626215452472984642137528910798843813060955526227208375186298370667872244" + - "30195793793786072107254277289071732854874374355781966511716618330881129120245204" + - "04868220007234403502544820283425418788465360259150644527165770004452109773558589" + - "76226554849416217149895323834216001140629507184904277892585527430352213968356790" + - "18076406042138307308774460170842688272261177180842664333651780002171903449234264" + - "26629226145600433738386833555534345300426481847398921562708609565062934040526494" + - "32442614456659212912256488935696550091543064261342526684725949143142393988454324" + - "86327461842846655985332312210466259890141712103446084271616619001257195870793217" + - "56969854401339762209674945418540711844643394699016269835160784892451405894094639" + - "52678073545797003070511636825194877011897640028276484141605872061841852971891540" + - "19688253289309149665345753571427318482016384644832499037886069008072709327673127" + - "58196656394114896171683298045513972950668760474091542042842999354102582911350224" + - "16907694316685742425225090269390348148564513030699251995904363840284292674125734" + - "22447765584177886171737265462085498294498946787350929581652632072258992368768457" + - "01782303809656788311228930580914057261086588484587310165815116753332767488701482" + - "91674197015125597825727074064318086014281490241467804723275976842696339357735429" + - "30186739439716388611764209004068663398856841681003872389214483176070116684503887" + - "21236436704331409115573328018297798873659091665961240202177855885487617616198937" + - "07943800566633648843650891448055710397652146960276625835990519870423001794655367" + - "9" -} diff --git a/src/pkg/compress/lzw/Makefile b/src/pkg/compress/lzw/Makefile new file mode 100644 index 000000000..28f5e6abc --- /dev/null +++ b/src/pkg/compress/lzw/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=compress/lzw +GOFILES=\ + reader.go\ + writer.go\ + +include ../../../Make.pkg diff --git a/src/pkg/compress/lzw/reader.go b/src/pkg/compress/lzw/reader.go new file mode 100644 index 000000000..8a540cbe6 --- /dev/null +++ b/src/pkg/compress/lzw/reader.go @@ -0,0 +1,210 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The lzw package implements the Lempel-Ziv-Welch compressed data format, +// described in T. A. Welch, ``A Technique for High-Performance Data +// Compression'', Computer, 17(6) (June 1984), pp 8-19. +// +// In particular, it implements LZW as used by the GIF, TIFF and PDF file +// formats, which means variable-width codes up to 12 bits and the first +// two non-literal codes are a clear code and an EOF code. +package lzw + +// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF, +// modulo LSB/MSB packing order. + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// Order specifies the bit ordering in an LZW data stream. +type Order int + +const ( + // LSB means Least Significant Bits first, as used in the GIF file format. + LSB Order = iota + // MSB means Most Significant Bits first, as used in the TIFF and PDF + // file formats. + MSB +) + +// decoder is the state from which the readXxx method converts a byte +// stream into a code stream. +type decoder struct { + r io.ByteReader + bits uint32 + nBits uint + width uint +} + +// readLSB returns the next code for "Least Significant Bits first" data. +func (d *decoder) readLSB() (uint16, os.Error) { + for d.nBits < d.width { + x, err := d.r.ReadByte() + if err != nil { + return 0, err + } + d.bits |= uint32(x) << d.nBits + d.nBits += 8 + } + code := uint16(d.bits & (1<<d.width - 1)) + d.bits >>= d.width + d.nBits -= d.width + return code, nil +} + +// readMSB returns the next code for "Most Significant Bits first" data. +func (d *decoder) readMSB() (uint16, os.Error) { + for d.nBits < d.width { + x, err := d.r.ReadByte() + if err != nil { + return 0, err + } + d.bits |= uint32(x) << (24 - d.nBits) + d.nBits += 8 + } + code := uint16(d.bits >> (32 - d.width)) + d.bits <<= d.width + d.nBits -= d.width + return code, nil +} + +// decode decompresses bytes from r and writes them to pw. +// read specifies how to decode bytes into codes. +// litWidth is the width in bits of literal codes. +func decode(r io.Reader, read func(*decoder) (uint16, os.Error), litWidth int, pw *io.PipeWriter) { + br, ok := r.(io.ByteReader) + if !ok { + br = bufio.NewReader(r) + } + pw.CloseWithError(decode1(pw, br, read, uint(litWidth))) +} + +func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os.Error), litWidth uint) os.Error { + const ( + maxWidth = 12 + invalidCode = 0xffff + ) + d := decoder{r, 0, 0, 1 + litWidth} + w := bufio.NewWriter(pw) + // The first 1<<litWidth codes are literal codes. + // The next two codes mean clear and EOF. + // Other valid codes are in the range [lo, hi] where lo := clear + 2, + // with the upper bound incrementing on each code seen. + clear := uint16(1) << litWidth + eof, hi := clear+1, clear+1 + // overflow is the code at which hi overflows the code width. + overflow := uint16(1) << d.width + var ( + // Each code c in [lo, hi] expands to two or more bytes. For c != hi: + // suffix[c] is the last of these bytes. + // prefix[c] is the code for all but the last byte. + // This code can either be a literal code or another code in [lo, c). + // The c == hi case is a special case. + suffix [1 << maxWidth]uint8 + prefix [1 << maxWidth]uint16 + // buf is a scratch buffer for reconstituting the bytes that a code expands to. + // Code suffixes are written right-to-left from the end of the buffer. + buf [1 << maxWidth]byte + ) + + // Loop over the code stream, converting codes into decompressed bytes. + last := uint16(invalidCode) + for { + code, err := read(&d) + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + switch { + case code < clear: + // We have a literal code. + if err := w.WriteByte(uint8(code)); err != nil { + return err + } + if last != invalidCode { + // Save what the hi code expands to. + suffix[hi] = uint8(code) + prefix[hi] = last + } + case code == clear: + d.width = 1 + litWidth + hi = eof + overflow = 1 << d.width + last = invalidCode + continue + case code == eof: + return w.Flush() + case code <= hi: + c, i := code, len(buf)-1 + if code == hi { + // code == hi is a special case which expands to the last expansion + // followed by the head of the last expansion. To find the head, we walk + // the prefix chain until we find a literal code. + c = last + for c >= clear { + c = prefix[c] + } + buf[i] = uint8(c) + i-- + c = last + } + // Copy the suffix chain into buf and then write that to w. + for c >= clear { + buf[i] = suffix[c] + i-- + c = prefix[c] + } + buf[i] = uint8(c) + if _, err := w.Write(buf[i:]); err != nil { + return err + } + // Save what the hi code expands to. + suffix[hi] = uint8(c) + prefix[hi] = last + default: + return os.NewError("lzw: invalid code") + } + last, hi = code, hi+1 + if hi == overflow { + if d.width == maxWidth { + return os.NewError("lzw: missing clear code") + } + d.width++ + overflow <<= 1 + } + } + panic("unreachable") +} + +// NewReader creates a new io.ReadCloser that satisfies reads by decompressing +// the data read from r. +// It is the caller's responsibility to call Close on the ReadCloser when +// finished reading. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser { + pr, pw := io.Pipe() + var read func(*decoder) (uint16, os.Error) + switch order { + case LSB: + read = (*decoder).readLSB + case MSB: + read = (*decoder).readMSB + default: + pw.CloseWithError(os.NewError("lzw: unknown order")) + return pr + } + if litWidth < 2 || 8 < litWidth { + pw.CloseWithError(fmt.Errorf("lzw: litWidth %d out of range", litWidth)) + return pr + } + go decode(r, read, litWidth, pw) + return pr +} diff --git a/src/pkg/compress/lzw/reader_test.go b/src/pkg/compress/lzw/reader_test.go new file mode 100644 index 000000000..7795a4c14 --- /dev/null +++ b/src/pkg/compress/lzw/reader_test.go @@ -0,0 +1,132 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strconv" + "strings" + "testing" +) + +type lzwTest struct { + desc string + raw string + compressed string + err os.Error +} + +var lzwTests = []lzwTest{ + { + "empty;LSB;8", + "", + "\x01\x01", + nil, + }, + { + "empty;MSB;8", + "", + "\x80\x80", + nil, + }, + { + "tobe;LSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04\x12\x34\xb8\xb0\xe0\xc1\x84\x01\x01", + nil, + }, + { + "tobe;MSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;MSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x2a\x13\xc8\x44\x52\x79\x48\x9c\x4f\x2a\x40\xa0\x90\x68\x5c\x16\x0f\x09\x80\x80", + nil, + }, + { + "tobe-truncated;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04", + io.ErrUnexpectedEOF, + }, + // This example comes from http://en.wikipedia.org/wiki/Graphics_Interchange_Format. + { + "gif;LSB;8", + "\x28\xff\xff\xff\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x51\xfc\x1b\x28\x70\xa0\xc1\x83\x01\x01", + nil, + }, + // This example comes from http://compgroups.net/comp.lang.ruby/Decompressing-LZW-compression-from-PDF-file + { + "pdf;MSB;8", + "-----A---B", + "\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01", + nil, + }, +} + +func TestReader(t *testing.T) { + b := bytes.NewBuffer(nil) + for _, tt := range lzwTests { + d := strings.Split(tt.desc, ";", -1) + var order Order + switch d[1] { + case "LSB": + order = LSB + case "MSB": + order = MSB + default: + t.Errorf("%s: bad order %q", tt.desc, d[1]) + } + litWidth, _ := strconv.Atoi(d[2]) + rc := NewReader(strings.NewReader(tt.compressed), order, litWidth) + defer rc.Close() + b.Reset() + n, err := io.Copy(b, rc) + if err != nil { + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err) + } + continue + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw) + } + } +} + +type devNull struct{} + +func (devNull) Write(p []byte) (int, os.Error) { + return len(p), nil +} + +func BenchmarkDecoder(b *testing.B) { + b.StopTimer() + buf0, _ := ioutil.ReadFile("../testdata/e.txt") + compressed := bytes.NewBuffer(nil) + w := NewWriter(compressed, LSB, 8) + io.Copy(w, bytes.NewBuffer(buf0)) + w.Close() + buf1 := compressed.Bytes() + b.StartTimer() + for i := 0; i < b.N; i++ { + io.Copy(devNull{}, NewReader(bytes.NewBuffer(buf1), LSB, 8)) + } +} diff --git a/src/pkg/compress/lzw/writer.go b/src/pkg/compress/lzw/writer.go new file mode 100644 index 000000000..87143b7aa --- /dev/null +++ b/src/pkg/compress/lzw/writer.go @@ -0,0 +1,259 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// A writer is a buffered, flushable writer. +type writer interface { + WriteByte(byte) os.Error + Flush() os.Error +} + +// An errWriteCloser is an io.WriteCloser that always returns a given error. +type errWriteCloser struct { + err os.Error +} + +func (e *errWriteCloser) Write([]byte) (int, os.Error) { + return 0, e.err +} + +func (e *errWriteCloser) Close() os.Error { + return e.err +} + +const ( + // A code is a 12 bit value, stored as a uint32 when encoding to avoid + // type conversions when shifting bits. + maxCode = 1<<12 - 1 + invalidCode = 1<<32 - 1 + // There are 1<<12 possible codes, which is an upper bound on the number of + // valid hash table entries at any given point in time. tableSize is 4x that. + tableSize = 4 * 1 << 12 + tableMask = tableSize - 1 + // A hash table entry is a uint32. Zero is an invalid entry since the + // lower 12 bits of a valid entry must be a non-literal code. + invalidEntry = 0 +) + +// encoder is LZW compressor. +type encoder struct { + // w is the writer that compressed bytes are written to. + w writer + // write, bits, nBits and width are the state for converting a code stream + // into a byte stream. + write func(*encoder, uint32) os.Error + bits uint32 + nBits uint + width uint + // litWidth is the width in bits of literal codes. + litWidth uint + // hi is the code implied by the next code emission. + // overflow is the code at which hi overflows the code width. + hi, overflow uint32 + // savedCode is the accumulated code at the end of the most recent Write + // call. It is equal to invalidCode if there was no such call. + savedCode uint32 + // err is the first error encountered during writing. Closing the encoder + // will make any future Write calls return os.EINVAL. + err os.Error + // table is the hash table from 20-bit keys to 12-bit values. Each table + // entry contains key<<12|val and collisions resolve by linear probing. + // The keys consist of a 12-bit code prefix and an 8-bit byte suffix. + // The values are a 12-bit code. + table [tableSize]uint32 +} + +// writeLSB writes the code c for "Least Significant Bits first" data. +func (e *encoder) writeLSB(c uint32) os.Error { + e.bits |= c << e.nBits + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + e.bits >>= 8 + e.nBits -= 8 + } + return nil +} + +// writeMSB writes the code c for "Most Significant Bits first" data. +func (e *encoder) writeMSB(c uint32) os.Error { + e.bits |= c << (32 - e.width - e.nBits) + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits >> 24)); err != nil { + return err + } + e.bits <<= 8 + e.nBits -= 8 + } + return nil +} + +// errOutOfCodes is an internal error that means that the encoder has run out +// of unused codes and a clear code needs to be sent next. +var errOutOfCodes = os.NewError("lzw: out of codes") + +// incHi increments e.hi and checks for both overflow and running out of +// unused codes. In the latter case, incHi sends a clear code, resets the +// encoder state and returns errOutOfCodes. +func (e *encoder) incHi() os.Error { + e.hi++ + if e.hi == e.overflow { + e.width++ + e.overflow <<= 1 + } + if e.hi == maxCode { + clear := uint32(1) << e.litWidth + if err := e.write(e, clear); err != nil { + return err + } + e.width = uint(e.litWidth) + 1 + e.hi = clear + 1 + e.overflow = clear << 1 + for i := range e.table { + e.table[i] = invalidEntry + } + return errOutOfCodes + } + return nil +} + +// Write writes a compressed representation of p to e's underlying writer. +func (e *encoder) Write(p []byte) (int, os.Error) { + if e.err != nil { + return 0, e.err + } + if len(p) == 0 { + return 0, nil + } + litMask := uint32(1<<e.litWidth - 1) + code := e.savedCode + if code == invalidCode { + // The first code sent is always a literal code. + code, p = uint32(p[0])&litMask, p[1:] + } +loop: + for _, x := range p { + literal := uint32(x) & litMask + key := code<<8 | literal + // If there is a hash table hit for this key then we continue the loop + // and do not emit a code yet. + hash := (key>>12 ^ key) & tableMask + for h, t := hash, e.table[hash]; t != invalidEntry; { + if key == t>>12 { + code = t & maxCode + continue loop + } + h = (h + 1) & tableMask + t = e.table[h] + } + // Otherwise, write the current code, and literal becomes the start of + // the next emitted code. + if e.err = e.write(e, code); e.err != nil { + return 0, e.err + } + code = literal + // Increment e.hi, the next implied code. If we run out of codes, reset + // the encoder state (including clearing the hash table) and continue. + if err := e.incHi(); err != nil { + if err == errOutOfCodes { + continue + } + e.err = err + return 0, e.err + } + // Otherwise, insert key -> e.hi into the map that e.table represents. + for { + if e.table[hash] == invalidEntry { + e.table[hash] = (key << 12) | e.hi + break + } + hash = (hash + 1) & tableMask + } + } + e.savedCode = code + return len(p), nil +} + +// Close closes the encoder, flushing any pending output. It does not close or +// flush e's underlying writer. +func (e *encoder) Close() os.Error { + if e.err != nil { + if e.err == os.EINVAL { + return nil + } + return e.err + } + // Make any future calls to Write return os.EINVAL. + e.err = os.EINVAL + // Write the savedCode if valid. + if e.savedCode != invalidCode { + if err := e.write(e, e.savedCode); err != nil { + return err + } + if err := e.incHi(); err != nil && err != errOutOfCodes { + return err + } + } + // Write the eof code. + eof := uint32(1)<<e.litWidth + 1 + if err := e.write(e, eof); err != nil { + return err + } + // Write the final bits. + if e.nBits > 0 { + if e.write == (*encoder).writeMSB { + e.bits >>= 24 + } + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + } + return e.w.Flush() +} + +// NewWriter creates a new io.WriteCloser that satisfies writes by compressing +// the data and writing it to w. +// It is the caller's responsibility to call Close on the WriteCloser when +// finished writing. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewWriter(w io.Writer, order Order, litWidth int) io.WriteCloser { + var write func(*encoder, uint32) os.Error + switch order { + case LSB: + write = (*encoder).writeLSB + case MSB: + write = (*encoder).writeMSB + default: + return &errWriteCloser{os.NewError("lzw: unknown order")} + } + if litWidth < 2 || 8 < litWidth { + return &errWriteCloser{fmt.Errorf("lzw: litWidth %d out of range", litWidth)} + } + bw, ok := w.(writer) + if !ok { + bw = bufio.NewWriter(w) + } + lw := uint(litWidth) + return &encoder{ + w: bw, + write: write, + width: 1 + lw, + litWidth: lw, + hi: 1<<lw + 1, + overflow: 1 << (lw + 1), + savedCode: invalidCode, + } +} diff --git a/src/pkg/compress/lzw/writer_test.go b/src/pkg/compress/lzw/writer_test.go new file mode 100644 index 000000000..715b974aa --- /dev/null +++ b/src/pkg/compress/lzw/writer_test.go @@ -0,0 +1,111 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "io" + "io/ioutil" + "os" + "testing" +) + +var filenames = []string{ + "../testdata/e.txt", + "../testdata/pi.txt", +} + +// testFile tests that compressing and then decompressing the given file with +// the given options yields equivalent bytes to the original file. +func testFile(t *testing.T, fn string, order Order, litWidth int) { + // Read the file, as golden output. + golden, err := os.Open(fn, os.O_RDONLY, 0400) + if err != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err) + return + } + defer golden.Close() + + // Read the file again, and push it through a pipe that compresses at the write end, and decompresses at the read end. + raw, err := os.Open(fn, os.O_RDONLY, 0400) + if err != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err) + return + } + + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer raw.Close() + defer pipew.Close() + lzww := NewWriter(pipew, order, litWidth) + defer lzww.Close() + var b [4096]byte + for { + n, err0 := raw.Read(b[:]) + if err0 != nil && err0 != os.EOF { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err0) + return + } + _, err1 := lzww.Write(b[:n]) + if err1 == os.EPIPE { + // Fail, but do not report the error, as some other (presumably reportable) error broke the pipe. + return + } + if err1 != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1) + return + } + if err0 == os.EOF { + break + } + } + }() + lzwr := NewReader(piper, order, litWidth) + defer lzwr.Close() + + // Compare the two. + b0, err0 := ioutil.ReadAll(golden) + b1, err1 := ioutil.ReadAll(lzwr) + if err0 != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err0) + return + } + if err1 != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1) + return + } + if len(b0) != len(b1) { + t.Errorf("%s (order=%d litWidth=%d): length mismatch %d versus %d", fn, order, litWidth, len(b0), len(b1)) + return + } + for i := 0; i < len(b0); i++ { + if b0[i] != b1[i] { + t.Errorf("%s (order=%d litWidth=%d): mismatch at %d, 0x%02x versus 0x%02x\n", fn, order, litWidth, i, b0[i], b1[i]) + return + } + } +} + +func TestWriter(t *testing.T) { + for _, filename := range filenames { + for _, order := range [...]Order{LSB, MSB} { + // The test data "2.71828 etcetera" is ASCII text requiring at least 6 bits. + for _, litWidth := range [...]int{6, 7, 8} { + testFile(t, filename, order, litWidth) + } + } + } +} + +func BenchmarkEncoder(b *testing.B) { + b.StopTimer() + buf, _ := ioutil.ReadFile("../testdata/e.txt") + b.StartTimer() + for i := 0; i < b.N; i++ { + w := NewWriter(devNull{}, LSB, 8) + w.Write(buf) + w.Close() + } +} diff --git a/src/pkg/compress/zlib/testdata/e.txt b/src/pkg/compress/testdata/e.txt index 76cf2a7b6..76cf2a7b6 100644 --- a/src/pkg/compress/zlib/testdata/e.txt +++ b/src/pkg/compress/testdata/e.txt diff --git a/src/pkg/compress/zlib/testdata/pi.txt b/src/pkg/compress/testdata/pi.txt index 58d8f3b6d..58d8f3b6d 100644 --- a/src/pkg/compress/zlib/testdata/pi.txt +++ b/src/pkg/compress/testdata/pi.txt diff --git a/src/pkg/compress/zlib/writer_test.go b/src/pkg/compress/zlib/writer_test.go index fa9e78e8e..5a13ba825 100644 --- a/src/pkg/compress/zlib/writer_test.go +++ b/src/pkg/compress/zlib/writer_test.go @@ -12,8 +12,8 @@ import ( ) var filenames = []string{ - "testdata/e.txt", - "testdata/pi.txt", + "../testdata/e.txt", + "../testdata/pi.txt", } // Tests that compressing and then decompressing the given file at the given compression level diff --git a/src/pkg/crypto/openpgp/Makefile b/src/pkg/crypto/openpgp/Makefile new file mode 100644 index 000000000..b46ac2bba --- /dev/null +++ b/src/pkg/crypto/openpgp/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=crypto/openpgp +GOFILES=\ + canonical_text.go\ + keys.go\ + read.go\ + write.go\ + +include ../../../Make.pkg diff --git a/src/pkg/crypto/openpgp/canonical_text.go b/src/pkg/crypto/openpgp/canonical_text.go new file mode 100644 index 000000000..293eff354 --- /dev/null +++ b/src/pkg/crypto/openpgp/canonical_text.go @@ -0,0 +1,58 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "hash" + "os" +) + +// NewCanonicalTextHash reformats text written to it into the canonical +// form and then applies the hash h. See RFC 4880, section 5.2.1. +func NewCanonicalTextHash(h hash.Hash) hash.Hash { + return &canonicalTextHash{h, 0} +} + +type canonicalTextHash struct { + h hash.Hash + s int +} + +var newline = []byte{'\r', '\n'} + +func (cth *canonicalTextHash) Write(buf []byte) (int, os.Error) { + start := 0 + + for i, c := range buf { + switch cth.s { + case 0: + if c == '\r' { + cth.s = 1 + } else if c == '\n' { + cth.h.Write(buf[start:i]) + cth.h.Write(newline) + start = i + 1 + } + case 1: + cth.s = 0 + } + } + + cth.h.Write(buf[start:]) + return len(buf), nil +} + +func (cth *canonicalTextHash) Sum() []byte { + return cth.h.Sum() +} + +func (cth *canonicalTextHash) Reset() { + cth.h.Reset() + cth.s = 0 +} + +func (cth *canonicalTextHash) Size() int { + return cth.h.Size() +} diff --git a/src/pkg/crypto/openpgp/canonical_text_test.go b/src/pkg/crypto/openpgp/canonical_text_test.go new file mode 100644 index 000000000..69ecf91a8 --- /dev/null +++ b/src/pkg/crypto/openpgp/canonical_text_test.go @@ -0,0 +1,50 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "os" + "testing" +) + +type recordingHash struct { + buf *bytes.Buffer +} + +func (r recordingHash) Write(b []byte) (n int, err os.Error) { + return r.buf.Write(b) +} + +func (r recordingHash) Sum() []byte { + return r.buf.Bytes() +} + +func (r recordingHash) Reset() { + panic("shouldn't be called") +} + +func (r recordingHash) Size() int { + panic("shouldn't be called") +} + + +func testCanonicalText(t *testing.T, input, expected string) { + r := recordingHash{bytes.NewBuffer(nil)} + c := NewCanonicalTextHash(r) + c.Write([]byte(input)) + result := c.Sum() + if expected != string(result) { + t.Errorf("input: %x got: %x want: %x", input, result, expected) + } +} + +func TestCanonicalText(t *testing.T) { + testCanonicalText(t, "foo\n", "foo\r\n") + testCanonicalText(t, "foo", "foo") + testCanonicalText(t, "foo\r\n", "foo\r\n") + testCanonicalText(t, "foo\r\nbar", "foo\r\nbar") + testCanonicalText(t, "foo\r\nbar\n\n", "foo\r\nbar\r\n\r\n") +} diff --git a/src/pkg/crypto/openpgp/keys.go b/src/pkg/crypto/openpgp/keys.go new file mode 100644 index 000000000..ecaa86f28 --- /dev/null +++ b/src/pkg/crypto/openpgp/keys.go @@ -0,0 +1,280 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto/openpgp/error" + "crypto/openpgp/packet" + "io" + "os" +) + +// PublicKeyType is the armor type for a PGP public key. +var PublicKeyType = "PGP PUBLIC KEY BLOCK" + +// An Entity represents the components of an OpenPGP key: a primary public key +// (which must be a signing key), one or more identities claimed by that key, +// and zero or more subkeys, which may be encryption keys. +type Entity struct { + PrimaryKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Identities map[string]*Identity // indexed by Identity.Name + Subkeys []Subkey +} + +// An Identity represents an identity claimed by an Entity and zero or more +// assertions by other entities about that claim. +type Identity struct { + Name string // by convention, has the form "Full Name (comment) <email@example.com>" + UserId *packet.UserId + SelfSignature *packet.Signature + Signatures []*packet.Signature +} + +// A Subkey is an additional public key in an Entity. Subkeys can be used for +// encryption. +type Subkey struct { + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Sig *packet.Signature +} + +// A Key identifies a specific public key in an Entity. This is either the +// Entity's primary key or a subkey. +type Key struct { + Entity *Entity + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + SelfSignature *packet.Signature +} + +// A KeyRing provides access to public and private keys. +type KeyRing interface { + // KeysById returns the set of keys that have the given key id. + KeysById(id uint64) []Key + // DecryptionKeys returns all private keys that are valid for + // decryption. + DecryptionKeys() []Key +} + +// An EntityList contains one or more Entities. +type EntityList []*Entity + +// KeysById returns the set of keys that have the given key id. +func (el EntityList) KeysById(id uint64) (keys []Key) { + for _, e := range el { + if e.PrimaryKey.KeyId == id { + var selfSig *packet.Signature + for _, ident := range e.Identities { + if selfSig == nil { + selfSig = ident.SelfSignature + } else if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + selfSig = ident.SelfSignature + break + } + } + keys = append(keys, Key{e, e.PrimaryKey, e.PrivateKey, selfSig}) + } + + for _, subKey := range e.Subkeys { + if subKey.PublicKey.KeyId == id { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// DecryptionKeys returns all private keys that are valid for decryption. +func (el EntityList) DecryptionKeys() (keys []Key) { + for _, e := range el { + for _, subKey := range e.Subkeys { + if subKey.PrivateKey != nil && (!subKey.Sig.FlagsValid || subKey.Sig.FlagEncryptStorage || subKey.Sig.FlagEncryptCommunications) { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// ReadArmoredKeyRing reads one or more public/private keys from an armor keyring file. +func ReadArmoredKeyRing(r io.Reader) (EntityList, os.Error) { + body, err := readArmored(r, PublicKeyType) + if err != nil { + return nil, err + } + + return ReadKeyRing(body) +} + +// ReadKeyRing reads one or more public/private keys, ignoring unsupported keys. +func ReadKeyRing(r io.Reader) (el EntityList, err os.Error) { + packets := packet.NewReader(r) + + for { + var e *Entity + e, err = readEntity(packets) + if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + err = readToNextPublicKey(packets) + } + if err == os.EOF { + err = nil + return + } + if err != nil { + el = nil + return + } + } else { + el = append(el, e) + } + } + return +} + +// readToNextPublicKey reads packets until the start of the entity and leaves +// the first packet of the new entity in the Reader. +func readToNextPublicKey(packets *packet.Reader) (err os.Error) { + var p packet.Packet + for { + p, err = packets.Next() + if err == os.EOF { + return + } else if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + err = nil + continue + } + return + } + + if pk, ok := p.(*packet.PublicKey); ok && !pk.IsSubkey { + packets.Unread(p) + return + } + } + + panic("unreachable") +} + +// readEntity reads an entity (public key, identities, subkeys etc) from the +// given Reader. +func readEntity(packets *packet.Reader) (*Entity, os.Error) { + e := new(Entity) + e.Identities = make(map[string]*Identity) + + p, err := packets.Next() + if err != nil { + return nil, err + } + + var ok bool + if e.PrimaryKey, ok = p.(*packet.PublicKey); !ok { + if e.PrivateKey, ok = p.(*packet.PrivateKey); !ok { + packets.Unread(p) + return nil, error.StructuralError("first packet was not a public/private key") + } else { + e.PrimaryKey = &e.PrivateKey.PublicKey + } + } + + var current *Identity +EachPacket: + for { + p, err := packets.Next() + if err == os.EOF { + break + } else if err != nil { + return nil, err + } + + switch pkt := p.(type) { + case *packet.UserId: + current = new(Identity) + current.Name = pkt.Id + current.UserId = pkt + e.Identities[pkt.Id] = current + p, err = packets.Next() + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + return nil, err + } + return nil, error.StructuralError("identity self-signature invalid: " + err.String()) + } + current.SelfSignature, ok = p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("user ID packet not followed by self signature") + } + if current.SelfSignature.SigType != packet.SigTypePositiveCert { + return nil, error.StructuralError("user ID self-signature with wrong type") + } + if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, current.SelfSignature); err != nil { + return nil, error.StructuralError("user ID self-signature invalid: " + err.String()) + } + case *packet.Signature: + if current == nil { + return nil, error.StructuralError("signature packet found before user id packet") + } + current.Signatures = append(current.Signatures, pkt) + case *packet.PrivateKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, &pkt.PublicKey, pkt) + if err != nil { + return nil, err + } + case *packet.PublicKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, pkt, nil) + if err != nil { + return nil, err + } + default: + // we ignore unknown packets + } + } + + if len(e.Identities) == 0 { + return nil, error.StructuralError("entity without any identities") + } + + return e, nil +} + +func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) os.Error { + var subKey Subkey + subKey.PublicKey = pub + subKey.PrivateKey = priv + p, err := packets.Next() + if err == os.EOF { + return io.ErrUnexpectedEOF + } + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + var ok bool + subKey.Sig, ok = p.(*packet.Signature) + if !ok { + return error.StructuralError("subkey packet not followed by signature") + } + if subKey.Sig.SigType != packet.SigTypeSubkeyBinding { + return error.StructuralError("subkey signature with wrong type") + } + err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig) + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + e.Subkeys = append(e.Subkeys, subKey) + return nil +} diff --git a/src/pkg/crypto/openpgp/packet/encrypted_key.go b/src/pkg/crypto/openpgp/packet/encrypted_key.go index 4a926cdb1..b11a9b830 100644 --- a/src/pkg/crypto/openpgp/packet/encrypted_key.go +++ b/src/pkg/crypto/openpgp/packet/encrypted_key.go @@ -20,8 +20,8 @@ type EncryptedKey struct { KeyId uint64 Algo PublicKeyAlgorithm Encrypted []byte - CipherFunc CipherFunction // only valid after a sucessful Decrypt - Key []byte // only valid after a sucessful Decrypt + CipherFunc CipherFunction // only valid after a successful Decrypt + Key []byte // only valid after a successful Decrypt } func (e *EncryptedKey) parse(r io.Reader) (err os.Error) { diff --git a/src/pkg/crypto/openpgp/packet/literal.go b/src/pkg/crypto/openpgp/packet/literal.go index 5f72d6a2c..04f50e53e 100644 --- a/src/pkg/crypto/openpgp/packet/literal.go +++ b/src/pkg/crypto/openpgp/packet/literal.go @@ -14,11 +14,11 @@ import ( type LiteralData struct { IsBinary bool FileName string - Time uint32 // Unix epoc time. Either creation time or modification time. 0 means undefined. + Time uint32 // Unix epoch time. Either creation time or modification time. 0 means undefined. Body io.Reader } -// ForEyesOnly return whether the contents of the LiteralData have been marked +// ForEyesOnly returns whether the contents of the LiteralData have been marked // as especially sensitive. func (l *LiteralData) ForEyesOnly() bool { return l.FileName == "_CONSOLE" diff --git a/src/pkg/crypto/openpgp/packet/packet.go b/src/pkg/crypto/openpgp/packet/packet.go index 80e25e250..269603ba4 100644 --- a/src/pkg/crypto/openpgp/packet/packet.go +++ b/src/pkg/crypto/openpgp/packet/packet.go @@ -261,13 +261,13 @@ func Read(r io.Reader) (p Packet, err os.Error) { case packetTypePrivateKey, packetTypePrivateSubkey: pk := new(PrivateKey) if tag == packetTypePrivateSubkey { - pk.IsSubKey = true + pk.IsSubkey = true } p = pk case packetTypePublicKey, packetTypePublicSubkey: pk := new(PublicKey) if tag == packetTypePublicSubkey { - pk.IsSubKey = true + pk.IsSubkey = true } p = pk case packetTypeCompressed: @@ -300,7 +300,7 @@ type SignatureType uint8 const ( SigTypeBinary SignatureType = 0 - SigTypeText SignatureType = 1 + SigTypeText = 1 SigTypeGenericCert = 0x10 SigTypePersonaCert = 0x11 SigTypeCasualCert = 0x12 diff --git a/src/pkg/crypto/openpgp/packet/public_key.go b/src/pkg/crypto/openpgp/packet/public_key.go index 4a2ed0aca..8866bdaaa 100644 --- a/src/pkg/crypto/openpgp/packet/public_key.go +++ b/src/pkg/crypto/openpgp/packet/public_key.go @@ -23,7 +23,7 @@ type PublicKey struct { PublicKey interface{} // Either a *rsa.PublicKey or *dsa.PublicKey Fingerprint [20]byte KeyId uint64 - IsSubKey bool + IsSubkey bool n, e, p, q, g, y parsedMPI } diff --git a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go index ee5a30d32..5543b2029 100644 --- a/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go +++ b/src/pkg/crypto/openpgp/packet/symmetrically_encrypted_test.go @@ -54,7 +54,7 @@ func testMDCReader(t *testing.T) { err = mdcReader.Close() if err != nil { - t.Errorf("stride: %d, error on Close: %s", err) + t.Errorf("stride: %d, error on Close: %s", stride, err) } } diff --git a/src/pkg/crypto/openpgp/read.go b/src/pkg/crypto/openpgp/read.go new file mode 100644 index 000000000..ac6998f0d --- /dev/null +++ b/src/pkg/crypto/openpgp/read.go @@ -0,0 +1,413 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This openpgp package implements high level operations on OpenPGP messages. +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/rsa" + _ "crypto/sha256" + "hash" + "io" + "os" + "strconv" +) + +// SignatureType is the armor type for a PGP signature. +var SignatureType = "PGP SIGNATURE" + +// readArmored reads an armored block with the given type. +func readArmored(r io.Reader, expectedType string) (body io.Reader, err os.Error) { + block, err := armor.Decode(r) + if err != nil { + return + } + + if block.Type != expectedType { + return nil, error.InvalidArgumentError("expected '" + expectedType + "', got: " + block.Type) + } + + return block.Body, nil +} + +// MessageDetails contains the result of parsing an OpenPGP encrypted and/or +// signed message. +type MessageDetails struct { + IsEncrypted bool // true if the message was encrypted. + EncryptedToKeyIds []uint64 // the list of recipient key ids. + IsSymmetricallyEncrypted bool // true if a passphrase could have decrypted the message. + DecryptedWith Key // the private key used to decrypt the message, if any. + IsSigned bool // true if the message is signed. + SignedByKeyId uint64 // the key id of the signer, if any. + SignedBy *Key // the key of the signer, if availible. + LiteralData *packet.LiteralData // the metadata of the contents + UnverifiedBody io.Reader // the contents of the message. + + // If IsSigned is true and SignedBy is non-zero then the signature will + // be verified as UnverifiedBody is read. The signature cannot be + // checked until the whole of UnverifiedBody is read so UnverifiedBody + // must be consumed until EOF before the data can trusted. Even if a + // message isn't signed (or the signer is unknown) the data may contain + // an authentication code that is only checked once UnverifiedBody has + // been consumed. Once EOF has been seen, the following fields are + // valid. (An authentication code failure is reported as a + // SignatureError error when reading from UnverifiedBody.) + + SignatureError os.Error // nil if the signature is good. + Signature *packet.Signature // the signature packet itself. + + decrypted io.ReadCloser +} + +// A PromptFunction is used as a callback by functions that may need to decrypt +// a private key, or prompt for a passphrase. It is called with a list of +// acceptable, encrypted private keys and a boolean that indicates whether a +// passphrase is usable. It should either decrypt a private key or return a +// passphrase to try. If the decrypted private key or given passphrase isn't +// correct, the function will be called again, forever. Any error returned will +// be passed up. +type PromptFunction func(keys []Key, symmetric bool) ([]byte, os.Error) + +// A keyEnvelopePair is used to store a private key with the envelope that +// contains a symmetric key, encrypted with that key. +type keyEnvelopePair struct { + key Key + encryptedKey *packet.EncryptedKey +} + +// ReadMessage parses an OpenPGP message that may be signed and/or encrypted. +// The given KeyRing should contain both public keys (for signature +// verification) and, possibly encrypted, private keys for decrypting. +func ReadMessage(r io.Reader, keyring KeyRing, prompt PromptFunction) (md *MessageDetails, err os.Error) { + var p packet.Packet + + var symKeys []*packet.SymmetricKeyEncrypted + var pubKeys []keyEnvelopePair + var se *packet.SymmetricallyEncrypted + + packets := packet.NewReader(r) + md = new(MessageDetails) + md.IsEncrypted = true + + // The message, if encrypted, starts with a number of packets + // containing an encrypted decryption key. The decryption key is either + // encrypted to a public key, or with a passphrase. This loop + // collects these packets. +ParsePackets: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.SymmetricKeyEncrypted: + // This packet contains the decryption key encrypted with a passphrase. + md.IsSymmetricallyEncrypted = true + symKeys = append(symKeys, p) + case *packet.EncryptedKey: + // This packet contains the decryption key encrypted to a public key. + md.EncryptedToKeyIds = append(md.EncryptedToKeyIds, p.KeyId) + if p.Algo != packet.PubKeyAlgoRSA && p.Algo != packet.PubKeyAlgoRSAEncryptOnly { + continue + } + var keys []Key + if p.KeyId == 0 { + keys = keyring.DecryptionKeys() + } else { + keys = keyring.KeysById(p.KeyId) + } + for _, k := range keys { + pubKeys = append(pubKeys, keyEnvelopePair{k, p}) + } + case *packet.SymmetricallyEncrypted: + se = p + break ParsePackets + case *packet.Compressed, *packet.LiteralData, *packet.OnePassSignature: + // This message isn't encrypted. + if len(symKeys) != 0 || len(pubKeys) != 0 { + return nil, error.StructuralError("key material not followed by encrypted message") + } + packets.Unread(p) + return readSignedMessage(packets, nil, keyring) + } + } + + var candidates []Key + var decrypted io.ReadCloser + + // Now that we have the list of encrypted keys we need to decrypt at + // least one of them or, if we cannot, we need to call the prompt + // function so that it can decrypt a key or give us a passphrase. +FindKey: + for { + // See if any of the keys already have a private key availible + candidates = candidates[:0] + candidateFingerprints := make(map[string]bool) + + for _, pk := range pubKeys { + if pk.key.PrivateKey == nil { + continue + } + if !pk.key.PrivateKey.Encrypted { + if len(pk.encryptedKey.Key) == 0 { + pk.encryptedKey.DecryptRSA(pk.key.PrivateKey.PrivateKey.(*rsa.PrivateKey)) + } + if len(pk.encryptedKey.Key) == 0 { + continue + } + decrypted, err = se.Decrypt(pk.encryptedKey.CipherFunc, pk.encryptedKey.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + md.DecryptedWith = pk.key + break FindKey + } + } else { + fpr := string(pk.key.PublicKey.Fingerprint[:]) + if v := candidateFingerprints[fpr]; v { + continue + } + candidates = append(candidates, pk.key) + candidateFingerprints[fpr] = true + } + } + + if len(candidates) == 0 && len(symKeys) == 0 { + return nil, error.KeyIncorrectError + } + + if prompt == nil { + return nil, error.KeyIncorrectError + } + + passphrase, err := prompt(candidates, len(symKeys) != 0) + if err != nil { + return nil, err + } + + // Try the symmetric passphrase first + if len(symKeys) != 0 && passphrase != nil { + for _, s := range symKeys { + err = s.Decrypt(passphrase) + if err == nil && !s.Encrypted { + decrypted, err = se.Decrypt(s.CipherFunc, s.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + break FindKey + } + } + + } + } + } + + md.decrypted = decrypted + packets.Push(decrypted) + return readSignedMessage(packets, md, keyring) +} + +// readSignedMessage reads a possibily signed message if mdin is non-zero then +// that structure is updated and returned. Otherwise a fresh MessageDetails is +// used. +func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err os.Error) { + if mdin == nil { + mdin = new(MessageDetails) + } + md = mdin + + var p packet.Packet + var h hash.Hash + var wrappedHash hash.Hash +FindLiteralData: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.Compressed: + packets.Push(p.Body) + case *packet.OnePassSignature: + if !p.IsLast { + return nil, error.UnsupportedError("nested signatures") + } + + h, wrappedHash, err = hashForSignature(p.Hash, p.SigType) + if err != nil { + md = nil + return + } + + md.IsSigned = true + md.SignedByKeyId = p.KeyId + keys := keyring.KeysById(p.KeyId) + for _, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + md.SignedBy = &key + } + case *packet.LiteralData: + md.LiteralData = p + break FindLiteralData + } + } + + if md.SignedBy != nil { + md.UnverifiedBody = &signatureCheckReader{packets, h, wrappedHash, md} + } else if md.decrypted != nil { + md.UnverifiedBody = checkReader{md} + } else { + md.UnverifiedBody = md.LiteralData.Body + } + + return md, nil +} + +// hashForSignature returns a pair of hashes that can be used to verify a +// signature. The signature may specify that the contents of the signed message +// should be preprocessed (i.e. to normalise line endings). Thus this function +// returns two hashes. The second should be used to hash the message itself and +// performs any needed preprocessing. +func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, os.Error) { + h := hashId.New() + if h == nil { + return nil, nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hashId))) + } + + switch sigType { + case packet.SigTypeBinary: + return h, h, nil + case packet.SigTypeText: + return h, NewCanonicalTextHash(h), nil + } + + return nil, nil, error.UnsupportedError("unsupported signature type: " + strconv.Itoa(int(sigType))) +} + +// checkReader wraps an io.Reader from a LiteralData packet. When it sees EOF +// it closes the ReadCloser from any SymmetricallyEncrypted packet to trigger +// MDC checks. +type checkReader struct { + md *MessageDetails +} + +func (cr checkReader) Read(buf []byte) (n int, err os.Error) { + n, err = cr.md.LiteralData.Body.Read(buf) + if err == os.EOF { + mdcErr := cr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + return +} + +// signatureCheckReader wraps an io.Reader from a LiteralData packet and hashes +// the data as it is read. When it sees an EOF from the underlying io.Reader +// it parses and checks a trailing Signature packet and triggers any MDC checks. +type signatureCheckReader struct { + packets *packet.Reader + h, wrappedHash hash.Hash + md *MessageDetails +} + +func (scr *signatureCheckReader) Read(buf []byte) (n int, err os.Error) { + n, err = scr.md.LiteralData.Body.Read(buf) + scr.wrappedHash.Write(buf[:n]) + if err == os.EOF { + var p packet.Packet + p, scr.md.SignatureError = scr.packets.Next() + if scr.md.SignatureError != nil { + return + } + + var ok bool + if scr.md.Signature, ok = p.(*packet.Signature); !ok { + scr.md.SignatureError = error.StructuralError("LiteralData not followed by Signature") + return + } + + scr.md.SignatureError = scr.md.SignedBy.PublicKey.VerifySignature(scr.h, scr.md.Signature) + + // The SymmetricallyEncrypted packet, if any, might have an + // unsigned hash of its own. In order to check this we need to + // close that Reader. + if scr.md.decrypted != nil { + mdcErr := scr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + } + return +} + +// CheckDetachedSignature takes a signed file and a detached signature and +// returns the signer if the signature is valid. If the signer isn't know, +// UnknownIssuerError is returned. +func CheckDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + p, err := packet.Read(signature) + if err != nil { + return + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("non signature packet found") + } + + if sig.IssuerKeyId == nil { + return nil, error.StructuralError("signature doesn't have an issuer") + } + + keys := keyring.KeysById(*sig.IssuerKeyId) + if len(keys) == 0 { + return nil, error.UnknownIssuerError + } + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + + _, err = io.Copy(wrappedHash, signed) + if err != nil && err != os.EOF { + return + } + + for _, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + err = key.PublicKey.VerifySignature(h, sig) + if err == nil { + return key.Entity, nil + } + } + + if err != nil { + return + } + + return nil, error.UnknownIssuerError +} + +// CheckArmoredDetachedSignature performs the same actions as +// CheckDetachedSignature but expects the signature to be armored. +func CheckArmoredDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + body, err := readArmored(signature, SignatureType) + if err != nil { + return + } + + return CheckDetachedSignature(keyring, signed, body) +} diff --git a/src/pkg/crypto/openpgp/read_test.go b/src/pkg/crypto/openpgp/read_test.go new file mode 100644 index 000000000..58199e132 --- /dev/null +++ b/src/pkg/crypto/openpgp/read_test.go @@ -0,0 +1,237 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func TestReadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadPrivateKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B || kring[0].PrimaryKey == nil { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestGetKeyById(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + keys := kring.KeysById(0xa34d7e18c20c31bb) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } + + keys = kring.KeysById(0xfd94408d4543314f) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } +} + +func checkSignedMessage(t *testing.T, signedHex, expected string) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + md, err := ReadMessage(readerFromHex(signedHex), kring, nil) + if err != nil { + t.Error(err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) != 0 || md.IsSymmetricallyEncrypted { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestSignedMessage(t *testing.T) { + checkSignedMessage(t, signedMessageHex, signedInput) +} + +func TestTextSignedMessage(t *testing.T) { + checkSignedMessage(t, signedTextMessageHex, signedTextInput) +} + +func TestSignedEncryptedMessage(t *testing.T) { + expected := "Signed and encrypted message\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if symmetric { + t.Errorf("prompt: message was marked as symmetrically encrypted") + return nil, error.KeyIncorrectError + } + + if len(keys) == 0 { + t.Error("prompt: no keys requested") + return nil, error.KeyIncorrectError + } + + err := keys[0].PrivateKey.Decrypt([]byte("passphrase")) + if err != nil { + t.Errorf("prompt: error decrypting key: %s", err) + return nil, error.KeyIncorrectError + } + + return nil, nil + } + + md, err := ReadMessage(readerFromHex(signedEncryptedMessageHex), kring, prompt) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || !md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) == 0 || md.EncryptedToKeyIds[0] != 0x2a67d68660df41c7 { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestUnspecifiedRecipient(t *testing.T) { + expected := "Recipient unspecified\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + + md, err := ReadMessage(readerFromHex(recipientUnspecifiedHex), kring, nil) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } +} + +func TestSymmetricallyEncrypted(t *testing.T) { + expected := "Symmetrically encrypted.\n" + + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if len(keys) != 0 { + t.Errorf("prompt: len(keys) = %d (want 0)", len(keys)) + } + + if !symmetric { + t.Errorf("symmetric is not set") + } + + return []byte("password"), nil + } + + md, err := ReadMessage(readerFromHex(symmetricallyEncryptedCompressedHex), nil, prompt) + if err != nil { + t.Errorf("ReadMessage: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("ReadAll: %s", err) + } + + expectedCreatationTime := uint32(1295992998) + if md.LiteralData.Time != expectedCreatationTime { + t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreatationTime) + } + + if string(contents) != expected { + t.Errorf("contents got: %s want: %s", string(contents), expected) + } +} + +func testDetachedSignature(t *testing.T, kring KeyRing, signature io.Reader, sigInput, tag string) { + signed := bytes.NewBufferString(sigInput) + signer, err := CheckDetachedSignature(kring, signed, signature) + if err != nil { + t.Errorf("%s: signature error: %s", tag, err) + return + } + if signer == nil { + t.Errorf("%s: signer is nil", tag) + return + } + expectedSignerKeyId := uint64(0xa34d7e18c20c31bb) + if signer.PrimaryKey.KeyId != expectedSignerKeyId { + t.Errorf("%s: wrong signer got:%x want:%x", tag, signer.PrimaryKey.KeyId, expectedSignerKeyId) + } +} + +func TestDetachedSignature(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureHex), signedInput, "binary") + testDetachedSignature(t, kring, readerFromHex(detachedSignatureTextHex), signedInput, "text") +} + +const signedInput = "Signed message\nline 2\nline 3\n" +const signedTextInput = "Signed message\r\nline 2\r\nline 3\r\n" + +const recipientUnspecifiedHex = "848c0300000000000000000103ff62d4d578d03cf40c3da998dfe216c074fa6ddec5e31c197c9666ba292830d91d18716a80f699f9d897389a90e6d62d0238f5f07a5248073c0f24920e4bc4a30c2d17ee4e0cae7c3d4aaa4e8dced50e3010a80ee692175fa0385f62ecca4b56ee6e9980aa3ec51b61b077096ac9e800edaf161268593eedb6cc7027ff5cb32745d250010d407a6221ae22ef18469b444f2822478c4d190b24d36371a95cb40087cdd42d9399c3d06a53c0673349bfb607927f20d1e122bde1e2bf3aa6cae6edf489629bcaa0689539ae3b718914d88ededc3b" + +const detachedSignatureHex = "889c04000102000605024d449cd1000a0910a34d7e18c20c31bb167603ff57718d09f28a519fdc7b5a68b6a3336da04df85e38c5cd5d5bd2092fa4629848a33d85b1729402a2aab39c3ac19f9d573f773cc62c264dc924c067a79dfd8a863ae06c7c8686120760749f5fd9b1e03a64d20a7df3446ddc8f0aeadeaeba7cbaee5c1e366d65b6a0c6cc749bcb912d2f15013f812795c2e29eb7f7b77f39ce77" + +const detachedSignatureTextHex = "889c04010102000605024d449d21000a0910a34d7e18c20c31bbc8c60400a24fbef7342603a41cb1165767bd18985d015fb72fe05db42db36cfb2f1d455967f1e491194fbf6cf88146222b23bf6ffbd50d17598d976a0417d3192ff9cc0034fd00f287b02e90418bbefe609484b09231e4e7a5f3562e199bf39909ab5276c4d37382fe088f6b5c3426fc1052865da8b3ab158672d58b6264b10823dc4b39" + +const testKeys1And2Hex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b0020003b88d044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f0011010001889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab0020003988d044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b0020003b88d044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020003" + +const testKeys1And2PrivateHex = "9501d8044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd00110100010003ff4d91393b9a8e3430b14d6209df42f98dc927425b881f1209f319220841273a802a97c7bdb8b3a7740b3ab5866c4d1d308ad0d3a79bd1e883aacf1ac92dfe720285d10d08752a7efe3c609b1d00f17f2805b217be53999a7da7e493bfc3e9618fd17018991b8128aea70a05dbce30e4fbe626aa45775fa255dd9177aabf4df7cf0200c1ded12566e4bc2bb590455e5becfb2e2c9796482270a943343a7835de41080582c2be3caf5981aa838140e97afa40ad652a0b544f83eb1833b0957dce26e47b0200eacd6046741e9ce2ec5beb6fb5e6335457844fb09477f83b050a96be7da043e17f3a9523567ed40e7a521f818813a8b8a72209f1442844843ccc7eb9805442570200bdafe0438d97ac36e773c7162028d65844c4d463e2420aa2228c6e50dc2743c3d6c72d0d782a5173fe7be2169c8a9f4ef8a7cf3e37165e8c61b89c346cdc6c1799d2b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b00200009d01d8044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f00110100010003fd17a7490c22a79c59281fb7b20f5e6553ec0c1637ae382e8adaea295f50241037f8997cf42c1ce26417e015091451b15424b2c59eb8d4161b0975630408e394d3b00f88d4b4e18e2cc85e8251d4753a27c639c83f5ad4a571c4f19d7cd460b9b73c25ade730c99df09637bd173d8e3e981ac64432078263bb6dc30d3e974150dd0200d0ee05be3d4604d2146fb0457f31ba17c057560785aa804e8ca5530a7cd81d3440d0f4ba6851efcfd3954b7e68908fc0ba47f7ac37bf559c6c168b70d3a7c8cd0200da1c677c4bce06a068070f2b3733b0a714e88d62aa3f9a26c6f5216d48d5c2b5624144f3807c0df30be66b3268eeeca4df1fbded58faf49fc95dc3c35f134f8b01fd1396b6c0fc1b6c4f0eb8f5e44b8eace1e6073e20d0b8bc5385f86f1cf3f050f66af789f3ef1fc107b7f4421e19e0349c730c68f0a226981f4e889054fdb4dc149e8e889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab00200009501fe044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001fe030302e9030f3c783e14856063f16938530e148bc57a7aa3f3e4f90df9dceccdc779bc0835e1ad3d006e4a8d7b36d08b8e0de5a0d947254ecfbd22037e6572b426bcfdc517796b224b0036ff90bc574b5509bede85512f2eefb520fb4b02aa523ba739bff424a6fe81c5041f253f8d757e69a503d3563a104d0d49e9e890b9d0c26f96b55b743883b472caa7050c4acfd4a21f875bdf1258d88bd61224d303dc9df77f743137d51e6d5246b88c406780528fd9a3e15bab5452e5b93970d9dcc79f48b38651b9f15bfbcf6da452837e9cc70683d1bdca94507870f743e4ad902005812488dd342f836e72869afd00ce1850eea4cfa53ce10e3608e13d3c149394ee3cbd0e23d018fcbcb6e2ec5a1a22972d1d462ca05355d0d290dd2751e550d5efb38c6c89686344df64852bf4ff86638708f644e8ec6bd4af9b50d8541cb91891a431326ab2e332faa7ae86cfb6e0540aa63160c1e5cdd5a4add518b303fff0a20117c6bc77f7cfbaf36b04c865c6c2b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b00200009d01fe044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001fe030302e9030f3c783e148560f936097339ae381d63116efcf802ff8b1c9360767db5219cc987375702a4123fd8657d3e22700f23f95020d1b261eda5257e9a72f9a918e8ef22dd5b3323ae03bbc1923dd224db988cadc16acc04b120a9f8b7e84da9716c53e0334d7b66586ddb9014df604b41be1e960dcfcbc96f4ed150a1a0dd070b9eb14276b9b6be413a769a75b519a53d3ecc0c220e85cd91ca354d57e7344517e64b43b6e29823cbd87eae26e2b2e78e6dedfbb76e3e9f77bcb844f9a8932eb3db2c3f9e44316e6f5d60e9e2a56e46b72abe6b06dc9a31cc63f10023d1f5e12d2a3ee93b675c96f504af0001220991c88db759e231b3320dcedf814dcf723fd9857e3d72d66a0f2af26950b915abdf56c1596f46a325bf17ad4810d3535fb02a259b247ac3dbd4cc3ecf9c51b6c07cebb009c1506fba0a89321ec8683e3fd009a6e551d50243e2d5092fefb3321083a4bad91320dc624bd6b5dddf93553e3d53924c05bfebec1fb4bd47e89a1a889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020000" + +const signedMessageHex = "a3019bc0cbccc0c4b8d8b74ee2108fe16ec6d3ca490cbe362d3f8333d3f352531472538b8b13d353b97232f352158c20943157c71c16064626063656269052062e4e01987e9b6fccff4b7df3a34c534b23e679cbec3bc0f8f6e64dfb4b55fe3f8efa9ce110ddb5cd79faf1d753c51aecfa669f7e7aa043436596cccc3359cb7dd6bbe9ecaa69e5989d9e57209571edc0b2fa7f57b9b79a64ee6e99ce1371395fee92fec2796f7b15a77c386ff668ee27f6d38f0baa6c438b561657377bf6acff3c5947befd7bf4c196252f1d6e5c524d0300" + +const signedTextMessageHex = "a3019bc0cbccc8c4b8d8b74ee2108fe16ec6d36a250cbece0c178233d3f352531472538b8b13d35379b97232f352158ca0b4312f57c71c1646462606365626906a062e4e019811591798ff99bf8afee860b0d8a8c2a85c3387e3bcf0bb3b17987f2bbcfab2aa526d930cbfd3d98757184df3995c9f3e7790e36e3e9779f06089d4c64e9e47dd6202cb6e9bc73c5d11bb59fbaf89d22d8dc7cf199ddf17af96e77c5f65f9bbed56f427bd8db7af37f6c9984bf9385efaf5f184f986fb3e6adb0ecfe35bbf92d16a7aa2a344fb0bc52fb7624f0200" + +const signedEncryptedMessageHex = "848c032a67d68660df41c70103ff5789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8d2c03b018bd210b1d3791e1aba74b0f1034e122ab72e760492c192383cf5e20b5628bd043272d63df9b923f147eb6091cd897553204832aba48fec54aa447547bb16305a1024713b90e77fd0065f1918271947549205af3c74891af22ee0b56cd29bfec6d6e351901cd4ab3ece7c486f1e32a792d4e474aed98ee84b3f591c7dff37b64e0ecd68fd036d517e412dcadf85840ce184ad7921ad446c4ee28db80447aea1ca8d4f574db4d4e37688158ddd19e14ee2eab4873d46947d65d14a23e788d912cf9a19624ca7352469b72a83866b7c23cb5ace3deab3c7018061b0ba0f39ed2befe27163e5083cf9b8271e3e3d52cc7ad6e2a3bd81d4c3d7022f8d" + +const symmetricallyEncryptedCompressedHex = "8c0d04030302eb4a03808145d0d260c92f714339e13de5a79881216431925bf67ee2898ea61815f07894cd0703c50d0a76ef64d482196f47a8bc729af9b80bb6" diff --git a/src/pkg/crypto/openpgp/write.go b/src/pkg/crypto/openpgp/write.go new file mode 100644 index 000000000..1a2e2bf04 --- /dev/null +++ b/src/pkg/crypto/openpgp/write.go @@ -0,0 +1,92 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/rsa" + _ "crypto/sha256" + "io" + "os" + "strconv" + "time" +) + +// DetachSign signs message with the private key from signer (which must +// already have been decrypted) and writes the signature to w. +func DetachSign(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeBinary) +} + +// ArmoredDetachSign signs message with the private key from signer (which +// must already have been decrypted) and writes an armored signature to w. +func ArmoredDetachSign(w io.Writer, signer *Entity, message io.Reader) (err os.Error) { + return armoredDetachSign(w, signer, message, packet.SigTypeBinary) +} + +// DetachSignText signs message (after canonicalising the line endings) with +// the private key from signer (which must already have been decrypted) and +// writes the signature to w. +func DetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeText) +} + +// ArmoredDetachSignText signs message (after canonicalising the line endings) +// with the private key from signer (which must already have been decrypted) +// and writes an armored signature to w. +func SignTextDetachedArmored(w io.Writer, signer *Entity, message io.Reader) os.Error { + return armoredDetachSign(w, signer, message, packet.SigTypeText) +} + +func armoredDetachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + out, err := armor.Encode(w, SignatureType, nil) + if err != nil { + return + } + err = detachSign(out, signer, message, sigType) + if err != nil { + return + } + return out.Close() +} + +func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing key doesn't have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing key is encrypted") + } + + sig := new(packet.Signature) + sig.SigType = sigType + sig.PubKeyAlgo = signer.PrivateKey.PubKeyAlgo + sig.Hash = crypto.SHA256 + sig.CreationTime = uint32(time.Seconds()) + sig.IssuerKeyId = &signer.PrivateKey.KeyId + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + io.Copy(wrappedHash, message) + + switch signer.PrivateKey.PubKeyAlgo { + case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSASignOnly: + priv := signer.PrivateKey.PrivateKey.(*rsa.PrivateKey) + err = sig.SignRSA(h, priv) + default: + err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) + } + + if err != nil { + return + } + + return sig.Serialize(w) +} diff --git a/src/pkg/crypto/openpgp/write_test.go b/src/pkg/crypto/openpgp/write_test.go new file mode 100644 index 000000000..33e8809f2 --- /dev/null +++ b/src/pkg/crypto/openpgp/write_test.go @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "testing" +) + +func TestSignDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check") +} + +func TestSignTextDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSignText(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check") +} diff --git a/src/pkg/crypto/rand/rand_unix.go b/src/pkg/crypto/rand/rand_unix.go index 900b57330..66b72c076 100644 --- a/src/pkg/crypto/rand/rand_unix.go +++ b/src/pkg/crypto/rand/rand_unix.go @@ -8,6 +8,7 @@ package rand import ( + "bufio" "crypto/aes" "io" "os" @@ -23,7 +24,7 @@ func init() { Reader = &devReader{name: "/dev/urandom"} } // A devReader satisfies reads by reading the file named name. type devReader struct { name string - f *os.File + f io.Reader mu sync.Mutex } @@ -35,7 +36,7 @@ func (r *devReader) Read(b []byte) (n int, err os.Error) { if f == nil { return 0, err } - r.f = f + r.f = bufio.NewReader(f) } return r.f.Read(b) } diff --git a/src/pkg/crypto/rsa/rsa.go b/src/pkg/crypto/rsa/rsa.go index c7a8d2053..faf914991 100644 --- a/src/pkg/crypto/rsa/rsa.go +++ b/src/pkg/crypto/rsa/rsa.go @@ -274,6 +274,14 @@ func EncryptOAEP(hash hash.Hash, rand io.Reader, pub *PublicKey, msg []byte, lab m.SetBytes(em) c := encrypt(new(big.Int), pub, m) out = c.Bytes() + + if len(out) < k { + // If the output is too small, we need to left-pad with zeros. + t := make([]byte, k) + copy(t[k-len(out):], out) + out = t + } + return } diff --git a/src/pkg/crypto/rsa/rsa_test.go b/src/pkg/crypto/rsa/rsa_test.go index df1f17f17..22d4576e8 100644 --- a/src/pkg/crypto/rsa/rsa_test.go +++ b/src/pkg/crypto/rsa/rsa_test.go @@ -66,7 +66,7 @@ func TestEncryptOAEP(t *testing.T) { t.Errorf("#%d,%d error: %s", i, j, err) } if bytes.Compare(out, message.out) != 0 { - t.Errorf("#%d,%d bad result: %s (want %s)", i, j, out, message.out) + t.Errorf("#%d,%d bad result: %x (want %x)", i, j, out, message.out) } } } diff --git a/src/pkg/crypto/tls/handshake_client.go b/src/pkg/crypto/tls/handshake_client.go index 19d2bfa3b..a325a9b95 100644 --- a/src/pkg/crypto/tls/handshake_client.go +++ b/src/pkg/crypto/tls/handshake_client.go @@ -57,7 +57,7 @@ func (c *Conn) clientHandshake() os.Error { vers, ok := mutualVersion(serverHello.vers) if !ok { - c.sendAlert(alertProtocolVersion) + return c.sendAlert(alertProtocolVersion) } c.vers = vers c.haveVers = true diff --git a/src/pkg/crypto/tls/handshake_client_test.go b/src/pkg/crypto/tls/handshake_client_test.go index e5c9684b9..fd1f145cf 100644 --- a/src/pkg/crypto/tls/handshake_client_test.go +++ b/src/pkg/crypto/tls/handshake_client_test.go @@ -61,7 +61,7 @@ func TestRunClient(t *testing.T) { // Script of interaction with gnutls implementation. // The values for this test are obtained by building and running in client mode: -// % gotest -match "TestRunClient" -connect +// % gotest -test.run "TestRunClient" -connect // and then: // % gnutls-serv -p 10443 --debug 100 --x509keyfile key.pem --x509certfile cert.pem -a > /tmp/log 2>&1 // % python parse-gnutls-cli-debug-log.py < /tmp/log diff --git a/src/pkg/crypto/tls/handshake_server_test.go b/src/pkg/crypto/tls/handshake_server_test.go index 5cf3ae049..6beb6a9f6 100644 --- a/src/pkg/crypto/tls/handshake_server_test.go +++ b/src/pkg/crypto/tls/handshake_server_test.go @@ -194,7 +194,7 @@ var testPrivateKey = &rsa.PrivateKey{ // Script of interaction with gnutls implementation. // The values for this test are obtained by building and running in server mode: -// % gotest -match "TestRunServer" -serve +// % gotest -test.run "TestRunServer" -serve // and then: // % gnutls-cli --insecure --debug 100 -p 10443 localhost > /tmp/log 2>&1 // % python parse-gnutls-cli-debug-log.py < /tmp/log diff --git a/src/pkg/exp/eval/stmt_test.go b/src/pkg/exp/eval/stmt_test.go index a14a288d9..4a883ef5e 100644 --- a/src/pkg/exp/eval/stmt_test.go +++ b/src/pkg/exp/eval/stmt_test.go @@ -217,7 +217,7 @@ var stmtTests = []test{ Val2("if false { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), Val2("if i == i2 { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), // Omit optional parts - Val2("if { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), + Val2("if true { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), Val2("if true { i = 2 }; i2 = 4", "i", 2, "i2", 4), Val2("if false { i = 2 }; i2 = 4", "i", 1, "i2", 4), // Init @@ -243,11 +243,11 @@ var stmtTests = []test{ CErr("fn1 := func() int { if true { return 1 } }", "return"), CErr("fn1 := func() int { if true { } }", "return"), Run("fn1 := func() int { if true { }; return 1 }"), - CErr("fn1 := func() int { if { } }", "return"), - CErr("fn1 := func() int { if { } else { return 2 } }", "return"), - Run("fn1 := func() int { if { return 1 } }"), - Run("fn1 := func() int { if { return 1 } else { } }"), - Run("fn1 := func() int { if { return 1 } else { } }"), + CErr("fn1 := func() int { if true { } }", "return"), + CErr("fn1 := func() int { if true { } else { return 2 } }", "return"), + Run("fn1 := func() int { if true { return 1 }; return 0 }"), + Run("fn1 := func() int { if true { return 1 } else { }; return 0 }"), + Run("fn1 := func() int { if true { return 1 } else { }; return 0 }"), // Switch Val1("switch { case false: i += 2; case true: i += 4; default: i += 8 }", "i", 1+4), diff --git a/src/pkg/exp/wingui/Makefile b/src/pkg/exp/wingui/Makefile index e9d44d2bc..983a8270b 100644 --- a/src/pkg/exp/wingui/Makefile +++ b/src/pkg/exp/wingui/Makefile @@ -6,6 +6,8 @@ GOOS=windows include ../../../Make.inc +LD:=$(LD) -Hwindowsgui + TARG=wingui GOFILES=\ diff --git a/src/pkg/fmt/doc.go b/src/pkg/fmt/doc.go index b40e265ae..77ee62bb1 100644 --- a/src/pkg/fmt/doc.go +++ b/src/pkg/fmt/doc.go @@ -120,7 +120,7 @@ An analogous set of functions scans formatted text to yield values. Scan, Scanf and Scanln read from os.Stdin; Fscan, Fscanf and Fscanln read from a specified os.Reader; Sscan, - Sscanf and Sscanln read from an argument string. Sscanln, + Sscanf and Sscanln read from an argument string. Scanln, Fscanln and Sscanln stop scanning at a newline and require that the items be followed by one; Sscanf, Fscanf and Sscanf require newlines in the input to match newlines in the format; the other @@ -164,13 +164,15 @@ All arguments to be scanned must be either pointers to basic types or implementations of the Scanner interface. - Note: Fscan etc. can read one character (rune) past the - input they return, which means that a loop calling a scan - routine may skip some of the input. This is usually a - problem only when there is no space between input values. - However, if the reader provided to Fscan implements UnreadRune, + Note: Fscan etc. can read one character (rune) past the input + they return, which means that a loop calling a scan routine + may skip some of the input. This is usually a problem only + when there is no space between input values. If the reader + provided to Fscan implements ReadRune, that method will be used + to read characters. If the reader also implements UnreadRune, that method will be used to save the character and successive - calls will not lose data. To attach an UnreadRune method - to a reader without that capability, use bufio.NewReader. + calls will not lose data. To attach ReadRune and UnreadRune + methods to a reader without that capability, use + bufio.NewReader. */ package fmt diff --git a/src/pkg/fmt/fmt_test.go b/src/pkg/fmt/fmt_test.go index 3f085b722..c8aa6090b 100644 --- a/src/pkg/fmt/fmt_test.go +++ b/src/pkg/fmt/fmt_test.go @@ -311,9 +311,9 @@ var fmttests = []struct { // go syntax {"%#v", A{1, 2, "a", []int{1, 2}}, `fmt_test.A{i:1, j:0x2, s:"a", x:[]int{1, 2}}`}, - {"%#v", &b, "(*uint8)(PTR)"}, - {"%#v", TestFmtInterface, "(func(*testing.T))(PTR)"}, - {"%#v", make(chan int), "(chan int)(PTR)"}, + {"%#v", &b, "(*uint8)(0xPTR)"}, + {"%#v", TestFmtInterface, "(func(*testing.T))(0xPTR)"}, + {"%#v", make(chan int), "(chan int)(0xPTR)"}, {"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"}, {"%#v", 1000000000, "1000000000"}, {"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`}, @@ -365,14 +365,15 @@ var fmttests = []struct { {"%6T", &intVal, " *int"}, // %p - {"p0=%p", new(int), "p0=PTR"}, + {"p0=%p", new(int), "p0=0xPTR"}, {"p1=%s", &pValue, "p1=String(p)"}, // String method... - {"p2=%p", &pValue, "p2=PTR"}, // ... not called with %p + {"p2=%p", &pValue, "p2=0xPTR"}, // ... not called with %p + {"p4=%#p", new(int), "p4=PTR"}, // %p on non-pointers - {"%p", make(chan int), "PTR"}, - {"%p", make(map[int]int), "PTR"}, - {"%p", make([]int, 1), "PTR"}, + {"%p", make(chan int), "0xPTR"}, + {"%p", make(map[int]int), "0xPTR"}, + {"%p", make([]int, 1), "0xPTR"}, {"%p", 27, "%!p(int=27)"}, // not a pointer at all // erroneous things @@ -388,8 +389,8 @@ var fmttests = []struct { func TestSprintf(t *testing.T) { for _, tt := range fmttests { s := Sprintf(tt.fmt, tt.val) - if i := strings.Index(s, "0x"); i >= 0 && strings.Contains(tt.out, "PTR") { - j := i + 2 + if i := strings.Index(tt.out, "PTR"); i >= 0 { + j := i for ; j < len(s); j++ { c := s[j] if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { diff --git a/src/pkg/fmt/format.go b/src/pkg/fmt/format.go index 86057bf69..caaa7ac1a 100644 --- a/src/pkg/fmt/format.go +++ b/src/pkg/fmt/format.go @@ -107,7 +107,7 @@ func (f *fmt) writePadding(n int, padding []byte) { } // Append b to f.buf, padded on left (w > 0) or right (w < 0 or f.minus) -// clear flags aftewards. +// clear flags afterwards. func (f *fmt) pad(b []byte) { var padding []byte var left, right int @@ -124,7 +124,7 @@ func (f *fmt) pad(b []byte) { } // append s to buf, padded on left (w > 0) or right (w < 0 or f.minus). -// clear flags aftewards. +// clear flags afterwards. func (f *fmt) padString(s string) { var padding []byte var left, right int diff --git a/src/pkg/fmt/print.go b/src/pkg/fmt/print.go index d6dc8eb3d..4e14fdaa4 100644 --- a/src/pkg/fmt/print.go +++ b/src/pkg/fmt/print.go @@ -348,11 +348,11 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}) { } } -// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x by -// temporarily turning on the sharp flag. -func (p *pp) fmt0x64(v uint64) { +// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x or +// not, as requested, by temporarily setting the sharp flag. +func (p *pp) fmt0x64(v uint64, leading0x bool) { sharp := p.fmt.sharp - p.fmt.sharp = true // turn on 0x + p.fmt.sharp = leading0x p.fmt.integer(int64(v), 16, unsigned, ldigits) p.fmt.sharp = sharp } @@ -384,7 +384,7 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { p.fmt.integer(int64(v), 10, unsigned, ldigits) case 'v': if goSyntax { - p.fmt0x64(v) + p.fmt0x64(v, true) } else { p.fmt.integer(int64(v), 10, unsigned, ldigits) } @@ -534,11 +534,11 @@ func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSynt if u == 0 { p.buf.Write(nilBytes) } else { - p.fmt0x64(uint64(v.Get())) + p.fmt0x64(uint64(v.Get()), true) } p.add(')') } else { - p.fmt0x64(uint64(u)) + p.fmt0x64(uint64(u), !p.fmt.sharp) } } @@ -801,7 +801,7 @@ BigSwitch: if v == 0 { p.buf.Write(nilBytes) } else { - p.fmt0x64(uint64(v)) + p.fmt0x64(uint64(v), true) } p.buf.WriteByte(')') break @@ -810,7 +810,7 @@ BigSwitch: p.buf.Write(nilAngleBytes) break } - p.fmt0x64(uint64(v)) + p.fmt0x64(uint64(v), true) case uintptrGetter: p.fmtPointer(field, value, verb, goSyntax) default: diff --git a/src/pkg/fmt/scan.go b/src/pkg/fmt/scan.go index 53d88d574..c0f2bacb6 100644 --- a/src/pkg/fmt/scan.go +++ b/src/pkg/fmt/scan.go @@ -28,23 +28,30 @@ type runeUnreader interface { // Scanners may do rune-at-a-time scanning or ask the ScanState // to discover the next space-delimited token. type ScanState interface { - // GetRune reads the next rune (Unicode code point) from the input. - GetRune() (rune int, err os.Error) - // UngetRune causes the next call to GetRune to return the same rune. - UngetRune() - // Width returns the value of the width option and whether it has been set. - // The unit is Unicode code points. - Width() (wid int, ok bool) + // ReadRune reads the next rune (Unicode code point) from the input. + // If invoked during Scanln, Fscanln, or Sscanln, ReadRune() will + // return EOF after returning the first '\n' or when reading beyond + // the specified width. + ReadRune() (rune int, size int, err os.Error) + // UnreadRune causes the next call to ReadRune to return the same rune. + UnreadRune() os.Error // Token returns the next space-delimited token from the input. If // a width has been specified, the returned token will be no longer // than the width. Token() (token string, err os.Error) + // Width returns the value of the width option and whether it has been set. + // The unit is Unicode code points. + Width() (wid int, ok bool) + // Because ReadRune is implemented by the interface, Read should never be + // called by the scanning routines and a valid implementation of + // ScanState may choose always to return an error from Read. + Read(buf []byte) (n int, err os.Error) } // Scanner is implemented by any value that has a Scan method, which scans // the input for the representation of a value and stores the result in the // receiver, which must be a pointer to be useful. The Scan method is called -// for any argument to Scan or Scanln that implements it. +// for any argument to Scan, Scanf, or Scanln that implements it. type Scanner interface { Scan(state ScanState, verb int) os.Error } @@ -96,18 +103,18 @@ func Sscanf(str string, format string, a ...interface{}) (n int, err os.Error) { // returns the number of items successfully scanned. If that is less // than the number of arguments, err will report why. func Fscan(r io.Reader, a ...interface{}) (n int, err os.Error) { - s := newScanState(r, true) + s, old := newScanState(r, true, false) n, err = s.doScan(a) - s.free() + s.free(old) return } // Fscanln is similar to Fscan, but stops scanning at a newline and // after the final item there must be a newline or EOF. func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { - s := newScanState(r, false) + s, old := newScanState(r, false, true) n, err = s.doScan(a) - s.free() + s.free(old) return } @@ -115,9 +122,9 @@ func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { // values into successive arguments as determined by the format. It // returns the number of items successfully parsed. func Fscanf(r io.Reader, format string, a ...interface{}) (n int, err os.Error) { - s := newScanState(r, false) + s, old := newScanState(r, false, false) n, err = s.doScanf(format, a) - s.free() + s.free(old) return } @@ -131,53 +138,70 @@ const EOF = -1 // ss is the internal implementation of ScanState. type ss struct { - rr io.RuneReader // where to read input - buf bytes.Buffer // token accumulator - nlIsSpace bool // whether newline counts as white space - peekRune int // one-rune lookahead - prevRune int // last rune returned by GetRune - atEOF bool // already read EOF - maxWid int // max width of field, in runes - widPresent bool // width was specified - wid int // width consumed so far; used in accept() + rr io.RuneReader // where to read input + buf bytes.Buffer // token accumulator + peekRune int // one-rune lookahead + prevRune int // last rune returned by ReadRune + count int // runes consumed so far. + atEOF bool // already read EOF + ssave } -func (s *ss) GetRune() (rune int, err os.Error) { +// ssave holds the parts of ss that need to be +// saved and restored on recursive scans. +type ssave struct { + validSave bool // is or was a part of an actual ss. + nlIsEnd bool // whether newline terminates scan + nlIsSpace bool // whether newline counts as white space + fieldLimit int // max value of ss.count for this field; fieldLimit <= limit + limit int // max value of ss.count. + maxWid int // width of this field. +} + +// The Read method is only in ScanState so that ScanState +// satisfies io.Reader. It will never be called when used as +// intended, so there is no need to make it actually work. +func (s *ss) Read(buf []byte) (n int, err os.Error) { + return 0, os.ErrorString("ScanState's Read should not be called. Use ReadRune") +} + +func (s *ss) ReadRune() (rune int, size int, err os.Error) { if s.peekRune >= 0 { + s.count++ rune = s.peekRune + size = utf8.RuneLen(rune) s.prevRune = rune s.peekRune = -1 return } - rune, _, err = s.rr.ReadRune() + if s.atEOF || s.nlIsEnd && s.prevRune == '\n' || s.count >= s.fieldLimit { + err = os.EOF + return + } + + rune, size, err = s.rr.ReadRune() if err == nil { + s.count++ s.prevRune = rune + } else if err == os.EOF { + s.atEOF = true } return } func (s *ss) Width() (wid int, ok bool) { - return s.maxWid, s.widPresent + if s.maxWid == hugeWid { + return 0, false + } + return s.maxWid, true } // The public method returns an error; this private one panics. // If getRune reaches EOF, the return value is EOF (-1). func (s *ss) getRune() (rune int) { - if s.atEOF { - return EOF - } - if s.peekRune >= 0 { - rune = s.peekRune - s.prevRune = rune - s.peekRune = -1 - return - } - rune, _, err := s.rr.ReadRune() - if err == nil { - s.prevRune = rune - } else if err != nil { + rune, _, err := s.ReadRune() + if err != nil { if err == os.EOF { - s.atEOF = true return EOF } s.error(err) @@ -185,35 +209,25 @@ func (s *ss) getRune() (rune int) { return } -// mustGetRune turns os.EOF into a panic(io.ErrUnexpectedEOF). +// mustReadRune turns os.EOF into a panic(io.ErrUnexpectedEOF). // It is called in cases such as string scanning where an EOF is a // syntax error. -func (s *ss) mustGetRune() (rune int) { - if s.atEOF { +func (s *ss) mustReadRune() (rune int) { + rune = s.getRune() + if rune == EOF { s.error(io.ErrUnexpectedEOF) } - if s.peekRune >= 0 { - rune = s.peekRune - s.peekRune = -1 - return - } - rune, _, err := s.rr.ReadRune() - if err != nil { - if err == os.EOF { - err = io.ErrUnexpectedEOF - } - s.error(err) - } return } - -func (s *ss) UngetRune() { +func (s *ss) UnreadRune() os.Error { if u, ok := s.rr.(runeUnreader); ok { u.UnreadRune() } else { s.peekRune = s.prevRune } + s.count-- + return nil } func (s *ss) error(err os.Error) { @@ -300,23 +314,43 @@ func (r *readRune) ReadRune() (rune int, size int, err os.Error) { var ssFree = newCache(func() interface{} { return new(ss) }) // Allocate a new ss struct or grab a cached one. -func newScanState(r io.Reader, nlIsSpace bool) *ss { - s := ssFree.get().(*ss) +func newScanState(r io.Reader, nlIsSpace, nlIsEnd bool) (s *ss, old ssave) { + // If the reader is a *ss, then we've got a recursive + // call to Scan, so re-use the scan state. + s, ok := r.(*ss) + if ok { + old = s.ssave + s.limit = s.fieldLimit + s.nlIsEnd = nlIsEnd || s.nlIsEnd + s.nlIsSpace = nlIsSpace + return + } + + s = ssFree.get().(*ss) if rr, ok := r.(io.RuneReader); ok { s.rr = rr } else { s.rr = &readRune{reader: r} } s.nlIsSpace = nlIsSpace + s.nlIsEnd = nlIsEnd + s.prevRune = -1 s.peekRune = -1 s.atEOF = false - s.maxWid = 0 - s.widPresent = false - return s + s.limit = hugeWid + s.fieldLimit = hugeWid + s.maxWid = hugeWid + s.validSave = true + return } // Save used ss structs in ssFree; avoid an allocation per invocation. -func (s *ss) free() { +func (s *ss) free(old ssave) { + // If it was used recursively, just restore the old state. + if old.validSave { + s.ssave = old + return + } // Don't hold on to ss structs with large buffers. if cap(s.buf.Bytes()) > 1024 { return @@ -344,7 +378,7 @@ func (s *ss) skipSpace(stopAtNewline bool) { return } if !unicode.IsSpace(rune) { - s.UngetRune() + s.UnreadRune() break } } @@ -356,13 +390,13 @@ func (s *ss) skipSpace(stopAtNewline bool) { func (s *ss) token() string { s.skipSpace(false) // read until white space or newline - for nrunes := 0; !s.widPresent || nrunes < s.maxWid; nrunes++ { + for { rune := s.getRune() if rune == EOF { break } if unicode.IsSpace(rune) { - s.UngetRune() + s.UnreadRune() break } s.buf.WriteRune(rune) @@ -381,9 +415,6 @@ var boolError = os.ErrorString("syntax error scanning boolean") // consume reads the next rune in the input and reports whether it is in the ok string. // If accept is true, it puts the character into the input token. func (s *ss) consume(ok string, accept bool) bool { - if s.wid >= s.maxWid { - return false - } rune := s.getRune() if rune == EOF { return false @@ -391,12 +422,11 @@ func (s *ss) consume(ok string, accept bool) bool { if strings.IndexRune(ok, rune) >= 0 { if accept { s.buf.WriteRune(rune) - s.wid++ } return true } if rune != EOF && accept { - s.UngetRune() + s.UnreadRune() } return false } @@ -405,7 +435,7 @@ func (s *ss) consume(ok string, accept bool) bool { func (s *ss) peek(ok string) bool { rune := s.getRune() if rune != EOF { - s.UngetRune() + s.UnreadRune() } return strings.IndexRune(ok, rune) >= 0 } @@ -433,7 +463,7 @@ func (s *ss) scanBool(verb int) bool { return false } // Syntax-checking a boolean is annoying. We're not fastidious about case. - switch s.mustGetRune() { + switch s.mustReadRune() { case '0': return false case '1': @@ -494,7 +524,7 @@ func (s *ss) scanNumber(digits string, haveDigits bool) string { // scanRune returns the next rune value in the input. func (s *ss) scanRune(bitSize int) int64 { - rune := int64(s.mustGetRune()) + rune := int64(s.mustReadRune()) n := uint(bitSize) x := (rune << (64 - n)) >> (64 - n) if x != rune { @@ -710,12 +740,12 @@ func (s *ss) convertString(verb int) (str string) { // quotedString returns the double- or back-quoted string represented by the next input characters. func (s *ss) quotedString() string { - quote := s.mustGetRune() + quote := s.mustReadRune() switch quote { case '`': // Back-quoted: Anything goes until EOF or back quote. for { - rune := s.mustGetRune() + rune := s.mustReadRune() if rune == quote { break } @@ -726,13 +756,13 @@ func (s *ss) quotedString() string { // Double-quoted: Include the quotes and let strconv.Unquote do the backslash escapes. s.buf.WriteRune(quote) for { - rune := s.mustGetRune() + rune := s.mustReadRune() s.buf.WriteRune(rune) if rune == '\\' { // In a legal backslash escape, no matter how long, only the character // immediately after the escape can itself be a backslash or quote. // Thus we only need to protect the first character after the backslash. - rune := s.mustGetRune() + rune := s.mustReadRune() s.buf.WriteRune(rune) } else if rune == '"' { break @@ -771,10 +801,10 @@ func (s *ss) hexByte() (b byte, ok bool) { return } if unicode.IsSpace(rune1) { - s.UngetRune() + s.UnreadRune() return } - rune2 := s.mustGetRune() + rune2 := s.mustReadRune() return byte(s.hexDigit(rune1)<<4 | s.hexDigit(rune2)), true } @@ -796,6 +826,8 @@ func (s *ss) hexString() string { const floatVerbs = "beEfFgGv" +const hugeWid = 1 << 30 + // scanOne scans a single value, deriving the scanner from the type of the argument. func (s *ss) scanOne(verb int, field interface{}) { s.buf.Reset() @@ -804,14 +836,13 @@ func (s *ss) scanOne(verb int, field interface{}) { if v, ok := field.(Scanner); ok { err = v.Scan(s, verb) if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } s.error(err) } return } - if !s.widPresent { - s.maxWid = 1 << 30 // Huge - } - s.wid = 0 switch v := field.(type) { case *bool: *v = s.scanBool(verb) @@ -912,7 +943,6 @@ func errorHandler(errp *os.Error) { } // doScan does the real work for scanning without a format string. -// At the moment, it handles only pointers to basic types. func (s *ss) doScan(a []interface{}) (numProcessed int, err os.Error) { defer errorHandler(&err) for _, field := range a { @@ -973,9 +1003,9 @@ func (s *ss) advance(format string) (i int) { s.skipSpace(true) continue } - inputc := s.mustGetRune() + inputc := s.mustReadRune() if fmtc != inputc { - s.UngetRune() + s.UnreadRune() return -1 } i += w @@ -1007,7 +1037,15 @@ func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.E i++ // % is one byte // do we have 20 (width)? - s.maxWid, s.widPresent, i = parsenum(format, i, end) + var widPresent bool + s.maxWid, widPresent, i = parsenum(format, i, end) + if !widPresent { + s.maxWid = hugeWid + } + s.fieldLimit = s.limit + if f := s.count + s.maxWid; f < s.fieldLimit { + s.fieldLimit = f + } c, w := utf8.DecodeRuneInString(format[i:]) i += w @@ -1020,6 +1058,7 @@ func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.E s.scanOne(c, field) numProcessed++ + s.fieldLimit = s.limit } if numProcessed < len(a) { s.errorString("too many operands") diff --git a/src/pkg/fmt/scan_test.go b/src/pkg/fmt/scan_test.go index 478b10923..cab86dd98 100644 --- a/src/pkg/fmt/scan_test.go +++ b/src/pkg/fmt/scan_test.go @@ -6,6 +6,7 @@ package fmt_test import ( "bufio" + "bytes" . "fmt" "io" "math" @@ -87,21 +88,7 @@ type FloatTest struct { type Xs string func (x *Xs) Scan(state ScanState, verb int) os.Error { - var tok string - var c int - var err os.Error - wid, present := state.Width() - if !present { - tok, err = state.Token() - } else { - for i := 0; i < wid; i++ { - c, err = state.GetRune() - if err != nil { - break - } - tok += string(c) - } - } + tok, err := state.Token() if err != nil { return err } @@ -114,6 +101,26 @@ func (x *Xs) Scan(state ScanState, verb int) os.Error { var xVal Xs +// IntString accepts an integer followed immediately by a string. +// It tests the embedding of a scan within a scan. +type IntString struct { + i int + s string +} + +func (s *IntString) Scan(state ScanState, verb int) os.Error { + if _, err := Fscan(state, &s.i); err != nil { + return err + } + + if _, err := Fscan(state, &s.s); err != nil { + return err + } + return nil +} + +var intStringVal IntString + // myStringReader implements Read but not ReadRune, allowing us to test our readRune wrapper // type that creates something that can read runes given only Read(). type myStringReader struct { @@ -200,8 +207,9 @@ var scanTests = []ScanTest{ {"114\n", &renamedStringVal, renamedString("114")}, {"115\n", &renamedBytesVal, renamedBytes([]byte("115"))}, - // Custom scanner. + // Custom scanners. {" vvv ", &xVal, Xs("vvv")}, + {" 1234hello", &intStringVal, IntString{1234, "hello"}}, // Fixed bugs {"2147483648\n", &int64Val, int64(2147483648)}, // was: integer overflow @@ -308,6 +316,7 @@ var f float64 var s, t string var c complex128 var x, y Xs +var z IntString var multiTests = []ScanfMultiTest{ {"", "", nil, nil, ""}, @@ -321,8 +330,9 @@ var multiTests = []ScanfMultiTest{ {"%d%s", "123abc", args(&i, &s), args(123, "abc"), ""}, {"%c%c%c", "2\u50c2X", args(&i, &j, &k), args('2', '\u50c2', 'X'), ""}, - // Custom scanner. + // Custom scanners. {"%2e%f", "eefffff", args(&x, &y), args(Xs("ee"), Xs("fffff")), ""}, + {"%4v%s", "12abcd", args(&z, &s), args(IntString{12, "ab"}, "cd"), ""}, // Errors {"%t", "23 18", args(&i), nil, "bad verb"}, @@ -345,7 +355,11 @@ func testScan(name string, t *testing.T, scan func(r io.Reader, a ...interface{} } n, err := scan(r, test.in) if err != nil { - t.Errorf("%s got error scanning %q: %s", name, test.text, err) + m := "" + if n > 0 { + m = Sprintf(" (%d fields ok)", n) + } + t.Errorf("%s got error scanning %q: %s%s", name, test.text, err, m) continue } if n != 1 { @@ -462,22 +476,12 @@ func verifyInf(str string, t *testing.T) { } } - func TestInf(t *testing.T) { for _, s := range []string{"inf", "+inf", "-inf", "INF", "-INF", "+INF", "Inf", "-Inf", "+Inf"} { verifyInf(s, t) } } -// TODO: there's no conversion from []T to ...T, but we can fake it. These -// functions do the faking. We index the table by the length of the param list. -var fscanf = []func(io.Reader, string, []interface{}) (int, os.Error){ - 0: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f) }, - 1: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f, i[0]) }, - 2: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f, i[0], i[1]) }, - 3: func(r io.Reader, f string, i []interface{}) (int, os.Error) { return Fscanf(r, f, i[0], i[1], i[2]) }, -} - func testScanfMulti(name string, t *testing.T) { sliceType := reflect.Typeof(make([]interface{}, 1)).(*reflect.SliceType) for _, test := range multiTests { @@ -487,7 +491,7 @@ func testScanfMulti(name string, t *testing.T) { } else { r = newReader(test.text) } - n, err := fscanf[len(test.in)](r, test.format, test.in) + n, err := Fscanf(r, test.format, test.in...) if err != nil { if test.err == "" { t.Errorf("got error scanning (%q, %q): %q", test.format, test.text, err) @@ -673,3 +677,178 @@ func TestUnreadRuneWithBufio(t *testing.T) { t.Errorf("expected αb; got %q", a) } } + +type TwoLines string + +// Attempt to read two lines into the object. Scanln should prevent this +// because it stops at newline; Scan and Scanf should be fine. +func (t *TwoLines) Scan(state ScanState, verb int) os.Error { + chars := make([]int, 0, 100) + for nlCount := 0; nlCount < 2; { + c, _, err := state.ReadRune() + if err != nil { + return err + } + chars = append(chars, c) + if c == '\n' { + nlCount++ + } + } + *t = TwoLines(string(chars)) + return nil +} + +func TestMultiLine(t *testing.T) { + input := "abc\ndef\n" + // Sscan should work + var tscan TwoLines + n, err := Sscan(input, &tscan) + if n != 1 { + t.Errorf("Sscan: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscan: expected no error; got %s", err) + } + if string(tscan) != input { + t.Errorf("Sscan: expected %q; got %q", input, tscan) + } + // Sscanf should work + var tscanf TwoLines + n, err = Sscanf(input, "%s", &tscanf) + if n != 1 { + t.Errorf("Sscanf: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscanf: expected no error; got %s", err) + } + if string(tscanf) != input { + t.Errorf("Sscanf: expected %q; got %q", input, tscanf) + } + // Sscanln should not work + var tscanln TwoLines + n, err = Sscanln(input, &tscanln) + if n != 0 { + t.Errorf("Sscanln: expected 0 items; got %d: %q", n, tscanln) + } + if err == nil { + t.Error("Sscanln: expected error; got none") + } else if err != io.ErrUnexpectedEOF { + t.Errorf("Sscanln: expected io.ErrUnexpectedEOF (ha!); got %s", err) + } +} + +// RecursiveInt accepts an string matching %d.%d.%d.... +// and parses it into a linked list. +// It allows us to benchmark recursive descent style scanners. +type RecursiveInt struct { + i int + next *RecursiveInt +} + +func (r *RecursiveInt) Scan(state ScanState, verb int) (err os.Error) { + _, err = Fscan(state, &r.i) + if err != nil { + return + } + next := new(RecursiveInt) + _, err = Fscanf(state, ".%v", next) + if err != nil { + if err == os.ErrorString("input does not match format") || err == io.ErrUnexpectedEOF { + err = nil + } + return + } + r.next = next + return +} + +// Perform the same scanning task as RecursiveInt.Scan +// but without recurring through scanner, so we can compare +// performance more directly. +func scanInts(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + r.next = nil + _, err = Fscan(b, &r.i) + if err != nil { + return + } + var c int + c, _, err = b.ReadRune() + if err != nil { + if err == os.EOF { + err = nil + } + return + } + if c != '.' { + return + } + next := new(RecursiveInt) + err = scanInts(next, b) + if err == nil { + r.next = next + } + return +} + +func makeInts(n int) []byte { + var buf bytes.Buffer + Fprintf(&buf, "1") + for i := 1; i < n; i++ { + Fprintf(&buf, ".%d", i+1) + } + return buf.Bytes() +} + +func TestScanInts(t *testing.T) { + testScanInts(t, scanInts) + testScanInts(t, func(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + _, err = Fscan(b, r) + return + }) +} + +const intCount = 1000 + +func testScanInts(t *testing.T, scan func(*RecursiveInt, *bytes.Buffer) os.Error) { + r := new(RecursiveInt) + ints := makeInts(intCount) + buf := bytes.NewBuffer(ints) + err := scan(r, buf) + if err != nil { + t.Error("unexpected error", err) + } + i := 1 + for ; r != nil; r = r.next { + if r.i != i { + t.Fatal("bad scan: expected %d got %d", i, r.i) + } + i++ + } + if i-1 != intCount { + t.Fatal("bad scan count: expected %d got %d", intCount, i-1) + } +} + +func BenchmarkScanInts(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + scanInts(&r, buf) + b.StopTimer() + } +} + +func BenchmarkScanRecursiveInt(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + Fscan(buf, &r) + b.StopTimer() + } +} diff --git a/src/pkg/go/ast/ast.go b/src/pkg/go/ast/ast.go index 2e8f0973f..abafb5663 100644 --- a/src/pkg/go/ast/ast.go +++ b/src/pkg/go/ast/ast.go @@ -597,7 +597,7 @@ type ( IfStmt struct { If token.Pos // position of "if" keyword Init Stmt // initalization statement; or nil - Cond Expr // condition; or nil + Cond Expr // condition Body *BlockStmt Else Stmt // else branch; or nil } diff --git a/src/pkg/go/ast/walk.go b/src/pkg/go/ast/walk.go index a77f8ee5e..20c337c3b 100644 --- a/src/pkg/go/ast/walk.go +++ b/src/pkg/go/ast/walk.go @@ -227,9 +227,7 @@ func Walk(v Visitor, node Node) { if n.Init != nil { Walk(v, n.Init) } - if n.Cond != nil { - Walk(v, n.Cond) - } + Walk(v, n.Cond) Walk(v, n.Body) if n.Else != nil { Walk(v, n.Else) diff --git a/src/pkg/go/parser/interface.go b/src/pkg/go/parser/interface.go index 84d699a67..6f35b495e 100644 --- a/src/pkg/go/parser/interface.go +++ b/src/pkg/go/parser/interface.go @@ -14,7 +14,7 @@ import ( "io" "io/ioutil" "os" - pathutil "path" + "path/filepath" ) @@ -198,7 +198,7 @@ func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, for i := 0; i < len(list); i++ { d := &list[i] if filter == nil || filter(d) { - filenames[n] = pathutil.Join(path, d.Name) + filenames[n] = filepath.Join(path, d.Name) n++ } } diff --git a/src/pkg/go/parser/parser.go b/src/pkg/go/parser/parser.go index 2395b8158..7c5843f36 100644 --- a/src/pkg/go/parser/parser.go +++ b/src/pkg/go/parser/parser.go @@ -1327,44 +1327,34 @@ func (p *parser) makeExpr(s ast.Stmt) ast.Expr { } -func (p *parser) parseControlClause(isForStmt bool) (s1, s2, s3 ast.Stmt) { - if p.tok != token.LBRACE { +func (p *parser) parseIfStmt() *ast.IfStmt { + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + + var s ast.Stmt + var x ast.Expr + { prevLev := p.exprLev p.exprLev = -1 - - if p.tok != token.SEMICOLON { - s1 = p.parseSimpleStmt(false) - } if p.tok == token.SEMICOLON { p.next() - if p.tok != token.LBRACE && p.tok != token.SEMICOLON { - s2 = p.parseSimpleStmt(false) - } - if isForStmt { - // for statements have a 3rd section - p.expectSemi() - if p.tok != token.LBRACE { - s3 = p.parseSimpleStmt(false) - } - } + x = p.parseExpr() } else { - s1, s2 = nil, s1 + s = p.parseSimpleStmt(false) + if p.tok == token.SEMICOLON { + p.next() + x = p.parseExpr() + } else { + x = p.makeExpr(s) + s = nil + } } - p.exprLev = prevLev } - return s1, s2, s3 -} - - -func (p *parser) parseIfStmt() *ast.IfStmt { - if p.trace { - defer un(trace(p, "IfStmt")) - } - - pos := p.expect(token.IF) - s1, s2, _ := p.parseControlClause(false) body := p.parseBlockStmt() var else_ ast.Stmt if p.tok == token.ELSE { @@ -1374,7 +1364,7 @@ func (p *parser) parseIfStmt() *ast.IfStmt { p.expectSemi() } - return &ast.IfStmt{pos, s1, p.makeExpr(s2), body, else_} + return &ast.IfStmt{pos, s, x, body, else_} } @@ -1457,7 +1447,24 @@ func (p *parser) parseSwitchStmt() ast.Stmt { } pos := p.expect(token.SWITCH) - s1, s2, _ := p.parseControlClause(false) + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + s2 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } if isExprSwitch(s2) { lbrace := p.expect(token.LBRACE) @@ -1575,7 +1582,29 @@ func (p *parser) parseForStmt() ast.Stmt { } pos := p.expect(token.FOR) - s1, s2, s3 := p.parseControlClause(true) + + var s1, s2, s3 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } + body := p.parseBlockStmt() p.expectSemi() diff --git a/src/pkg/go/parser/parser_test.go b/src/pkg/go/parser/parser_test.go index 5a7f05ca8..38535627a 100644 --- a/src/pkg/go/parser/parser_test.go +++ b/src/pkg/go/parser/parser_test.go @@ -18,6 +18,9 @@ var illegalInputs = []interface{}{ 3.14, []byte(nil), "foo!", + `package p; func f() { if /* should have condition */ {} };`, + `package p; func f() { if ; /* should have condition */ {} };`, + `package p; func f() { if f(); /* should have condition */ {} };`, } @@ -32,21 +35,23 @@ func TestParseIllegalInputs(t *testing.T) { var validPrograms = []interface{}{ - "package main\n", - `package main;`, - `package main; import "fmt"; func main() { fmt.Println("Hello, World!") };`, - `package main; func main() { if f(T{}) {} };`, - `package main; func main() { _ = (<-chan int)(x) };`, - `package main; func main() { _ = (<-chan <-chan int)(x) };`, - `package main; func f(func() func() func());`, - `package main; func f(...T);`, - `package main; func f(float, ...int);`, - `package main; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, - `package main; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, - `package main; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, - `package main; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, - `package main; var a = T{{1, 2}, {3, 4}}`, - `package main; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + "package p\n", + `package p;`, + `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, + `package p; func f() { if f(T{}) {} };`, + `package p; func f() { _ = (<-chan int)(x) };`, + `package p; func f() { _ = (<-chan <-chan int)(x) };`, + `package p; func f(func() func() func());`, + `package p; func f(...T);`, + `package p; func f(float, ...int);`, + `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, + `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, + `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, + `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, + `package p; var a = T{{1, 2}, {3, 4}}`, + `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + `package p; func f() { if ; true {} };`, + `package p; func f() { switch ; {} };`, } diff --git a/src/pkg/go/printer/printer.go b/src/pkg/go/printer/printer.go index 48e2af1b7..90d9784ac 100644 --- a/src/pkg/go/printer/printer.go +++ b/src/pkg/go/printer/printer.go @@ -12,7 +12,7 @@ import ( "go/token" "io" "os" - "path" + "path/filepath" "runtime" "tabwriter" ) @@ -244,7 +244,7 @@ func (p *printer) writeItem(pos token.Position, data []byte) { } if debug { // do not update p.pos - use write0 - _, filename := path.Split(pos.Filename) + _, filename := filepath.Split(pos.Filename) p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column))) } p.write(data) diff --git a/src/pkg/go/printer/printer_test.go b/src/pkg/go/printer/printer_test.go index 565075aa2..62b726913 100644 --- a/src/pkg/go/printer/printer_test.go +++ b/src/pkg/go/printer/printer_test.go @@ -11,7 +11,7 @@ import ( "go/ast" "go/parser" "go/token" - "path" + "path/filepath" "testing" ) @@ -129,8 +129,8 @@ var data = []entry{ func TestFiles(t *testing.T) { for _, e := range data { - source := path.Join(dataDir, e.source) - golden := path.Join(dataDir, e.golden) + source := filepath.Join(dataDir, e.source) + golden := filepath.Join(dataDir, e.golden) check(t, source, golden, e.mode) // TODO(gri) check that golden is idempotent //check(t, golden, golden, e.mode); diff --git a/src/pkg/go/printer/testdata/statements.golden b/src/pkg/go/printer/testdata/statements.golden index 5eceb7dd5..290060269 100644 --- a/src/pkg/go/printer/testdata/statements.golden +++ b/src/pkg/go/printer/testdata/statements.golden @@ -10,9 +10,9 @@ func use(x interface{}) {} // Formatting of if-statement headers. func _() { - if { + if true { } - if { + if true { } // no semicolon printed if expr { } @@ -22,7 +22,7 @@ func _() { } // no parens printed if expr { } // no semicolon and parens printed - if x := expr; { + if x := expr; true { use(x) } if x := expr; expr { @@ -354,14 +354,14 @@ func _() { func _() { - if { + if true { _ = 0 } _ = 0 // the indentation here should not be affected by the long label name AnOverlongLabel: _ = 0 - if { + if true { _ = 0 } _ = 0 diff --git a/src/pkg/go/printer/testdata/statements.input b/src/pkg/go/printer/testdata/statements.input index 7819820ed..21e61efc4 100644 --- a/src/pkg/go/printer/testdata/statements.input +++ b/src/pkg/go/printer/testdata/statements.input @@ -10,13 +10,13 @@ func use(x interface{}) {} // Formatting of if-statement headers. func _() { - if {} - if;{} // no semicolon printed + if true {} + if; true {} // no semicolon printed if expr{} if;expr{} // no semicolon printed if (expr){} // no parens printed if;((expr)){} // no semicolon and parens printed - if x:=expr;{ + if x:=expr;true{ use(x)} if x:=expr; expr {use(x)} } @@ -271,14 +271,14 @@ func _() { func _() { - if { + if true { _ = 0 } _ = 0 // the indentation here should not be affected by the long label name AnOverlongLabel: _ = 0 - if { + if true { _ = 0 } _ = 0 diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go index 8c3205230..153707f59 100644 --- a/src/pkg/go/scanner/scanner.go +++ b/src/pkg/go/scanner/scanner.go @@ -8,7 +8,8 @@ // // var s Scanner // fset := token.NewFileSet() // position information is relative to fset -// s.Init(fset, filename, src, nil /* no error handler */, 0) +// file := fset.AddFile(filename, fset.Base(), len(src)) // register file +// s.Init(file, src, nil /* no error handler */, 0) // for { // pos, tok, lit := s.Scan() // if tok == token.EOF { @@ -22,7 +23,7 @@ package scanner import ( "bytes" "go/token" - "path" + "path/filepath" "strconv" "unicode" "utf8" @@ -117,7 +118,7 @@ func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint panic("file size does not match src len") } S.file = file - S.dir, _ = path.Split(file.Name()) + S.dir, _ = filepath.Split(file.Name()) S.src = src S.err = err S.mode = mode @@ -179,10 +180,10 @@ func (S *Scanner) interpretLineComment(text []byte) { if i := bytes.Index(text, []byte{':'}); i > 0 { if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 { // valid //line filename:line comment; - filename := path.Clean(string(text[len(prefix):i])) + filename := filepath.Clean(string(text[len(prefix):i])) if filename[0] != '/' { // make filename relative to current directory - filename = path.Join(S.dir, filename) + filename = filepath.Join(S.dir, filename) } // update scanner position S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line diff --git a/src/pkg/gob/codec_test.go b/src/pkg/gob/codec_test.go index fe1f60ba7..4562e1930 100644 --- a/src/pkg/gob/codec_test.go +++ b/src/pkg/gob/codec_test.go @@ -303,7 +303,7 @@ func TestScalarEncInstructions(t *testing.T) { } } -func execDec(typ string, instr *decInstr, state *decodeState, t *testing.T, p unsafe.Pointer) { +func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, p unsafe.Pointer) { defer testError(t) v := int(state.decodeUint()) if v+state.fieldnum != 6 { @@ -313,7 +313,7 @@ func execDec(typ string, instr *decInstr, state *decodeState, t *testing.T, p un state.fieldnum = 6 } -func newDecodeStateFromData(data []byte) *decodeState { +func newDecodeStateFromData(data []byte) *decoderState { b := bytes.NewBuffer(data) state := newDecodeState(nil, b) state.fieldnum = -1 @@ -342,7 +342,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a int } - instr := &decInstr{decOpMap[reflect.Int], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Int], 6, 0, 0, ovfl} state := newDecodeStateFromData(signedResult) execDec("int", instr, state, t, unsafe.Pointer(&data)) if data.a != 17 { @@ -355,7 +355,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a uint } - instr := &decInstr{decOpMap[reflect.Uint], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Uint], 6, 0, 0, ovfl} state := newDecodeStateFromData(unsignedResult) execDec("uint", instr, state, t, unsafe.Pointer(&data)) if data.a != 17 { @@ -446,7 +446,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a uintptr } - instr := &decInstr{decOpMap[reflect.Uintptr], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Uintptr], 6, 0, 0, ovfl} state := newDecodeStateFromData(unsignedResult) execDec("uintptr", instr, state, t, unsafe.Pointer(&data)) if data.a != 17 { @@ -511,7 +511,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a complex64 } - instr := &decInstr{decOpMap[reflect.Complex64], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Complex64], 6, 0, 0, ovfl} state := newDecodeStateFromData(complexResult) execDec("complex", instr, state, t, unsafe.Pointer(&data)) if data.a != 17+19i { @@ -524,7 +524,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a complex128 } - instr := &decInstr{decOpMap[reflect.Complex128], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Complex128], 6, 0, 0, ovfl} state := newDecodeStateFromData(complexResult) execDec("complex", instr, state, t, unsafe.Pointer(&data)) if data.a != 17+19i { @@ -973,18 +973,32 @@ func TestIgnoredFields(t *testing.T) { } } + +func TestBadRecursiveType(t *testing.T) { + type Rec ***Rec + var rec Rec + b := new(bytes.Buffer) + err := NewEncoder(b).Encode(&rec) + if err == nil { + t.Error("expected error; got none") + } else if strings.Index(err.String(), "recursive") < 0 { + t.Error("expected recursive type error; got", err) + } + // Can't test decode easily because we can't encode one, so we can't pass one to a Decoder. +} + type Bad0 struct { - ch chan int - c float64 + CH chan int + C float64 } -var nilEncoder *Encoder func TestInvalidField(t *testing.T) { var bad0 Bad0 - bad0.ch = make(chan int) + bad0.CH = make(chan int) b := new(bytes.Buffer) - err := nilEncoder.encode(b, reflect.NewValue(&bad0)) + var nilEncoder *Encoder + err := nilEncoder.encode(b, reflect.NewValue(&bad0), userType(reflect.Typeof(&bad0))) if err == nil { t.Error("expected error; got none") } else if strings.Index(err.String(), "type") < 0 { diff --git a/src/pkg/gob/debug.go b/src/pkg/gob/debug.go index e4583901e..69c83bda7 100644 --- a/src/pkg/gob/debug.go +++ b/src/pkg/gob/debug.go @@ -155,6 +155,16 @@ func (deb *debugger) dump(format string, args ...interface{}) { // Debug prints a human-readable representation of the gob data read from r. func Debug(r io.Reader) { + err := debug(r) + if err != nil { + fmt.Fprintf(os.Stderr, "gob debug: %s\n", err) + } +} + +// debug implements Debug, but catches panics and returns +// them as errors to be printed by Debug. +func debug(r io.Reader) (err os.Error) { + defer catchError(&err) fmt.Fprintln(os.Stderr, "Start of debugging") deb := &debugger{ r: newPeekReader(r), @@ -166,6 +176,7 @@ func Debug(r io.Reader) { deb.remainingKnown = true } deb.gobStream() + return } // note that we've consumed some bytes @@ -386,11 +397,15 @@ func (deb *debugger) typeDefinition(indent tab, id typeId) { // Field number 1 is type Id of key deb.delta(1) keyId := deb.typeId() - wire.SliceT = &sliceType{com, id} // Field number 2 is type Id of elem deb.delta(1) elemId := deb.typeId() wire.MapT = &mapType{com, keyId, elemId} + case 4: // GobEncoder type, one field of {{Common}} + // Field number 0 is CommonType + deb.delta(1) + com := deb.common() + wire.GobEncoderT = &gobEncoderType{com} default: errorf("bad field in type %d", fieldNum) } @@ -507,6 +522,8 @@ func (deb *debugger) printWireType(indent tab, wire *wireType) { for i, field := range wire.StructT.Field { fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\tid=%d\n", indent+1, i, field.Name, field.Id) } + case wire.GobEncoderT != nil: + deb.printCommonType(indent, "GobEncoder", &wire.GobEncoderT.CommonType) } indent-- fmt.Fprintf(os.Stderr, "%s}\n", indent) @@ -538,6 +555,8 @@ func (deb *debugger) fieldValue(indent tab, id typeId) { deb.sliceValue(indent, wire) case wire.StructT != nil: deb.structValue(indent, id) + case wire.GobEncoderT != nil: + deb.gobEncoderValue(indent, id) default: panic("bad wire type for field") } @@ -654,3 +673,17 @@ func (deb *debugger) structValue(indent tab, id typeId) { fmt.Fprintf(os.Stderr, "%s} // end %s struct\n", indent, id.name()) deb.dump(">> End of struct value of type %d %q", id, id.name()) } + +// GobEncoderValue: +// uint(n) byte*n +func (deb *debugger) gobEncoderValue(indent tab, id typeId) { + len := deb.uint64() + deb.dump("GobEncoder value of %q id=%d, length %d\n", id.name(), id, len) + fmt.Fprintf(os.Stderr, "%s%s (implements GobEncoder)\n", indent, id.name()) + data := make([]byte, len) + _, err := deb.r.Read(data) + if err != nil { + errorf("gobEncoder data read: %s", err) + } + fmt.Fprintf(os.Stderr, "%s[% .2x]\n", indent+1, data) +} diff --git a/src/pkg/gob/decode.go b/src/pkg/gob/decode.go index 9667f6157..b7ae78200 100644 --- a/src/pkg/gob/decode.go +++ b/src/pkg/gob/decode.go @@ -13,9 +13,7 @@ import ( "math" "os" "reflect" - "unicode" "unsafe" - "utf8" ) var ( @@ -24,9 +22,9 @@ var ( errRange = os.ErrorString("gob: internal error: field numbers out of bounds") ) -// The execution state of an instance of the decoder. A new state +// decoderState is the execution state of an instance of the decoder. A new state // is created for nested objects. -type decodeState struct { +type decoderState struct { dec *Decoder // The buffer is stored with an extra indirection because it may be replaced // if we load a type during decode (when reading an interface value). @@ -37,8 +35,8 @@ type decodeState struct { // We pass the bytes.Buffer separately for easier testing of the infrastructure // without requiring a full Decoder. -func newDecodeState(dec *Decoder, buf *bytes.Buffer) *decodeState { - d := new(decodeState) +func newDecodeState(dec *Decoder, buf *bytes.Buffer) *decoderState { + d := new(decoderState) d.dec = dec d.b = buf d.buf = make([]byte, uint64Size) @@ -85,7 +83,7 @@ func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Erro // decodeUint reads an encoded unsigned integer from state.r. // Does not check for overflow. -func (state *decodeState) decodeUint() (x uint64) { +func (state *decoderState) decodeUint() (x uint64) { b, err := state.b.ReadByte() if err != nil { error(err) @@ -112,7 +110,7 @@ func (state *decodeState) decodeUint() (x uint64) { // decodeInt reads an encoded signed integer from state.r. // Does not check for overflow. -func (state *decodeState) decodeInt() int64 { +func (state *decoderState) decodeInt() int64 { x := state.decodeUint() if x&1 != 0 { return ^int64(x >> 1) @@ -120,7 +118,8 @@ func (state *decodeState) decodeInt() int64 { return int64(x >> 1) } -type decOp func(i *decInstr, state *decodeState, p unsafe.Pointer) +// decOp is the signature of a decoding operator for a given type. +type decOp func(i *decInstr, state *decoderState, p unsafe.Pointer) // The 'instructions' of the decoding machine type decInstr struct { @@ -150,26 +149,31 @@ func decIndirect(p unsafe.Pointer, indir int) unsafe.Pointer { return p } -func ignoreUint(i *decInstr, state *decodeState, p unsafe.Pointer) { +// ignoreUint discards a uint value with no destination. +func ignoreUint(i *decInstr, state *decoderState, p unsafe.Pointer) { state.decodeUint() } -func ignoreTwoUints(i *decInstr, state *decodeState, p unsafe.Pointer) { +// ignoreTwoUints discards a uint value with no destination. It's used to skip +// complex values. +func ignoreTwoUints(i *decInstr, state *decoderState, p unsafe.Pointer) { state.decodeUint() state.decodeUint() } -func decBool(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decBool decodes a uiint and stores it as a boolean through p. +func decBool(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(bool)) } p = *(*unsafe.Pointer)(p) } - *(*bool)(p) = state.decodeInt() != 0 + *(*bool)(p) = state.decodeUint() != 0 } -func decInt8(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decInt8 decodes an integer and stores it as an int8 through p. +func decInt8(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int8)) @@ -184,7 +188,8 @@ func decInt8(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decUint8(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decUint8 decodes an unsigned integer and stores it as a uint8 through p. +func decUint8(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint8)) @@ -199,7 +204,8 @@ func decUint8(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decInt16(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decInt16 decodes an integer and stores it as an int16 through p. +func decInt16(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int16)) @@ -214,7 +220,8 @@ func decInt16(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decUint16(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decUint16 decodes an unsigned integer and stores it as a uint16 through p. +func decUint16(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint16)) @@ -229,7 +236,8 @@ func decUint16(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decInt32(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decInt32 decodes an integer and stores it as an int32 through p. +func decInt32(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int32)) @@ -244,7 +252,8 @@ func decInt32(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decUint32(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decUint32 decodes an unsigned integer and stores it as a uint32 through p. +func decUint32(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint32)) @@ -259,7 +268,8 @@ func decUint32(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decInt64(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decInt64 decodes an integer and stores it as an int64 through p. +func decInt64(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int64)) @@ -269,7 +279,8 @@ func decInt64(i *decInstr, state *decodeState, p unsafe.Pointer) { *(*int64)(p) = int64(state.decodeInt()) } -func decUint64(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decUint64 decodes an unsigned integer and stores it as a uint64 through p. +func decUint64(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint64)) @@ -294,7 +305,9 @@ func floatFromBits(u uint64) float64 { return math.Float64frombits(v) } -func storeFloat32(i *decInstr, state *decodeState, p unsafe.Pointer) { +// storeFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point +// number, and stores it through p. It's a helper function for float32 and complex64. +func storeFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) { v := floatFromBits(state.decodeUint()) av := v if av < 0 { @@ -308,7 +321,9 @@ func storeFloat32(i *decInstr, state *decodeState, p unsafe.Pointer) { } } -func decFloat32(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point +// number, and stores it through p. +func decFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(float32)) @@ -318,7 +333,9 @@ func decFloat32(i *decInstr, state *decodeState, p unsafe.Pointer) { storeFloat32(i, state, p) } -func decFloat64(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decFloat64 decodes an unsigned integer, treats it as a 64-bit floating-point +// number, and stores it through p. +func decFloat64(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(float64)) @@ -328,8 +345,10 @@ func decFloat64(i *decInstr, state *decodeState, p unsafe.Pointer) { *(*float64)(p) = floatFromBits(uint64(state.decodeUint())) } -// Complex numbers are just a pair of floating-point numbers, real part first. -func decComplex64(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decComplex64 decodes a pair of unsigned integers, treats them as a +// pair of floating point numbers, and stores them as a complex64 through p. +// The real part comes first. +func decComplex64(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex64)) @@ -340,7 +359,10 @@ func decComplex64(i *decInstr, state *decodeState, p unsafe.Pointer) { storeFloat32(i, state, unsafe.Pointer(uintptr(p)+uintptr(unsafe.Sizeof(float32(0))))) } -func decComplex128(i *decInstr, state *decodeState, p unsafe.Pointer) { +// decComplex128 decodes a pair of unsigned integers, treats them as a +// pair of floating point numbers, and stores them as a complex128 through p. +// The real part comes first. +func decComplex128(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex128)) @@ -352,8 +374,10 @@ func decComplex128(i *decInstr, state *decodeState, p unsafe.Pointer) { *(*complex128)(p) = complex(real, imag) } +// decUint8Array decodes byte array and stores through p a slice header +// describing the data. // uint8 arrays are encoded as an unsigned count followed by the raw bytes. -func decUint8Array(i *decInstr, state *decodeState, p unsafe.Pointer) { +func decUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new([]uint8)) @@ -365,8 +389,10 @@ func decUint8Array(i *decInstr, state *decodeState, p unsafe.Pointer) { *(*[]uint8)(p) = b } +// decString decodes byte array and stores through p a string header +// describing the data. // Strings are encoded as an unsigned count followed by the raw bytes. -func decString(i *decInstr, state *decodeState, p unsafe.Pointer) { +func decString(i *decInstr, state *decoderState, p unsafe.Pointer) { if i.indir > 0 { if *(*unsafe.Pointer)(p) == nil { *(*unsafe.Pointer)(p) = unsafe.Pointer(new([]byte)) @@ -378,7 +404,8 @@ func decString(i *decInstr, state *decodeState, p unsafe.Pointer) { *(*string)(p) = string(b) } -func ignoreUint8Array(i *decInstr, state *decodeState, p unsafe.Pointer) { +// ignoreUint8Array skips over the data for a byte slice value with no destination. +func ignoreUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) { b := make([]byte, state.decodeUint()) state.b.Read(b) } @@ -409,9 +436,15 @@ func allocate(rtyp reflect.Type, p uintptr, indir int) uintptr { return *(*uintptr)(up) } -func (dec *Decoder) decodeSingle(engine *decEngine, rtyp reflect.Type, p uintptr, indir int) (err os.Error) { - defer catchError(&err) - p = allocate(rtyp, p, indir) +// decodeSingle decodes a top-level value that is not a struct and stores it through p. +// Such values are preceded by a zero, making them have the memory layout of a +// struct field (although with an illegal field number). +func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, p uintptr) (err os.Error) { + indir := ut.indir + if ut.isGobDecoder { + indir = int(ut.decIndir) + } + p = allocate(ut.base, p, indir) state := newDecodeState(dec, &dec.buf) state.fieldnum = singletonField basep := p @@ -428,9 +461,13 @@ func (dec *Decoder) decodeSingle(engine *decEngine, rtyp reflect.Type, p uintptr return nil } -func (dec *Decoder) decodeStruct(engine *decEngine, rtyp *reflect.StructType, p uintptr, indir int) (err os.Error) { - defer catchError(&err) - p = allocate(rtyp, p, indir) +// decodeSingle decodes a top-level struct and stores it through p. +// Indir is for the value, not the type. At the time of the call it may +// differ from ut.indir, which was computed when the engine was built. +// This state cannot arise for decodeSingle, which is called directly +// from the user's value, not from the innards of an engine. +func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p uintptr, indir int) (err os.Error) { + p = allocate(ut.base.(*reflect.StructType), p, indir) state := newDecodeState(dec, &dec.buf) state.fieldnum = -1 basep := p @@ -458,8 +495,8 @@ func (dec *Decoder) decodeStruct(engine *decEngine, rtyp *reflect.StructType, p return nil } +// ignoreStruct discards the data for a struct with no destination. func (dec *Decoder) ignoreStruct(engine *decEngine) (err os.Error) { - defer catchError(&err) state := newDecodeState(dec, &dec.buf) state.fieldnum = -1 for state.b.Len() > 0 { @@ -481,8 +518,9 @@ func (dec *Decoder) ignoreStruct(engine *decEngine) (err os.Error) { return nil } +// ignoreSingle discards the data for a top-level non-struct value with no +// destination. It's used when calling Decode with a nil value. func (dec *Decoder) ignoreSingle(engine *decEngine) (err os.Error) { - defer catchError(&err) state := newDecodeState(dec, &dec.buf) state.fieldnum = singletonField delta := int(state.decodeUint()) @@ -494,7 +532,8 @@ func (dec *Decoder) ignoreSingle(engine *decEngine) (err os.Error) { return nil } -func (dec *Decoder) decodeArrayHelper(state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.ErrorString) { +// decodeArrayHelper does the work for decoding arrays and slices. +func (dec *Decoder) decodeArrayHelper(state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.ErrorString) { instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl} for i := 0; i < length; i++ { up := unsafe.Pointer(p) @@ -506,7 +545,10 @@ func (dec *Decoder) decodeArrayHelper(state *decodeState, p uintptr, elemOp decO } } -func (dec *Decoder) decodeArray(atyp *reflect.ArrayType, state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, length, indir, elemIndir int, ovfl os.ErrorString) { +// decodeArray decodes an array and stores it through p, that is, p points to the zeroth element. +// The length is an unsigned integer preceding the elements. Even though the length is redundant +// (it's part of the type), it's a useful check and is included in the encoding. +func (dec *Decoder) decodeArray(atyp *reflect.ArrayType, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, length, indir, elemIndir int, ovfl os.ErrorString) { if indir > 0 { p = allocate(atyp, p, 1) // All but the last level has been allocated by dec.Indirect } @@ -516,9 +558,11 @@ func (dec *Decoder) decodeArray(atyp *reflect.ArrayType, state *decodeState, p u dec.decodeArrayHelper(state, p, elemOp, elemWid, length, elemIndir, ovfl) } -func decodeIntoValue(state *decodeState, op decOp, indir int, v reflect.Value, ovfl os.ErrorString) reflect.Value { +// decodeIntoValue is a helper for map decoding. Since maps are decoded using reflection, +// unlike the other items we can't use a pointer directly. +func decodeIntoValue(state *decoderState, op decOp, indir int, v reflect.Value, ovfl os.ErrorString) reflect.Value { instr := &decInstr{op, 0, indir, 0, ovfl} - up := unsafe.Pointer(v.Addr()) + up := unsafe.Pointer(v.UnsafeAddr()) if indir > 1 { up = decIndirect(up, indir) } @@ -526,7 +570,11 @@ func decodeIntoValue(state *decodeState, op decOp, indir int, v reflect.Value, o return v } -func (dec *Decoder) decodeMap(mtyp *reflect.MapType, state *decodeState, p uintptr, keyOp, elemOp decOp, indir, keyIndir, elemIndir int, ovfl os.ErrorString) { +// decodeMap decodes a map and stores its header through p. +// Maps are encoded as a length followed by key:value pairs. +// Because the internals of maps are not visible to us, we must +// use reflection rather than pointer magic. +func (dec *Decoder) decodeMap(mtyp *reflect.MapType, state *decoderState, p uintptr, keyOp, elemOp decOp, indir, keyIndir, elemIndir int, ovfl os.ErrorString) { if indir > 0 { p = allocate(mtyp, p, 1) // All but the last level has been allocated by dec.Indirect } @@ -538,7 +586,7 @@ func (dec *Decoder) decodeMap(mtyp *reflect.MapType, state *decodeState, p uintp // Maps cannot be accessed by moving addresses around the way // that slices etc. can. We must recover a full reflection value for // the iteration. - v := reflect.NewValue(unsafe.Unreflect(mtyp, unsafe.Pointer((p)))).(*reflect.MapValue) + v := reflect.NewValue(unsafe.Unreflect(mtyp, unsafe.Pointer(p))).(*reflect.MapValue) n := int(state.decodeUint()) for i := 0; i < n; i++ { key := decodeIntoValue(state, keyOp, keyIndir, reflect.MakeZero(mtyp.Key()), ovfl) @@ -547,21 +595,24 @@ func (dec *Decoder) decodeMap(mtyp *reflect.MapType, state *decodeState, p uintp } } -func (dec *Decoder) ignoreArrayHelper(state *decodeState, elemOp decOp, length int) { +// ignoreArrayHelper does the work for discarding arrays and slices. +func (dec *Decoder) ignoreArrayHelper(state *decoderState, elemOp decOp, length int) { instr := &decInstr{elemOp, 0, 0, 0, os.ErrorString("no error")} for i := 0; i < length; i++ { elemOp(instr, state, nil) } } -func (dec *Decoder) ignoreArray(state *decodeState, elemOp decOp, length int) { +// ignoreArray discards the data for an array value with no destination. +func (dec *Decoder) ignoreArray(state *decoderState, elemOp decOp, length int) { if n := state.decodeUint(); n != uint64(length) { errorf("gob: length mismatch in ignoreArray") } dec.ignoreArrayHelper(state, elemOp, length) } -func (dec *Decoder) ignoreMap(state *decodeState, keyOp, elemOp decOp) { +// ignoreMap discards the data for a map value with no destination. +func (dec *Decoder) ignoreMap(state *decoderState, keyOp, elemOp decOp) { n := int(state.decodeUint()) keyInstr := &decInstr{keyOp, 0, 0, 0, os.ErrorString("no error")} elemInstr := &decInstr{elemOp, 0, 0, 0, os.ErrorString("no error")} @@ -571,7 +622,9 @@ func (dec *Decoder) ignoreMap(state *decodeState, keyOp, elemOp decOp) { } } -func (dec *Decoder) decodeSlice(atyp *reflect.SliceType, state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl os.ErrorString) { +// decodeSlice decodes a slice and stores the slice header through p. +// Slices are encoded as an unsigned length followed by the elements. +func (dec *Decoder) decodeSlice(atyp *reflect.SliceType, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl os.ErrorString) { n := int(uintptr(state.decodeUint())) if indir > 0 { up := unsafe.Pointer(p) @@ -590,7 +643,8 @@ func (dec *Decoder) decodeSlice(atyp *reflect.SliceType, state *decodeState, p u dec.decodeArrayHelper(state, hdrp.Data, elemOp, elemWid, n, elemIndir, ovfl) } -func (dec *Decoder) ignoreSlice(state *decodeState, elemOp decOp) { +// ignoreSlice skips over the data for a slice value with no destination. +func (dec *Decoder) ignoreSlice(state *decoderState, elemOp decOp) { dec.ignoreArrayHelper(state, elemOp, int(state.decodeUint())) } @@ -609,9 +663,10 @@ func setInterfaceValue(ivalue *reflect.InterfaceValue, value reflect.Value) { ivalue.Set(value) } -// decodeInterface receives the name of a concrete type followed by its value. +// decodeInterface decodes an interface value and stores it through p. +// Interfaces are encoded as the name of a concrete type followed by a value. // If the name is empty, the value is nil and no value is sent. -func (dec *Decoder) decodeInterface(ityp *reflect.InterfaceType, state *decodeState, p uintptr, indir int) { +func (dec *Decoder) decodeInterface(ityp *reflect.InterfaceType, state *decoderState, p uintptr, indir int) { // Create an interface reflect.Value. We need one even for the nil case. ivalue := reflect.MakeZero(ityp).(*reflect.InterfaceValue) // Read the name of the concrete type. @@ -655,7 +710,8 @@ func (dec *Decoder) decodeInterface(ityp *reflect.InterfaceType, state *decodeSt *(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.Get() } -func (dec *Decoder) ignoreInterface(state *decodeState) { +// ignoreInterface discards the data for an interface value with no destination. +func (dec *Decoder) ignoreInterface(state *decoderState) { // Read the name of the concrete type. b := make([]byte, state.decodeUint()) _, err := state.b.Read(b) @@ -670,8 +726,34 @@ func (dec *Decoder) ignoreInterface(state *decodeState) { state.b.Next(int(state.decodeUint())) } +// decodeGobDecoder decodes something implementing the GobDecoder interface. +// The data is encoded as a byte slice. +func (dec *Decoder) decodeGobDecoder(state *decoderState, v reflect.Value, index int) { + // Read the bytes for the value. + b := make([]byte, state.decodeUint()) + _, err := state.b.Read(b) + if err != nil { + error(err) + } + // We know it's a GobDecoder, so just call the method directly. + err = v.Interface().(GobDecoder).GobDecode(b) + if err != nil { + error(err) + } +} + +// ignoreGobDecoder discards the data for a GobDecoder value with no destination. +func (dec *Decoder) ignoreGobDecoder(state *decoderState) { + // Read the bytes for the value. + b := make([]byte, state.decodeUint()) + _, err := state.b.Read(b) + if err != nil { + error(err) + } +} + // Index by Go types. -var decOpMap = []decOp{ +var decOpTable = [...]decOp{ reflect.Bool: decBool, reflect.Int8: decInt8, reflect.Int16: decInt16, @@ -699,37 +781,49 @@ var decIgnoreOpMap = map[typeId]decOp{ tComplex: ignoreTwoUints, } -// Return the decoding op for the base type under rt and +// decOpFor returns the decoding op for the base type under rt and // the indirection count to reach it. -func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp, int) { - typ, indir := indirect(rt) +func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) (*decOp, int) { + ut := userType(rt) + // If the type implements GobEncoder, we handle it without further processing. + if ut.isGobDecoder { + return dec.gobDecodeOpFor(ut) + } + // If this type is already in progress, it's a recursive type (e.g. map[string]*T). + // Return the pointer to the op we're already building. + if opPtr := inProgress[rt]; opPtr != nil { + return opPtr, ut.indir + } + typ := ut.base + indir := ut.indir var op decOp k := typ.Kind() - if int(k) < len(decOpMap) { - op = decOpMap[k] + if int(k) < len(decOpTable) { + op = decOpTable[k] } if op == nil { + inProgress[rt] = &op // Special cases switch t := typ.(type) { case *reflect.ArrayType: name = "element of " + name elemId := dec.wireType[wireId].ArrayT.Elem - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name) + elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - state.dec.decodeArray(t, state, uintptr(p), elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl) + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + state.dec.decodeArray(t, state, uintptr(p), *elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl) } case *reflect.MapType: name = "element of " + name keyId := dec.wireType[wireId].MapT.Key elemId := dec.wireType[wireId].MapT.Elem - keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name) - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name) + keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name, inProgress) + elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { up := unsafe.Pointer(p) - state.dec.decodeMap(t, state, uintptr(up), keyOp, elemOp, i.indir, keyIndir, elemIndir, ovfl) + state.dec.decodeMap(t, state, uintptr(up), *keyOp, *elemOp, i.indir, keyIndir, elemIndir, ovfl) } case *reflect.SliceType: @@ -744,46 +838,46 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp } else { elemId = dec.wireType[wireId].SliceT.Elem } - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name) + elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - state.dec.decodeSlice(t, state, uintptr(p), elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + state.dec.decodeSlice(t, state, uintptr(p), *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) } case *reflect.StructType: // Generate a closure that calls out to the engine for the nested type. - enginePtr, err := dec.getDecEnginePtr(wireId, typ) + enginePtr, err := dec.getDecEnginePtr(wireId, userType(typ)) if err != nil { error(err) } - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - // indirect through enginePtr to delay evaluation for recursive structs - err = dec.decodeStruct(*enginePtr, t, uintptr(p), i.indir) + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + // indirect through enginePtr to delay evaluation for recursive structs. + err = dec.decodeStruct(*enginePtr, userType(typ), uintptr(p), i.indir) if err != nil { error(err) } } case *reflect.InterfaceType: - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - dec.decodeInterface(t, state, uintptr(p), i.indir) + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + state.dec.decodeInterface(t, state, uintptr(p), i.indir) } } } if op == nil { errorf("gob: decode can't handle type %s", rt.String()) } - return op, indir + return &op, indir } -// Return the decoding op for a field that has no destination. +// decIgnoreOpFor returns the decoding op for a field that has no destination. func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { op, ok := decIgnoreOpMap[wireId] if !ok { if wireId == tInterface { // Special case because it's a method: the ignored item might // define types and we need to record their state in the decoder. - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - dec.ignoreInterface(state) + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + state.dec.ignoreInterface(state) } return op } @@ -795,7 +889,7 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { case wire.ArrayT != nil: elemId := wire.ArrayT.Elem elemOp := dec.decIgnoreOpFor(elemId) - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { state.dec.ignoreArray(state, elemOp, wire.ArrayT.Len) } @@ -804,14 +898,14 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { elemId := dec.wireType[wireId].MapT.Elem keyOp := dec.decIgnoreOpFor(keyId) elemOp := dec.decIgnoreOpFor(elemId) - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { state.dec.ignoreMap(state, keyOp, elemOp) } case wire.SliceT != nil: elemId := wire.SliceT.Elem elemOp := dec.decIgnoreOpFor(elemId) - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { state.dec.ignoreSlice(state, elemOp) } @@ -821,10 +915,15 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { if err != nil { error(err) } - op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { // indirect through enginePtr to delay evaluation for recursive structs state.dec.ignoreStruct(*enginePtr) } + + case wire.GobEncoderT != nil: + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + state.dec.ignoreGobDecoder(state) + } } } if op == nil { @@ -833,14 +932,58 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { return op } -// Are these two gob Types compatible? -// Answers the question for basic types, arrays, and slices. +// gobDecodeOpFor returns the op for a type that is known to implement +// GobDecoder. +func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) (*decOp, int) { + rt := ut.user + if ut.decIndir != 0 { + errorf("gob: TODO: can't handle indirection to reach GobDecoder") + } + index := -1 + for i := 0; i < rt.NumMethod(); i++ { + if rt.Method(i).Name == gobDecodeMethodName { + index = i + break + } + } + if index < 0 { + panic("can't find GobDecode method") + } + var op decOp + op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + // Allocate the underlying data, but hold on to the address we have, + // since it's known to be the receiver's address. + // TODO: fix this up when decIndir can be non-zero. + allocate(ut.base, uintptr(p), ut.indir) + v := reflect.NewValue(unsafe.Unreflect(rt, p)) + state.dec.decodeGobDecoder(state, v, index) + } + return &op, int(ut.decIndir) + +} + +// compatibleType asks: Are these two gob Types compatible? +// Answers the question for basic types, arrays, maps and slices, plus +// GobEncoder/Decoder pairs. // Structs are considered ok; fields will be checked later. -func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool { - fr, _ = indirect(fr) - switch t := fr.(type) { +func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId, inProgress map[reflect.Type]typeId) bool { + if rhs, ok := inProgress[fr]; ok { + return rhs == fw + } + inProgress[fr] = fw + ut := userType(fr) + wire, ok := dec.wireType[fw] + // If fr is a GobDecoder, the wire type must be GobEncoder. + // And if fr is not a GobDecoder, the wire type must not be either. + if ut.isGobDecoder != (ok && wire.GobEncoderT != nil) { // the parentheses look odd but are correct. + return false + } + if ut.isGobDecoder { // This test trumps all others. + return true + } + switch t := ut.base.(type) { default: - // map, chan, etc: cannot handle. + // chan, etc: cannot handle. return false case *reflect.BoolType: return fw == tBool @@ -857,19 +1000,17 @@ func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool { case *reflect.InterfaceType: return fw == tInterface case *reflect.ArrayType: - wire, ok := dec.wireType[fw] if !ok || wire.ArrayT == nil { return false } array := wire.ArrayT - return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem) + return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem, inProgress) case *reflect.MapType: - wire, ok := dec.wireType[fw] if !ok || wire.MapT == nil { return false } MapType := wire.MapT - return dec.compatibleType(t.Key(), MapType.Key) && dec.compatibleType(t.Elem(), MapType.Elem) + return dec.compatibleType(t.Key(), MapType.Key, inProgress) && dec.compatibleType(t.Elem(), MapType.Elem, inProgress) case *reflect.SliceType: // Is it an array of bytes? if t.Elem().Kind() == reflect.Uint8 { @@ -882,8 +1023,8 @@ func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool { } else { sw = dec.wireType[fw].SliceT } - elem, _ := indirect(t.Elem()) - return sw != nil && dec.compatibleType(elem, sw.Elem) + elem := userType(t.Elem()).base + return sw != nil && dec.compatibleType(elem, sw.Elem, inProgress) case *reflect.StructType: return true } @@ -899,21 +1040,27 @@ func (dec *Decoder) typeString(remoteId typeId) string { return dec.wireType[remoteId].string() } - -func (dec *Decoder) compileSingle(remoteId typeId, rt reflect.Type) (engine *decEngine, err os.Error) { +// compileSingle compiles the decoder engine for a non-struct top-level value, including +// GobDecoders. +func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) { + rt := ut.base + if ut.isGobDecoder { + rt = ut.user + } engine = new(decEngine) engine.instr = make([]decInstr, 1) // one item name := rt.String() // best we can do - if !dec.compatibleType(rt, remoteId) { + if !dec.compatibleType(rt, remoteId, make(map[reflect.Type]typeId)) { return nil, os.ErrorString("gob: wrong type received for local value " + name + ": " + dec.typeString(remoteId)) } - op, indir := dec.decOpFor(remoteId, rt, name) + op, indir := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp)) ovfl := os.ErrorString(`value for "` + name + `" out of range`) - engine.instr[singletonField] = decInstr{op, singletonField, indir, 0, ovfl} + engine.instr[singletonField] = decInstr{*op, singletonField, indir, 0, ovfl} engine.numInstr = 1 return } +// compileIgnoreSingle compiles the decoder engine for a non-struct top-level value that will be discarded. func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err os.Error) { engine = new(decEngine) engine.instr = make([]decInstr, 1) // one item @@ -924,17 +1071,13 @@ func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err return } -// Is this an exported - upper case - name? -func isExported(name string) bool { - rune, _ := utf8.DecodeRuneInString(name) - return unicode.IsUpper(rune) -} - -func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEngine, err os.Error) { - defer catchError(&err) +// compileDec compiles the decoder engine for a value. If the value is not a struct, +// it calls out to compileSingle. +func (dec *Decoder) compileDec(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err os.Error) { + rt := ut.base srt, ok := rt.(*reflect.StructType) - if !ok { - return dec.compileSingle(remoteId, rt) + if !ok || ut.isGobDecoder { + return dec.compileSingle(remoteId, ut) } var wireStruct *structType // Builtin types can come from global pool; the rest must be defined by the decoder. @@ -953,6 +1096,7 @@ func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEng } engine = new(decEngine) engine.instr = make([]decInstr, len(wireStruct.Field)) + seen := make(map[reflect.Type]*decOp) // Loop over the fields of the wire type. for fieldnum := 0; fieldnum < len(wireStruct.Field); fieldnum++ { wireField := wireStruct.Field[fieldnum] @@ -968,17 +1112,19 @@ func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEng engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl} continue } - if !dec.compatibleType(localField.Type, wireField.Id) { + if !dec.compatibleType(localField.Type, wireField.Id, make(map[reflect.Type]typeId)) { errorf("gob: wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name) } - op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name) - engine.instr[fieldnum] = decInstr{op, fieldnum, indir, uintptr(localField.Offset), ovfl} + op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen) + engine.instr[fieldnum] = decInstr{*op, fieldnum, indir, uintptr(localField.Offset), ovfl} engine.numInstr++ } return } -func (dec *Decoder) getDecEnginePtr(remoteId typeId, rt reflect.Type) (enginePtr **decEngine, err os.Error) { +// getDecEnginePtr returns the engine for the specified type. +func (dec *Decoder) getDecEnginePtr(remoteId typeId, ut *userTypeInfo) (enginePtr **decEngine, err os.Error) { + rt := ut.base decoderMap, ok := dec.decoderCache[rt] if !ok { decoderMap = make(map[typeId]**decEngine) @@ -988,7 +1134,7 @@ func (dec *Decoder) getDecEnginePtr(remoteId typeId, rt reflect.Type) (enginePtr // To handle recursive types, mark this engine as underway before compiling. enginePtr = new(*decEngine) decoderMap[remoteId] = enginePtr - *enginePtr, err = dec.compileDec(remoteId, rt) + *enginePtr, err = dec.compileDec(remoteId, ut) if err != nil { decoderMap[remoteId] = nil, false } @@ -996,11 +1142,12 @@ func (dec *Decoder) getDecEnginePtr(remoteId typeId, rt reflect.Type) (enginePtr return } -// When ignoring struct data, in effect we compile it into this type +// emptyStruct is the type we compile into when ignoring a struct value. type emptyStruct struct{} var emptyStructType = reflect.Typeof(emptyStruct{}) +// getDecEnginePtr returns the engine for the specified type when the value is to be discarded. func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, err os.Error) { var ok bool if enginePtr, ok = dec.ignorerCache[wireId]; !ok { @@ -1009,7 +1156,7 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er dec.ignorerCache[wireId] = enginePtr wire := dec.wireType[wireId] if wire != nil && wire.StructT != nil { - *enginePtr, err = dec.compileDec(wireId, emptyStructType) + *enginePtr, err = dec.compileDec(wireId, userType(emptyStructType)) } else { *enginePtr, err = dec.compileIgnoreSingle(wireId) } @@ -1020,28 +1167,39 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er return } -func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) os.Error { +// decodeValue decodes the data stream representing a value and stores it in val. +func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) (err os.Error) { + defer catchError(&err) // If the value is nil, it means we should just ignore this item. if val == nil { return dec.decodeIgnoredValue(wireId) } // Dereference down to the underlying struct type. - rt, indir := indirect(val.Type()) - enginePtr, err := dec.getDecEnginePtr(wireId, rt) + ut := userType(val.Type()) + base := ut.base + indir := ut.indir + if ut.isGobDecoder { + indir = int(ut.decIndir) + if indir != 0 { + errorf("TODO: can't handle indirection in GobDecoder value") + } + } + enginePtr, err := dec.getDecEnginePtr(wireId, ut) if err != nil { return err } engine := *enginePtr - if st, ok := rt.(*reflect.StructType); ok { + if st, ok := base.(*reflect.StructType); ok && !ut.isGobDecoder { if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 { - name := rt.Name() + name := base.Name() return os.ErrorString("gob: type mismatch: no fields matched compiling decoder for " + name) } - return dec.decodeStruct(engine, st, uintptr(val.Addr()), indir) + return dec.decodeStruct(engine, ut, uintptr(val.UnsafeAddr()), indir) } - return dec.decodeSingle(engine, rt, uintptr(val.Addr()), indir) + return dec.decodeSingle(engine, ut, uintptr(val.UnsafeAddr())) } +// decodeIgnoredValue decodes the data stream representing a value of the specified type and discards it. func (dec *Decoder) decodeIgnoredValue(wireId typeId) os.Error { enginePtr, err := dec.getIgnoreEnginePtr(wireId) if err != nil { @@ -1066,8 +1224,8 @@ func init() { default: panic("gob: unknown size of int/uint") } - decOpMap[reflect.Int] = iop - decOpMap[reflect.Uint] = uop + decOpTable[reflect.Int] = iop + decOpTable[reflect.Uint] = uop // Finally uintptr switch reflect.Typeof(uintptr(0)).Bits() { @@ -1078,5 +1236,5 @@ func init() { default: panic("gob: unknown size of uintptr") } - decOpMap[reflect.Uintptr] = uop + decOpTable[reflect.Uintptr] = uop } diff --git a/src/pkg/gob/decoder.go b/src/pkg/gob/decoder.go index f7c994ffa..719274583 100644 --- a/src/pkg/gob/decoder.go +++ b/src/pkg/gob/decoder.go @@ -21,7 +21,7 @@ type Decoder struct { wireType map[typeId]*wireType // map from remote ID to local description decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines ignorerCache map[typeId]**decEngine // ditto for ignored objects - countState *decodeState // reads counts from wire + countState *decoderState // reads counts from wire countBuf []byte // used for decoding integers while parsing messages tmp []byte // temporary storage for i/o; saves reallocating err os.Error diff --git a/src/pkg/gob/encode.go b/src/pkg/gob/encode.go index 2e5ba2487..9190d9203 100644 --- a/src/pkg/gob/encode.go +++ b/src/pkg/gob/encode.go @@ -15,7 +15,7 @@ import ( const uint64Size = unsafe.Sizeof(uint64(0)) -// The global execution state of an instance of the encoder. +// encoderState is the global execution state of an instance of the encoder. // Field numbers are delta encoded and always increase. The field // number is initialized to -1 so 0 comes out as delta(1). A delta of // 0 terminates the structure. @@ -72,6 +72,7 @@ func (state *encoderState) encodeInt(i int64) { state.encodeUint(uint64(x)) } +// encOp is the signature of an encoding operator for a given type. type encOp func(i *encInstr, state *encoderState, p unsafe.Pointer) // The 'instructions' of the encoding machine @@ -82,8 +83,8 @@ type encInstr struct { offset uintptr // offset in the structure of the field to encode } -// Emit a field number and update the state to record its value for delta encoding. -// If the instruction pointer is nil, do nothing +// update emits a field number and updates the state to record its value for delta encoding. +// If the instruction pointer is nil, it does nothing func (state *encoderState) update(instr *encInstr) { if instr != nil { state.encodeUint(uint64(instr.field - state.fieldnum)) @@ -97,6 +98,7 @@ func (state *encoderState) update(instr *encInstr) { // Otherwise, the output (for a scalar) is the field number, as an encoded integer, // followed by the field data in its appropriate format. +// encIndirect dereferences p indir times and returns the result. func encIndirect(p unsafe.Pointer, indir int) unsafe.Pointer { for ; indir > 0; indir-- { p = *(*unsafe.Pointer)(p) @@ -107,6 +109,7 @@ func encIndirect(p unsafe.Pointer, indir int) unsafe.Pointer { return p } +// encBool encodes the bool with address p as an unsigned 0 or 1. func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) { b := *(*bool)(p) if b || state.sendZero { @@ -119,6 +122,7 @@ func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encInt encodes the int with address p. func encInt(i *encInstr, state *encoderState, p unsafe.Pointer) { v := int64(*(*int)(p)) if v != 0 || state.sendZero { @@ -127,6 +131,7 @@ func encInt(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encUint encodes the uint with address p. func encUint(i *encInstr, state *encoderState, p unsafe.Pointer) { v := uint64(*(*uint)(p)) if v != 0 || state.sendZero { @@ -135,6 +140,7 @@ func encUint(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encInt8 encodes the int8 with address p. func encInt8(i *encInstr, state *encoderState, p unsafe.Pointer) { v := int64(*(*int8)(p)) if v != 0 || state.sendZero { @@ -143,6 +149,7 @@ func encInt8(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encUint8 encodes the uint8 with address p. func encUint8(i *encInstr, state *encoderState, p unsafe.Pointer) { v := uint64(*(*uint8)(p)) if v != 0 || state.sendZero { @@ -151,6 +158,7 @@ func encUint8(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encInt16 encodes the int16 with address p. func encInt16(i *encInstr, state *encoderState, p unsafe.Pointer) { v := int64(*(*int16)(p)) if v != 0 || state.sendZero { @@ -159,6 +167,7 @@ func encInt16(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encUint16 encodes the uint16 with address p. func encUint16(i *encInstr, state *encoderState, p unsafe.Pointer) { v := uint64(*(*uint16)(p)) if v != 0 || state.sendZero { @@ -167,6 +176,7 @@ func encUint16(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encInt32 encodes the int32 with address p. func encInt32(i *encInstr, state *encoderState, p unsafe.Pointer) { v := int64(*(*int32)(p)) if v != 0 || state.sendZero { @@ -175,6 +185,7 @@ func encInt32(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encUint encodes the uint32 with address p. func encUint32(i *encInstr, state *encoderState, p unsafe.Pointer) { v := uint64(*(*uint32)(p)) if v != 0 || state.sendZero { @@ -183,6 +194,7 @@ func encUint32(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encInt64 encodes the int64 with address p. func encInt64(i *encInstr, state *encoderState, p unsafe.Pointer) { v := *(*int64)(p) if v != 0 || state.sendZero { @@ -191,6 +203,7 @@ func encInt64(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encInt64 encodes the uint64 with address p. func encUint64(i *encInstr, state *encoderState, p unsafe.Pointer) { v := *(*uint64)(p) if v != 0 || state.sendZero { @@ -199,6 +212,7 @@ func encUint64(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encUintptr encodes the uintptr with address p. func encUintptr(i *encInstr, state *encoderState, p unsafe.Pointer) { v := uint64(*(*uintptr)(p)) if v != 0 || state.sendZero { @@ -207,6 +221,7 @@ func encUintptr(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// floatBits returns a uint64 holding the bits of a floating-point number. // Floating-point numbers are transmitted as uint64s holding the bits // of the underlying representation. They are sent byte-reversed, with // the exponent end coming out first, so integer floating point numbers @@ -223,6 +238,7 @@ func floatBits(f float64) uint64 { return v } +// encFloat32 encodes the float32 with address p. func encFloat32(i *encInstr, state *encoderState, p unsafe.Pointer) { f := *(*float32)(p) if f != 0 || state.sendZero { @@ -232,6 +248,7 @@ func encFloat32(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encFloat64 encodes the float64 with address p. func encFloat64(i *encInstr, state *encoderState, p unsafe.Pointer) { f := *(*float64)(p) if f != 0 || state.sendZero { @@ -241,6 +258,7 @@ func encFloat64(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encComplex64 encodes the complex64 with address p. // Complex numbers are just a pair of floating-point numbers, real part first. func encComplex64(i *encInstr, state *encoderState, p unsafe.Pointer) { c := *(*complex64)(p) @@ -253,6 +271,7 @@ func encComplex64(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encComplex128 encodes the complex128 with address p. func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) { c := *(*complex128)(p) if c != 0+0i || state.sendZero { @@ -264,6 +283,7 @@ func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encUint8Array encodes the byte slice whose header has address p. // Byte arrays are encoded as an unsigned count followed by the raw bytes. func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) { b := *(*[]byte)(p) @@ -274,6 +294,7 @@ func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) { } } +// encString encodes the string whose header has address p. // Strings are encoded as an unsigned count followed by the raw bytes. func encString(i *encInstr, state *encoderState, p unsafe.Pointer) { s := *(*string)(p) @@ -284,14 +305,15 @@ func encString(i *encInstr, state *encoderState, p unsafe.Pointer) { } } -// The end of a struct is marked by a delta field number of 0. +// encStructTerminator encodes the end of an encoded struct +// as delta field number of 0. func encStructTerminator(i *encInstr, state *encoderState, p unsafe.Pointer) { state.encodeUint(0) } // Execution engine -// The encoder engine is an array of instructions indexed by field number of the encoding +// encEngine an array of instructions indexed by field number of the encoding // data, typically a struct. It is executed top to bottom, walking the struct. type encEngine struct { instr []encInstr @@ -299,6 +321,7 @@ type encEngine struct { const singletonField = 0 +// encodeSingle encodes a single top-level non-struct value. func (enc *Encoder) encodeSingle(b *bytes.Buffer, engine *encEngine, basep uintptr) { state := newEncoderState(enc, b) state.fieldnum = singletonField @@ -315,6 +338,7 @@ func (enc *Encoder) encodeSingle(b *bytes.Buffer, engine *encEngine, basep uintp instr.op(instr, state, p) } +// encodeStruct encodes a single struct value. func (enc *Encoder) encodeStruct(b *bytes.Buffer, engine *encEngine, basep uintptr) { state := newEncoderState(enc, b) state.fieldnum = -1 @@ -330,6 +354,7 @@ func (enc *Encoder) encodeStruct(b *bytes.Buffer, engine *encEngine, basep uintp } } +// encodeArray encodes the array whose 0th element is at p. func (enc *Encoder) encodeArray(b *bytes.Buffer, p uintptr, op encOp, elemWid uintptr, elemIndir int, length int) { state := newEncoderState(enc, b) state.fieldnum = -1 @@ -349,6 +374,7 @@ func (enc *Encoder) encodeArray(b *bytes.Buffer, p uintptr, op encOp, elemWid ui } } +// encodeReflectValue is a helper for maps. It encodes the value v. func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir int) { for i := 0; i < indir && v != nil; i++ { v = reflect.Indirect(v) @@ -356,9 +382,12 @@ func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir in if v == nil { errorf("gob: encodeReflectValue: nil element") } - op(nil, state, unsafe.Pointer(v.Addr())) + op(nil, state, unsafe.Pointer(v.UnsafeAddr())) } +// encodeMap encodes a map as unsigned count followed by key:value pairs. +// Because map internals are not exposed, we must use reflection rather than +// addresses. func (enc *Encoder) encodeMap(b *bytes.Buffer, mv *reflect.MapValue, keyOp, elemOp encOp, keyIndir, elemIndir int) { state := newEncoderState(enc, b) state.fieldnum = -1 @@ -371,6 +400,7 @@ func (enc *Encoder) encodeMap(b *bytes.Buffer, mv *reflect.MapValue, keyOp, elem } } +// encodeInterface encodes the interface value iv. // To send an interface, we send a string identifying the concrete type, followed // by the type identifier (which might require defining that type right now), followed // by the concrete value. A nil value gets sent as the empty string for the name, @@ -384,10 +414,10 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv *reflect.InterfaceValue) return } - typ, _ := indirect(iv.Elem().Type()) - name, ok := concreteTypeToName[typ] + ut := userType(iv.Elem().Type()) + name, ok := concreteTypeToName[ut.base] if !ok { - errorf("gob: type not registered for interface: %s", typ) + errorf("gob: type not registered for interface: %s", ut.base) } // Send the name. state.encodeUint(uint64(len(name))) @@ -396,14 +426,14 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv *reflect.InterfaceValue) error(err) } // Define the type id if necessary. - enc.sendTypeDescriptor(enc.writer(), state, typ) + enc.sendTypeDescriptor(enc.writer(), state, ut) // Send the type id. - enc.sendTypeId(state, typ) + enc.sendTypeId(state, ut) // Encode the value into a new buffer. Any nested type definitions // should be written to b, before the encoded value. enc.pushWriter(b) data := new(bytes.Buffer) - err = enc.encode(data, iv.Elem()) + err = enc.encode(data, iv.Elem(), ut) if err != nil { error(err) } @@ -414,7 +444,22 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv *reflect.InterfaceValue) } } -var encOpMap = []encOp{ +// encGobEncoder encodes a value that implements the GobEncoder interface. +// The data is sent as a byte array. +func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, v reflect.Value, index int) { + // TODO: should we catch panics from the called method? + // We know it's a GobEncoder, so just call the method directly. + data, err := v.Interface().(GobEncoder).GobEncode() + if err != nil { + error(err) + } + state := newEncoderState(enc, b) + state.fieldnum = -1 + state.encodeUint(uint64(len(data))) + state.b.Write(data) +} + +var encOpTable = [...]encOp{ reflect.Bool: encBool, reflect.Int: encInt, reflect.Int8: encInt8, @@ -434,16 +479,28 @@ var encOpMap = []encOp{ reflect.String: encString, } -// Return the encoding op for the base type under rt and +// encOpFor returns (a pointer to) the encoding op for the base type under rt and // the indirection count to reach it. -func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { - typ, indir := indirect(rt) - var op encOp +func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp) (*encOp, int) { + ut := userType(rt) + // If the type implements GobEncoder, we handle it without further processing. + if ut.isGobEncoder { + return enc.gobEncodeOpFor(ut) + } + // If this type is already in progress, it's a recursive type (e.g. map[string]*T). + // Return the pointer to the op we're already building. + if opPtr := inProgress[rt]; opPtr != nil { + return opPtr, ut.indir + } + typ := ut.base + indir := ut.indir k := typ.Kind() - if int(k) < len(encOpMap) { - op = encOpMap[k] + var op encOp + if int(k) < len(encOpTable) { + op = encOpTable[k] } if op == nil { + inProgress[rt] = &op // Special cases switch t := typ.(type) { case *reflect.SliceType: @@ -452,40 +509,40 @@ func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { break } // Slices have a header; we decode it to find the underlying array. - elemOp, indir := enc.encOpFor(t.Elem()) + elemOp, indir := enc.encOpFor(t.Elem(), inProgress) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { slice := (*reflect.SliceHeader)(p) if !state.sendZero && slice.Len == 0 { return } state.update(i) - state.enc.encodeArray(state.b, slice.Data, elemOp, t.Elem().Size(), indir, int(slice.Len)) + state.enc.encodeArray(state.b, slice.Data, *elemOp, t.Elem().Size(), indir, int(slice.Len)) } case *reflect.ArrayType: // True arrays have size in the type. - elemOp, indir := enc.encOpFor(t.Elem()) + elemOp, indir := enc.encOpFor(t.Elem(), inProgress) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { state.update(i) - state.enc.encodeArray(state.b, uintptr(p), elemOp, t.Elem().Size(), indir, t.Len()) + state.enc.encodeArray(state.b, uintptr(p), *elemOp, t.Elem().Size(), indir, t.Len()) } case *reflect.MapType: - keyOp, keyIndir := enc.encOpFor(t.Key()) - elemOp, elemIndir := enc.encOpFor(t.Elem()) + keyOp, keyIndir := enc.encOpFor(t.Key(), inProgress) + elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { // Maps cannot be accessed by moving addresses around the way // that slices etc. can. We must recover a full reflection value for // the iteration. - v := reflect.NewValue(unsafe.Unreflect(t, unsafe.Pointer((p)))) + v := reflect.NewValue(unsafe.Unreflect(t, unsafe.Pointer(p))) mv := reflect.Indirect(v).(*reflect.MapValue) if !state.sendZero && mv.Len() == 0 { return } state.update(i) - state.enc.encodeMap(state.b, mv, keyOp, elemOp, keyIndir, elemIndir) + state.enc.encodeMap(state.b, mv, *keyOp, *elemOp, keyIndir, elemIndir) } case *reflect.StructType: // Generate a closure that calls out to the engine for the nested type. - enc.getEncEngine(typ) + enc.getEncEngine(userType(typ)) info := mustGetTypeInfo(typ) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { state.update(i) @@ -496,7 +553,7 @@ func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { // Interfaces transmit the name and contents of the concrete // value they contain. - v := reflect.NewValue(unsafe.Unreflect(t, unsafe.Pointer((p)))) + v := reflect.NewValue(unsafe.Unreflect(t, unsafe.Pointer(p))) iv := reflect.Indirect(v).(*reflect.InterfaceValue) if !state.sendZero && (iv == nil || iv.IsNil()) { return @@ -509,21 +566,54 @@ func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { if op == nil { errorf("gob enc: can't happen: encode type %s", rt.String()) } - return op, indir + return &op, indir } -// The local Type was compiled from the actual value, so we know it's compatible. -func (enc *Encoder) compileEnc(rt reflect.Type) *encEngine { - srt, isStruct := rt.(*reflect.StructType) +// gobEncodeOpFor returns the op for a type that is known to implement +// GobEncoder. +func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { + rt := ut.user + if ut.encIndir != 0 { + errorf("gob: TODO: can't handle indirection to reach GobEncoder") + } + index := -1 + for i := 0; i < rt.NumMethod(); i++ { + if rt.Method(i).Name == gobEncodeMethodName { + index = i + break + } + } + if index < 0 { + panic("can't find GobEncode method") + } + var op encOp + op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { + // TODO: this will need fixing when ut.encIndr != 0. + v := reflect.NewValue(unsafe.Unreflect(rt, p)) + state.update(i) + state.enc.encodeGobEncoder(state.b, v, index) + } + return &op, int(ut.encIndir) +} + +// compileEnc returns the engine to compile the type. +func (enc *Encoder) compileEnc(ut *userTypeInfo) *encEngine { + srt, isStruct := ut.base.(*reflect.StructType) engine := new(encEngine) - if isStruct { - for fieldNum := 0; fieldNum < srt.NumField(); fieldNum++ { + seen := make(map[reflect.Type]*encOp) + rt := ut.base + if ut.isGobEncoder { + rt = ut.user + } + if !ut.isGobEncoder && isStruct { + for fieldNum, wireFieldNum := 0, 0; fieldNum < srt.NumField(); fieldNum++ { f := srt.Field(fieldNum) if !isExported(f.Name) { continue } - op, indir := enc.encOpFor(f.Type) - engine.instr = append(engine.instr, encInstr{op, fieldNum, indir, uintptr(f.Offset)}) + op, indir := enc.encOpFor(f.Type, seen) + engine.instr = append(engine.instr, encInstr{*op, wireFieldNum, indir, uintptr(f.Offset)}) + wireFieldNum++ } if srt.NumField() > 0 && len(engine.instr) == 0 { errorf("type %s has no exported fields", rt) @@ -531,46 +621,52 @@ func (enc *Encoder) compileEnc(rt reflect.Type) *encEngine { engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, 0, 0}) } else { engine.instr = make([]encInstr, 1) - op, indir := enc.encOpFor(rt) - engine.instr[0] = encInstr{op, singletonField, indir, 0} // offset is zero + op, indir := enc.encOpFor(rt, seen) + engine.instr[0] = encInstr{*op, singletonField, indir, 0} // offset is zero } return engine } +// getEncEngine returns the engine to compile the type. // typeLock must be held (or we're in initialization and guaranteed single-threaded). -// The reflection type must have all its indirections processed out. -func (enc *Encoder) getEncEngine(rt reflect.Type) *encEngine { - info, err1 := getTypeInfo(rt) +func (enc *Encoder) getEncEngine(ut *userTypeInfo) *encEngine { + info, err1 := getTypeInfo(ut) if err1 != nil { error(err1) } if info.encoder == nil { // mark this engine as underway before compiling to handle recursive types. info.encoder = new(encEngine) - info.encoder = enc.compileEnc(rt) + info.encoder = enc.compileEnc(ut) } return info.encoder } -// Put this in a function so we can hold the lock only while compiling, not when encoding. -func (enc *Encoder) lockAndGetEncEngine(rt reflect.Type) *encEngine { +// lockAndGetEncEngine is a function that locks and compiles. +// This lets us hold the lock only while compiling, not when encoding. +func (enc *Encoder) lockAndGetEncEngine(ut *userTypeInfo) *encEngine { typeLock.Lock() defer typeLock.Unlock() - return enc.getEncEngine(rt) + return enc.getEncEngine(ut) } -func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value) (err os.Error) { +func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInfo) (err os.Error) { defer catchError(&err) - // Dereference down to the underlying object. - rt, indir := indirect(value.Type()) + engine := enc.lockAndGetEncEngine(ut) + indir := ut.indir + if ut.isGobEncoder { + indir = int(ut.encIndir) + if indir != 0 { + errorf("TODO: can't handle indirection in GobEncoder value") + } + } for i := 0; i < indir; i++ { value = reflect.Indirect(value) } - engine := enc.lockAndGetEncEngine(rt) - if value.Type().Kind() == reflect.Struct { - enc.encodeStruct(b, engine, value.Addr()) + if !ut.isGobEncoder && value.Type().Kind() == reflect.Struct { + enc.encodeStruct(b, engine, value.UnsafeAddr()) } else { - enc.encodeSingle(b, engine, value.Addr()) + enc.encodeSingle(b, engine, value.UnsafeAddr()) } return nil } diff --git a/src/pkg/gob/encoder.go b/src/pkg/gob/encoder.go index 29ba44057..4bfcf15c7 100644 --- a/src/pkg/gob/encoder.go +++ b/src/pkg/gob/encoder.go @@ -78,11 +78,57 @@ func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) { } } +// sendActualType sends the requested type, without further investigation, unless +// it's been sent before. +func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTypeInfo, actual reflect.Type) (sent bool) { + if _, alreadySent := enc.sent[actual]; alreadySent { + return false + } + typeLock.Lock() + info, err := getTypeInfo(ut) + typeLock.Unlock() + if err != nil { + enc.setError(err) + return + } + // Send the pair (-id, type) + // Id: + state.encodeInt(-int64(info.id)) + // Type: + enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo) + enc.writeMessage(w, state.b) + if enc.err != nil { + return + } + + // Remember we've sent this type, both what the user gave us and the base type. + enc.sent[ut.base] = info.id + if ut.user != ut.base { + enc.sent[ut.user] = info.id + } + // Now send the inner types + switch st := actual.(type) { + case *reflect.StructType: + for i := 0; i < st.NumField(); i++ { + enc.sendType(w, state, st.Field(i).Type) + } + case reflect.ArrayOrSliceType: + enc.sendType(w, state, st.Elem()) + } + return true +} + +// sendType sends the type info to the other side, if necessary. func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) { - // Drill down to the base type. - rt, _ := indirect(origt) + ut := userType(origt) + if ut.isGobEncoder { + // The rules are different: regardless of the underlying type's representation, + // we need to tell the other side that this exact type is a GobEncoder. + return enc.sendActualType(w, state, ut, ut.user) + } - switch rt := rt.(type) { + // It's a concrete value, so drill down to the base type. + switch rt := ut.base.(type) { default: // Basic types and interfaces do not need to be described. return @@ -108,43 +154,7 @@ func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Typ return } - // Have we already sent this type? This time we ask about the base type. - if _, alreadySent := enc.sent[rt]; alreadySent { - return - } - - // Need to send it. - typeLock.Lock() - info, err := getTypeInfo(rt) - typeLock.Unlock() - if err != nil { - enc.setError(err) - return - } - // Send the pair (-id, type) - // Id: - state.encodeInt(-int64(info.id)) - // Type: - enc.encode(state.b, reflect.NewValue(info.wire)) - enc.writeMessage(w, state.b) - if enc.err != nil { - return - } - - // Remember we've sent this type. - enc.sent[rt] = info.id - // Remember we've sent the top-level, possibly indirect type too. - enc.sent[origt] = info.id - // Now send the inner types - switch st := rt.(type) { - case *reflect.StructType: - for i := 0; i < st.NumField(); i++ { - enc.sendType(w, state, st.Field(i).Type) - } - case reflect.ArrayOrSliceType: - enc.sendType(w, state, st.Elem()) - } - return true + return enc.sendActualType(w, state, ut, ut.base) } // Encode transmits the data item represented by the empty interface value, @@ -153,12 +163,19 @@ func (enc *Encoder) Encode(e interface{}) os.Error { return enc.EncodeValue(reflect.NewValue(e)) } -// sendTypeId makes sure the remote side knows about this type. +// sendTypeDescriptor makes sure the remote side knows about this type. // It will send a descriptor if this is the first time the type has been // sent. -func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, rt reflect.Type) { +func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) { // Make sure the type is known to the other side. // First, have we already sent this type? + rt := ut.base + if ut.isGobEncoder { + rt = ut.user + if ut.encIndir != 0 { + panic("TODO: can't handle non-zero encIndir") + } + } if _, alreadySent := enc.sent[rt]; !alreadySent { // No, so send it. sent := enc.sendType(w, state, rt) @@ -170,7 +187,7 @@ func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, rt refl // need to send the type info but we do need to update enc.sent. if !sent { typeLock.Lock() - info, err := getTypeInfo(rt) + info, err := getTypeInfo(ut) typeLock.Unlock() if err != nil { enc.setError(err) @@ -182,9 +199,9 @@ func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, rt refl } // sendTypeId sends the id, which must have already been defined. -func (enc *Encoder) sendTypeId(state *encoderState, rt reflect.Type) { +func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) { // Identify the type of this top-level value. - state.encodeInt(int64(enc.sent[rt])) + state.encodeInt(int64(enc.sent[ut.base])) } // EncodeValue transmits the data item represented by the reflection value, @@ -198,19 +215,22 @@ func (enc *Encoder) EncodeValue(value reflect.Value) os.Error { // Remove any nested writers remaining due to previous errors. enc.w = enc.w[0:1] - enc.err = nil - rt, _ := indirect(value.Type()) + ut, err := validUserType(value.Type()) + if err != nil { + return err + } + enc.err = nil state := newEncoderState(enc, new(bytes.Buffer)) - enc.sendTypeDescriptor(enc.writer(), state, rt) - enc.sendTypeId(state, rt) + enc.sendTypeDescriptor(enc.writer(), state, ut) + enc.sendTypeId(state, ut) if enc.err != nil { return enc.err } // Encode the object. - err := enc.encode(state.b, value) + err = enc.encode(state.b, value, ut) if err != nil { enc.setError(err) } else { diff --git a/src/pkg/gob/encoder_test.go b/src/pkg/gob/encoder_test.go index 3e06db727..a0c713b81 100644 --- a/src/pkg/gob/encoder_test.go +++ b/src/pkg/gob/encoder_test.go @@ -249,6 +249,24 @@ func TestArray(t *testing.T) { } } +func TestRecursiveMapType(t *testing.T) { + type recursiveMap map[string]recursiveMap + r1 := recursiveMap{"A": recursiveMap{"B": nil, "C": nil}, "D": nil} + r2 := make(recursiveMap) + if err := encAndDec(r1, &r2); err != nil { + t.Error(err) + } +} + +func TestRecursiveSliceType(t *testing.T) { + type recursiveSlice []recursiveSlice + r1 := recursiveSlice{0: recursiveSlice{0: nil}, 1: nil} + r2 := make(recursiveSlice, 0) + if err := encAndDec(r1, &r2); err != nil { + t.Error(err) + } +} + // Regression test for bug: must send zero values inside arrays func TestDefaultsInArray(t *testing.T) { type Type7 struct { diff --git a/src/pkg/gob/gobencdec_test.go b/src/pkg/gob/gobencdec_test.go new file mode 100644 index 000000000..82ca68170 --- /dev/null +++ b/src/pkg/gob/gobencdec_test.go @@ -0,0 +1,331 @@ +// Copyright 20011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains tests of the GobEncoder/GobDecoder support. + +package gob + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" +) + +// Types that implement the GobEncoder/Decoder interfaces. + +type ByteStruct struct { + a byte // not an exported field +} + +type StringStruct struct { + s string // not an exported field +} + +type Gobber int + +type ValueGobber string // encodes with a value, decodes with a pointer. + +// The relevant methods + +func (g *ByteStruct) GobEncode() ([]byte, os.Error) { + b := make([]byte, 3) + b[0] = g.a + b[1] = g.a + 1 + b[2] = g.a + 2 + return b, nil +} + +func (g *ByteStruct) GobDecode(data []byte) os.Error { + if g == nil { + return os.ErrorString("NIL RECEIVER") + } + // Expect N sequential-valued bytes. + if len(data) == 0 { + return os.EOF + } + g.a = data[0] + for i, c := range data { + if c != g.a+byte(i) { + return os.ErrorString("invalid data sequence") + } + } + return nil +} + +func (g *StringStruct) GobEncode() ([]byte, os.Error) { + return []byte(g.s), nil +} + +func (g *StringStruct) GobDecode(data []byte) os.Error { + // Expect N sequential-valued bytes. + if len(data) == 0 { + return os.EOF + } + a := data[0] + for i, c := range data { + if c != a+byte(i) { + return os.ErrorString("invalid data sequence") + } + } + g.s = string(data) + return nil +} + +func (g *Gobber) GobEncode() ([]byte, os.Error) { + return []byte(fmt.Sprintf("VALUE=%d", *g)), nil +} + +func (g *Gobber) GobDecode(data []byte) os.Error { + _, err := fmt.Sscanf(string(data), "VALUE=%d", (*int)(g)) + return err +} + +func (v ValueGobber) GobEncode() ([]byte, os.Error) { + return []byte(fmt.Sprintf("VALUE=%s", v)), nil +} + +func (v *ValueGobber) GobDecode(data []byte) os.Error { + _, err := fmt.Sscanf(string(data), "VALUE=%s", (*string)(v)) + return err +} + +// Structs that include GobEncodable fields. + +type GobTest0 struct { + X int // guarantee we have something in common with GobTest* + G *ByteStruct +} + +type GobTest1 struct { + X int // guarantee we have something in common with GobTest* + G *StringStruct +} + +type GobTest2 struct { + X int // guarantee we have something in common with GobTest* + G string // not a GobEncoder - should give us errors +} + +type GobTest3 struct { + X int // guarantee we have something in common with GobTest* + G *Gobber // TODO: should be able to satisfy interface without a pointer +} + +type GobTest4 struct { + X int // guarantee we have something in common with GobTest* + V ValueGobber +} + +type GobTest5 struct { + X int // guarantee we have something in common with GobTest* + V *ValueGobber +} + +type GobTestIgnoreEncoder struct { + X int // guarantee we have something in common with GobTest* +} + +func TestGobEncoderField(t *testing.T) { + b := new(bytes.Buffer) + // First a field that's a structure. + enc := NewEncoder(b) + err := enc.Encode(GobTest0{17, &ByteStruct{'A'}}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := new(GobTest0) + err = dec.Decode(x) + if err != nil { + t.Fatal("decode error:", err) + } + if x.G.a != 'A' { + t.Errorf("expected 'A' got %c", x.G.a) + } + // Now a field that's not a structure. + b.Reset() + gobber := Gobber(23) + err = enc.Encode(GobTest3{17, &gobber}) + if err != nil { + t.Fatal("encode error:", err) + } + y := new(GobTest3) + err = dec.Decode(y) + if err != nil { + t.Fatal("decode error:", err) + } + if *y.G != 23 { + t.Errorf("expected '23 got %d", *y.G) + } +} + +// As long as the fields have the same name and implement the +// interface, we can cross-connect them. Not sure it's useful +// and may even be bad but it works and it's hard to prevent +// without exposing the contents of the object, which would +// defeat the purpose. +func TestGobEncoderFieldsOfDifferentType(t *testing.T) { + // first, string in field to byte in field + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := new(GobTest0) + err = dec.Decode(x) + if err != nil { + t.Fatal("decode error:", err) + } + if x.G.a != 'A' { + t.Errorf("expected 'A' got %c", x.G.a) + } + // now the other direction, byte in field to string in field + b.Reset() + err = enc.Encode(GobTest0{17, &ByteStruct{'X'}}) + if err != nil { + t.Fatal("encode error:", err) + } + y := new(GobTest1) + err = dec.Decode(y) + if err != nil { + t.Fatal("decode error:", err) + } + if y.G.s != "XYZ" { + t.Fatalf("expected `XYZ` got %c", y.G.s) + } +} + +// Test that we can encode a value and decode into a pointer. +func TestGobEncoderValueEncoder(t *testing.T) { + // first, string in field to byte in field + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(GobTest4{17, ValueGobber("hello")}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := new(GobTest5) + err = dec.Decode(x) + if err != nil { + t.Fatal("decode error:", err) + } + if *x.V != "hello" { + t.Errorf("expected `hello` got %s", x.V) + } +} + +func TestGobEncoderFieldTypeError(t *testing.T) { + // GobEncoder to non-decoder: error + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := &GobTest2{} + err = dec.Decode(x) + if err == nil { + t.Fatal("expected decode error for mismatched fields (encoder to non-decoder)") + } + if strings.Index(err.String(), "type") < 0 { + t.Fatal("expected type error; got", err) + } + // Non-encoder to GobDecoder: error + b.Reset() + err = enc.Encode(GobTest2{17, "ABC"}) + if err != nil { + t.Fatal("encode error:", err) + } + y := &GobTest1{} + err = dec.Decode(y) + if err == nil { + t.Fatal("expected decode error for mistmatched fields (non-encoder to decoder)") + } + if strings.Index(err.String(), "type") < 0 { + t.Fatal("expected type error; got", err) + } +} + +// Even though ByteStruct is a struct, it's treated as a singleton at the top level. +func TestGobEncoderStructSingleton(t *testing.T) { + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(&ByteStruct{'A'}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := new(ByteStruct) + err = dec.Decode(x) + if err != nil { + t.Fatal("decode error:", err) + } + if x.a != 'A' { + t.Errorf("expected 'A' got %c", x.a) + } +} + +func TestGobEncoderNonStructSingleton(t *testing.T) { + b := new(bytes.Buffer) + enc := NewEncoder(b) + g := Gobber(1234) // TODO: shouldn't need to take the address here. + err := enc.Encode(&g) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + var x Gobber + err = dec.Decode(&x) + if err != nil { + t.Fatal("decode error:", err) + } + if x != 1234 { + t.Errorf("expected 1234 got %c", x) + } +} + +func TestGobEncoderIgnoreStructField(t *testing.T) { + b := new(bytes.Buffer) + // First a field that's a structure. + enc := NewEncoder(b) + err := enc.Encode(GobTest0{17, &ByteStruct{'A'}}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := new(GobTestIgnoreEncoder) + err = dec.Decode(x) + if err != nil { + t.Fatal("decode error:", err) + } + if x.X != 17 { + t.Errorf("expected 17 got %c", x.X) + } +} + +func TestGobEncoderIgnoreNonStructField(t *testing.T) { + b := new(bytes.Buffer) + // First a field that's a structure. + enc := NewEncoder(b) + gobber := Gobber(23) + err := enc.Encode(GobTest3{17, &gobber}) + if err != nil { + t.Fatal("encode error:", err) + } + dec := NewDecoder(b) + x := new(GobTestIgnoreEncoder) + err = dec.Decode(x) + if err != nil { + t.Fatal("decode error:", err) + } + if x.X != 17 { + t.Errorf("expected 17 got %c", x.X) + } +} diff --git a/src/pkg/gob/type.go b/src/pkg/gob/type.go index f613f6e8a..a43813941 100644 --- a/src/pkg/gob/type.go +++ b/src/pkg/gob/type.go @@ -9,15 +9,157 @@ import ( "os" "reflect" "sync" + "unicode" + "utf8" ) -// Reflection types are themselves interface values holding structs -// describing the type. Each type has a different struct so that struct can -// be the kind. For example, if typ is the reflect type for an int8, typ is -// a pointer to a reflect.Int8Type struct; if typ is the reflect type for a -// function, typ is a pointer to a reflect.FuncType struct; we use the type -// of that pointer as the kind. +// userTypeInfo stores the information associated with a type the user has handed +// to the package. It's computed once and stored in a map keyed by reflection +// type. +type userTypeInfo struct { + user reflect.Type // the type the user handed us + base reflect.Type // the base type after all indirections + indir int // number of indirections to reach the base type + isGobEncoder bool // does the type implement GobEncoder? + isGobDecoder bool // does the type implement GobDecoder? + encIndir int8 // number of indirections to reach the receiver type; may be negative + decIndir int8 // number of indirections to reach the receiver type; may be negative +} + +var ( + // Protected by an RWMutex because we read it a lot and write + // it only when we see a new type, typically when compiling. + userTypeLock sync.RWMutex + userTypeCache = make(map[reflect.Type]*userTypeInfo) +) + +// validType returns, and saves, the information associated with user-provided type rt. +// If the user type is not valid, err will be non-nil. To be used when the error handler +// is not set up. +func validUserType(rt reflect.Type) (ut *userTypeInfo, err os.Error) { + userTypeLock.RLock() + ut = userTypeCache[rt] + userTypeLock.RUnlock() + if ut != nil { + return + } + // Now set the value under the write lock. + userTypeLock.Lock() + defer userTypeLock.Unlock() + if ut = userTypeCache[rt]; ut != nil { + // Lost the race; not a problem. + return + } + ut = new(userTypeInfo) + ut.base = rt + ut.user = rt + // A type that is just a cycle of pointers (such as type T *T) cannot + // be represented in gobs, which need some concrete data. We use a + // cycle detection algorithm from Knuth, Vol 2, Section 3.1, Ex 6, + // pp 539-540. As we step through indirections, run another type at + // half speed. If they meet up, there's a cycle. + slowpoke := ut.base // walks half as fast as ut.base + for { + pt, ok := ut.base.(*reflect.PtrType) + if !ok { + break + } + ut.base = pt.Elem() + if ut.base == slowpoke { // ut.base lapped slowpoke + // recursive pointer type. + return nil, os.ErrorString("can't represent recursive pointer type " + ut.base.String()) + } + if ut.indir%2 == 0 { + slowpoke = slowpoke.(*reflect.PtrType).Elem() + } + ut.indir++ + } + ut.isGobEncoder, ut.encIndir = implementsGobEncoder(ut.user) + ut.isGobDecoder, ut.decIndir = implementsGobDecoder(ut.user) + userTypeCache[rt] = ut + if ut.encIndir != 0 || ut.decIndir != 0 { + // There are checks in lots of other places, but putting this here means we won't even + // attempt to encode/decode this type. + // TODO: make it possible to handle types that are indirect to the implementation, + // such as a structure field of type T when *T implements GobDecoder. + return nil, os.ErrorString("TODO: gob can't handle indirections to GobEncoder/Decoder") + } + return +} + +const ( + gobEncodeMethodName = "GobEncode" + gobDecodeMethodName = "GobDecode" +) + +// implementsGobEncoder reports whether the type implements the interface. It also +// returns the number of indirections required to get to the implementation. +// TODO: when reflection makes it possible, should also be prepared to climb up +// one level if we're not on a pointer (implementation could be on *T for our T). +// That will mean that indir could be < 0, which is sure to cause problems, but +// we ignore them now as indir is always >= 0 now. +func implementsGobEncoder(rt reflect.Type) (implements bool, indir int8) { + if rt == nil { + return + } + // The type might be a pointer, or it might not, and we need to keep + // dereferencing to the base type until we find an implementation. + for { + if rt.NumMethod() > 0 { // avoid allocations etc. unless there's some chance + if _, ok := reflect.MakeZero(rt).Interface().(GobEncoder); ok { + return true, indir + } + } + if p, ok := rt.(*reflect.PtrType); ok { + indir++ + if indir > 100 { // insane number of indirections + return false, 0 + } + rt = p.Elem() + continue + } + break + } + return false, 0 +} + +// implementsGobDecoder reports whether the type implements the interface. It also +// returns the number of indirections required to get to the implementation. +// TODO: see comment on implementsGobEncoder. +func implementsGobDecoder(rt reflect.Type) (implements bool, indir int8) { + if rt == nil { + return + } + // The type might be a pointer, or it might not, and we need to keep + // dereferencing to the base type until we find an implementation. + for { + if rt.NumMethod() > 0 { // avoid allocations etc. unless there's some chance + if _, ok := reflect.MakeZero(rt).Interface().(GobDecoder); ok { + return true, indir + } + } + if p, ok := rt.(*reflect.PtrType); ok { + indir++ + if indir > 100 { // insane number of indirections + return false, 0 + } + rt = p.Elem() + continue + } + break + } + return false, 0 +} +// userType returns, and saves, the information associated with user-provided type rt. +// If the user type is not valid, it calls error. +func userType(rt reflect.Type) *userTypeInfo { + ut, err := validUserType(rt) + if err != nil { + error(err) + } + return ut +} // A typeId represents a gob Type as an integer that can be passed on the wire. // Internally, typeIds are used as keys to a map to recover the underlying type info. type typeId int32 @@ -110,6 +252,7 @@ var ( // Predefined because it's needed by the Decoder var tWireType = mustGetTypeInfo(reflect.Typeof(wireType{})).id +var wireTypeUserInfo *userTypeInfo // userTypeInfo of (*wireType) func init() { // Some magic numbers to make sure there are no surprises. @@ -133,6 +276,7 @@ func init() { } nextId = firstUserId registerBasics() + wireTypeUserInfo = userType(reflect.Typeof((*wireType)(nil))) } // Array type @@ -142,12 +286,18 @@ type arrayType struct { Len int } -func newArrayType(name string, elem gobType, length int) *arrayType { - a := &arrayType{CommonType{Name: name}, elem.id(), length} - setTypeId(a) +func newArrayType(name string) *arrayType { + a := &arrayType{CommonType{Name: name}, 0, 0} return a } +func (a *arrayType) init(elem gobType, len int) { + // Set our type id before evaluating the element's, in case it's our own. + setTypeId(a) + a.Elem = elem.id() + a.Len = len +} + func (a *arrayType) safeString(seen map[typeId]bool) string { if seen[a.Id] { return a.Name @@ -158,6 +308,23 @@ func (a *arrayType) safeString(seen map[typeId]bool) string { func (a *arrayType) string() string { return a.safeString(make(map[typeId]bool)) } +// GobEncoder type (something that implements the GobEncoder interface) +type gobEncoderType struct { + CommonType +} + +func newGobEncoderType(name string) *gobEncoderType { + g := &gobEncoderType{CommonType{Name: name}} + setTypeId(g) + return g +} + +func (g *gobEncoderType) safeString(seen map[typeId]bool) string { + return g.Name +} + +func (g *gobEncoderType) string() string { return g.Name } + // Map type type mapType struct { CommonType @@ -165,12 +332,18 @@ type mapType struct { Elem typeId } -func newMapType(name string, key, elem gobType) *mapType { - m := &mapType{CommonType{Name: name}, key.id(), elem.id()} - setTypeId(m) +func newMapType(name string) *mapType { + m := &mapType{CommonType{Name: name}, 0, 0} return m } +func (m *mapType) init(key, elem gobType) { + // Set our type id before evaluating the element's, in case it's our own. + setTypeId(m) + m.Key = key.id() + m.Elem = elem.id() +} + func (m *mapType) safeString(seen map[typeId]bool) string { if seen[m.Id] { return m.Name @@ -189,12 +362,17 @@ type sliceType struct { Elem typeId } -func newSliceType(name string, elem gobType) *sliceType { - s := &sliceType{CommonType{Name: name}, elem.id()} - setTypeId(s) +func newSliceType(name string) *sliceType { + s := &sliceType{CommonType{Name: name}, 0} return s } +func (s *sliceType) init(elem gobType) { + // Set our type id before evaluating the element's, in case it's our own. + setTypeId(s) + s.Elem = elem.id() +} + func (s *sliceType) safeString(seen map[typeId]bool) string { if seen[s.Id] { return s.Name @@ -236,26 +414,31 @@ func (s *structType) string() string { return s.safeString(make(map[typeId]bool) func newStructType(name string) *structType { s := &structType{CommonType{Name: name}, nil} + // For historical reasons we set the id here rather than init. + // Se the comment in newTypeObject for details. setTypeId(s) return s } -// Step through the indirections on a type to discover the base type. -// Return the base type and the number of indirections. -func indirect(t reflect.Type) (rt reflect.Type, count int) { - rt = t - for { - pt, ok := rt.(*reflect.PtrType) - if !ok { - break - } - rt = pt.Elem() - count++ +// newTypeObject allocates a gobType for the reflection type rt. +// Unless ut represents a GobEncoder, rt should be the base type +// of ut. +// This is only called from the encoding side. The decoding side +// works through typeIds and userTypeInfos alone. +func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os.Error) { + // Does this type implement GobEncoder? + if ut.isGobEncoder { + return newGobEncoderType(name), nil } - return -} - -func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { + var err os.Error + var type0, type1 gobType + defer func() { + if err != nil { + types[rt] = nil, false + } + }() + // Install the top-level type before the subtypes (e.g. struct before + // fields) so recursive types can be constructed safely. switch t := rt.(type) { // All basic types are easy: they are predefined. case *reflect.BoolType: @@ -280,57 +463,73 @@ func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { return tInterface.gobType(), nil case *reflect.ArrayType: - gt, err := getType("", t.Elem()) + at := newArrayType(name) + types[rt] = at + type0, err = getBaseType("", t.Elem()) if err != nil { return nil, err } - return newArrayType(name, gt, t.Len()), nil + // Historical aside: + // For arrays, maps, and slices, we set the type id after the elements + // are constructed. This is to retain the order of type id allocation after + // a fix made to handle recursive types, which changed the order in + // which types are built. Delaying the setting in this way preserves + // type ids while allowing recursive types to be described. Structs, + // done below, were already handling recursion correctly so they + // assign the top-level id before those of the field. + at.init(type0, t.Len()) + return at, nil case *reflect.MapType: - kt, err := getType("", t.Key()) + mt := newMapType(name) + types[rt] = mt + type0, err = getBaseType("", t.Key()) if err != nil { return nil, err } - vt, err := getType("", t.Elem()) + type1, err = getBaseType("", t.Elem()) if err != nil { return nil, err } - return newMapType(name, kt, vt), nil + mt.init(type0, type1) + return mt, nil case *reflect.SliceType: // []byte == []uint8 is a special case if t.Elem().Kind() == reflect.Uint8 { return tBytes.gobType(), nil } - gt, err := getType(t.Elem().Name(), t.Elem()) + st := newSliceType(name) + types[rt] = st + type0, err = getBaseType(t.Elem().Name(), t.Elem()) if err != nil { return nil, err } - return newSliceType(name, gt), nil + st.init(type0) + return st, nil case *reflect.StructType: - // Install the struct type itself before the fields so recursive - // structures can be constructed safely. - strType := newStructType(name) - types[rt] = strType - idToType[strType.id()] = strType - field := make([]*fieldType, t.NumField()) + st := newStructType(name) + types[rt] = st + idToType[st.id()] = st for i := 0; i < t.NumField(); i++ { f := t.Field(i) - typ, _ := indirect(f.Type) + if !isExported(f.Name) { + continue + } + typ := userType(f.Type).base tname := typ.Name() if tname == "" { - t, _ := indirect(f.Type) + t := userType(f.Type).base tname = t.String() } - gt, err := getType(tname, f.Type) + gt, err := getBaseType(tname, f.Type) if err != nil { return nil, err } - field[i] = &fieldType{f.Name, gt.id()} + st.Field = append(st.Field, &fieldType{f.Name, gt.id()}) } - strType.Field = field - return strType, nil + return st, nil default: return nil, os.ErrorString("gob NewTypeObject can't handle type: " + rt.String()) @@ -338,15 +537,30 @@ func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { return nil, nil } +// isExported reports whether this is an exported - upper case - name. +func isExported(name string) bool { + rune, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(rune) +} + +// getBaseType returns the Gob type describing the given reflect.Type's base type. +// typeLock must be held. +func getBaseType(name string, rt reflect.Type) (gobType, os.Error) { + ut := userType(rt) + return getType(name, ut, ut.base) +} + // getType returns the Gob type describing the given reflect.Type. +// Should be called only when handling GobEncoders/Decoders, +// which may be pointers. All other types are handled through the +// base type, never a pointer. // typeLock must be held. -func getType(name string, rt reflect.Type) (gobType, os.Error) { - rt, _ = indirect(rt) +func getType(name string, ut *userTypeInfo, rt reflect.Type) (gobType, os.Error) { typ, present := types[rt] if present { return typ, nil } - typ, err := newTypeObject(name, rt) + typ, err := newTypeObject(name, ut, rt) if err == nil { types[rt] = typ } @@ -371,6 +585,7 @@ func bootstrapType(name string, e interface{}, expect typeId) typeId { types[rt] = typ setTypeId(typ) checkId(expect, nextId) + userType(rt) // might as well cache it now return nextId } @@ -381,15 +596,16 @@ func bootstrapType(name string, e interface{}, expect typeId) typeId { // For bootstrapping purposes, we assume that the recipient knows how // to decode a wireType; it is exactly the wireType struct here, interpreted // using the gob rules for sending a structure, except that we assume the -// ids for wireType and structType are known. The relevant pieces +// ids for wireType and structType etc. are known. The relevant pieces // are built in encode.go's init() function. // To maintain binary compatibility, if you extend this type, always put // the new fields last. type wireType struct { - ArrayT *arrayType - SliceT *sliceType - StructT *structType - MapT *mapType + ArrayT *arrayType + SliceT *sliceType + StructT *structType + MapT *mapType + GobEncoderT *gobEncoderType } func (w *wireType) string() string { @@ -406,6 +622,8 @@ func (w *wireType) string() string { return w.StructT.Name case w.MapT != nil: return w.MapT.Name + case w.GobEncoderT != nil: + return w.GobEncoderT.Name } return unknown } @@ -418,49 +636,96 @@ type typeInfo struct { var typeInfoMap = make(map[reflect.Type]*typeInfo) // protected by typeLock -// The reflection type must have all its indirections processed out. // typeLock must be held. -func getTypeInfo(rt reflect.Type) (*typeInfo, os.Error) { - if rt.Kind() == reflect.Ptr { - panic("pointer type in getTypeInfo: " + rt.String()) +func getTypeInfo(ut *userTypeInfo) (*typeInfo, os.Error) { + rt := ut.base + if ut.isGobEncoder { + // We want the user type, not the base type. + rt = ut.user } info, ok := typeInfoMap[rt] - if !ok { - info = new(typeInfo) - name := rt.Name() - gt, err := getType(name, rt) + if ok { + return info, nil + } + info = new(typeInfo) + gt, err := getBaseType(rt.Name(), rt) + if err != nil { + return nil, err + } + info.id = gt.id() + + if ut.isGobEncoder { + userType, err := getType(rt.Name(), ut, rt) if err != nil { return nil, err } - info.id = gt.id() - t := info.id.gobType() - switch typ := rt.(type) { - case *reflect.ArrayType: - info.wire = &wireType{ArrayT: t.(*arrayType)} - case *reflect.MapType: - info.wire = &wireType{MapT: t.(*mapType)} - case *reflect.SliceType: - // []byte == []uint8 is a special case handled separately - if typ.Elem().Kind() != reflect.Uint8 { - info.wire = &wireType{SliceT: t.(*sliceType)} - } - case *reflect.StructType: - info.wire = &wireType{StructT: t.(*structType)} + info.wire = &wireType{GobEncoderT: userType.id().gobType().(*gobEncoderType)} + typeInfoMap[ut.user] = info + return info, nil + } + + t := info.id.gobType() + switch typ := rt.(type) { + case *reflect.ArrayType: + info.wire = &wireType{ArrayT: t.(*arrayType)} + case *reflect.MapType: + info.wire = &wireType{MapT: t.(*mapType)} + case *reflect.SliceType: + // []byte == []uint8 is a special case handled separately + if typ.Elem().Kind() != reflect.Uint8 { + info.wire = &wireType{SliceT: t.(*sliceType)} } - typeInfoMap[rt] = info + case *reflect.StructType: + info.wire = &wireType{StructT: t.(*structType)} } + typeInfoMap[rt] = info return info, nil } // Called only when a panic is acceptable and unexpected. func mustGetTypeInfo(rt reflect.Type) *typeInfo { - t, err := getTypeInfo(rt) + t, err := getTypeInfo(userType(rt)) if err != nil { panic("getTypeInfo: " + err.String()) } return t } +// GobEncoder is the interface describing data that provides its own +// representation for encoding values for transmission to a GobDecoder. +// A type that implements GobEncoder and GobDecoder has complete +// control over the representation of its data and may therefore +// contain things such as private fields, channels, and functions, +// which are not usually transmissable in gob streams. +// +// Note: Since gobs can be stored permanently, It is good design +// to guarantee the encoding used by a GobEncoder is stable as the +// software evolves. For instance, it might make sense for GobEncode +// to include a version number in the encoding. +// +// Note: At the moment, the type implementing GobEncoder must +// be exactly the type passed to Encode. For example, if *T implements +// GobEncoder, the data item must be of type *T, not T or **T. +type GobEncoder interface { + // GobEncode returns a byte slice representing the encoding of the + // receiver for transmission to a GobDecoder, usually of the same + // concrete type. + GobEncode() ([]byte, os.Error) +} + +// GobDecoder is the interface describing data that provides its own +// routine for decoding transmitted values sent by a GobEncoder. +// +// Note: At the moment, the type implementing GobDecoder must +// be exactly the type passed to Decode. For example, if *T implements +// GobDecoder, the data item must be of type *T, not T or **T. +type GobDecoder interface { + // GobDecode overwrites the receiver, which must be a pointer, + // with the value represented by the byte slice, which was written + // by GobEncode, usually for the same concrete type. + GobDecode([]byte) os.Error +} + var ( nameToConcreteType = make(map[string]reflect.Type) concreteTypeToName = make(map[reflect.Type]string) @@ -473,18 +738,18 @@ func RegisterName(name string, value interface{}) { // reserved for nil panic("attempt to register empty name") } - rt, _ := indirect(reflect.Typeof(value)) + base := userType(reflect.Typeof(value)).base // Check for incompatible duplicates. - if t, ok := nameToConcreteType[name]; ok && t != rt { + if t, ok := nameToConcreteType[name]; ok && t != base { panic("gob: registering duplicate types for " + name) } - if n, ok := concreteTypeToName[rt]; ok && n != name { - panic("gob: registering duplicate names for " + rt.String()) + if n, ok := concreteTypeToName[base]; ok && n != name { + panic("gob: registering duplicate names for " + base.String()) } // Store the name and type provided by the user.... nameToConcreteType[name] = reflect.Typeof(value) // but the flattened type in the type table, since that's what decode needs. - concreteTypeToName[rt] = name + concreteTypeToName[base] = name } // Register records a type, identified by a value for that type, under its diff --git a/src/pkg/gob/type_test.go b/src/pkg/gob/type_test.go index 5aecde103..ffd1345e5 100644 --- a/src/pkg/gob/type_test.go +++ b/src/pkg/gob/type_test.go @@ -26,7 +26,7 @@ var basicTypes = []typeT{ func getTypeUnlocked(name string, rt reflect.Type) gobType { typeLock.Lock() defer typeLock.Unlock() - t, err := getType(name, rt) + t, err := getBaseType(name, rt) if err != nil { panic("getTypeUnlocked: " + err.String()) } @@ -126,27 +126,27 @@ func TestMapType(t *testing.T) { } type Bar struct { - x string + X string } // This structure has pointers and refers to itself, making it a good test case. type Foo struct { - a int - b int32 // will become int - c string - d []byte - e *float64 // will become float64 - f ****float64 // will become float64 - g *Bar - h *Bar // should not interpolate the definition of Bar again - i *Foo // will not explode + A int + B int32 // will become int + C string + D []byte + E *float64 // will become float64 + F ****float64 // will become float64 + G *Bar + H *Bar // should not interpolate the definition of Bar again + I *Foo // will not explode } func TestStructType(t *testing.T) { sstruct := getTypeUnlocked("Foo", reflect.Typeof(Foo{})) str := sstruct.string() // If we can print it correctly, we built it correctly. - expected := "Foo = struct { a int; b int; c string; d bytes; e float; f float; g Bar = struct { x string; }; h Bar; i Foo; }" + expected := "Foo = struct { A int; B int; C string; D bytes; E float; F float; G Bar = struct { X string; }; H Bar; I Foo; }" if str != expected { t.Errorf("struct printed as %q; expected %q", str, expected) } diff --git a/src/pkg/html/doc.go b/src/pkg/html/doc.go index c5338d078..4f5dee72d 100644 --- a/src/pkg/html/doc.go +++ b/src/pkg/html/doc.go @@ -69,6 +69,9 @@ call to Next. For example, to extract an HTML page's anchor text: } } +A Tokenizer typically skips over HTML comments. To return comment tokens, set +Tokenizer.ReturnComments to true before looping over calls to Next. + Parsing is done by calling Parse with an io.Reader, which returns the root of the parse tree (the document element) as a *Node. It is the caller's responsibility to ensure that the Reader provides UTF-8 encoded HTML. For diff --git a/src/pkg/html/token.go b/src/pkg/html/token.go index d63883850..ad03241ed 100644 --- a/src/pkg/html/token.go +++ b/src/pkg/html/token.go @@ -25,6 +25,8 @@ const ( EndTagToken // A SelfClosingTagToken tag looks like <br/>. SelfClosingTagToken + // A CommentToken looks like <!--x-->. + CommentToken ) // String returns a string representation of the TokenType. @@ -40,6 +42,8 @@ func (t TokenType) String() string { return "EndTag" case SelfClosingTagToken: return "SelfClosingTag" + case CommentToken: + return "Comment" } return "Invalid(" + strconv.Itoa(int(t)) + ")" } @@ -52,8 +56,8 @@ type Attribute struct { } // A Token consists of a TokenType and some Data (tag name for start and end -// tags, content for text). A tag Token may also contain a slice of Attributes. -// Data is unescaped for both tag and text Tokens (it looks like "a<b" rather +// tags, content for text and comments). A tag Token may also contain a slice +// of Attributes. Data is unescaped for all Tokens (it looks like "a<b" rather // than "a<b"). type Token struct { Type TokenType @@ -91,12 +95,18 @@ func (t Token) String() string { return "</" + t.tagString() + ">" case SelfClosingTagToken: return "<" + t.tagString() + "/>" + case CommentToken: + return "<!--" + EscapeString(t.Data) + "-->" } return "Invalid(" + strconv.Itoa(int(t.Type)) + ")" } // A Tokenizer returns a stream of HTML Tokens. type Tokenizer struct { + // If ReturnComments is set, Next returns comment tokens; + // otherwise it skips over comments (default). + ReturnComments bool + // r is the source of the HTML text. r io.Reader // tt is the TokenType of the most recently read token. If tt == Error @@ -176,6 +186,39 @@ func (z *Tokenizer) readTo(x uint8) os.Error { panic("unreachable") } +// nextMarkupDeclaration returns the next TokenType starting with "<!". +func (z *Tokenizer) nextMarkupDeclaration() (TokenType, os.Error) { + // TODO: check for <!DOCTYPE ... >, don't just assume that it's a comment. + for i := 0; i < 2; i++ { + c, err := z.readByte() + if err != nil { + return TextToken, err + } + if c != '-' { + return z.nextText(), nil + } + } + // <!--> is a valid comment. + for dashCount := 2; ; { + c, err := z.readByte() + if err != nil { + return TextToken, err + } + switch c { + case '-': + dashCount++ + case '>': + if dashCount >= 2 { + return CommentToken, nil + } + fallthrough + default: + dashCount = 0 + } + } + panic("unreachable") +} + // nextTag returns the next TokenType starting from the tag open state. func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) { c, err := z.readByte() @@ -189,7 +232,7 @@ func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) { case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': tt = StartTagToken case c == '!': - return ErrorToken, os.NewError("html: TODO(nigeltao): implement comments") + return z.nextMarkupDeclaration() case c == '?': return ErrorToken, os.NewError("html: TODO(nigeltao): implement XML processing instructions") default: @@ -221,22 +264,8 @@ func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) { panic("unreachable") } -// Next scans the next token and returns its type. -func (z *Tokenizer) Next() TokenType { - if z.err != nil { - z.tt = ErrorToken - return z.tt - } - z.p0 = z.p1 - c, err := z.readByte() - if err != nil { - z.tt, z.err = ErrorToken, err - return z.tt - } - if c == '<' { - z.tt, z.err = z.nextTag() - return z.tt - } +// nextText reads all text up until an '<'. +func (z *Tokenizer) nextText() TokenType { for { c, err := z.readByte() if err != nil { @@ -255,6 +284,31 @@ func (z *Tokenizer) Next() TokenType { panic("unreachable") } +// Next scans the next token and returns its type. +func (z *Tokenizer) Next() TokenType { + for { + if z.err != nil { + z.tt = ErrorToken + return z.tt + } + z.p0 = z.p1 + c, err := z.readByte() + if err != nil { + z.tt, z.err = ErrorToken, err + return z.tt + } + if c == '<' { + z.tt, z.err = z.nextTag() + if z.tt == CommentToken && !z.ReturnComments { + continue + } + return z.tt + } + return z.nextText() + } + panic("unreachable") +} + // trim returns the largest j such that z.buf[i:j] contains only white space, // or only white space plus the final ">" or "/>" of the raw data. func (z *Tokenizer) trim(i int) int { @@ -299,18 +353,33 @@ loop: return z.buf[i0:i], z.trim(i) } -// Text returns the raw data after unescaping. +// Text returns the unescaped text of a TextToken or a CommentToken. // The contents of the returned slice may change on the next call to Next. func (z *Tokenizer) Text() []byte { - s := unescape(z.Raw()) - z.p0 = z.p1 - return s + switch z.tt { + case TextToken: + s := unescape(z.Raw()) + z.p0 = z.p1 + return s + case CommentToken: + // We trim the "<!--" from the left and the "-->" from the right. + // "<!-->" is a valid comment, so the adjusted endpoints might overlap. + i0 := z.p0 + 4 + i1 := z.p1 - 3 + z.p0 = z.p1 + var s []byte + if i0 < i1 { + s = unescape(z.buf[i0:i1]) + } + return s + } + return nil } // TagName returns the lower-cased name of a tag token (the `img` out of -// `<IMG SRC="foo">`), and whether the tag has attributes. +// `<IMG SRC="foo">`) and whether the tag has attributes. // The contents of the returned slice may change on the next call to Next. -func (z *Tokenizer) TagName() (name []byte, remaining bool) { +func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { i := z.p0 + 1 if i >= z.p1 { z.p0 = z.p1 @@ -320,14 +389,14 @@ func (z *Tokenizer) TagName() (name []byte, remaining bool) { i++ } name, z.p0 = z.lower(i) - remaining = z.p0 != z.p1 + hasAttr = z.p0 != z.p1 return } // TagAttr returns the lower-cased key and unescaped value of the next unparsed -// attribute for the current tag token, and whether there are more attributes. +// attribute for the current tag token and whether there are more attributes. // The contents of the returned slices may change on the next call to Next. -func (z *Tokenizer) TagAttr() (key, val []byte, remaining bool) { +func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { key, i := z.lower(z.p0) // Get past the "=\"". if i == z.p1 || z.buf[i] != '=' { @@ -363,7 +432,7 @@ loop: } } val, z.p0 = z.buf[i:dst], z.trim(src) - remaining = z.p0 != z.p1 + moreAttr = z.p0 != z.p1 return } @@ -372,14 +441,14 @@ loop: func (z *Tokenizer) Token() Token { t := Token{Type: z.tt} switch z.tt { - case TextToken: + case TextToken, CommentToken: t.Data = string(z.Text()) case StartTagToken, EndTagToken, SelfClosingTagToken: var attr []Attribute - name, remaining := z.TagName() - for remaining { + name, moreAttr := z.TagName() + for moreAttr { var key, val []byte - key, val, remaining = z.TagAttr() + key, val, moreAttr = z.TagAttr() attr = append(attr, Attribute{string(key), string(val)}) } t.Data = string(name) diff --git a/src/pkg/html/token_test.go b/src/pkg/html/token_test.go index e07999ca5..5cf1f6dac 100644 --- a/src/pkg/html/token_test.go +++ b/src/pkg/html/token_test.go @@ -7,6 +7,7 @@ package html import ( "bytes" "os" + "strings" "testing" ) @@ -15,8 +16,8 @@ type tokenTest struct { desc string // The HTML to parse. html string - // The string representations of the expected tokens. - tokens []string + // The string representations of the expected tokens, joined by '$'. + golden string } var tokenTests = []tokenTest{ @@ -25,61 +26,86 @@ var tokenTests = []tokenTest{ { "text", "foo bar", - []string{ - "foo bar", - }, + "foo bar", }, // An entity. { "entity", "one < two", - []string{ - "one < two", - }, + "one < two", }, // A start, self-closing and end tag. The tokenizer does not care if the start // and end tokens don't match; that is the job of the parser. { "tags", "<a>b<c/>d</e>", - []string{ - "<a>", - "b", - "<c/>", - "d", - "</e>", - }, + "<a>$b$<c/>$d$</e>", + }, + // Comments. + { + "comment0", + "abc<b><!-- skipme --></b>def", + "abc$<b>$</b>$def", + }, + { + "comment1", + "a<!-->z", + "a$z", + }, + { + "comment2", + "a<!--->z", + "a$z", + }, + { + "comment3", + "a<!--x>-->z", + "a$z", + }, + { + "comment4", + "a<!--x->-->z", + "a$z", + }, + { + "comment5", + "a<!>z", + "a$<!>z", + }, + { + "comment6", + "a<!->z", + "a$<!->z", + }, + { + "comment7", + "a<!---<>z", + "a$<!---<>z", + }, + { + "comment8", + "a<!--z", + "a$<!--z", }, // An attribute with a backslash. { "backslash", `<p id="a\"b">`, - []string{ - `<p id="a"b">`, - }, + `<p id="a"b">`, }, // Entities, tag name and attribute key lower-casing, and whitespace // normalization within a tag. { "tricky", "<p \t\n iD=\"a"B\" foo=\"bar\"><EM>te<&;xt</em></p>", - []string{ - `<p id="a"B" foo="bar">`, - "<em>", - "te<&;xt", - "</em>", - "</p>", - }, + `<p id="a"B" foo="bar">$<em>$te<&;xt$</em>$</p>`, }, // A non-existant entity. Tokenizing and converting back to a string should // escape the "&" to become "&". { "noSuchEntity", `<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`, - []string{ - `<a b="c&noSuchEntity;d">`, - "<&alsoDoesntExist;&", - }, + `<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`, }, } @@ -87,7 +113,7 @@ func TestTokenizer(t *testing.T) { loop: for _, tt := range tokenTests { z := NewTokenizer(bytes.NewBuffer([]byte(tt.html))) - for i, s := range tt.tokens { + for i, s := range strings.Split(tt.golden, "$", -1) { if z.Next() == ErrorToken { t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error()) continue loop diff --git a/src/pkg/http/Makefile b/src/pkg/http/Makefile index 7e4f80c28..389b04222 100644 --- a/src/pkg/http/Makefile +++ b/src/pkg/http/Makefile @@ -8,8 +8,10 @@ TARG=http GOFILES=\ chunked.go\ client.go\ + cookie.go\ dump.go\ fs.go\ + header.go\ lex.go\ persist.go\ request.go\ @@ -17,6 +19,7 @@ GOFILES=\ server.go\ status.go\ transfer.go\ + transport.go\ url.go\ include ../../Make.pkg diff --git a/src/pkg/http/cgi/Makefile b/src/pkg/http/cgi/Makefile new file mode 100644 index 000000000..02f6cfc9e --- /dev/null +++ b/src/pkg/http/cgi/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=http/cgi +GOFILES=\ + cgi.go\ + +include ../../../Make.pkg diff --git a/src/pkg/http/cgi/cgi.go b/src/pkg/http/cgi/cgi.go new file mode 100644 index 000000000..dba59efa2 --- /dev/null +++ b/src/pkg/http/cgi/cgi.go @@ -0,0 +1,201 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cgi implements CGI (Common Gateway Interface) as specified +// in RFC 3875. +// +// Note that using CGI means starting a new process to handle each +// request, which is typically less efficient than using a +// long-running server. This package is intended primarily for +// compatibility with existing systems. +package cgi + +import ( + "encoding/line" + "exec" + "fmt" + "http" + "io" + "log" + "os" + "path" + "regexp" + "strconv" + "strings" +) + +var trailingPort = regexp.MustCompile(`:([0-9]+)$`) + +// Handler runs an executable in a subprocess with a CGI environment. +type Handler struct { + Path string // path to the CGI executable + Root string // root URI prefix of handler or empty for "/" + Env []string // extra environment variables to set, if any + Logger *log.Logger // optional log for errors or nil to use log.Print +} + +func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + root := h.Root + if root == "" { + root = "/" + } + + if len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" { + rw.WriteHeader(http.StatusBadRequest) + rw.Write([]byte("Chunked request bodies are not supported by CGI.")) + return + } + + pathInfo := req.URL.Path + if root != "/" && strings.HasPrefix(pathInfo, root) { + pathInfo = pathInfo[len(root):] + } + + port := "80" + if matches := trailingPort.FindStringSubmatch(req.Host); len(matches) != 0 { + port = matches[1] + } + + env := []string{ + "SERVER_SOFTWARE=go", + "SERVER_NAME=" + req.Host, + "HTTP_HOST=" + req.Host, + "GATEWAY_INTERFACE=CGI/1.1", + "REQUEST_METHOD=" + req.Method, + "QUERY_STRING=" + req.URL.RawQuery, + "REQUEST_URI=" + req.URL.RawPath, + "PATH_INFO=" + pathInfo, + "SCRIPT_NAME=" + root, + "SCRIPT_FILENAME=" + h.Path, + "REMOTE_ADDR=" + rw.RemoteAddr(), + "REMOTE_HOST=" + rw.RemoteAddr(), + "SERVER_PORT=" + port, + } + + for k, _ := range req.Header { + k = strings.Map(upperCaseAndUnderscore, k) + env = append(env, "HTTP_"+k+"="+req.Header.Get(k)) + } + + if req.ContentLength > 0 { + env = append(env, fmt.Sprintf("CONTENT_LENGTH=%d", req.ContentLength)) + } + if ctype := req.Header.Get("Content-Type"); ctype != "" { + env = append(env, "CONTENT_TYPE="+ctype) + } + + if h.Env != nil { + env = append(env, h.Env...) + } + + // TODO: use filepath instead of path when available + cwd, pathBase := path.Split(h.Path) + if cwd == "" { + cwd = "." + } + + cmd, err := exec.Run( + pathBase, + []string{h.Path}, + env, + cwd, + exec.Pipe, // stdin + exec.Pipe, // stdout + exec.PassThrough, // stderr (for now) + ) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("CGI error: %v", err) + return + } + defer func() { + cmd.Stdin.Close() + cmd.Stdout.Close() + cmd.Wait(0) // no zombies + }() + + if req.ContentLength != 0 { + go io.Copy(cmd.Stdin, req.Body) + } + + linebody := line.NewReader(cmd.Stdout, 1024) + headers := make(map[string]string) + statusCode := http.StatusOK + for { + line, isPrefix, err := linebody.ReadLine() + if isPrefix { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("CGI: long header line from subprocess.") + return + } + if err == os.EOF { + break + } + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("CGI: error reading headers: %v", err) + return + } + if len(line) == 0 { + break + } + parts := strings.Split(string(line), ":", 2) + if len(parts) < 2 { + h.printf("CGI: bogus header line: %s", string(line)) + continue + } + header, val := parts[0], parts[1] + header = strings.TrimSpace(header) + val = strings.TrimSpace(val) + switch { + case header == "Status": + if len(val) < 3 { + h.printf("CGI: bogus status (short): %q", val) + return + } + code, err := strconv.Atoi(val[0:3]) + if err != nil { + h.printf("CGI: bogus status: %q", val) + h.printf("CGI: line was %q", line) + return + } + statusCode = code + default: + headers[header] = val + } + } + for h, v := range headers { + rw.SetHeader(h, v) + } + rw.WriteHeader(statusCode) + + _, err = io.Copy(rw, linebody) + if err != nil { + h.printf("CGI: copy error: %v", err) + } +} + +func (h *Handler) printf(format string, v ...interface{}) { + if h.Logger != nil { + h.Logger.Printf(format, v...) + } else { + log.Printf(format, v...) + } +} + +func upperCaseAndUnderscore(rune int) int { + switch { + case rune >= 'a' && rune <= 'z': + return rune - ('a' - 'A') + case rune == '-': + return '_' + case rune == '=': + // Maybe not part of the CGI 'spec' but would mess up + // the environment in any case, as Go represents the + // environment as a slice of "key=value" strings. + return '_' + } + // TODO: other transformations in spec or practice? + return rune +} diff --git a/src/pkg/http/cgi/cgi_test.go b/src/pkg/http/cgi/cgi_test.go new file mode 100644 index 000000000..daf9a2cb3 --- /dev/null +++ b/src/pkg/http/cgi/cgi_test.go @@ -0,0 +1,247 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for package cgi + +package cgi + +import ( + "bufio" + "exec" + "fmt" + "http" + "http/httptest" + "os" + "strings" + "testing" +) + +var cgiScriptWorks = canRun("./testdata/test.cgi") + +func canRun(s string) bool { + c, err := exec.Run(s, []string{s}, nil, ".", exec.DevNull, exec.DevNull, exec.DevNull) + if err != nil { + return false + } + w, err := c.Wait(0) + if err != nil { + return false + } + return w.Exited() && w.ExitStatus() == 0 +} + +func newRequest(httpreq string) *http.Request { + buf := bufio.NewReader(strings.NewReader(httpreq)) + req, err := http.ReadRequest(buf) + if err != nil { + panic("cgi: bogus http request in test: " + httpreq) + } + return req +} + +func runCgiTest(t *testing.T, h *Handler, httpreq string, expectedMap map[string]string) *httptest.ResponseRecorder { + rw := httptest.NewRecorder() + req := newRequest(httpreq) + h.ServeHTTP(rw, req) + + // Make a map to hold the test map that the CGI returns. + m := make(map[string]string) +readlines: + for { + line, err := rw.Body.ReadString('\n') + switch { + case err == os.EOF: + break readlines + case err != nil: + t.Fatalf("unexpected error reading from CGI: %v", err) + } + line = strings.TrimRight(line, "\r\n") + split := strings.Split(line, "=", 2) + if len(split) != 2 { + t.Fatalf("Unexpected %d parts from invalid line: %q", len(split), line) + } + m[split[0]] = split[1] + } + + for key, expected := range expectedMap { + if got := m[key]; got != expected { + t.Errorf("for key %q got %q; expected %q", key, got, expected) + } + } + return rw +} + +func skipTest(t *testing.T) bool { + if !cgiScriptWorks { + // No Perl on Windows, needed by test.cgi + // TODO: make the child process be Go, not Perl. + t.Logf("Skipping test: test.cgi failed.") + return true + } + return false +} + + +func TestCGIBasicGet(t *testing.T) { + if skipTest(t) { + return + } + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "test": "Hello CGI", + "param-a": "b", + "param-foo": "bar", + "env-GATEWAY_INTERFACE": "CGI/1.1", + "env-HTTP_HOST": "example.com", + "env-PATH_INFO": "", + "env-QUERY_STRING": "foo=bar&a=b", + "env-REMOTE_ADDR": "1.2.3.4", + "env-REMOTE_HOST": "1.2.3.4", + "env-REQUEST_METHOD": "GET", + "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/test.cgi", + "env-SERVER_NAME": "example.com", + "env-SERVER_PORT": "80", + "env-SERVER_SOFTWARE": "go", + } + replay := runCgiTest(t, h, "GET /test.cgi?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) + + if expected, got := "text/html", replay.Header.Get("Content-Type"); got != expected { + t.Errorf("got a Content-Type of %q; expected %q", got, expected) + } + if expected, got := "X-Test-Value", replay.Header.Get("X-Test-Header"); got != expected { + t.Errorf("got a X-Test-Header of %q; expected %q", got, expected) + } +} + +func TestCGIBasicGetAbsPath(t *testing.T) { + if skipTest(t) { + return + } + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd error: %v", err) + } + h := &Handler{ + Path: pwd + "/testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", + "env-SCRIPT_FILENAME": pwd + "/testdata/test.cgi", + "env-SCRIPT_NAME": "/test.cgi", + } + runCgiTest(t, h, "GET /test.cgi?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestPathInfo(t *testing.T) { + if skipTest(t) { + return + } + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "param-a": "b", + "env-PATH_INFO": "/extrapath", + "env-QUERY_STRING": "a=b", + "env-REQUEST_URI": "/test.cgi/extrapath?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/test.cgi", + } + runCgiTest(t, h, "GET /test.cgi/extrapath?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestPathInfoDirRoot(t *testing.T) { + if skipTest(t) { + return + } + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/myscript/", + } + expectedMap := map[string]string{ + "env-PATH_INFO": "bar", + "env-QUERY_STRING": "a=b", + "env-REQUEST_URI": "/myscript/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/myscript/", + } + runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestPathInfoNoRoot(t *testing.T) { + if skipTest(t) { + return + } + h := &Handler{ + Path: "testdata/test.cgi", + Root: "", + } + expectedMap := map[string]string{ + "env-PATH_INFO": "/bar", + "env-QUERY_STRING": "a=b", + "env-REQUEST_URI": "/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/", + } + runCgiTest(t, h, "GET /bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestCGIBasicPost(t *testing.T) { + if skipTest(t) { + return + } + postReq := `POST /test.cgi?a=b HTTP/1.0 +Host: example.com +Content-Type: application/x-www-form-urlencoded +Content-Length: 15 + +postfoo=postbar` + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "test": "Hello CGI", + "param-postfoo": "postbar", + "env-REQUEST_METHOD": "POST", + "env-CONTENT_LENGTH": "15", + "env-REQUEST_URI": "/test.cgi?a=b", + } + runCgiTest(t, h, postReq, expectedMap) +} + +func chunk(s string) string { + return fmt.Sprintf("%x\r\n%s\r\n", len(s), s) +} + +// The CGI spec doesn't allow chunked requests. +func TestCGIPostChunked(t *testing.T) { + if skipTest(t) { + return + } + postReq := `POST /test.cgi?a=b HTTP/1.1 +Host: example.com +Content-Type: application/x-www-form-urlencoded +Transfer-Encoding: chunked + +` + chunk("postfoo") + chunk("=") + chunk("postbar") + chunk("") + + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{} + resp := runCgiTest(t, h, postReq, expectedMap) + if got, expected := resp.Code, http.StatusBadRequest; got != expected { + t.Fatalf("Expected %v response code from chunked request body; got %d", + expected, got) + } +} diff --git a/src/pkg/http/cgi/testdata/test.cgi b/src/pkg/http/cgi/testdata/test.cgi new file mode 100755 index 000000000..b931b04c5 --- /dev/null +++ b/src/pkg/http/cgi/testdata/test.cgi @@ -0,0 +1,34 @@ +#!/usr/bin/perl +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +# +# Test script run as a child process under cgi_test.go + +use strict; +use CGI; + +my $q = CGI->new; +my $params = $q->Vars; + +my $NL = "\r\n"; +$NL = "\n" if 1 || $params->{mode} eq "NL"; + +my $p = sub { + print "$_[0]$NL"; +}; + +# With carriage returns +$p->("Content-Type: text/html"); +$p->("X-Test-Header: X-Test-Value"); +$p->(""); + +print "test=Hello CGI\n"; + +foreach my $k (sort keys %$params) { + print "param-$k=$params->{$k}\n"; +} + +foreach my $k (sort keys %ENV) { + print "env-$k=$ENV{$k}\n"; +} diff --git a/src/pkg/http/client.go b/src/pkg/http/client.go index 022f4f124..c24eea581 100644 --- a/src/pkg/http/client.go +++ b/src/pkg/http/client.go @@ -7,18 +7,41 @@ package http import ( - "bufio" "bytes" - "crypto/tls" "encoding/base64" "fmt" "io" - "net" "os" "strconv" "strings" ) +// A Client is an HTTP client. Its zero value (DefaultClient) is a usable client +// that uses DefaultTransport. +// Client is not yet very configurable. +type Client struct { + Transport Transport // if nil, DefaultTransport is used +} + +// DefaultClient is the default Client and is used by Get, Head, and Post. +var DefaultClient = &Client{} + +// Transport is an interface representing the ability to execute a +// single HTTP transaction, obtaining the Response for a given Request. +type Transport interface { + // Do executes a single HTTP transaction, returning the Response for the + // request req. Do should not attempt to interpret the response. + // In particular, Do must return err == nil if it obtained a response, + // regardless of the response's HTTP status code. A non-nil err should + // be reserved for failure to obtain a response. Similarly, Do should + // not attempt to handle higher-level protocol details such as redirects, + // authentication, or cookies. + // + // Transports may modify the request. The request Headers field is + // guaranteed to be initalized. + Do(req *Request) (resp *Response, err os.Error) +} + // Given a string of the form "host", "host:port", or "[ipv6::address]:port", // return true if the string includes a port. func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } @@ -31,67 +54,83 @@ type readClose struct { io.Closer } -// Send issues an HTTP request. Caller should close resp.Body when done reading it. +// matchNoProxy returns true if requests to addr should not use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +func matchNoProxy(addr string) bool { + if len(addr) == 0 { + return false + } + no_proxy := os.Getenv("NO_PROXY") + if len(no_proxy) == 0 { + no_proxy = os.Getenv("no_proxy") + } + if no_proxy == "*" { + return true + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(no_proxy, ",", -1) { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p || (p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:])) { + return true + } + } + return false +} + +// Do sends an HTTP request and returns an HTTP response, following +// policy (e.g. redirects, cookies, auth) as configured on the client. +// +// Callers should close resp.Body when done reading from it. +// +// Generally Get, Post, or PostForm will be used instead of Do. +func (c *Client) Do(req *Request) (resp *Response, err os.Error) { + return send(req, c.Transport) +} + + +// send issues an HTTP request. Caller should close resp.Body when done reading from it. // // TODO: support persistent connections (multiple requests on a single connection). // send() method is nonpublic because, when we refactor the code for persistent // connections, it may no longer make sense to have a method with this signature. -func send(req *Request) (resp *Response, err os.Error) { - if req.URL.Scheme != "http" && req.URL.Scheme != "https" { - return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} +func send(req *Request, t Transport) (resp *Response, err os.Error) { + if t == nil { + t = DefaultTransport + if t == nil { + err = os.NewError("no http.Client.Transport or http.DefaultTransport") + return + } } - addr := req.URL.Host - if !hasPort(addr) { - addr += ":" + req.URL.Scheme + // Most the callers of send (Get, Post, et al) don't need + // Headers, leaving it uninitialized. We guarantee to the + // Transport that this has been initialized, though. + if req.Header == nil { + req.Header = make(Header) } + info := req.URL.RawUserinfo if len(info) > 0 { enc := base64.URLEncoding encoded := make([]byte, enc.EncodedLen(len(info))) enc.Encode(encoded, []byte(info)) if req.Header == nil { - req.Header = make(map[string]string) + req.Header = make(Header) } - req.Header["Authorization"] = "Basic " + string(encoded) - } - - var conn io.ReadWriteCloser - if req.URL.Scheme == "http" { - conn, err = net.Dial("tcp", "", addr) - if err != nil { - return nil, err - } - } else { // https - conn, err = tls.Dial("tcp", "", addr, nil) - if err != nil { - return nil, err - } - h := req.URL.Host - if hasPort(h) { - h = h[0:strings.LastIndex(h, ":")] - } - if err := conn.(*tls.Conn).VerifyHostname(h); err != nil { - return nil, err - } - } - - err = req.Write(conn) - if err != nil { - conn.Close() - return nil, err + req.Header.Set("Authorization", "Basic "+string(encoded)) } - - reader := bufio.NewReader(conn) - resp, err = ReadResponse(reader, req.Method) - if err != nil { - conn.Close() - return nil, err - } - - resp.Body = readClose{resp.Body, conn} - - return + return t.Do(req) } // True if the specified HTTP status code is one for which the Get utility should @@ -115,12 +154,32 @@ func shouldRedirect(statusCode int) bool { // finalURL is the URL from which the response was fetched -- identical to the // input URL unless redirects were followed. // -// Caller should close r.Body when done reading it. +// Caller should close r.Body when done reading from it. +// +// Get is a convenience wrapper around DefaultClient.Get. func Get(url string) (r *Response, finalURL string, err os.Error) { + return DefaultClient.Get(url) +} + +// Get issues a GET to the specified URL. If the response is one of the following +// redirect codes, it follows the redirect, up to a maximum of 10 redirects: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// +// finalURL is the URL from which the response was fetched -- identical to the +// input URL unless redirects were followed. +// +// Caller should close r.Body when done reading from it. +func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) { // TODO: if/when we add cookie support, the redirected request shouldn't // necessarily supply the same cookies as the original. // TODO: set referrer header on redirects. var base *URL + // TODO: remove this hard-coded 10 and use the Client's policy + // (ClientConfig) instead. for redirect := 0; ; redirect++ { if redirect >= 10 { err = os.ErrorString("stopped after 10 redirects") @@ -128,6 +187,9 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { } var req Request + req.Method = "GET" + req.ProtoMajor = 1 + req.ProtoMinor = 1 if base == nil { req.URL, err = ParseURL(url) } else { @@ -137,12 +199,12 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { break } url = req.URL.String() - if r, err = send(&req); err != nil { + if r, err = send(&req, c.Transport); err != nil { break } if shouldRedirect(r.StatusCode) { r.Body.Close() - if url = r.GetHeader("Location"); url == "" { + if url = r.Header.Get("Location"); url == "" { err = os.ErrorString(fmt.Sprintf("%d response missing Location header", r.StatusCode)) break } @@ -159,16 +221,25 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { // Post issues a POST to the specified URL. // -// Caller should close r.Body when done reading it. +// Caller should close r.Body when done reading from it. +// +// Post is a wrapper around DefaultClient.Post func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { + return DefaultClient.Post(url, bodyType, body) +} + +// Post issues a POST to the specified URL. +// +// Caller should close r.Body when done reading from it. +func (c *Client) Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { var req Request req.Method = "POST" req.ProtoMajor = 1 req.ProtoMinor = 1 req.Close = true req.Body = nopCloser{body} - req.Header = map[string]string{ - "Content-Type": bodyType, + req.Header = Header{ + "Content-Type": {bodyType}, } req.TransferEncoding = []string{"chunked"} @@ -177,14 +248,24 @@ func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Erro return nil, err } - return send(&req) + return send(&req, c.Transport) } // PostForm issues a POST to the specified URL, // with data's keys and values urlencoded as the request body. // -// Caller should close r.Body when done reading it. +// Caller should close r.Body when done reading from it. +// +// PostForm is a wrapper around DefaultClient.PostForm func PostForm(url string, data map[string]string) (r *Response, err os.Error) { + return DefaultClient.PostForm(url, data) +} + +// PostForm issues a POST to the specified URL, +// with data's keys and values urlencoded as the request body. +// +// Caller should close r.Body when done reading from it. +func (c *Client) PostForm(url string, data map[string]string) (r *Response, err os.Error) { var req Request req.Method = "POST" req.ProtoMajor = 1 @@ -192,9 +273,9 @@ func PostForm(url string, data map[string]string) (r *Response, err os.Error) { req.Close = true body := urlencode(data) req.Body = nopCloser{body} - req.Header = map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - "Content-Length": strconv.Itoa(body.Len()), + req.Header = Header{ + "Content-Type": {"application/x-www-form-urlencoded"}, + "Content-Length": {strconv.Itoa(body.Len())}, } req.ContentLength = int64(body.Len()) @@ -203,7 +284,7 @@ func PostForm(url string, data map[string]string) (r *Response, err os.Error) { return nil, err } - return send(&req) + return send(&req, c.Transport) } // TODO: remove this function when PostForm takes a multimap. @@ -216,17 +297,20 @@ func urlencode(data map[string]string) (b *bytes.Buffer) { } // Head issues a HEAD to the specified URL. +// +// Head is a wrapper around DefaultClient.Head func Head(url string) (r *Response, err os.Error) { + return DefaultClient.Head(url) +} + +// Head issues a HEAD to the specified URL. +func (c *Client) Head(url string) (r *Response, err os.Error) { var req Request req.Method = "HEAD" if req.URL, err = ParseURL(url); err != nil { return } - url = req.URL.String() - if r, err = send(&req); err != nil { - return - } - return + return send(&req, c.Transport) } type nopCloser struct { diff --git a/src/pkg/http/client_test.go b/src/pkg/http/client_test.go index 013653a82..c89ecbce2 100644 --- a/src/pkg/http/client_test.go +++ b/src/pkg/http/client_test.go @@ -8,6 +8,7 @@ package http import ( "io/ioutil" + "os" "strings" "testing" ) @@ -38,3 +39,28 @@ func TestClientHead(t *testing.T) { t.Error("Last-Modified header not found.") } } + +type recordingTransport struct { + req *Request +} + +func (t *recordingTransport) Do(req *Request) (resp *Response, err os.Error) { + t.req = req + return nil, os.NewError("dummy impl") +} + +func TestGetRequestFormat(t *testing.T) { + tr := &recordingTransport{} + client := &Client{Transport: tr} + url := "http://dummy.faketld/" + client.Get(url) // Note: doesn't hit network + if tr.req.Method != "GET" { + t.Errorf("expected method %q; got %q", "GET", tr.req.Method) + } + if tr.req.URL.String() != url { + t.Errorf("expected URL %q; got %q", url, tr.req.URL.String()) + } + if tr.req.Header == nil { + t.Errorf("expected non-nil request Header") + } +} diff --git a/src/pkg/http/cookie.go b/src/pkg/http/cookie.go new file mode 100644 index 000000000..ff75c47c9 --- /dev/null +++ b/src/pkg/http/cookie.go @@ -0,0 +1,336 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "fmt" + "io" + "os" + "sort" + "strconv" + "strings" + "time" +) + +// A note on Version=0 vs. Version=1 cookies +// +// The difference between Set-Cookie and Set-Cookie2 is hard to discern from the +// RFCs as it is not stated explicitly. There seem to be three standards +// lingering on the web: Netscape, RFC 2109 (aka Version=0) and RFC 2965 (aka +// Version=1). It seems that Netscape and RFC 2109 are the same thing, hereafter +// Version=0 cookies. +// +// In general, Set-Cookie2 is a superset of Set-Cookie. It has a few new +// attributes like HttpOnly and Secure. To be meticulous, if a server intends +// to use these, it needs to send a Set-Cookie2. However, it is most likely +// most modern browsers will not complain seeing an HttpOnly attribute in a +// Set-Cookie header. +// +// Both RFC 2109 and RFC 2965 use Cookie in the same way - two send cookie +// values from clients to servers - and the allowable attributes seem to be the +// same. +// +// The Cookie2 header is used for a different purpose. If a client suspects that +// the server speaks Version=1 (RFC 2965) then along with the Cookie header +// lines, you can also send: +// +// Cookie2: $Version="1" +// +// in order to suggest to the server that you understand Version=1 cookies. At +// which point the server may continue responding with Set-Cookie2 headers. +// When a client sends the (above) Cookie2 header line, it must be prepated to +// understand incoming Set-Cookie2. +// +// This implementation of cookies supports neither Set-Cookie2 nor Cookie2 +// headers. However, it parses Version=1 Cookies (along with Version=0) as well +// as Set-Cookie headers which utilize the full Set-Cookie2 syntax. + +// TODO(petar): Explicitly forbid parsing of Set-Cookie attributes +// starting with '$', which have been used to hack into broken +// servers using the eventual Request headers containing those +// invalid attributes that may overwrite intended $Version, $Path, +// etc. attributes. +// TODO(petar): Read 'Set-Cookie2' headers and prioritize them over equivalent +// 'Set-Cookie' headers. 'Set-Cookie2' headers are still extremely rare. + +// A Cookie represents an RFC 2965 HTTP cookie as sent in +// the Set-Cookie header of an HTTP response or the Cookie header +// of an HTTP request. +// The Set-Cookie2 and Cookie2 headers are unimplemented. +type Cookie struct { + Name string + Value string + Path string + Domain string + Comment string + Version int + Expires time.Time + RawExpires string + MaxAge int // Max age in seconds + Secure bool + HttpOnly bool + Raw string + Unparsed []string // Raw text of unparsed attribute-value pairs +} + +// readSetCookies parses all "Set-Cookie" values from +// the header h, removes the successfully parsed values from the +// "Set-Cookie" key in h and returns the parsed Cookies. +func readSetCookies(h Header) []*Cookie { + cookies := []*Cookie{} + var unparsedLines []string + for _, line := range h["Set-Cookie"] { + parts := strings.Split(strings.TrimSpace(line), ";", -1) + if len(parts) == 1 && parts[0] == "" { + continue + } + parts[0] = strings.TrimSpace(parts[0]) + j := strings.Index(parts[0], "=") + if j < 0 { + unparsedLines = append(unparsedLines, line) + continue + } + name, value := parts[0][:j], parts[0][j+1:] + value, err := URLUnescape(value) + if err != nil { + unparsedLines = append(unparsedLines, line) + continue + } + c := &Cookie{ + Name: name, + Value: value, + MaxAge: -1, // Not specified + Raw: line, + } + for i := 1; i < len(parts); i++ { + parts[i] = strings.TrimSpace(parts[i]) + if len(parts[i]) == 0 { + continue + } + + attr, val := parts[i], "" + if j := strings.Index(attr, "="); j >= 0 { + attr, val = attr[:j], attr[j+1:] + val, err = URLUnescape(val) + if err != nil { + c.Unparsed = append(c.Unparsed, parts[i]) + continue + } + } + switch strings.ToLower(attr) { + case "secure": + c.Secure = true + continue + case "httponly": + c.HttpOnly = true + continue + case "comment": + c.Comment = val + continue + case "domain": + c.Domain = val + // TODO: Add domain parsing + continue + case "max-age": + secs, err := strconv.Atoi(val) + if err != nil || secs < 0 { + break + } + c.MaxAge = secs + continue + case "expires": + c.RawExpires = val + exptime, err := time.Parse(time.RFC1123, val) + if err != nil { + c.Expires = time.Time{} + break + } + c.Expires = *exptime + continue + case "path": + c.Path = val + // TODO: Add path parsing + continue + case "version": + c.Version, err = strconv.Atoi(val) + if err != nil { + c.Version = 0 + break + } + continue + } + c.Unparsed = append(c.Unparsed, parts[i]) + } + cookies = append(cookies, c) + } + h["Set-Cookie"] = unparsedLines, unparsedLines != nil + return cookies +} + +// writeSetCookies writes the wire representation of the set-cookies +// to w. Each cookie is written on a separate "Set-Cookie: " line. +// This choice is made because HTTP parsers tend to have a limit on +// line-length, so it seems safer to place cookies on separate lines. +func writeSetCookies(w io.Writer, kk []*Cookie) os.Error { + if kk == nil { + return nil + } + lines := make([]string, 0, len(kk)) + var b bytes.Buffer + for _, c := range kk { + b.Reset() + // TODO(petar): c.Value (below) should be unquoted if it is recognized as quoted + fmt.Fprintf(&b, "%s=%s", CanonicalHeaderKey(c.Name), c.Value) + if c.Version > 0 { + fmt.Fprintf(&b, "Version=%d; ", c.Version) + } + if len(c.Path) > 0 { + fmt.Fprintf(&b, "; Path=%s", URLEscape(c.Path)) + } + if len(c.Domain) > 0 { + fmt.Fprintf(&b, "; Domain=%s", URLEscape(c.Domain)) + } + if len(c.Expires.Zone) > 0 { + fmt.Fprintf(&b, "; Expires=%s", c.Expires.Format(time.RFC1123)) + } + if c.MaxAge >= 0 { + fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge) + } + if c.HttpOnly { + fmt.Fprintf(&b, "; HttpOnly") + } + if c.Secure { + fmt.Fprintf(&b, "; Secure") + } + if len(c.Comment) > 0 { + fmt.Fprintf(&b, "; Comment=%s", URLEscape(c.Comment)) + } + lines = append(lines, "Set-Cookie: "+b.String()+"\r\n") + } + sort.SortStrings(lines) + for _, l := range lines { + if _, err := io.WriteString(w, l); err != nil { + return err + } + } + return nil +} + +// readCookies parses all "Cookie" values from +// the header h, removes the successfully parsed values from the +// "Cookie" key in h and returns the parsed Cookies. +func readCookies(h Header) []*Cookie { + cookies := []*Cookie{} + lines, ok := h["Cookie"] + if !ok { + return cookies + } + unparsedLines := []string{} + for _, line := range lines { + parts := strings.Split(strings.TrimSpace(line), ";", -1) + if len(parts) == 1 && parts[0] == "" { + continue + } + // Per-line attributes + var lineCookies = make(map[string]string) + var version int + var path string + var domain string + var comment string + var httponly bool + for i := 0; i < len(parts); i++ { + parts[i] = strings.TrimSpace(parts[i]) + if len(parts[i]) == 0 { + continue + } + attr, val := parts[i], "" + var err os.Error + if j := strings.Index(attr, "="); j >= 0 { + attr, val = attr[:j], attr[j+1:] + val, err = URLUnescape(val) + if err != nil { + continue + } + } + switch strings.ToLower(attr) { + case "$httponly": + httponly = true + case "$version": + version, err = strconv.Atoi(val) + if err != nil { + version = 0 + continue + } + case "$domain": + domain = val + // TODO: Add domain parsing + case "$path": + path = val + // TODO: Add path parsing + case "$comment": + comment = val + default: + lineCookies[attr] = val + } + } + if len(lineCookies) == 0 { + unparsedLines = append(unparsedLines, line) + } + for n, v := range lineCookies { + cookies = append(cookies, &Cookie{ + Name: n, + Value: v, + Path: path, + Domain: domain, + Comment: comment, + Version: version, + HttpOnly: httponly, + MaxAge: -1, + Raw: line, + }) + } + } + h["Cookie"] = unparsedLines, len(unparsedLines) > 0 + return cookies +} + +// writeCookies writes the wire representation of the cookies +// to w. Each cookie is written on a separate "Cookie: " line. +// This choice is made because HTTP parsers tend to have a limit on +// line-length, so it seems safer to place cookies on separate lines. +func writeCookies(w io.Writer, kk []*Cookie) os.Error { + lines := make([]string, 0, len(kk)) + var b bytes.Buffer + for _, c := range kk { + b.Reset() + n := c.Name + if c.Version > 0 { + fmt.Fprintf(&b, "$Version=%d; ", c.Version) + } + // TODO(petar): c.Value (below) should be unquoted if it is recognized as quoted + fmt.Fprintf(&b, "%s=%s", CanonicalHeaderKey(n), c.Value) + if len(c.Path) > 0 { + fmt.Fprintf(&b, "; $Path=%s", URLEscape(c.Path)) + } + if len(c.Domain) > 0 { + fmt.Fprintf(&b, "; $Domain=%s", URLEscape(c.Domain)) + } + if c.HttpOnly { + fmt.Fprintf(&b, "; $HttpOnly") + } + if len(c.Comment) > 0 { + fmt.Fprintf(&b, "; $Comment=%s", URLEscape(c.Comment)) + } + lines = append(lines, "Cookie: "+b.String()+"\r\n") + } + sort.SortStrings(lines) + for _, l := range lines { + if _, err := io.WriteString(w, l); err != nil { + return err + } + } + return nil +} diff --git a/src/pkg/http/cookie_test.go b/src/pkg/http/cookie_test.go new file mode 100644 index 000000000..363c841bb --- /dev/null +++ b/src/pkg/http/cookie_test.go @@ -0,0 +1,96 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "reflect" + "testing" +) + + +var writeSetCookiesTests = []struct { + Cookies []*Cookie + Raw string +}{ + { + []*Cookie{&Cookie{Name: "cookie-1", Value: "v$1", MaxAge: -1}}, + "Set-Cookie: Cookie-1=v$1\r\n", + }, +} + +func TestWriteSetCookies(t *testing.T) { + for i, tt := range writeSetCookiesTests { + var w bytes.Buffer + writeSetCookies(&w, tt.Cookies) + seen := string(w.Bytes()) + if seen != tt.Raw { + t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, tt.Raw, seen) + continue + } + } +} + +var writeCookiesTests = []struct { + Cookies []*Cookie + Raw string +}{ + { + []*Cookie{&Cookie{Name: "cookie-1", Value: "v$1", MaxAge: -1}}, + "Cookie: Cookie-1=v$1\r\n", + }, +} + +func TestWriteCookies(t *testing.T) { + for i, tt := range writeCookiesTests { + var w bytes.Buffer + writeCookies(&w, tt.Cookies) + seen := string(w.Bytes()) + if seen != tt.Raw { + t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, tt.Raw, seen) + continue + } + } +} + +var readSetCookiesTests = []struct { + Header Header + Cookies []*Cookie +}{ + { + Header{"Set-Cookie": {"Cookie-1=v$1"}}, + []*Cookie{&Cookie{Name: "Cookie-1", Value: "v$1", MaxAge: -1, Raw: "Cookie-1=v$1"}}, + }, +} + +func TestReadSetCookies(t *testing.T) { + for i, tt := range readSetCookiesTests { + c := readSetCookies(tt.Header) + if !reflect.DeepEqual(c, tt.Cookies) { + t.Errorf("#%d readSetCookies: have\n%#v\nwant\n%#v\n", i, c, tt.Cookies) + continue + } + } +} + +var readCookiesTests = []struct { + Header Header + Cookies []*Cookie +}{ + { + Header{"Cookie": {"Cookie-1=v$1"}}, + []*Cookie{&Cookie{Name: "Cookie-1", Value: "v$1", MaxAge: -1, Raw: "Cookie-1=v$1"}}, + }, +} + +func TestReadCookies(t *testing.T) { + for i, tt := range readCookiesTests { + c := readCookies(tt.Header) + if !reflect.DeepEqual(c, tt.Cookies) { + t.Errorf("#%d readCookies: have\n%#v\nwant\n%#v\n", i, c, tt.Cookies) + continue + } + } +} diff --git a/src/pkg/http/fs.go b/src/pkg/http/fs.go index bbfa58d26..a4cd7072e 100644 --- a/src/pkg/http/fs.go +++ b/src/pkg/http/fs.go @@ -11,7 +11,7 @@ import ( "io" "mime" "os" - "path" + "path/filepath" "strconv" "strings" "time" @@ -104,7 +104,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { } } - if t, _ := time.Parse(TimeFormat, r.Header["If-Modified-Since"]); t != nil && d.Mtime_ns/1e9 <= t.Seconds() { + if t, _ := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); t != nil && d.Mtime_ns/1e9 <= t.Seconds() { w.WriteHeader(StatusNotModified) return } @@ -112,7 +112,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { // use contents of index.html for directory, if present if d.IsDirectory() { - index := name + indexPage + index := name + filepath.FromSlash(indexPage) ff, err := os.Open(index, os.O_RDONLY, 0) if err == nil { defer ff.Close() @@ -135,7 +135,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { code := StatusOK // use extension to find content type. - ext := path.Ext(name) + ext := filepath.Ext(name) if ctype := mime.TypeByExtension(ext); ctype != "" { w.SetHeader("Content-Type", ctype) } else { @@ -153,7 +153,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { // handle Content-Range header. // TODO(adg): handle multiple ranges - ranges, err := parseRange(r.Header["Range"], size) + ranges, err := parseRange(r.Header.Get("Range"), size) if err != nil || len(ranges) > 1 { Error(w, err.String(), StatusRequestedRangeNotSatisfiable) return @@ -202,7 +202,7 @@ func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { return } path = path[len(f.prefix):] - serveFile(w, r, f.root+"/"+path, true) + serveFile(w, r, filepath.Join(f.root, filepath.FromSlash(path)), true) } // httpRange specifies the byte range to be sent to the client. diff --git a/src/pkg/http/fs_test.go b/src/pkg/http/fs_test.go index 0a5636b88..a89c76d0b 100644 --- a/src/pkg/http/fs_test.go +++ b/src/pkg/http/fs_test.go @@ -2,89 +2,22 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package http +package http_test import ( "fmt" + . "http" + "http/httptest" "io/ioutil" - "net" "os" - "sync" "testing" ) -var ParseRangeTests = []struct { - s string - length int64 - r []httpRange -}{ - {"", 0, nil}, - {"foo", 0, nil}, - {"bytes=", 0, nil}, - {"bytes=5-4", 10, nil}, - {"bytes=0-2,5-4", 10, nil}, - {"bytes=0-9", 10, []httpRange{{0, 10}}}, - {"bytes=0-", 10, []httpRange{{0, 10}}}, - {"bytes=5-", 10, []httpRange{{5, 5}}}, - {"bytes=0-20", 10, []httpRange{{0, 10}}}, - {"bytes=15-,0-5", 10, nil}, - {"bytes=-5", 10, []httpRange{{5, 5}}}, - {"bytes=-15", 10, []httpRange{{0, 10}}}, - {"bytes=0-499", 10000, []httpRange{{0, 500}}}, - {"bytes=500-999", 10000, []httpRange{{500, 500}}}, - {"bytes=-500", 10000, []httpRange{{9500, 500}}}, - {"bytes=9500-", 10000, []httpRange{{9500, 500}}}, - {"bytes=0-0,-1", 10000, []httpRange{{0, 1}, {9999, 1}}}, - {"bytes=500-600,601-999", 10000, []httpRange{{500, 101}, {601, 399}}}, - {"bytes=500-700,601-999", 10000, []httpRange{{500, 201}, {601, 399}}}, -} - -func TestParseRange(t *testing.T) { - for _, test := range ParseRangeTests { - r := test.r - ranges, err := parseRange(test.s, test.length) - if err != nil && r != nil { - t.Errorf("parseRange(%q) returned error %q", test.s, err) - } - if len(ranges) != len(r) { - t.Errorf("len(parseRange(%q)) = %d, want %d", test.s, len(ranges), len(r)) - continue - } - for i := range r { - if ranges[i].start != r[i].start { - t.Errorf("parseRange(%q)[%d].start = %d, want %d", test.s, i, ranges[i].start, r[i].start) - } - if ranges[i].length != r[i].length { - t.Errorf("parseRange(%q)[%d].length = %d, want %d", test.s, i, ranges[i].length, r[i].length) - } - } - } -} - const ( testFile = "testdata/file" testFileLength = 11 ) -var ( - serverOnce sync.Once - serverAddr string -) - -func startServer(t *testing.T) { - serverOnce.Do(func() { - HandleFunc("/ServeFile", func(w ResponseWriter, r *Request) { - ServeFile(w, r, "testdata/file") - }) - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("listen:", err) - } - serverAddr = l.Addr().String() - go Serve(l, nil) - }) -} - var ServeFileRangeTests = []struct { start, end int r string @@ -99,7 +32,11 @@ var ServeFileRangeTests = []struct { } func TestServeFile(t *testing.T) { - startServer(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, "testdata/file") + })) + defer ts.Close() + var err os.Error file, err := ioutil.ReadFile(testFile) @@ -109,8 +46,8 @@ func TestServeFile(t *testing.T) { // set up the Request (re-used for all tests) var req Request - req.Header = make(map[string]string) - if req.URL, err = ParseURL("http://" + serverAddr + "/ServeFile"); err != nil { + req.Header = make(Header) + if req.URL, err = ParseURL(ts.URL); err != nil { t.Fatal("ParseURL:", err) } req.Method = "GET" @@ -123,9 +60,9 @@ func TestServeFile(t *testing.T) { // Range tests for _, rt := range ServeFileRangeTests { - req.Header["Range"] = "bytes=" + rt.r + req.Header.Set("Range", "bytes="+rt.r) if rt.r == "" { - req.Header["Range"] = "" + req.Header["Range"] = nil } r, body := getBody(t, req) if r.StatusCode != rt.code { @@ -138,8 +75,9 @@ func TestServeFile(t *testing.T) { if rt.r == "" { h = "" } - if r.Header["Content-Range"] != h { - t.Errorf("header mismatch: range=%q: got %q, want %q", rt.r, r.Header["Content-Range"], h) + cr := r.Header.Get("Content-Range") + if cr != h { + t.Errorf("header mismatch: range=%q: got %q, want %q", rt.r, cr, h) } if !equal(body, file[rt.start:rt.end]) { t.Errorf("body mismatch: range=%q: got %q, want %q", rt.r, body, file[rt.start:rt.end]) @@ -148,7 +86,7 @@ func TestServeFile(t *testing.T) { } func getBody(t *testing.T, req Request) (*Response, []byte) { - r, err := send(&req) + r, err := DefaultClient.Do(&req) if err != nil { t.Fatal(req.URL.String(), "send:", err) } diff --git a/src/pkg/http/header.go b/src/pkg/http/header.go new file mode 100644 index 000000000..95b0f3db6 --- /dev/null +++ b/src/pkg/http/header.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import "net/textproto" + +// A Header represents the key-value pairs in an HTTP header. +type Header map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +func (h Header) Add(key, value string) { + textproto.MIMEHeader(h).Add(key, value) +} + +// Set sets the header entries associated with key to +// the single element value. It replaces any existing +// values associated with key. +func (h Header) Set(key, value string) { + textproto.MIMEHeader(h).Set(key, value) +} + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +// Get is a convenience method. For more complex queries, +// access the map directly. +func (h Header) Get(key string) string { + return textproto.MIMEHeader(h).Get(key) +} + +// Del deletes the values associated with key. +func (h Header) Del(key string) { + textproto.MIMEHeader(h).Del(key) +} + +// CanonicalHeaderKey returns the canonical format of the +// header key s. The canonicalization converts the first +// letter and any letter following a hyphen to upper case; +// the rest are converted to lowercase. For example, the +// canonical key for "accept-encoding" is "Accept-Encoding". +func CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) } diff --git a/src/pkg/http/httptest/Makefile b/src/pkg/http/httptest/Makefile new file mode 100644 index 000000000..eb35d8aec --- /dev/null +++ b/src/pkg/http/httptest/Makefile @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=http/httptest +GOFILES=\ + recorder.go\ + server.go\ + +include ../../../Make.pkg diff --git a/src/pkg/http/httptest/recorder.go b/src/pkg/http/httptest/recorder.go new file mode 100644 index 000000000..ec7bde8aa --- /dev/null +++ b/src/pkg/http/httptest/recorder.go @@ -0,0 +1,79 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The httptest package provides utilities for HTTP testing. +package httptest + +import ( + "bytes" + "http" + "os" +) + +// ResponseRecorder is an implementation of http.ResponseWriter that +// records its mutations for later inspection in tests. +type ResponseRecorder struct { + Code int // the HTTP response code from WriteHeader + Header http.Header // if non-nil, the headers to populate + Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to + Flushed bool + + FakeRemoteAddr string // the fake RemoteAddr to return, or "" for DefaultRemoteAddr + FakeUsingTLS bool // whether to return true from the UsingTLS method +} + +// NewRecorder returns an initialized ResponseRecorder. +func NewRecorder() *ResponseRecorder { + return &ResponseRecorder{ + Header: http.Header(make(map[string][]string)), + Body: new(bytes.Buffer), + } +} + +// DefaultRemoteAddr is the default remote address to return in RemoteAddr if +// an explicit DefaultRemoteAddr isn't set on ResponseRecorder. +const DefaultRemoteAddr = "1.2.3.4" + +// RemoteAddr returns the value of rw.FakeRemoteAddr, if set, else +// returns DefaultRemoteAddr. +func (rw *ResponseRecorder) RemoteAddr() string { + if rw.FakeRemoteAddr != "" { + return rw.FakeRemoteAddr + } + return DefaultRemoteAddr +} + +// UsingTLS returns the fake value in rw.FakeUsingTLS +func (rw *ResponseRecorder) UsingTLS() bool { + return rw.FakeUsingTLS +} + +// SetHeader populates rw.Header, if non-nil. +func (rw *ResponseRecorder) SetHeader(k, v string) { + if rw.Header != nil { + if v == "" { + rw.Header.Del(k) + } else { + rw.Header.Set(k, v) + } + } +} + +// Write always succeeds and writes to rw.Body, if not nil. +func (rw *ResponseRecorder) Write(buf []byte) (int, os.Error) { + if rw.Body != nil { + rw.Body.Write(buf) + } + return len(buf), nil +} + +// WriteHeader sets rw.Code. +func (rw *ResponseRecorder) WriteHeader(code int) { + rw.Code = code +} + +// Flush sets rw.Flushed to true. +func (rw *ResponseRecorder) Flush() { + rw.Flushed = true +} diff --git a/src/pkg/http/httptest/server.go b/src/pkg/http/httptest/server.go new file mode 100644 index 000000000..86c9eb435 --- /dev/null +++ b/src/pkg/http/httptest/server.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implementation of Server + +package httptest + +import ( + "fmt" + "http" + "net" +) + +// A Server is an HTTP server listening on a system-chosen port on the +// local loopback interface, for use in end-to-end HTTP tests. +type Server struct { + URL string // base URL of form http://ipaddr:port with no trailing slash + Listener net.Listener +} + +// NewServer starts and returns a new Server. +// The caller should call Close when finished, to shut it down. +func NewServer(handler http.Handler) *Server { + ts := new(Server) + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) + } + } + ts.Listener = l + ts.URL = "http://" + l.Addr().String() + server := &http.Server{Handler: handler} + go server.Serve(l) + return ts +} + +// Close shuts down the server. +func (s *Server) Close() { + s.Listener.Close() +} diff --git a/src/pkg/http/persist.go b/src/pkg/http/persist.go index 000a4200e..53efd7c8c 100644 --- a/src/pkg/http/persist.go +++ b/src/pkg/http/persist.go @@ -25,15 +25,15 @@ var ( // i.e. requests can be read out of sync (but in the same order) while the // respective responses are sent. type ServerConn struct { + lk sync.Mutex // read-write protects the following fields c net.Conn r *bufio.Reader - clsd bool // indicates a graceful close re, we os.Error // read/write errors lastbody io.ReadCloser nread, nwritten int - pipe textproto.Pipeline pipereq map[*Request]uint - lk sync.Mutex // protected read/write to re,we + + pipe textproto.Pipeline } // NewServerConn returns a new ServerConn reading and writing c. If r is not @@ -90,15 +90,21 @@ func (sc *ServerConn) Read() (req *Request, err os.Error) { defer sc.lk.Unlock() return nil, sc.re } + if sc.r == nil { // connection closed by user in the meantime + defer sc.lk.Unlock() + return nil, os.EBADF + } + r := sc.r + lastbody := sc.lastbody + sc.lastbody = nil sc.lk.Unlock() // Make sure body is fully consumed, even if user does not call body.Close - if sc.lastbody != nil { + if lastbody != nil { // body.Close is assumed to be idempotent and multiple calls to // it should return the error that its first invokation // returned. - err = sc.lastbody.Close() - sc.lastbody = nil + err = lastbody.Close() if err != nil { sc.lk.Lock() defer sc.lk.Unlock() @@ -107,10 +113,10 @@ func (sc *ServerConn) Read() (req *Request, err os.Error) { } } - req, err = ReadRequest(sc.r) + req, err = ReadRequest(r) + sc.lk.Lock() + defer sc.lk.Unlock() if err != nil { - sc.lk.Lock() - defer sc.lk.Unlock() if err == io.ErrUnexpectedEOF { // A close from the opposing client is treated as a // graceful close, even if there was some unparse-able @@ -119,18 +125,16 @@ func (sc *ServerConn) Read() (req *Request, err os.Error) { return nil, sc.re } else { sc.re = err - return + return req, err } } sc.lastbody = req.Body sc.nread++ if req.Close { - sc.lk.Lock() - defer sc.lk.Unlock() sc.re = ErrPersistEOF return req, sc.re } - return + return req, err } // Pending returns the number of unanswered requests @@ -165,24 +169,27 @@ func (sc *ServerConn) Write(req *Request, resp *Response) os.Error { defer sc.lk.Unlock() return sc.we } - sc.lk.Unlock() + if sc.c == nil { // connection closed by user in the meantime + defer sc.lk.Unlock() + return os.EBADF + } + c := sc.c if sc.nread <= sc.nwritten { + defer sc.lk.Unlock() return os.NewError("persist server pipe count") } - if resp.Close { // After signaling a keep-alive close, any pipelined unread // requests will be lost. It is up to the user to drain them // before signaling. - sc.lk.Lock() sc.re = ErrPersistEOF - sc.lk.Unlock() } + sc.lk.Unlock() - err := resp.Write(sc.c) + err := resp.Write(c) + sc.lk.Lock() + defer sc.lk.Unlock() if err != nil { - sc.lk.Lock() - defer sc.lk.Unlock() sc.we = err return err } @@ -196,14 +203,15 @@ func (sc *ServerConn) Write(req *Request, resp *Response) os.Error { // responsible for closing the underlying connection. One must call Close to // regain control of that connection and deal with it as desired. type ClientConn struct { + lk sync.Mutex // read-write protects the following fields c net.Conn r *bufio.Reader re, we os.Error // read/write errors lastbody io.ReadCloser nread, nwritten int - pipe textproto.Pipeline pipereq map[*Request]uint - lk sync.Mutex // protects read/write to re,we,pipereq,etc. + + pipe textproto.Pipeline } // NewClientConn returns a new ClientConn reading and writing c. If r is not @@ -221,11 +229,11 @@ func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { // logic. The user should not call Close while Read or Write is in progress. func (cc *ClientConn) Close() (c net.Conn, r *bufio.Reader) { cc.lk.Lock() + defer cc.lk.Unlock() c = cc.c r = cc.r cc.c = nil cc.r = nil - cc.lk.Unlock() return } @@ -261,20 +269,22 @@ func (cc *ClientConn) Write(req *Request) (err os.Error) { defer cc.lk.Unlock() return cc.we } - cc.lk.Unlock() - + if cc.c == nil { // connection closed by user in the meantime + defer cc.lk.Unlock() + return os.EBADF + } + c := cc.c if req.Close { // We write the EOF to the write-side error, because there // still might be some pipelined reads - cc.lk.Lock() cc.we = ErrPersistEOF - cc.lk.Unlock() } + cc.lk.Unlock() - err = req.Write(cc.c) + err = req.Write(c) + cc.lk.Lock() + defer cc.lk.Unlock() if err != nil { - cc.lk.Lock() - defer cc.lk.Unlock() cc.we = err return err } @@ -316,15 +326,21 @@ func (cc *ClientConn) Read(req *Request) (resp *Response, err os.Error) { defer cc.lk.Unlock() return nil, cc.re } + if cc.r == nil { // connection closed by user in the meantime + defer cc.lk.Unlock() + return nil, os.EBADF + } + r := cc.r + lastbody := cc.lastbody + cc.lastbody = nil cc.lk.Unlock() // Make sure body is fully consumed, even if user does not call body.Close - if cc.lastbody != nil { + if lastbody != nil { // body.Close is assumed to be idempotent and multiple calls to // it should return the error that its first invokation // returned. - err = cc.lastbody.Close() - cc.lastbody = nil + err = lastbody.Close() if err != nil { cc.lk.Lock() defer cc.lk.Unlock() @@ -333,24 +349,22 @@ func (cc *ClientConn) Read(req *Request) (resp *Response, err os.Error) { } } - resp, err = ReadResponse(cc.r, req.Method) + resp, err = ReadResponse(r, req.Method) + cc.lk.Lock() + defer cc.lk.Unlock() if err != nil { - cc.lk.Lock() - defer cc.lk.Unlock() cc.re = err - return + return resp, err } cc.lastbody = resp.Body cc.nread++ if resp.Close { - cc.lk.Lock() - defer cc.lk.Unlock() cc.re = ErrPersistEOF // don't send any more requests return resp, cc.re } - return + return resp, err } // Do is convenience method that writes a request and reads a response. diff --git a/src/pkg/http/proxy_test.go b/src/pkg/http/proxy_test.go new file mode 100644 index 000000000..0f2ca458f --- /dev/null +++ b/src/pkg/http/proxy_test.go @@ -0,0 +1,45 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "os" + "testing" +) + +// TODO(mattn): +// test ProxyAuth + +var MatchNoProxyTests = []struct { + host string + match bool +}{ + {"localhost", true}, // match completely + {"barbaz.net", true}, // match as .barbaz.net + {"foobar.com:443", true}, // have a port but match + {"foofoobar.com", false}, // not match as a part of foobar.com + {"baz.com", false}, // not match as a part of barbaz.com + {"localhost.net", false}, // not match as suffix of address + {"local.localhost", false}, // not match as prefix as address + {"barbarbaz.net", false}, // not match because NO_PROXY have a '.' + {"www.foobar.com", false}, // not match because NO_PROXY is not .foobar.com +} + +func TestMatchNoProxy(t *testing.T) { + oldenv := os.Getenv("NO_PROXY") + no_proxy := "foobar.com, .barbaz.net , localhost" + os.Setenv("NO_PROXY", no_proxy) + defer os.Setenv("NO_PROXY", oldenv) + + for _, test := range MatchNoProxyTests { + if matchNoProxy(test.host) != test.match { + if test.match { + t.Errorf("matchNoProxy(%v) = %v, want %v", test.host, !test.match, test.match) + } else { + t.Errorf("not expected: '%s' shouldn't match as '%s'", test.host, no_proxy) + } + } + } +} diff --git a/src/pkg/http/range_test.go b/src/pkg/http/range_test.go new file mode 100644 index 000000000..5274a81fa --- /dev/null +++ b/src/pkg/http/range_test.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "testing" +) + +var ParseRangeTests = []struct { + s string + length int64 + r []httpRange +}{ + {"", 0, nil}, + {"foo", 0, nil}, + {"bytes=", 0, nil}, + {"bytes=5-4", 10, nil}, + {"bytes=0-2,5-4", 10, nil}, + {"bytes=0-9", 10, []httpRange{{0, 10}}}, + {"bytes=0-", 10, []httpRange{{0, 10}}}, + {"bytes=5-", 10, []httpRange{{5, 5}}}, + {"bytes=0-20", 10, []httpRange{{0, 10}}}, + {"bytes=15-,0-5", 10, nil}, + {"bytes=-5", 10, []httpRange{{5, 5}}}, + {"bytes=-15", 10, []httpRange{{0, 10}}}, + {"bytes=0-499", 10000, []httpRange{{0, 500}}}, + {"bytes=500-999", 10000, []httpRange{{500, 500}}}, + {"bytes=-500", 10000, []httpRange{{9500, 500}}}, + {"bytes=9500-", 10000, []httpRange{{9500, 500}}}, + {"bytes=0-0,-1", 10000, []httpRange{{0, 1}, {9999, 1}}}, + {"bytes=500-600,601-999", 10000, []httpRange{{500, 101}, {601, 399}}}, + {"bytes=500-700,601-999", 10000, []httpRange{{500, 201}, {601, 399}}}, +} + +func TestParseRange(t *testing.T) { + for _, test := range ParseRangeTests { + r := test.r + ranges, err := parseRange(test.s, test.length) + if err != nil && r != nil { + t.Errorf("parseRange(%q) returned error %q", test.s, err) + } + if len(ranges) != len(r) { + t.Errorf("len(parseRange(%q)) = %d, want %d", test.s, len(ranges), len(r)) + continue + } + for i := range r { + if ranges[i].start != r[i].start { + t.Errorf("parseRange(%q)[%d].start = %d, want %d", test.s, i, ranges[i].start, r[i].start) + } + if ranges[i].length != r[i].length { + t.Errorf("parseRange(%q)[%d].length = %d, want %d", test.s, i, ranges[i].length, r[i].length) + } + } + } +} diff --git a/src/pkg/http/readrequest_test.go b/src/pkg/http/readrequest_test.go index 5e1cbcbcb..19e2ff774 100644 --- a/src/pkg/http/readrequest_test.go +++ b/src/pkg/http/readrequest_test.go @@ -50,14 +50,14 @@ var reqTests = []reqTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip,deflate", - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Keep-Alive": "300", - "Proxy-Connection": "keep-alive", - "Content-Length": "7", + Header: Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "Accept-Language": {"en-us,en;q=0.5"}, + "Accept-Encoding": {"gzip,deflate"}, + "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, + "Keep-Alive": {"300"}, + "Proxy-Connection": {"keep-alive"}, + "Content-Length": {"7"}, }, Close: false, ContentLength: 7, @@ -93,7 +93,7 @@ var reqTests = []reqTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{}, + Header: Header{}, Close: false, ContentLength: -1, Host: "test", diff --git a/src/pkg/http/request.go b/src/pkg/http/request.go index 04bebaaf5..d8456bab3 100644 --- a/src/pkg/http/request.go +++ b/src/pkg/http/request.go @@ -11,13 +11,13 @@ package http import ( "bufio" - "bytes" "container/vector" "fmt" "io" "io/ioutil" "mime" "mime/multipart" + "net/textproto" "os" "strconv" "strings" @@ -90,7 +90,10 @@ type Request struct { // The request parser implements this by canonicalizing the // name, making the first character and any characters // following a hyphen uppercase and the rest lowercase. - Header map[string]string + Header Header + + // Cookie records the HTTP cookies sent with the request. + Cookie []*Cookie // The message body. Body io.ReadCloser @@ -133,7 +136,7 @@ type Request struct { // Trailer maps trailer keys to values. Like for Header, if the // response has multiple trailer lines with the same key, they will be // concatenated, delimited by commas. - Trailer map[string]string + Trailer Header } // ProtoAtLeast returns whether the HTTP protocol used @@ -146,8 +149,8 @@ func (r *Request) ProtoAtLeast(major, minor int) bool { // MultipartReader returns a MIME multipart reader if this is a // multipart/form-data POST request, else returns nil and an error. func (r *Request) MultipartReader() (multipart.Reader, os.Error) { - v, ok := r.Header["Content-Type"] - if !ok { + v := r.Header.Get("Content-Type") + if v == "" { return nil, ErrNotMultipart } d, params := mime.ParseMediaType(v) @@ -184,6 +187,19 @@ const defaultUserAgent = "Go http package" // If Body is present, Write forces "Transfer-Encoding: chunked" as a header // and then closes Body when finished sending it. func (req *Request) Write(w io.Writer) os.Error { + return req.write(w, false) +} + +// WriteProxy is like Write but writes the request in the form +// expected by an HTTP proxy. It includes the scheme and host +// name in the URI instead of using a separate Host: header line. +// If req.RawURL is non-empty, WriteProxy uses it unchanged +// instead of URL but still omits the Host: header. +func (req *Request) WriteProxy(w io.Writer) os.Error { + return req.write(w, true) +} + +func (req *Request) write(w io.Writer, usingProxy bool) os.Error { host := req.Host if host == "" { host = req.URL.Host @@ -195,12 +211,20 @@ func (req *Request) Write(w io.Writer) os.Error { if req.URL.RawQuery != "" { uri += "?" + req.URL.RawQuery } + if usingProxy { + if uri == "" || uri[0] != '/' { + uri = "/" + uri + } + uri = req.URL.Scheme + "://" + host + uri + } } fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), uri) // Header lines - fmt.Fprintf(w, "Host: %s\r\n", host) + if !usingProxy { + fmt.Fprintf(w, "Host: %s\r\n", host) + } fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent)) if req.Referer != "" { fmt.Fprintf(w, "Referer: %s\r\n", req.Referer) @@ -223,11 +247,15 @@ func (req *Request) Write(w io.Writer) os.Error { // from Request, and introduce Request methods along the lines of // Response.{GetHeader,AddHeader} and string constants for "Host", // "User-Agent" and "Referer". - err = writeSortedKeyValue(w, req.Header, reqExcludeHeader) + err = writeSortedHeader(w, req.Header, reqExcludeHeader) if err != nil { return err } + if err = writeCookies(w, req.Cookie); err != nil { + return err + } + io.WriteString(w, "\r\n") // Write body and trailer @@ -277,78 +305,6 @@ func readLine(b *bufio.Reader) (s string, err os.Error) { return string(p), nil } -var colon = []byte{':'} - -// Read a key/value pair from b. -// A key/value has the form Key: Value\r\n -// and the Value can continue on multiple lines if each continuation line -// starts with a space. -func readKeyValue(b *bufio.Reader) (key, value string, err os.Error) { - line, e := readLineBytes(b) - if e != nil { - return "", "", e - } - if len(line) == 0 { - return "", "", nil - } - - // Scan first line for colon. - i := bytes.Index(line, colon) - if i < 0 { - goto Malformed - } - - key = string(line[0:i]) - if strings.Contains(key, " ") { - // Key field has space - no good. - goto Malformed - } - - // Skip initial space before value. - for i++; i < len(line); i++ { - if line[i] != ' ' { - break - } - } - value = string(line[i:]) - - // Look for extension lines, which must begin with space. - for { - c, e := b.ReadByte() - if c != ' ' { - if e != os.EOF { - b.UnreadByte() - } - break - } - - // Eat leading space. - for c == ' ' { - if c, e = b.ReadByte(); e != nil { - if e == os.EOF { - e = io.ErrUnexpectedEOF - } - return "", "", e - } - } - b.UnreadByte() - - // Read the rest of the line and add to value. - if line, e = readLineBytes(b); e != nil { - return "", "", e - } - value += " " + string(line) - - if len(value) >= maxValueLength { - return "", "", &badStringError{"value too long for key", key} - } - } - return key, value, nil - -Malformed: - return "", "", &badStringError{"malformed header line", string(line)} -} - // Convert decimal at s[i:len(s)] to integer, // returning value, string position where the digits stopped, // and whether there was a valid number (digits, not too big). @@ -367,8 +323,9 @@ func atoi(s string, i int) (n, i1 int, ok bool) { return n, i, true } -// Parse HTTP version: "HTTP/1.2" -> (1, 2, true). -func parseHTTPVersion(vers string) (int, int, bool) { +// ParseHTTPVersion parses a HTTP version string. +// "HTTP/1.0" returns (1, 0, true). +func ParseHTTPVersion(vers string) (major, minor int, ok bool) { if len(vers) < 5 || vers[0:5] != "HTTP/" { return 0, 0, false } @@ -376,7 +333,6 @@ func parseHTTPVersion(vers string) (int, int, bool) { if !ok || i >= len(vers) || vers[i] != '.' { return 0, 0, false } - var minor int minor, i, ok = atoi(vers, i+1) if !ok || i != len(vers) { return 0, 0, false @@ -384,43 +340,6 @@ func parseHTTPVersion(vers string) (int, int, bool) { return major, minor, true } -// CanonicalHeaderKey returns the canonical format of the -// HTTP header key s. The canonicalization converts the first -// letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the -// canonical key for "accept-encoding" is "Accept-Encoding". -func CanonicalHeaderKey(s string) string { - // canonicalize: first letter upper case - // and upper case after each dash. - // (Host, User-Agent, If-Modified-Since). - // HTTP headers are ASCII only, so no Unicode issues. - var a []byte - upper := true - for i := 0; i < len(s); i++ { - v := s[i] - if upper && 'a' <= v && v <= 'z' { - if a == nil { - a = []byte(s) - } - a[i] = v + 'A' - 'a' - } - if !upper && 'A' <= v && v <= 'Z' { - if a == nil { - a = []byte(s) - } - a[i] = v + 'a' - 'A' - } - upper = false - if v == '-' { - upper = true - } - } - if a != nil { - return string(a) - } - return s -} - type chunkedReader struct { r *bufio.Reader n uint64 // unread bytes in chunk @@ -486,11 +405,16 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err os.Error) { // ReadRequest reads and parses a request from b. func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { + + tp := textproto.NewReader(b) req = new(Request) // First line: GET /index.html HTTP/1.0 var s string - if s, err = readLine(b); err != nil { + if s, err = tp.ReadLine(); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } return nil, err } @@ -500,7 +424,7 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { } req.Method, req.RawURL, req.Proto = f[0], f[1], f[2] var ok bool - if req.ProtoMajor, req.ProtoMinor, ok = parseHTTPVersion(req.Proto); !ok { + if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { return nil, &badStringError{"malformed HTTP version", req.Proto} } @@ -509,32 +433,11 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { } // Subsequent lines: Key: value. - nheader := 0 - req.Header = make(map[string]string) - for { - var key, value string - if key, value, err = readKeyValue(b); err != nil { - return nil, err - } - if key == "" { - break - } - if nheader++; nheader >= maxHeaderLines { - return nil, ErrHeaderTooLong - } - - key = CanonicalHeaderKey(key) - - // RFC 2616 says that if you send the same header key - // multiple times, it has to be semantically equivalent - // to concatenating the values separated by commas. - oldvalue, present := req.Header[key] - if present { - req.Header[key] = oldvalue + "," + value - } else { - req.Header[key] = value - } + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err } + req.Header = Header(mimeHeader) // RFC2616: Must treat // GET /index.html HTTP/1.1 @@ -545,18 +448,18 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { // the same. In the second case, any Host line is ignored. req.Host = req.URL.Host if req.Host == "" { - req.Host = req.Header["Host"] + req.Host = req.Header.Get("Host") } - req.Header["Host"] = "", false + req.Header.Del("Host") fixPragmaCacheControl(req.Header) // Pull out useful fields as a convenience to clients. - req.Referer = req.Header["Referer"] - req.Header["Referer"] = "", false + req.Referer = req.Header.Get("Referer") + req.Header.Del("Referer") - req.UserAgent = req.Header["User-Agent"] - req.Header["User-Agent"] = "", false + req.UserAgent = req.Header.Get("User-Agent") + req.Header.Del("User-Agent") // TODO: Parse specific header values: // Accept @@ -589,6 +492,8 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { return nil, err } + req.Cookie = readCookies(req.Header) + return req, nil } @@ -642,7 +547,7 @@ func (r *Request) ParseForm() (err os.Error) { if r.Body == nil { return os.ErrorString("missing form body") } - ct := r.Header["Content-Type"] + ct := r.Header.Get("Content-Type") switch strings.Split(ct, ";", 2)[0] { case "text/plain", "application/x-www-form-urlencoded", "": b, e := ioutil.ReadAll(r.Body) @@ -677,17 +582,12 @@ func (r *Request) FormValue(key string) string { } func (r *Request) expectsContinue() bool { - expectation, ok := r.Header["Expect"] - return ok && strings.ToLower(expectation) == "100-continue" + return strings.ToLower(r.Header.Get("Expect")) == "100-continue" } func (r *Request) wantsHttp10KeepAlive() bool { if r.ProtoMajor != 1 || r.ProtoMinor != 0 { return false } - value, exists := r.Header["Connection"] - if !exists { - return false - } - return strings.Contains(strings.ToLower(value), "keep-alive") + return strings.Contains(strings.ToLower(r.Header.Get("Connection")), "keep-alive") } diff --git a/src/pkg/http/request_test.go b/src/pkg/http/request_test.go index d25e5e5e7..ae1c4e982 100644 --- a/src/pkg/http/request_test.go +++ b/src/pkg/http/request_test.go @@ -74,7 +74,9 @@ func TestQuery(t *testing.T) { func TestPostQuery(t *testing.T) { req := &Request{Method: "POST"} req.URL, _ = ParseURL("http://www.google.com/search?q=foo&q=bar&both=x") - req.Header = map[string]string{"Content-Type": "application/x-www-form-urlencoded; boo!"} + req.Header = Header{ + "Content-Type": {"application/x-www-form-urlencoded; boo!"}, + } req.Body = nopCloser{strings.NewReader("z=post&both=y")} if q := req.FormValue("q"); q != "foo" { t.Errorf(`req.FormValue("q") = %q, want "foo"`, q) @@ -87,18 +89,18 @@ func TestPostQuery(t *testing.T) { } } -type stringMap map[string]string +type stringMap map[string][]string type parseContentTypeTest struct { contentType stringMap error bool } var parseContentTypeTests = []parseContentTypeTest{ - {contentType: stringMap{"Content-Type": "text/plain"}}, - {contentType: stringMap{"Content-Type": ""}}, - {contentType: stringMap{"Content-Type": "text/plain; boundary="}}, + {contentType: stringMap{"Content-Type": {"text/plain"}}}, + {contentType: stringMap{}}, // Non-existent keys are not placed. The value nil is illegal. + {contentType: stringMap{"Content-Type": {"text/plain; boundary="}}}, { - contentType: stringMap{"Content-Type": "application/unknown"}, + contentType: stringMap{"Content-Type": {"application/unknown"}}, error: true, }, } @@ -107,7 +109,7 @@ func TestPostContentTypeParsing(t *testing.T) { for i, test := range parseContentTypeTests { req := &Request{ Method: "POST", - Header: test.contentType, + Header: Header(test.contentType), Body: nopCloser{bytes.NewBufferString("body")}, } err := req.ParseForm() @@ -123,7 +125,7 @@ func TestPostContentTypeParsing(t *testing.T) { func TestMultipartReader(t *testing.T) { req := &Request{ Method: "POST", - Header: stringMap{"Content-Type": `multipart/form-data; boundary="foo123"`}, + Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}}, Body: nopCloser{new(bytes.Buffer)}, } multipart, err := req.MultipartReader() @@ -131,7 +133,7 @@ func TestMultipartReader(t *testing.T) { t.Errorf("expected multipart; error: %v", err) } - req.Header = stringMap{"Content-Type": "text/plain"} + req.Header = Header{"Content-Type": {"text/plain"}} multipart, err = req.MultipartReader() if multipart != nil { t.Errorf("unexpected multipart for text/plain") diff --git a/src/pkg/http/requestwrite_test.go b/src/pkg/http/requestwrite_test.go index 3ceabe4ee..03a766efd 100644 --- a/src/pkg/http/requestwrite_test.go +++ b/src/pkg/http/requestwrite_test.go @@ -10,8 +10,10 @@ import ( ) type reqWriteTest struct { - Req Request - Raw string + Req Request + Body []byte + Raw string + RawProxy string } var reqWriteTests = []reqWriteTest{ @@ -34,13 +36,13 @@ var reqWriteTests = []reqWriteTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Encoding": "gzip,deflate", - "Accept-Language": "en-us,en;q=0.5", - "Keep-Alive": "300", - "Proxy-Connection": "keep-alive", + Header: Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, + "Accept-Encoding": {"gzip,deflate"}, + "Accept-Language": {"en-us,en;q=0.5"}, + "Keep-Alive": {"300"}, + "Proxy-Connection": {"keep-alive"}, }, Body: nil, Close: false, @@ -50,13 +52,24 @@ var reqWriteTests = []reqWriteTest{ Form: map[string][]string{}, }, + nil, + "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + "Host: www.techcrunch.com\r\n" + "User-Agent: Fake\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + "Accept-Encoding: gzip,deflate\r\n" + "Accept-Language: en-us,en;q=0.5\r\n" + + "Keep-Alive: 300\r\n" + + "Proxy-Connection: keep-alive\r\n\r\n", + + "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + + "User-Agent: Fake\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + + "Accept-Encoding: gzip,deflate\r\n" + + "Accept-Language: en-us,en;q=0.5\r\n" + "Keep-Alive: 300\r\n" + "Proxy-Connection: keep-alive\r\n\r\n", }, @@ -71,16 +84,22 @@ var reqWriteTests = []reqWriteTest{ }, ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{}, - Body: nopCloser{bytes.NewBufferString("abcdef")}, + Header: Header{}, TransferEncoding: []string{"chunked"}, }, + []byte("abcdef"), + "GET /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "6\r\nabcdef\r\n0\r\n\r\n", + + "GET http://www.google.com/search HTTP/1.1\r\n" + + "User-Agent: Go http package\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + "6\r\nabcdef\r\n0\r\n\r\n", }, // HTTP/1.1 POST => chunked coding; body; empty trailer { @@ -93,18 +112,25 @@ var reqWriteTests = []reqWriteTest{ }, ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{}, + Header: Header{}, Close: true, - Body: nopCloser{bytes.NewBufferString("abcdef")}, TransferEncoding: []string{"chunked"}, }, + []byte("abcdef"), + "POST /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "Connection: close\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "6\r\nabcdef\r\n0\r\n\r\n", + + "POST http://www.google.com/search HTTP/1.1\r\n" + + "User-Agent: Go http package\r\n" + + "Connection: close\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + "6\r\nabcdef\r\n0\r\n\r\n", }, // default to HTTP/1.1 { @@ -114,16 +140,26 @@ var reqWriteTests = []reqWriteTest{ Host: "www.google.com", }, + nil, + "GET /search HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "User-Agent: Go http package\r\n" + "\r\n", + + // Looks weird but RawURL overrides what WriteProxy would choose. + "GET /search HTTP/1.1\r\n" + + "User-Agent: Go http package\r\n" + + "\r\n", }, } func TestRequestWrite(t *testing.T) { for i := range reqWriteTests { tt := &reqWriteTests[i] + if tt.Body != nil { + tt.Req.Body = nopCloser{bytes.NewBuffer(tt.Body)} + } var braw bytes.Buffer err := tt.Req.Write(&braw) if err != nil { @@ -135,5 +171,20 @@ func TestRequestWrite(t *testing.T) { t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, tt.Raw, sraw) continue } + + if tt.Body != nil { + tt.Req.Body = nopCloser{bytes.NewBuffer(tt.Body)} + } + var praw bytes.Buffer + err = tt.Req.WriteProxy(&praw) + if err != nil { + t.Errorf("error writing #%d: %s", i, err) + continue + } + sraw = praw.String() + if sraw != tt.RawProxy { + t.Errorf("Test Proxy %d, expecting:\n%s\nGot:\n%s\n", i, tt.RawProxy, sraw) + continue + } } } diff --git a/src/pkg/http/response.go b/src/pkg/http/response.go index a24726110..3d77c5555 100644 --- a/src/pkg/http/response.go +++ b/src/pkg/http/response.go @@ -10,6 +10,7 @@ import ( "bufio" "fmt" "io" + "net/textproto" "os" "sort" "strconv" @@ -43,7 +44,10 @@ type Response struct { // omitted from Header. // // Keys in the map are canonicalized (see CanonicalHeaderKey). - Header map[string]string + Header Header + + // SetCookie records the Set-Cookie requests sent with the response. + SetCookie []*Cookie // Body represents the response body. Body io.ReadCloser @@ -63,10 +67,9 @@ type Response struct { // ReadResponse nor Response.Write ever closes a connection. Close bool - // Trailer maps trailer keys to values. Like for Header, if the - // response has multiple trailer lines with the same key, they will be - // concatenated, delimited by commas. - Trailer map[string]string + // Trailer maps trailer keys to values, in the same + // format as the header. + Trailer Header } // ReadResponse reads and returns an HTTP response from r. The RequestMethod @@ -76,13 +79,17 @@ type Response struct { // key/value pairs included in the response trailer. func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) { + tp := textproto.NewReader(r) resp = new(Response) resp.RequestMethod = strings.ToUpper(requestMethod) // Parse the first line of the response. - line, err := readLine(r) + line, err := tp.ReadLine() if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } return nil, err } f := strings.Split(line, " ", 3) @@ -101,26 +108,16 @@ func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os resp.Proto = f[0] var ok bool - if resp.ProtoMajor, resp.ProtoMinor, ok = parseHTTPVersion(resp.Proto); !ok { + if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok { return nil, &badStringError{"malformed HTTP version", resp.Proto} } // Parse the response headers. - nheader := 0 - resp.Header = make(map[string]string) - for { - key, value, err := readKeyValue(r) - if err != nil { - return nil, err - } - if key == "" { - break // end of response header - } - if nheader++; nheader >= maxHeaderLines { - return nil, ErrHeaderTooLong - } - resp.AddHeader(key, value) + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err } + resp.Header = Header(mimeHeader) fixPragmaCacheControl(resp.Header) @@ -129,6 +126,8 @@ func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os return nil, err } + resp.SetCookie = readSetCookies(resp.Header) + return resp, nil } @@ -136,34 +135,14 @@ func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os // Pragma: no-cache // like // Cache-Control: no-cache -func fixPragmaCacheControl(header map[string]string) { - if header["Pragma"] == "no-cache" { +func fixPragmaCacheControl(header Header) { + if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" { if _, presentcc := header["Cache-Control"]; !presentcc { - header["Cache-Control"] = "no-cache" + header["Cache-Control"] = []string{"no-cache"} } } } -// AddHeader adds a value under the given key. Keys are not case sensitive. -func (r *Response) AddHeader(key, value string) { - key = CanonicalHeaderKey(key) - - oldValues, oldValuesPresent := r.Header[key] - if oldValuesPresent { - r.Header[key] = oldValues + "," + value - } else { - r.Header[key] = value - } -} - -// GetHeader returns the value of the response header with the given key. -// If there were multiple headers with this key, their values are concatenated, -// with a comma delimiter. If there were no response headers with the given -// key, GetHeader returns an empty string. Keys are not case sensitive. -func (r *Response) GetHeader(key string) (value string) { - return r.Header[CanonicalHeaderKey(key)] -} - // ProtoAtLeast returns whether the HTTP protocol used // in the response is at least major.minor. func (r *Response) ProtoAtLeast(major, minor int) bool { @@ -213,11 +192,15 @@ func (resp *Response) Write(w io.Writer) os.Error { } // Rest of header - err = writeSortedKeyValue(w, resp.Header, respExcludeHeader) + err = writeSortedHeader(w, resp.Header, respExcludeHeader) if err != nil { return err } + if err = writeSetCookies(w, resp.SetCookie); err != nil { + return err + } + // End-of-header io.WriteString(w, "\r\n") @@ -231,20 +214,19 @@ func (resp *Response) Write(w io.Writer) os.Error { return nil } -func writeSortedKeyValue(w io.Writer, kvm map[string]string, exclude map[string]bool) os.Error { - kva := make([]string, len(kvm)) - i := 0 - for k, v := range kvm { +func writeSortedHeader(w io.Writer, h Header, exclude map[string]bool) os.Error { + keys := make([]string, 0, len(h)) + for k := range h { if !exclude[k] { - kva[i] = fmt.Sprint(k + ": " + v + "\r\n") - i++ + keys = append(keys, k) } } - kva = kva[0:i] - sort.SortStrings(kva) - for _, l := range kva { - if _, err := io.WriteString(w, l); err != nil { - return err + sort.SortStrings(keys) + for _, k := range keys { + for _, v := range h[k] { + if _, err := fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { + return err + } } } return nil diff --git a/src/pkg/http/response_test.go b/src/pkg/http/response_test.go index 11bfdd08c..bf63ccb9e 100644 --- a/src/pkg/http/response_test.go +++ b/src/pkg/http/response_test.go @@ -34,8 +34,8 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{ - "Connection": "close", // TODO(rsc): Delete? + Header: Header{ + "Connection": {"close"}, // TODO(rsc): Delete? }, Close: true, ContentLength: -1, @@ -100,9 +100,9 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{ - "Connection": "close", // TODO(rsc): Delete? - "Content-Length": "10", // TODO(rsc): Delete? + Header: Header{ + "Connection": {"close"}, // TODO(rsc): Delete? + "Content-Length": {"10"}, // TODO(rsc): Delete? }, Close: true, ContentLength: 10, @@ -128,7 +128,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, TransferEncoding: []string{"chunked"}, @@ -155,7 +155,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, // TODO(rsc): Fix? TransferEncoding: []string{"chunked"}, @@ -175,7 +175,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, }, @@ -194,7 +194,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, }, diff --git a/src/pkg/http/responsewrite_test.go b/src/pkg/http/responsewrite_test.go index 9f10be562..228ed5f7d 100644 --- a/src/pkg/http/responsewrite_test.go +++ b/src/pkg/http/responsewrite_test.go @@ -22,7 +22,7 @@ var respWriteTests = []respWriteTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, ContentLength: 6, }, @@ -38,7 +38,7 @@ var respWriteTests = []respWriteTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, ContentLength: -1, }, @@ -53,7 +53,7 @@ var respWriteTests = []respWriteTest{ ProtoMajor: 1, ProtoMinor: 1, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, ContentLength: 6, TransferEncoding: []string{"chunked"}, diff --git a/src/pkg/http/serve_test.go b/src/pkg/http/serve_test.go index 5594d512a..86d64bdbb 100644 --- a/src/pkg/http/serve_test.go +++ b/src/pkg/http/serve_test.go @@ -4,16 +4,18 @@ // End-to-end serving tests -package http +package http_test import ( "bufio" "bytes" "fmt" - "io" + . "http" + "http/httptest" "io/ioutil" "os" "net" + "strings" "testing" "time" ) @@ -169,13 +171,10 @@ func TestHostHandlers(t *testing.T) { for _, h := range handlers { Handle(h.pattern, stringHandler(h.msg)) } - l, err := net.Listen("tcp", "127.0.0.1:0") // any port - if err != nil { - t.Fatal(err) - } - defer l.Close() - go Serve(l, nil) - conn, err := net.Dial("tcp", "", l.Addr().String()) + ts := httptest.NewServer(nil) + defer ts.Close() + + conn, err := net.Dial("tcp", "", ts.Listener.Addr().String()) if err != nil { t.Fatal(err) } @@ -197,53 +196,13 @@ func TestHostHandlers(t *testing.T) { t.Errorf("reading response: %v", err) continue } - s := r.Header["Result"] + s := r.Header.Get("Result") if s != vt.expected { t.Errorf("Get(%q) = %q, want %q", vt.url, s, vt.expected) } } } -type responseWriterMethodCall struct { - method string - headerKey, headerValue string // if method == "SetHeader" - bytesWritten []byte // if method == "Write" - responseCode int // if method == "WriteHeader" -} - -type recordingResponseWriter struct { - log []*responseWriterMethodCall -} - -func (rw *recordingResponseWriter) RemoteAddr() string { - return "1.2.3.4" -} - -func (rw *recordingResponseWriter) UsingTLS() bool { - return false -} - -func (rw *recordingResponseWriter) SetHeader(k, v string) { - rw.log = append(rw.log, &responseWriterMethodCall{method: "SetHeader", headerKey: k, headerValue: v}) -} - -func (rw *recordingResponseWriter) Write(buf []byte) (int, os.Error) { - rw.log = append(rw.log, &responseWriterMethodCall{method: "Write", bytesWritten: buf}) - return len(buf), nil -} - -func (rw *recordingResponseWriter) WriteHeader(code int) { - rw.log = append(rw.log, &responseWriterMethodCall{method: "WriteHeader", responseCode: code}) -} - -func (rw *recordingResponseWriter) Flush() { - rw.log = append(rw.log, &responseWriterMethodCall{method: "Flush"}) -} - -func (rw *recordingResponseWriter) Hijack() (io.ReadWriteCloser, *bufio.ReadWriter, os.Error) { - panic("Not supported") -} - // Tests for http://code.google.com/p/go/issues/detail?id=900 func TestMuxRedirectLeadingSlashes(t *testing.T) { paths := []string{"//foo.txt", "///foo.txt", "/../../foo.txt"} @@ -253,35 +212,17 @@ func TestMuxRedirectLeadingSlashes(t *testing.T) { t.Errorf("%s", err) } mux := NewServeMux() - resp := new(recordingResponseWriter) - resp.log = make([]*responseWriterMethodCall, 0) + resp := httptest.NewRecorder() mux.ServeHTTP(resp, req) - dumpLog := func() { - t.Logf("For path %q:", path) - for _, call := range resp.log { - t.Logf("Got call: %s, header=%s, value=%s, buf=%q, code=%d", call.method, - call.headerKey, call.headerValue, call.bytesWritten, call.responseCode) - } - } - - if len(resp.log) != 2 { - dumpLog() - t.Errorf("expected 2 calls to response writer; got %d", len(resp.log)) + if loc, expected := resp.Header.Get("Location"), "/foo.txt"; loc != expected { + t.Errorf("Expected Location header set to %q; got %q", expected, loc) return } - if resp.log[0].method != "SetHeader" || - resp.log[0].headerKey != "Location" || resp.log[0].headerValue != "/foo.txt" { - dumpLog() - t.Errorf("Expected SetHeader of Location to /foo.txt") - return - } - - if resp.log[1].method != "WriteHeader" || resp.log[1].responseCode != StatusMovedPermanently { - dumpLog() - t.Errorf("Expected WriteHeader of StatusMovedPermanently") + if code, expected := resp.Code, StatusMovedPermanently; code != expected { + t.Errorf("Expected response code of StatusMovedPermanently; got %d", code) return } } @@ -349,3 +290,78 @@ func TestServerTimeouts(t *testing.T) { l.Close() } + +// TestIdentityResponse verifies that a handler can unset +func TestIdentityResponse(t *testing.T) { + handler := HandlerFunc(func(rw ResponseWriter, req *Request) { + rw.SetHeader("Content-Length", "3") + rw.SetHeader("Transfer-Encoding", req.FormValue("te")) + switch { + case req.FormValue("overwrite") == "1": + _, err := rw.Write([]byte("foo TOO LONG")) + if err != ErrContentLength { + t.Errorf("expected ErrContentLength; got %v", err) + } + case req.FormValue("underwrite") == "1": + rw.SetHeader("Content-Length", "500") + rw.Write([]byte("too short")) + default: + rw.Write([]byte("foo")) + } + }) + + ts := httptest.NewServer(handler) + defer ts.Close() + + // Note: this relies on the assumption (which is true) that + // Get sends HTTP/1.1 or greater requests. Otherwise the + // server wouldn't have the choice to send back chunked + // responses. + for _, te := range []string{"", "identity"} { + url := ts.URL + "/?te=" + te + res, _, err := Get(url) + if err != nil { + t.Fatalf("error with Get of %s: %v", url, err) + } + if cl, expected := res.ContentLength, int64(3); cl != expected { + t.Errorf("for %s expected res.ContentLength of %d; got %d", url, expected, cl) + } + if cl, expected := res.Header.Get("Content-Length"), "3"; cl != expected { + t.Errorf("for %s expected Content-Length header of %q; got %q", url, expected, cl) + } + if tl, expected := len(res.TransferEncoding), 0; tl != expected { + t.Errorf("for %s expected len(res.TransferEncoding) of %d; got %d (%v)", + url, expected, tl, res.TransferEncoding) + } + } + + // Verify that ErrContentLength is returned + url := ts.URL + "/?overwrite=1" + _, _, err := Get(url) + if err != nil { + t.Fatalf("error with Get of %s: %v", url, err) + } + + // Verify that the connection is closed when the declared Content-Length + // is larger than what the handler wrote. + conn, err := net.Dial("tcp", "", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + _, err = conn.Write([]byte("GET /?underwrite=1 HTTP/1.1\r\nHost: foo\r\n\r\n")) + if err != nil { + t.Fatalf("error writing: %v", err) + } + // The next ReadAll will hang for a failing test, so use a Timer instead + // to fail more traditionally + timer := time.AfterFunc(2e9, func() { + t.Fatalf("Timeout expired in ReadAll.") + }) + defer timer.Stop() + got, _ := ioutil.ReadAll(conn) + expectedSuffix := "\r\n\r\ntoo short" + if !strings.HasSuffix(string(got), expectedSuffix) { + t.Fatalf("Expected output to end with %q; got response body %q", + expectedSuffix, string(got)) + } +} diff --git a/src/pkg/http/server.go b/src/pkg/http/server.go index 0be270ad3..5d623e696 100644 --- a/src/pkg/http/server.go +++ b/src/pkg/http/server.go @@ -31,6 +31,7 @@ var ( ErrWriteAfterFlush = os.NewError("Conn.Write called after Flush") ErrBodyNotAllowed = os.NewError("http: response status code does not allow body") ErrHijacked = os.NewError("Conn has been hijacked") + ErrContentLength = os.NewError("Conn.Write wrote more than the declared Content-Length") ) // Objects implementing the Handler interface can be @@ -60,10 +61,10 @@ type ResponseWriter interface { // // Content-Type: text/html; charset=utf-8 // - // being sent. UTF-8 encoded HTML is the default setting for + // being sent. UTF-8 encoded HTML is the default setting for // Content-Type in this library, so users need not make that - // particular call. Calls to SetHeader after WriteHeader (or Write) - // are ignored. + // particular call. Calls to SetHeader after WriteHeader (or Write) + // are ignored. An empty value removes the header if previously set. SetHeader(string, string) // Write writes the data to the connection as part of an HTTP reply. @@ -80,7 +81,10 @@ type ResponseWriter interface { // Flush sends any buffered data to the client. Flush() +} +// A Hijacker is an HTTP request which be taken over by an HTTP handler. +type Hijacker interface { // Hijack lets the caller take over the connection. // After a call to Hijack(), the HTTP server library // will not do anything else with the connection. @@ -108,6 +112,7 @@ type response struct { wroteContinue bool // 100 Continue response was written header map[string]string // reply header parameters written int64 // number of bytes written in body + contentLength int64 // explicitly-declared Content-Length; or -1 status int // status code passed to WriteHeader // close connection after this reply. set on request and @@ -170,33 +175,13 @@ func (c *conn) readRequest() (w *response, err os.Error) { w.conn = c w.req = req w.header = make(map[string]string) + w.contentLength = -1 // Expect 100 Continue support if req.expectsContinue() && req.ProtoAtLeast(1, 1) { // Wrap the Body reader with one that replies on the connection req.Body = &expectContinueReader{readCloser: req.Body, resp: w} } - - // Default output is HTML encoded in UTF-8. - w.SetHeader("Content-Type", "text/html; charset=utf-8") - w.SetHeader("Date", time.UTC().Format(TimeFormat)) - - if req.Method == "HEAD" { - // do nothing - } else if req.ProtoAtLeast(1, 1) { - // HTTP/1.1 or greater: use chunked transfer encoding - // to avoid closing the connection at EOF. - w.chunking = true - w.SetHeader("Transfer-Encoding", "chunked") - } else { - // HTTP version < 1.1: cannot do chunked transfer - // encoding, so signal EOF by closing connection. - // Will be overridden if the HTTP handler ends up - // writing a Content-Length and the client requested - // "Connection: keep-alive" - w.closeAfterReply = true - } - return w, nil } @@ -209,7 +194,10 @@ func (w *response) UsingTLS() bool { func (w *response) RemoteAddr() string { return w.conn.remoteAddr } // SetHeader implements the ResponseWriter.SetHeader method -func (w *response) SetHeader(hdr, val string) { w.header[CanonicalHeaderKey(hdr)] = val } +// An empty value removes the header from the map. +func (w *response) SetHeader(hdr, val string) { + w.header[CanonicalHeaderKey(hdr)] = val, val != "" +} // WriteHeader implements the ResponseWriter.WriteHeader method func (w *response) WriteHeader(code int) { @@ -225,13 +213,86 @@ func (w *response) WriteHeader(code int) { w.status = code if code == StatusNotModified { // Must not have body. - w.header["Content-Type"] = "", false - w.header["Transfer-Encoding"] = "", false + for _, header := range []string{"Content-Type", "Content-Length", "Transfer-Encoding"} { + if w.header[header] != "" { + // TODO: return an error if WriteHeader gets a return parameter + // or set a flag on w to make future Writes() write an error page? + // for now just log and drop the header. + log.Printf("http: StatusNotModified response with header %q defined", header) + w.header[header] = "", false + } + } + } else { + // Default output is HTML encoded in UTF-8. + if w.header["Content-Type"] == "" { + w.SetHeader("Content-Type", "text/html; charset=utf-8") + } + } + + if w.header["Date"] == "" { + w.SetHeader("Date", time.UTC().Format(TimeFormat)) + } + + // Check for a explicit (and valid) Content-Length header. + var hasCL bool + var contentLength int64 + if clenStr, ok := w.header["Content-Length"]; ok { + var err os.Error + contentLength, err = strconv.Atoi64(clenStr) + if err == nil { + hasCL = true + } else { + log.Printf("http: invalid Content-Length of %q sent", clenStr) + w.SetHeader("Content-Length", "") + } + } + + te, hasTE := w.header["Transfer-Encoding"] + if hasCL && hasTE && te != "identity" { + // TODO: return an error if WriteHeader gets a return parameter + // For now just ignore the Content-Length. + log.Printf("http: WriteHeader called with both Transfer-Encoding of %q and a Content-Length of %d", + te, contentLength) + w.SetHeader("Content-Length", "") + hasCL = false + } + + if w.req.Method == "HEAD" { + // do nothing + } else if hasCL { w.chunking = false + w.contentLength = contentLength + w.SetHeader("Transfer-Encoding", "") + } else if w.req.ProtoAtLeast(1, 1) { + // HTTP/1.1 or greater: use chunked transfer encoding + // to avoid closing the connection at EOF. + // TODO: this blows away any custom or stacked Transfer-Encoding they + // might have set. Deal with that as need arises once we have a valid + // use case. + w.chunking = true + w.SetHeader("Transfer-Encoding", "chunked") + } else { + // HTTP version < 1.1: cannot do chunked transfer + // encoding and we don't know the Content-Length so + // signal EOF by closing connection. + w.closeAfterReply = true + w.chunking = false // redundant + w.SetHeader("Transfer-Encoding", "") // in case already set } + + if w.req.wantsHttp10KeepAlive() && (w.req.Method == "HEAD" || hasCL) { + _, connectionHeaderSet := w.header["Connection"] + if !connectionHeaderSet { + w.SetHeader("Connection", "keep-alive") + } + } else if !w.req.ProtoAtLeast(1, 1) { + // Client did not ask to keep connection alive. + w.closeAfterReply = true + } + // Cannot use Content-Length with non-identity Transfer-Encoding. if w.chunking { - w.header["Content-Length"] = "", false + w.SetHeader("Content-Length", "") } if !w.req.ProtoAtLeast(1, 0) { return @@ -259,15 +320,6 @@ func (w *response) Write(data []byte) (n int, err os.Error) { return 0, ErrHijacked } if !w.wroteHeader { - if w.req.wantsHttp10KeepAlive() { - _, hasLength := w.header["Content-Length"] - if hasLength { - _, connectionHeaderSet := w.header["Connection"] - if !connectionHeaderSet { - w.header["Connection"] = "keep-alive" - } - } - } w.WriteHeader(StatusOK) } if len(data) == 0 { @@ -280,6 +332,9 @@ func (w *response) Write(data []byte) (n int, err os.Error) { } w.written += int64(len(data)) // ignoring errors, for errorKludge + if w.contentLength != -1 && w.written > w.contentLength { + return 0, ErrContentLength + } // TODO(rsc): if chunking happened after the buffering, // then there would be fewer chunk headers. @@ -369,6 +424,11 @@ func (w *response) finishRequest() { } w.conn.buf.Flush() w.req.Body.Close() + + if w.contentLength != -1 && w.contentLength != w.written { + // Did not write enough. Avoid getting out of sync. + w.closeAfterReply = true + } } // Flush implements the ResponseWriter.Flush method. @@ -657,10 +717,12 @@ func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Re // Handle registers the handler for the given pattern // in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } // HandleFunc registers the handler function for the given pattern // in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { DefaultServeMux.HandleFunc(pattern, handler) } diff --git a/src/pkg/http/transfer.go b/src/pkg/http/transfer.go index f80f0ac63..996e28973 100644 --- a/src/pkg/http/transfer.go +++ b/src/pkg/http/transfer.go @@ -21,7 +21,7 @@ type transferWriter struct { ContentLength int64 Close bool TransferEncoding []string - Trailer map[string]string + Trailer Header } func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) { @@ -159,7 +159,7 @@ func (t *transferWriter) WriteBody(w io.Writer) (err os.Error) { type transferReader struct { // Input - Header map[string]string + Header Header StatusCode int RequestMethod string ProtoMajor int @@ -169,7 +169,7 @@ type transferReader struct { ContentLength int64 TransferEncoding []string Close bool - Trailer map[string]string + Trailer Header } // bodyAllowedForStatus returns whether a given response status code @@ -289,14 +289,14 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err os.Error) { func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } // Sanitize transfer encoding -func fixTransferEncoding(header map[string]string) ([]string, os.Error) { +func fixTransferEncoding(header Header) ([]string, os.Error) { raw, present := header["Transfer-Encoding"] if !present { return nil, nil } - header["Transfer-Encoding"] = "", false - encodings := strings.Split(raw, ",", -1) + header["Transfer-Encoding"] = nil, false + encodings := strings.Split(raw[0], ",", -1) te := make([]string, 0, len(encodings)) // TODO: Even though we only support "identity" and "chunked" // encodings, the loop below is designed with foresight. One @@ -321,7 +321,7 @@ func fixTransferEncoding(header map[string]string) ([]string, os.Error) { // Chunked encoding trumps Content-Length. See RFC 2616 // Section 4.4. Currently len(te) > 0 implies chunked // encoding. - header["Content-Length"] = "", false + header["Content-Length"] = nil, false return te, nil } @@ -331,7 +331,7 @@ func fixTransferEncoding(header map[string]string) ([]string, os.Error) { // Determine the expected body length, using RFC 2616 Section 4.4. This // function is not a method, because ultimately it should be shared by // ReadResponse and ReadRequest. -func fixLength(status int, requestMethod string, header map[string]string, te []string) (int64, os.Error) { +func fixLength(status int, requestMethod string, header Header, te []string) (int64, os.Error) { // Logic based on response type or status if noBodyExpected(requestMethod) { @@ -351,23 +351,21 @@ func fixLength(status int, requestMethod string, header map[string]string, te [] } // Logic based on Content-Length - if cl, present := header["Content-Length"]; present { - cl = strings.TrimSpace(cl) - if cl != "" { - n, err := strconv.Atoi64(cl) - if err != nil || n < 0 { - return -1, &badStringError{"bad Content-Length", cl} - } - return n, nil - } else { - header["Content-Length"] = "", false + cl := strings.TrimSpace(header.Get("Content-Length")) + if cl != "" { + n, err := strconv.Atoi64(cl) + if err != nil || n < 0 { + return -1, &badStringError{"bad Content-Length", cl} } + return n, nil + } else { + header.Del("Content-Length") } // Logic based on media type. The purpose of the following code is just // to detect whether the unsupported "multipart/byteranges" is being // used. A proper Content-Type parser is needed in the future. - if strings.Contains(strings.ToLower(header["Content-Type"]), "multipart/byteranges") { + if strings.Contains(strings.ToLower(header.Get("Content-Type")), "multipart/byteranges") { return -1, ErrNotSupported } @@ -378,24 +376,19 @@ func fixLength(status int, requestMethod string, header map[string]string, te [] // Determine whether to hang up after sending a request and body, or // receiving a response and body // 'header' is the request headers -func shouldClose(major, minor int, header map[string]string) bool { +func shouldClose(major, minor int, header Header) bool { if major < 1 { return true } else if major == 1 && minor == 0 { - v, present := header["Connection"] - if !present { - return true - } - v = strings.ToLower(v) - if !strings.Contains(v, "keep-alive") { + if !strings.Contains(strings.ToLower(header.Get("Connection")), "keep-alive") { return true } return false - } else if v, present := header["Connection"]; present { + } else { // TODO: Should split on commas, toss surrounding white space, // and check each field. - if v == "close" { - header["Connection"] = "", false + if strings.ToLower(header.Get("Connection")) == "close" { + header.Del("Connection") return true } } @@ -403,14 +396,14 @@ func shouldClose(major, minor int, header map[string]string) bool { } // Parse the trailer header -func fixTrailer(header map[string]string, te []string) (map[string]string, os.Error) { - raw, present := header["Trailer"] - if !present { +func fixTrailer(header Header, te []string) (Header, os.Error) { + raw := header.Get("Trailer") + if raw == "" { return nil, nil } - header["Trailer"] = "", false - trailer := make(map[string]string) + header.Del("Trailer") + trailer := make(Header) keys := strings.Split(raw, ",", -1) for _, key := range keys { key = CanonicalHeaderKey(strings.TrimSpace(key)) @@ -418,7 +411,7 @@ func fixTrailer(header map[string]string, te []string) (map[string]string, os.Er case "Transfer-Encoding", "Trailer", "Content-Length": return nil, &badStringError{"bad trailer key", key} } - trailer[key] = "" + trailer.Del(key) } if len(trailer) == 0 { return nil, nil diff --git a/src/pkg/http/transport.go b/src/pkg/http/transport.go new file mode 100644 index 000000000..78d316a55 --- /dev/null +++ b/src/pkg/http/transport.go @@ -0,0 +1,151 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "fmt" + "net" + "os" + "strings" + "sync" +) + +// DefaultTransport is the default implementation of Transport and is +// used by DefaultClient. It establishes a new network connection for +// each call to Do and uses HTTP proxies as directed by the +// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy) +// environment variables. +var DefaultTransport Transport = &transport{} + +// transport implements Tranport for the default case, using TCP +// connections to either the host or a proxy, serving http or https +// schemes. In the future this may become public and support options +// on keep-alive connection duration, pipelining controls, etc. For +// now this is simply a port of the old Go code client code to the +// Transport interface. +type transport struct { + // TODO: keep-alives, pipelining, etc using a map from + // scheme/host to a connection. Something like: + l sync.Mutex + hostConn map[string]*ClientConn +} + +func (ct *transport) Do(req *Request) (resp *Response, err os.Error) { + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + } + + addr := req.URL.Host + if !hasPort(addr) { + addr += ":" + req.URL.Scheme + } + + var proxyURL *URL + proxyAuth := "" + proxy := "" + if !matchNoProxy(addr) { + proxy = os.Getenv("HTTP_PROXY") + if proxy == "" { + proxy = os.Getenv("http_proxy") + } + } + + var write = (*Request).Write + + if proxy != "" { + write = (*Request).WriteProxy + proxyURL, err = ParseRequestURL(proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + if proxyURL.Host == "" { + proxyURL, err = ParseRequestURL("http://" + proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + } + addr = proxyURL.Host + proxyInfo := proxyURL.RawUserinfo + if proxyInfo != "" { + enc := base64.URLEncoding + encoded := make([]byte, enc.EncodedLen(len(proxyInfo))) + enc.Encode(encoded, []byte(proxyInfo)) + proxyAuth = "Basic " + string(encoded) + } + } + + // Connect to server or proxy + conn, err := net.Dial("tcp", "", addr) + if err != nil { + return nil, err + } + + if req.URL.Scheme == "http" { + // Include proxy http header if needed. + if proxyAuth != "" { + req.Header.Set("Proxy-Authorization", proxyAuth) + } + } else { // https + if proxyURL != nil { + // Ask proxy for direct connection to server. + // addr defaults above to ":https" but we need to use numbers + addr = req.URL.Host + if !hasPort(addr) { + addr += ":443" + } + fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\n", addr) + fmt.Fprintf(conn, "Host: %s\r\n", addr) + if proxyAuth != "" { + fmt.Fprintf(conn, "Proxy-Authorization: %s\r\n", proxyAuth) + } + fmt.Fprintf(conn, "\r\n") + + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := ReadResponse(br, "CONNECT") + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + f := strings.Split(resp.Status, " ", 2) + return nil, os.ErrorString(f[1]) + } + } + + // Initiate TLS and check remote host name against certificate. + conn = tls.Client(conn, nil) + if err = conn.(*tls.Conn).Handshake(); err != nil { + return nil, err + } + h := req.URL.Host + if hasPort(h) { + h = h[:strings.LastIndex(h, ":")] + } + if err = conn.(*tls.Conn).VerifyHostname(h); err != nil { + return nil, err + } + } + + err = write(req, conn) + if err != nil { + conn.Close() + return nil, err + } + + reader := bufio.NewReader(conn) + resp, err = ReadResponse(reader, req.Method) + if err != nil { + conn.Close() + return nil, err + } + + resp.Body = readClose{resp.Body, conn} + return +} diff --git a/src/pkg/image/decode_test.go b/src/pkg/image/decode_test.go new file mode 100644 index 000000000..5b87344c5 --- /dev/null +++ b/src/pkg/image/decode_test.go @@ -0,0 +1,89 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package image_test + +import ( + "bufio" + "image" + "os" + "testing" + + // TODO(nigeltao): implement bmp, gif and tiff decoders. + _ "image/jpeg" + _ "image/png" +) + +const goldenFile = "testdata/video-001.png" + +type imageTest struct { + filename string + tolerance int +} + +var imageTests = []imageTest{ + //{"testdata/video-001.bmp", 0}, + // GIF images are restricted to a 256-color palette and the conversion + // to GIF loses significant image quality. + //{"testdata/video-001.gif", 64<<8}, + // JPEG is a lossy format and hence needs a non-zero tolerance. + {"testdata/video-001.jpeg", 8 << 8}, + {"testdata/video-001.png", 0}, + //{"testdata/video-001.tiff", 0}, +} + +func decode(filename string) (image.Image, string, os.Error) { + f, err := os.Open(filename, os.O_RDONLY, 0400) + if err != nil { + return nil, "", err + } + defer f.Close() + return image.Decode(bufio.NewReader(f)) +} + +func delta(u0, u1 uint32) int { + d := int(u0) - int(u1) + if d < 0 { + return -d + } + return d +} + +func withinTolerance(c0, c1 image.Color, tolerance int) bool { + r0, g0, b0, a0 := c0.RGBA() + r1, g1, b1, a1 := c1.RGBA() + r := delta(r0, r1) + g := delta(g0, g1) + b := delta(b0, b1) + a := delta(a0, a1) + return r <= tolerance && g <= tolerance && b <= tolerance && a <= tolerance +} + +func TestDecode(t *testing.T) { + golden, _, err := decode(goldenFile) + if err != nil { + t.Errorf("%s: %v", goldenFile, err) + } +loop: + for _, it := range imageTests { + m, _, err := decode(it.filename) + if err != nil { + t.Errorf("%s: %v", it.filename, err) + continue loop + } + b := golden.Bounds() + if !b.Eq(m.Bounds()) { + t.Errorf("%s: want bounds %v got %v", it.filename, b, m.Bounds()) + continue loop + } + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + if !withinTolerance(golden.At(x, y), m.At(x, y), it.tolerance) { + t.Errorf("%s: at (%d, %d), want %v got %v", it.filename, x, y, golden.At(x, y), m.At(x, y)) + continue loop + } + } + } + } +} diff --git a/src/pkg/image/png/reader.go b/src/pkg/image/png/reader.go index e2d679bb4..eee4eac2e 100644 --- a/src/pkg/image/png/reader.go +++ b/src/pkg/image/png/reader.go @@ -29,11 +29,19 @@ const ( // A cb is a combination of color type and bit depth. const ( cbInvalid = iota + cbG1 + cbG2 + cbG4 cbG8 + cbGA8 cbTC8 + cbP1 + cbP2 + cbP4 cbP8 cbTCA8 cbG16 + cbGA16 cbTC16 cbTCA16 ) @@ -70,6 +78,7 @@ type imgOrErr struct { type decoder struct { width, height int + depth int palette image.PalettedColorModel cb int stage int @@ -138,7 +147,29 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro return UnsupportedError("dimension overflow") } d.cb = cbInvalid - switch d.tmp[8] { + d.depth = int(d.tmp[8]) + switch d.depth { + case 1: + switch d.tmp[9] { + case ctGrayscale: + d.cb = cbG1 + case ctPaletted: + d.cb = cbP1 + } + case 2: + switch d.tmp[9] { + case ctGrayscale: + d.cb = cbG2 + case ctPaletted: + d.cb = cbP2 + } + case 4: + switch d.tmp[9] { + case ctGrayscale: + d.cb = cbG4 + case ctPaletted: + d.cb = cbP4 + } case 8: switch d.tmp[9] { case ctGrayscale: @@ -147,6 +178,8 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro d.cb = cbTC8 case ctPaletted: d.cb = cbP8 + case ctGrayscaleAlpha: + d.cb = cbGA8 case ctTrueColorAlpha: d.cb = cbTCA8 } @@ -156,6 +189,8 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro d.cb = cbG16 case ctTrueColor: d.cb = cbTC16 + case ctGrayscaleAlpha: + d.cb = cbGA16 case ctTrueColorAlpha: d.cb = cbTCA16 } @@ -169,7 +204,7 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Error { np := int(length / 3) // The number of palette entries. - if length%3 != 0 || np <= 0 || np > 256 { + if length%3 != 0 || np <= 0 || np > 256 || np > 1<<uint(d.depth) { return FormatError("bad PLTE length") } n, err := io.ReadFull(r, d.tmp[0:3*np]) @@ -178,7 +213,7 @@ func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Erro } crc.Write(d.tmp[0:n]) switch d.cb { - case cbP8: + case cbP1, cbP2, cbP4, cbP8: d.palette = image.PalettedColorModel(make([]image.Color, np)) for i := 0; i < np; i++ { d.palette[i] = image.RGBAColor{d.tmp[3*i+0], d.tmp[3*i+1], d.tmp[3*i+2], 0xff} @@ -206,7 +241,7 @@ func (d *decoder) parsetRNS(r io.Reader, crc hash.Hash32, length uint32) os.Erro return UnsupportedError("grayscale transparency") case cbTC8, cbTC16: return UnsupportedError("truecolor transparency") - case cbP8: + case cbP1, cbP2, cbP4, cbP8: if n > len(d.palette) { return FormatError("bad tRNS length") } @@ -214,7 +249,7 @@ func (d *decoder) parsetRNS(r io.Reader, crc hash.Hash32, length uint32) os.Erro rgba := d.palette[i].(image.RGBAColor) d.palette[i] = image.RGBAColor{rgba.R, rgba.G, rgba.B, d.tmp[i]} } - case cbTCA8, cbTCA16: + case cbGA8, cbGA16, cbTCA8, cbTCA16: return FormatError("tRNS, color type mismatch") } return nil @@ -240,7 +275,7 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { return nil, err } defer r.Close() - bpp := 0 // Bytes per pixel. + bitsPerPixel := 0 maxPalette := uint8(0) var ( gray *image.Gray @@ -253,40 +288,50 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { img image.Image ) switch d.cb { - case cbG8: - bpp = 1 + case cbG1, cbG2, cbG4, cbG8: + bitsPerPixel = d.depth gray = image.NewGray(d.width, d.height) img = gray + case cbGA8: + bitsPerPixel = 16 + nrgba = image.NewNRGBA(d.width, d.height) + img = nrgba case cbTC8: - bpp = 3 + bitsPerPixel = 24 rgba = image.NewRGBA(d.width, d.height) img = rgba - case cbP8: - bpp = 1 + case cbP1, cbP2, cbP4, cbP8: + bitsPerPixel = d.depth paletted = image.NewPaletted(d.width, d.height, d.palette) img = paletted maxPalette = uint8(len(d.palette) - 1) case cbTCA8: - bpp = 4 + bitsPerPixel = 32 nrgba = image.NewNRGBA(d.width, d.height) img = nrgba case cbG16: - bpp = 2 + bitsPerPixel = 16 gray16 = image.NewGray16(d.width, d.height) img = gray16 + case cbGA16: + bitsPerPixel = 32 + nrgba64 = image.NewNRGBA64(d.width, d.height) + img = nrgba64 case cbTC16: - bpp = 6 + bitsPerPixel = 48 rgba64 = image.NewRGBA64(d.width, d.height) img = rgba64 case cbTCA16: - bpp = 8 + bitsPerPixel = 64 nrgba64 = image.NewNRGBA64(d.width, d.height) img = nrgba64 } + bytesPerPixel := (bitsPerPixel + 7) / 8 + // cr and pr are the bytes for the current and previous row. // The +1 is for the per-row filter type, which is at cr[0]. - cr := make([]uint8, 1+bpp*d.width) - pr := make([]uint8, 1+bpp*d.width) + cr := make([]uint8, 1+(bitsPerPixel*d.width+7)/8) + pr := make([]uint8, 1+(bitsPerPixel*d.width+7)/8) for y := 0; y < d.height; y++ { // Read the decompressed bytes. @@ -302,26 +347,26 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { case ftNone: // No-op. case ftSub: - for i := bpp; i < len(cdat); i++ { - cdat[i] += cdat[i-bpp] + for i := bytesPerPixel; i < len(cdat); i++ { + cdat[i] += cdat[i-bytesPerPixel] } case ftUp: for i := 0; i < len(cdat); i++ { cdat[i] += pdat[i] } case ftAverage: - for i := 0; i < bpp; i++ { + for i := 0; i < bytesPerPixel; i++ { cdat[i] += pdat[i] / 2 } - for i := bpp; i < len(cdat); i++ { - cdat[i] += uint8((int(cdat[i-bpp]) + int(pdat[i])) / 2) + for i := bytesPerPixel; i < len(cdat); i++ { + cdat[i] += uint8((int(cdat[i-bytesPerPixel]) + int(pdat[i])) / 2) } case ftPaeth: - for i := 0; i < bpp; i++ { + for i := 0; i < bytesPerPixel; i++ { cdat[i] += paeth(0, pdat[i], 0) } - for i := bpp; i < len(cdat); i++ { - cdat[i] += paeth(cdat[i-bpp], pdat[i], pdat[i-bpp]) + for i := bytesPerPixel; i < len(cdat); i++ { + cdat[i] += paeth(cdat[i-bytesPerPixel], pdat[i], pdat[i-bytesPerPixel]) } default: return nil, FormatError("bad filter type") @@ -329,14 +374,79 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { // Convert from bytes to colors. switch d.cb { + case cbG1: + for x := 0; x < d.width; x += 8 { + b := cdat[x/8] + for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ { + gray.Set(x+x2, y, image.GrayColor{(b >> 7) * 0xff}) + b <<= 1 + } + } + case cbG2: + for x := 0; x < d.width; x += 4 { + b := cdat[x/4] + for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ { + gray.Set(x+x2, y, image.GrayColor{(b >> 6) * 0x55}) + b <<= 2 + } + } + case cbG4: + for x := 0; x < d.width; x += 2 { + b := cdat[x/2] + for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ { + gray.Set(x+x2, y, image.GrayColor{(b >> 4) * 0x11}) + b <<= 4 + } + } case cbG8: for x := 0; x < d.width; x++ { gray.Set(x, y, image.GrayColor{cdat[x]}) } + case cbGA8: + for x := 0; x < d.width; x++ { + ycol := cdat[2*x+0] + nrgba.Set(x, y, image.NRGBAColor{ycol, ycol, ycol, cdat[2*x+1]}) + } case cbTC8: for x := 0; x < d.width; x++ { rgba.Set(x, y, image.RGBAColor{cdat[3*x+0], cdat[3*x+1], cdat[3*x+2], 0xff}) } + case cbP1: + for x := 0; x < d.width; x += 8 { + b := cdat[x/8] + for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ { + idx := b >> 7 + if idx > maxPalette { + return nil, FormatError("palette index out of range") + } + paletted.SetColorIndex(x+x2, y, idx) + b <<= 1 + } + } + case cbP2: + for x := 0; x < d.width; x += 4 { + b := cdat[x/4] + for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ { + idx := b >> 6 + if idx > maxPalette { + return nil, FormatError("palette index out of range") + } + paletted.SetColorIndex(x+x2, y, idx) + b <<= 2 + } + } + case cbP4: + for x := 0; x < d.width; x += 2 { + b := cdat[x/2] + for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ { + idx := b >> 4 + if idx > maxPalette { + return nil, FormatError("palette index out of range") + } + paletted.SetColorIndex(x+x2, y, idx) + b <<= 4 + } + } case cbP8: for x := 0; x < d.width; x++ { if cdat[x] > maxPalette { @@ -353,6 +463,12 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { ycol := uint16(cdat[2*x+0])<<8 | uint16(cdat[2*x+1]) gray16.Set(x, y, image.Gray16Color{ycol}) } + case cbGA16: + for x := 0; x < d.width; x++ { + ycol := uint16(cdat[4*x+0])<<8 | uint16(cdat[4*x+1]) + acol := uint16(cdat[4*x+2])<<8 | uint16(cdat[4*x+3]) + nrgba64.Set(x, y, image.NRGBA64Color{ycol, ycol, ycol, acol}) + } case cbTC16: for x := 0; x < d.width; x++ { rcol := uint16(cdat[6*x+0])<<8 | uint16(cdat[6*x+1]) @@ -565,16 +681,20 @@ func DecodeConfig(r io.Reader) (image.Config, os.Error) { } var cm image.ColorModel switch d.cb { - case cbG8: + case cbG1, cbG2, cbG4, cbG8: cm = image.GrayColorModel + case cbGA8: + cm = image.NRGBAColorModel case cbTC8: cm = image.RGBAColorModel - case cbP8: + case cbP1, cbP2, cbP4, cbP8: cm = d.palette case cbTCA8: cm = image.NRGBAColorModel case cbG16: cm = image.Gray16ColorModel + case cbGA16: + cm = image.NRGBA64ColorModel case cbTC16: cm = image.RGBA64ColorModel case cbTCA16: diff --git a/src/pkg/image/png/reader_test.go b/src/pkg/image/png/reader_test.go index fefceee3a..8314a8338 100644 --- a/src/pkg/image/png/reader_test.go +++ b/src/pkg/image/png/reader_test.go @@ -13,23 +13,23 @@ import ( "testing" ) -// The go PNG library currently supports only a subset of the full PNG specification. -// In particular, bit depths other than 8 or 16 are not supported, nor are grayscale- -// alpha images. var filenames = []string{ - //"basn0g01", // bit depth is not 8 or 16 - //"basn0g02", // bit depth is not 8 or 16 - //"basn0g04", // bit depth is not 8 or 16 + "basn0g01", + "basn0g01-30", + "basn0g02", + "basn0g02-29", + "basn0g04", + "basn0g04-31", "basn0g08", "basn0g16", "basn2c08", "basn2c16", - //"basn3p01", // bit depth is not 8 or 16 - //"basn3p02", // bit depth is not 8 or 16 - //"basn3p04", // bit depth is not 8 or 16 + "basn3p01", + "basn3p02", + "basn3p04", "basn3p08", - //"basn4a08", // grayscale-alpha color model - //"basn4a16", // grayscale-alpha color model + "basn4a08", + "basn4a16", "basn6a08", "basn6a16", } @@ -58,7 +58,16 @@ func sng(w io.WriteCloser, filename string, png image.Image) { cpm, _ := cm.(image.PalettedColorModel) var paletted *image.Paletted if cpm != nil { - bitdepth = 8 + switch { + case len(cpm) <= 2: + bitdepth = 1 + case len(cpm) <= 4: + bitdepth = 2 + case len(cpm) <= 16: + bitdepth = 4 + default: + bitdepth = 8 + } paletted = png.(*image.Paletted) } @@ -131,8 +140,15 @@ func sng(w io.WriteCloser, filename string, png image.Image) { fmt.Fprintf(w, "%04x%04x%04x%04x ", nrgba64.R, nrgba64.G, nrgba64.B, nrgba64.A) } case cpm != nil: + var b, c int for x := bounds.Min.X; x < bounds.Max.X; x++ { - fmt.Fprintf(w, "%02x", paletted.ColorIndexAt(x, y)) + b = b<<uint(bitdepth) | int(paletted.ColorIndexAt(x, y)) + c++ + if c == 8/bitdepth { + fmt.Fprintf(w, "%02x", b) + b = 0 + c = 0 + } } } io.WriteString(w, "\n") @@ -143,14 +159,25 @@ func sng(w io.WriteCloser, filename string, png image.Image) { func TestReader(t *testing.T) { for _, fn := range filenames { // Read the .png file. - image, err := readPng("testdata/pngsuite/" + fn + ".png") + img, err := readPng("testdata/pngsuite/" + fn + ".png") if err != nil { t.Error(fn, err) continue } + + if fn == "basn4a16" { + // basn4a16.sng is gray + alpha but sng() will produce true color + alpha + // so we just check a single random pixel. + c := img.At(2, 1).(image.NRGBA64Color) + if c.R != 0x11a7 || c.G != 0x11a7 || c.B != 0x11a7 || c.A != 0x1085 { + t.Error(fn, fmt.Errorf("wrong pixel value at (2, 1): %x", c)) + } + continue + } + piper, pipew := io.Pipe() pb := bufio.NewReader(piper) - go sng(pipew, fn, image) + go sng(pipew, fn, img) defer piper.Close() // Read the .sng file. diff --git a/src/pkg/image/png/testdata/pngsuite/README b/src/pkg/image/png/testdata/pngsuite/README index 27e7c6558..abe3ecb20 100644 --- a/src/pkg/image/png/testdata/pngsuite/README +++ b/src/pkg/image/png/testdata/pngsuite/README @@ -5,5 +5,14 @@ README.original gives the following license for those files: Permission to use, copy, and distribute these images for any purpose and without fee is hereby granted. + +The files basn0g01-30.png, basn0g02-29.png and basn0g04-31.png are in fact +not part of pngsuite but were created from files in pngsuite. Their non-power- +of-two sizes makes them useful for testing bit-depths smaller than a byte. + The *.sng files in this directory were generated from the *.png files -by the sng command-line tool. +by the sng command-line tool and some hand editing. The files +basn0g0{1,2,4}.sng were actually generated by first converting the PNG +to a bitdepth of 8 and then running sng on them. basn4a08.sng was generated +by from a 16-bit rgba version of basn4a08.png rather than the original +gray + alpha. diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g01-30.png b/src/pkg/image/png/testdata/pngsuite/basn0g01-30.png Binary files differnew file mode 100644 index 000000000..007750c8c --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn0g01-30.png diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g01-30.sng b/src/pkg/image/png/testdata/pngsuite/basn0g01-30.sng new file mode 100644 index 000000000..7fa35710c --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn0g01-30.sng @@ -0,0 +1,39 @@ +#SNG: from basn0g01-30.png +IHDR { + width: 30; height: 30; bitdepth: 8; + using grayscale; +} +gAMA {1.0000} +IMAGE { + pixels hex +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffffff000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffff00000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffff0000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffffff000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffff00000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffff0000000000000000 +ffffffffff0000000000000000ffffffffffffffff000000000000000000 +ffffffffff0000000000000000ffffffffffffff00000000000000000000 +ffffffffffff0000ffff0000ffffffffffffff0000000000000000000000 +ffffffffffff0000ffff0000ffffffffffff000000000000000000000000 +ffffffffffffffffffffffffffffffffff00000000000000000000000000 +ffffffffffffffffffffffffffffffff0000000000000000000000000000 +ffffffffffffffffffffffffffffff000000000000000000000000000000 +ffffffffffffffffffffffffffff00000000000000000000000000000000 +ffffffffffffffffffffffffff00000000000000ffffffffffffff000000 +ffffffffffffffffffffffff0000000000000000ffffffffffffff000000 +ffffffffffffffffffffff000000000000000000ffff00000000ffff0000 +ffffffffffffffffffff00000000000000000000ffff00000000ffff0000 +ffffffffffffffffff0000000000000000000000ffffffffffffff000000 +ffffffffffffffff000000000000000000000000ffffffffffffff000000 +ffffffffffffff00000000000000000000000000ffff00000000ffff0000 +ffffffffffff0000000000000000000000000000ffff00000000ffff0000 +ffffffffff000000000000000000000000000000ffffffffffffff000000 +ffffffff00000000000000000000000000000000ffffffffffffff000000 +ffffff000000000000000000000000000000000000000000000000000000 +ffff00000000000000000000000000000000000000000000000000000000 +} diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g01.sng b/src/pkg/image/png/testdata/pngsuite/basn0g01.sng index e712d8ec6..2ce069de2 100644 --- a/src/pkg/image/png/testdata/pngsuite/basn0g01.sng +++ b/src/pkg/image/png/testdata/pngsuite/basn0g01.sng @@ -1,41 +1,41 @@ #SNG: from basn0g01.png IHDR { - width: 32; height: 32; bitdepth: 1; + width: 32; height: 32; bitdepth: 8; using grayscale; } gAMA {1.0000} IMAGE { pixels hex -fffffffe -fffffffc -fffffff8 -fffffff0 -f3f3ffe0 -f3f3ffc0 -f3f3ff80 -f333ff00 -f333fe00 -f333fc00 -f807f800 -f807f000 -fccfe000 -fccfc000 -ffff8000 -ffff0000 -fffe0000 -fffc0000 -fff80fe0 -fff00fe0 -ffe00c30 -ffc00c30 -ff800fe0 -ff000fe0 -fe000c30 -fc000c30 -f8000fe0 -f0000fe0 -e0000000 -c0000000 -80000000 -00000000 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff000000 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffff00000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffffff0000000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffff000000000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffff00000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffffff0000000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffff000000000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffff00000000000000000000 +ffffffffff0000000000000000ffffffffffffffff0000000000000000000000 +ffffffffff0000000000000000ffffffffffffff000000000000000000000000 +ffffffffffff0000ffff0000ffffffffffffff00000000000000000000000000 +ffffffffffff0000ffff0000ffffffffffff0000000000000000000000000000 +ffffffffffffffffffffffffffffffffff000000000000000000000000000000 +ffffffffffffffffffffffffffffffff00000000000000000000000000000000 +ffffffffffffffffffffffffffffff0000000000000000000000000000000000 +ffffffffffffffffffffffffffff000000000000000000000000000000000000 +ffffffffffffffffffffffffff00000000000000ffffffffffffff0000000000 +ffffffffffffffffffffffff0000000000000000ffffffffffffff0000000000 +ffffffffffffffffffffff000000000000000000ffff00000000ffff00000000 +ffffffffffffffffffff00000000000000000000ffff00000000ffff00000000 +ffffffffffffffffff0000000000000000000000ffffffffffffff0000000000 +ffffffffffffffff000000000000000000000000ffffffffffffff0000000000 +ffffffffffffff00000000000000000000000000ffff00000000ffff00000000 +ffffffffffff0000000000000000000000000000ffff00000000ffff00000000 +ffffffffff000000000000000000000000000000ffffffffffffff0000000000 +ffffffff00000000000000000000000000000000ffffffffffffff0000000000 +ffffff0000000000000000000000000000000000000000000000000000000000 +ffff000000000000000000000000000000000000000000000000000000000000 +ff00000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 } diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g02-29.png b/src/pkg/image/png/testdata/pngsuite/basn0g02-29.png Binary files differnew file mode 100644 index 000000000..d17d8f83c --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn0g02-29.png diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g02-29.sng b/src/pkg/image/png/testdata/pngsuite/basn0g02-29.sng new file mode 100644 index 000000000..afb5dba48 --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn0g02-29.sng @@ -0,0 +1,38 @@ +#SNG: from basn0g02-29.png +IHDR { + width: 29; height: 29; bitdepth: 8; + using grayscale; +} +gAMA {1.0000} +IMAGE { + pixels hex +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +} diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g02.sng b/src/pkg/image/png/testdata/pngsuite/basn0g02.sng index e7f2d7ea9..bb53d750d 100644 --- a/src/pkg/image/png/testdata/pngsuite/basn0g02.sng +++ b/src/pkg/image/png/testdata/pngsuite/basn0g02.sng @@ -1,41 +1,41 @@ #SNG: from basn0g02.png IHDR { - width: 32; height: 32; bitdepth: 2; + width: 32; height: 32; bitdepth: 8; using grayscale; } gAMA {1.0000} IMAGE { pixels hex -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa } diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g04-31.png b/src/pkg/image/png/testdata/pngsuite/basn0g04-31.png Binary files differnew file mode 100644 index 000000000..e30644d34 --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn0g04-31.png diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g04-31.sng b/src/pkg/image/png/testdata/pngsuite/basn0g04-31.sng new file mode 100644 index 000000000..7f7948e1f --- /dev/null +++ b/src/pkg/image/png/testdata/pngsuite/basn0g04-31.sng @@ -0,0 +1,40 @@ +#SNG: from basn0g04-31.png +IHDR { + width: 31; height: 31; bitdepth: 8; + using grayscale; +} +gAMA {1.0000} +IMAGE { + pixels hex +00000000111111112222222233333333444444445555555566666666777777 +00000000111111112222222233333333444444445555555566666666777777 +00000000111111112222222233333333444444445555555566666666777777 +00000000111111112222222233333333444444445555555566666666777777 +11111111222222223333333344444444555555556666666677777777888888 +11111111222222223333333344444444555555556666666677777777888888 +11111111222222223333333344444444555555556666666677777777888888 +11111111222222223333333344444444555555556666666677777777888888 +22222222333333334444444455555555666666667777777788888888999999 +22222222333333334444444455555555666666667777777788888888999999 +22222222333333334444444455555555666666667777777788888888999999 +22222222333333334444444455555555666666667777777788888888999999 +33333333444444445555555566666666777777778888888899999999aaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaa +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee +} diff --git a/src/pkg/image/png/testdata/pngsuite/basn0g04.sng b/src/pkg/image/png/testdata/pngsuite/basn0g04.sng index 396c508b2..a95ad01a3 100644 --- a/src/pkg/image/png/testdata/pngsuite/basn0g04.sng +++ b/src/pkg/image/png/testdata/pngsuite/basn0g04.sng @@ -1,41 +1,41 @@ #SNG: from basn0g04.png IHDR { - width: 32; height: 32; bitdepth: 4; + width: 32; height: 32; bitdepth: 8; using grayscale; } gAMA {1.0000} IMAGE { pixels hex -00001111222233334444555566667777 -00001111222233334444555566667777 -00001111222233334444555566667777 -00001111222233334444555566667777 -11112222333344445555666677778888 -11112222333344445555666677778888 -11112222333344445555666677778888 -11112222333344445555666677778888 -22223333444455556666777788889999 -22223333444455556666777788889999 -22223333444455556666777788889999 -22223333444455556666777788889999 -3333444455556666777788889999aaaa -3333444455556666777788889999aaaa -3333444455556666777788889999aaaa -3333444455556666777788889999aaaa -444455556666777788889999aaaabbbb -444455556666777788889999aaaabbbb -444455556666777788889999aaaabbbb -444455556666777788889999aaaabbbb -55556666777788889999aaaabbbbcccc -55556666777788889999aaaabbbbcccc -55556666777788889999aaaabbbbcccc -55556666777788889999aaaabbbbcccc -6666777788889999aaaabbbbccccdddd -6666777788889999aaaabbbbccccdddd -6666777788889999aaaabbbbccccdddd -6666777788889999aaaabbbbccccdddd -777788889999aaaabbbbccccddddeeee -777788889999aaaabbbbccccddddeeee -777788889999aaaabbbbccccddddeeee -777788889999aaaabbbbccccddddeeee +0000000011111111222222223333333344444444555555556666666677777777 +0000000011111111222222223333333344444444555555556666666677777777 +0000000011111111222222223333333344444444555555556666666677777777 +0000000011111111222222223333333344444444555555556666666677777777 +1111111122222222333333334444444455555555666666667777777788888888 +1111111122222222333333334444444455555555666666667777777788888888 +1111111122222222333333334444444455555555666666667777777788888888 +1111111122222222333333334444444455555555666666667777777788888888 +2222222233333333444444445555555566666666777777778888888899999999 +2222222233333333444444445555555566666666777777778888888899999999 +2222222233333333444444445555555566666666777777778888888899999999 +2222222233333333444444445555555566666666777777778888888899999999 +33333333444444445555555566666666777777778888888899999999aaaaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaaaa +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee } diff --git a/src/pkg/image/png/testdata/pngsuite/basn3p02.sng b/src/pkg/image/png/testdata/pngsuite/basn3p02.sng index f2321aa3d..ab3fb375f 100644 --- a/src/pkg/image/png/testdata/pngsuite/basn3p02.sng +++ b/src/pkg/image/png/testdata/pngsuite/basn3p02.sng @@ -4,14 +4,11 @@ IHDR { using color palette; } gAMA {1.0000} -sBIT { - red: 1; green: 1; blue: 1; -} PLTE { - ( 0,255, 0) # rgb = (0x00,0xff,0x00) green1 - (255, 0, 0) # rgb = (0xff,0x00,0x00) red1 - (255,255, 0) # rgb = (0xff,0xff,0x00) yellow1 - ( 0, 0,255) # rgb = (0x00,0x00,0xff) blue1 + ( 0,255, 0) # rgb = (0x00,0xff,0x00) + (255, 0, 0) # rgb = (0xff,0x00,0x00) + (255,255, 0) # rgb = (0xff,0xff,0x00) + ( 0, 0,255) # rgb = (0x00,0x00,0xff) } IMAGE { pixels hex diff --git a/src/pkg/image/png/testdata/pngsuite/basn3p04.sng b/src/pkg/image/png/testdata/pngsuite/basn3p04.sng index e52885dba..a2b2fb53c 100644 --- a/src/pkg/image/png/testdata/pngsuite/basn3p04.sng +++ b/src/pkg/image/png/testdata/pngsuite/basn3p04.sng @@ -4,19 +4,16 @@ IHDR { using color palette; } gAMA {1.0000} -sBIT { - red: 4; green: 4; blue: 4; -} PLTE { ( 34, 0,255) # rgb = (0x22,0x00,0xff) - ( 0,255,255) # rgb = (0x00,0xff,0xff) cyan1 + ( 0,255,255) # rgb = (0x00,0xff,0xff) (136, 0,255) # rgb = (0x88,0x00,0xff) ( 34,255, 0) # rgb = (0x22,0xff,0x00) ( 0,153,255) # rgb = (0x00,0x99,0xff) (255,102, 0) # rgb = (0xff,0x66,0x00) (221, 0,255) # rgb = (0xdd,0x00,0xff) (119,255, 0) # rgb = (0x77,0xff,0x00) - (255, 0, 0) # rgb = (0xff,0x00,0x00) red1 + (255, 0, 0) # rgb = (0xff,0x00,0x00) ( 0,255,153) # rgb = (0x00,0xff,0x99) (221,255, 0) # rgb = (0xdd,0xff,0x00) (255, 0,187) # rgb = (0xff,0x00,0xbb) diff --git a/src/pkg/image/png/testdata/pngsuite/basn4a08.sng b/src/pkg/image/png/testdata/pngsuite/basn4a08.sng index b760382a0..cc4096fac 100644 --- a/src/pkg/image/png/testdata/pngsuite/basn4a08.sng +++ b/src/pkg/image/png/testdata/pngsuite/basn4a08.sng @@ -1,41 +1,41 @@ #SNG: from basn4a08.png IHDR { width: 32; height: 32; bitdepth: 8; - using grayscale alpha; + using color alpha; } gAMA {1.0000} IMAGE { pixels hex -ff00 ff08 ff10 ff18 ff20 ff29 ff31 ff39 ff41 ff4a ff52 ff5a ff62 ff6a ff73 ff7b ff83 ff8b ff94 ff9c ffa4 ffac ffb4 ffbd ffc5 ffcd ffd5 ffde ffe6 ffee fff6 ffff -f600 f608 f610 f618 f620 f629 f631 f639 f641 f64a f652 f65a f662 f66a f673 f67b f683 f68b f694 f69c f6a4 f6ac f6b4 f6bd f6c5 f6cd f6d5 f6de f6e6 f6ee f6f6 f6ff -ee00 ee08 ee10 ee18 ee20 ee29 ee31 ee39 ee41 ee4a ee52 ee5a ee62 ee6a ee73 ee7b ee83 ee8b ee94 ee9c eea4 eeac eeb4 eebd eec5 eecd eed5 eede eee6 eeee eef6 eeff -e600 e608 e610 e618 e620 e629 e631 e639 e641 e64a e652 e65a e662 e66a e673 e67b e683 e68b e694 e69c e6a4 e6ac e6b4 e6bd e6c5 e6cd e6d5 e6de e6e6 e6ee e6f6 e6ff -de00 de08 de10 de18 de20 de29 de31 de39 de41 de4a de52 de5a de62 de6a de73 de7b de83 de8b de94 de9c dea4 deac deb4 debd dec5 decd ded5 dede dee6 deee def6 deff -d500 d508 d510 d518 d520 d529 d531 d539 d541 d54a d552 d55a d562 d56a d573 d57b d583 d58b d594 d59c d5a4 d5ac d5b4 d5bd d5c5 d5cd d5d5 d5de d5e6 d5ee d5f6 d5ff -cd00 cd08 cd10 cd18 cd20 cd29 cd31 cd39 cd41 cd4a cd52 cd5a cd62 cd6a cd73 cd7b cd83 cd8b cd94 cd9c cda4 cdac cdb4 cdbd cdc5 cdcd cdd5 cdde cde6 cdee cdf6 cdff -c500 c508 c510 c518 c520 c529 c531 c539 c541 c54a c552 c55a c562 c56a c573 c57b c583 c58b c594 c59c c5a4 c5ac c5b4 c5bd c5c5 c5cd c5d5 c5de c5e6 c5ee c5f6 c5ff -bd00 bd08 bd10 bd18 bd20 bd29 bd31 bd39 bd41 bd4a bd52 bd5a bd62 bd6a bd73 bd7b bd83 bd8b bd94 bd9c bda4 bdac bdb4 bdbd bdc5 bdcd bdd5 bdde bde6 bdee bdf6 bdff -b400 b408 b410 b418 b420 b429 b431 b439 b441 b44a b452 b45a b462 b46a b473 b47b b483 b48b b494 b49c b4a4 b4ac b4b4 b4bd b4c5 b4cd b4d5 b4de b4e6 b4ee b4f6 b4ff -ac00 ac08 ac10 ac18 ac20 ac29 ac31 ac39 ac41 ac4a ac52 ac5a ac62 ac6a ac73 ac7b ac83 ac8b ac94 ac9c aca4 acac acb4 acbd acc5 accd acd5 acde ace6 acee acf6 acff -a400 a408 a410 a418 a420 a429 a431 a439 a441 a44a a452 a45a a462 a46a a473 a47b a483 a48b a494 a49c a4a4 a4ac a4b4 a4bd a4c5 a4cd a4d5 a4de a4e6 a4ee a4f6 a4ff -9c00 9c08 9c10 9c18 9c20 9c29 9c31 9c39 9c41 9c4a 9c52 9c5a 9c62 9c6a 9c73 9c7b 9c83 9c8b 9c94 9c9c 9ca4 9cac 9cb4 9cbd 9cc5 9ccd 9cd5 9cde 9ce6 9cee 9cf6 9cff -9400 9408 9410 9418 9420 9429 9431 9439 9441 944a 9452 945a 9462 946a 9473 947b 9483 948b 9494 949c 94a4 94ac 94b4 94bd 94c5 94cd 94d5 94de 94e6 94ee 94f6 94ff -8b00 8b08 8b10 8b18 8b20 8b29 8b31 8b39 8b41 8b4a 8b52 8b5a 8b62 8b6a 8b73 8b7b 8b83 8b8b 8b94 8b9c 8ba4 8bac 8bb4 8bbd 8bc5 8bcd 8bd5 8bde 8be6 8bee 8bf6 8bff -8300 8308 8310 8318 8320 8329 8331 8339 8341 834a 8352 835a 8362 836a 8373 837b 8383 838b 8394 839c 83a4 83ac 83b4 83bd 83c5 83cd 83d5 83de 83e6 83ee 83f6 83ff -7b00 7b08 7b10 7b18 7b20 7b29 7b31 7b39 7b41 7b4a 7b52 7b5a 7b62 7b6a 7b73 7b7b 7b83 7b8b 7b94 7b9c 7ba4 7bac 7bb4 7bbd 7bc5 7bcd 7bd5 7bde 7be6 7bee 7bf6 7bff -7300 7308 7310 7318 7320 7329 7331 7339 7341 734a 7352 735a 7362 736a 7373 737b 7383 738b 7394 739c 73a4 73ac 73b4 73bd 73c5 73cd 73d5 73de 73e6 73ee 73f6 73ff -6a00 6a08 6a10 6a18 6a20 6a29 6a31 6a39 6a41 6a4a 6a52 6a5a 6a62 6a6a 6a73 6a7b 6a83 6a8b 6a94 6a9c 6aa4 6aac 6ab4 6abd 6ac5 6acd 6ad5 6ade 6ae6 6aee 6af6 6aff -6200 6208 6210 6218 6220 6229 6231 6239 6241 624a 6252 625a 6262 626a 6273 627b 6283 628b 6294 629c 62a4 62ac 62b4 62bd 62c5 62cd 62d5 62de 62e6 62ee 62f6 62ff -5a00 5a08 5a10 5a18 5a20 5a29 5a31 5a39 5a41 5a4a 5a52 5a5a 5a62 5a6a 5a73 5a7b 5a83 5a8b 5a94 5a9c 5aa4 5aac 5ab4 5abd 5ac5 5acd 5ad5 5ade 5ae6 5aee 5af6 5aff -5200 5208 5210 5218 5220 5229 5231 5239 5241 524a 5252 525a 5262 526a 5273 527b 5283 528b 5294 529c 52a4 52ac 52b4 52bd 52c5 52cd 52d5 52de 52e6 52ee 52f6 52ff -4a00 4a08 4a10 4a18 4a20 4a29 4a31 4a39 4a41 4a4a 4a52 4a5a 4a62 4a6a 4a73 4a7b 4a83 4a8b 4a94 4a9c 4aa4 4aac 4ab4 4abd 4ac5 4acd 4ad5 4ade 4ae6 4aee 4af6 4aff -4100 4108 4110 4118 4120 4129 4131 4139 4141 414a 4152 415a 4162 416a 4173 417b 4183 418b 4194 419c 41a4 41ac 41b4 41bd 41c5 41cd 41d5 41de 41e6 41ee 41f6 41ff -3900 3908 3910 3918 3920 3929 3931 3939 3941 394a 3952 395a 3962 396a 3973 397b 3983 398b 3994 399c 39a4 39ac 39b4 39bd 39c5 39cd 39d5 39de 39e6 39ee 39f6 39ff -3100 3108 3110 3118 3120 3129 3131 3139 3141 314a 3152 315a 3162 316a 3173 317b 3183 318b 3194 319c 31a4 31ac 31b4 31bd 31c5 31cd 31d5 31de 31e6 31ee 31f6 31ff -2900 2908 2910 2918 2920 2929 2931 2939 2941 294a 2952 295a 2962 296a 2973 297b 2983 298b 2994 299c 29a4 29ac 29b4 29bd 29c5 29cd 29d5 29de 29e6 29ee 29f6 29ff -2000 2008 2010 2018 2020 2029 2031 2039 2041 204a 2052 205a 2062 206a 2073 207b 2083 208b 2094 209c 20a4 20ac 20b4 20bd 20c5 20cd 20d5 20de 20e6 20ee 20f6 20ff -1800 1808 1810 1818 1820 1829 1831 1839 1841 184a 1852 185a 1862 186a 1873 187b 1883 188b 1894 189c 18a4 18ac 18b4 18bd 18c5 18cd 18d5 18de 18e6 18ee 18f6 18ff -1000 1008 1010 1018 1020 1029 1031 1039 1041 104a 1052 105a 1062 106a 1073 107b 1083 108b 1094 109c 10a4 10ac 10b4 10bd 10c5 10cd 10d5 10de 10e6 10ee 10f6 10ff -0800 0808 0810 0818 0820 0829 0831 0839 0841 084a 0852 085a 0862 086a 0873 087b 0883 088b 0894 089c 08a4 08ac 08b4 08bd 08c5 08cd 08d5 08de 08e6 08ee 08f6 08ff -0000 0008 0010 0018 0020 0029 0031 0039 0041 004a 0052 005a 0062 006a 0073 007b 0083 008b 0094 009c 00a4 00ac 00b4 00bd 00c5 00cd 00d5 00de 00e6 00ee 00f6 00ff +ffffff00 ffffff08 ffffff10 ffffff18 ffffff20 ffffff29 ffffff31 ffffff39 ffffff41 ffffff4a ffffff52 ffffff5a ffffff62 ffffff6a ffffff73 ffffff7b ffffff83 ffffff8b ffffff94 ffffff9c ffffffa4 ffffffac ffffffb4 ffffffbd ffffffc5 ffffffcd ffffffd5 ffffffde ffffffe6 ffffffee fffffff6 ffffffff +f6f6f600 f6f6f608 f6f6f610 f6f6f618 f6f6f620 f6f6f629 f6f6f631 f6f6f639 f6f6f641 f6f6f64a f6f6f652 f6f6f65a f6f6f662 f6f6f66a f6f6f673 f6f6f67b f6f6f683 f6f6f68b f6f6f694 f6f6f69c f6f6f6a4 f6f6f6ac f6f6f6b4 f6f6f6bd f6f6f6c5 f6f6f6cd f6f6f6d5 f6f6f6de f6f6f6e6 f6f6f6ee f6f6f6f6 f6f6f6ff +eeeeee00 eeeeee08 eeeeee10 eeeeee18 eeeeee20 eeeeee29 eeeeee31 eeeeee39 eeeeee41 eeeeee4a eeeeee52 eeeeee5a eeeeee62 eeeeee6a eeeeee73 eeeeee7b eeeeee83 eeeeee8b eeeeee94 eeeeee9c eeeeeea4 eeeeeeac eeeeeeb4 eeeeeebd eeeeeec5 eeeeeecd eeeeeed5 eeeeeede eeeeeee6 eeeeeeee eeeeeef6 eeeeeeff +e6e6e600 e6e6e608 e6e6e610 e6e6e618 e6e6e620 e6e6e629 e6e6e631 e6e6e639 e6e6e641 e6e6e64a e6e6e652 e6e6e65a e6e6e662 e6e6e66a e6e6e673 e6e6e67b e6e6e683 e6e6e68b e6e6e694 e6e6e69c e6e6e6a4 e6e6e6ac e6e6e6b4 e6e6e6bd e6e6e6c5 e6e6e6cd e6e6e6d5 e6e6e6de e6e6e6e6 e6e6e6ee e6e6e6f6 e6e6e6ff +dedede00 dedede08 dedede10 dedede18 dedede20 dedede29 dedede31 dedede39 dedede41 dedede4a dedede52 dedede5a dedede62 dedede6a dedede73 dedede7b dedede83 dedede8b dedede94 dedede9c dededea4 dededeac dededeb4 dededebd dededec5 dededecd dededed5 dededede dededee6 dededeee dededef6 dededeff +d5d5d500 d5d5d508 d5d5d510 d5d5d518 d5d5d520 d5d5d529 d5d5d531 d5d5d539 d5d5d541 d5d5d54a d5d5d552 d5d5d55a d5d5d562 d5d5d56a d5d5d573 d5d5d57b d5d5d583 d5d5d58b d5d5d594 d5d5d59c d5d5d5a4 d5d5d5ac d5d5d5b4 d5d5d5bd d5d5d5c5 d5d5d5cd d5d5d5d5 d5d5d5de d5d5d5e6 d5d5d5ee d5d5d5f6 d5d5d5ff +cdcdcd00 cdcdcd08 cdcdcd10 cdcdcd18 cdcdcd20 cdcdcd29 cdcdcd31 cdcdcd39 cdcdcd41 cdcdcd4a cdcdcd52 cdcdcd5a cdcdcd62 cdcdcd6a cdcdcd73 cdcdcd7b cdcdcd83 cdcdcd8b cdcdcd94 cdcdcd9c cdcdcda4 cdcdcdac cdcdcdb4 cdcdcdbd cdcdcdc5 cdcdcdcd cdcdcdd5 cdcdcdde cdcdcde6 cdcdcdee cdcdcdf6 cdcdcdff +c5c5c500 c5c5c508 c5c5c510 c5c5c518 c5c5c520 c5c5c529 c5c5c531 c5c5c539 c5c5c541 c5c5c54a c5c5c552 c5c5c55a c5c5c562 c5c5c56a c5c5c573 c5c5c57b c5c5c583 c5c5c58b c5c5c594 c5c5c59c c5c5c5a4 c5c5c5ac c5c5c5b4 c5c5c5bd c5c5c5c5 c5c5c5cd c5c5c5d5 c5c5c5de c5c5c5e6 c5c5c5ee c5c5c5f6 c5c5c5ff +bdbdbd00 bdbdbd08 bdbdbd10 bdbdbd18 bdbdbd20 bdbdbd29 bdbdbd31 bdbdbd39 bdbdbd41 bdbdbd4a bdbdbd52 bdbdbd5a bdbdbd62 bdbdbd6a bdbdbd73 bdbdbd7b bdbdbd83 bdbdbd8b bdbdbd94 bdbdbd9c bdbdbda4 bdbdbdac bdbdbdb4 bdbdbdbd bdbdbdc5 bdbdbdcd bdbdbdd5 bdbdbdde bdbdbde6 bdbdbdee bdbdbdf6 bdbdbdff +b4b4b400 b4b4b408 b4b4b410 b4b4b418 b4b4b420 b4b4b429 b4b4b431 b4b4b439 b4b4b441 b4b4b44a b4b4b452 b4b4b45a b4b4b462 b4b4b46a b4b4b473 b4b4b47b b4b4b483 b4b4b48b b4b4b494 b4b4b49c b4b4b4a4 b4b4b4ac b4b4b4b4 b4b4b4bd b4b4b4c5 b4b4b4cd b4b4b4d5 b4b4b4de b4b4b4e6 b4b4b4ee b4b4b4f6 b4b4b4ff +acacac00 acacac08 acacac10 acacac18 acacac20 acacac29 acacac31 acacac39 acacac41 acacac4a acacac52 acacac5a acacac62 acacac6a acacac73 acacac7b acacac83 acacac8b acacac94 acacac9c acacaca4 acacacac acacacb4 acacacbd acacacc5 acacaccd acacacd5 acacacde acacace6 acacacee acacacf6 acacacff +a4a4a400 a4a4a408 a4a4a410 a4a4a418 a4a4a420 a4a4a429 a4a4a431 a4a4a439 a4a4a441 a4a4a44a a4a4a452 a4a4a45a a4a4a462 a4a4a46a a4a4a473 a4a4a47b a4a4a483 a4a4a48b a4a4a494 a4a4a49c a4a4a4a4 a4a4a4ac a4a4a4b4 a4a4a4bd a4a4a4c5 a4a4a4cd a4a4a4d5 a4a4a4de a4a4a4e6 a4a4a4ee a4a4a4f6 a4a4a4ff +9c9c9c00 9c9c9c08 9c9c9c10 9c9c9c18 9c9c9c20 9c9c9c29 9c9c9c31 9c9c9c39 9c9c9c41 9c9c9c4a 9c9c9c52 9c9c9c5a 9c9c9c62 9c9c9c6a 9c9c9c73 9c9c9c7b 9c9c9c83 9c9c9c8b 9c9c9c94 9c9c9c9c 9c9c9ca4 9c9c9cac 9c9c9cb4 9c9c9cbd 9c9c9cc5 9c9c9ccd 9c9c9cd5 9c9c9cde 9c9c9ce6 9c9c9cee 9c9c9cf6 9c9c9cff +94949400 94949408 94949410 94949418 94949420 94949429 94949431 94949439 94949441 9494944a 94949452 9494945a 94949462 9494946a 94949473 9494947b 94949483 9494948b 94949494 9494949c 949494a4 949494ac 949494b4 949494bd 949494c5 949494cd 949494d5 949494de 949494e6 949494ee 949494f6 949494ff +8b8b8b00 8b8b8b08 8b8b8b10 8b8b8b18 8b8b8b20 8b8b8b29 8b8b8b31 8b8b8b39 8b8b8b41 8b8b8b4a 8b8b8b52 8b8b8b5a 8b8b8b62 8b8b8b6a 8b8b8b73 8b8b8b7b 8b8b8b83 8b8b8b8b 8b8b8b94 8b8b8b9c 8b8b8ba4 8b8b8bac 8b8b8bb4 8b8b8bbd 8b8b8bc5 8b8b8bcd 8b8b8bd5 8b8b8bde 8b8b8be6 8b8b8bee 8b8b8bf6 8b8b8bff +83838300 83838308 83838310 83838318 83838320 83838329 83838331 83838339 83838341 8383834a 83838352 8383835a 83838362 8383836a 83838373 8383837b 83838383 8383838b 83838394 8383839c 838383a4 838383ac 838383b4 838383bd 838383c5 838383cd 838383d5 838383de 838383e6 838383ee 838383f6 838383ff +7b7b7b00 7b7b7b08 7b7b7b10 7b7b7b18 7b7b7b20 7b7b7b29 7b7b7b31 7b7b7b39 7b7b7b41 7b7b7b4a 7b7b7b52 7b7b7b5a 7b7b7b62 7b7b7b6a 7b7b7b73 7b7b7b7b 7b7b7b83 7b7b7b8b 7b7b7b94 7b7b7b9c 7b7b7ba4 7b7b7bac 7b7b7bb4 7b7b7bbd 7b7b7bc5 7b7b7bcd 7b7b7bd5 7b7b7bde 7b7b7be6 7b7b7bee 7b7b7bf6 7b7b7bff +73737300 73737308 73737310 73737318 73737320 73737329 73737331 73737339 73737341 7373734a 73737352 7373735a 73737362 7373736a 73737373 7373737b 73737383 7373738b 73737394 7373739c 737373a4 737373ac 737373b4 737373bd 737373c5 737373cd 737373d5 737373de 737373e6 737373ee 737373f6 737373ff +6a6a6a00 6a6a6a08 6a6a6a10 6a6a6a18 6a6a6a20 6a6a6a29 6a6a6a31 6a6a6a39 6a6a6a41 6a6a6a4a 6a6a6a52 6a6a6a5a 6a6a6a62 6a6a6a6a 6a6a6a73 6a6a6a7b 6a6a6a83 6a6a6a8b 6a6a6a94 6a6a6a9c 6a6a6aa4 6a6a6aac 6a6a6ab4 6a6a6abd 6a6a6ac5 6a6a6acd 6a6a6ad5 6a6a6ade 6a6a6ae6 6a6a6aee 6a6a6af6 6a6a6aff +62626200 62626208 62626210 62626218 62626220 62626229 62626231 62626239 62626241 6262624a 62626252 6262625a 62626262 6262626a 62626273 6262627b 62626283 6262628b 62626294 6262629c 626262a4 626262ac 626262b4 626262bd 626262c5 626262cd 626262d5 626262de 626262e6 626262ee 626262f6 626262ff +5a5a5a00 5a5a5a08 5a5a5a10 5a5a5a18 5a5a5a20 5a5a5a29 5a5a5a31 5a5a5a39 5a5a5a41 5a5a5a4a 5a5a5a52 5a5a5a5a 5a5a5a62 5a5a5a6a 5a5a5a73 5a5a5a7b 5a5a5a83 5a5a5a8b 5a5a5a94 5a5a5a9c 5a5a5aa4 5a5a5aac 5a5a5ab4 5a5a5abd 5a5a5ac5 5a5a5acd 5a5a5ad5 5a5a5ade 5a5a5ae6 5a5a5aee 5a5a5af6 5a5a5aff +52525200 52525208 52525210 52525218 52525220 52525229 52525231 52525239 52525241 5252524a 52525252 5252525a 52525262 5252526a 52525273 5252527b 52525283 5252528b 52525294 5252529c 525252a4 525252ac 525252b4 525252bd 525252c5 525252cd 525252d5 525252de 525252e6 525252ee 525252f6 525252ff +4a4a4a00 4a4a4a08 4a4a4a10 4a4a4a18 4a4a4a20 4a4a4a29 4a4a4a31 4a4a4a39 4a4a4a41 4a4a4a4a 4a4a4a52 4a4a4a5a 4a4a4a62 4a4a4a6a 4a4a4a73 4a4a4a7b 4a4a4a83 4a4a4a8b 4a4a4a94 4a4a4a9c 4a4a4aa4 4a4a4aac 4a4a4ab4 4a4a4abd 4a4a4ac5 4a4a4acd 4a4a4ad5 4a4a4ade 4a4a4ae6 4a4a4aee 4a4a4af6 4a4a4aff +41414100 41414108 41414110 41414118 41414120 41414129 41414131 41414139 41414141 4141414a 41414152 4141415a 41414162 4141416a 41414173 4141417b 41414183 4141418b 41414194 4141419c 414141a4 414141ac 414141b4 414141bd 414141c5 414141cd 414141d5 414141de 414141e6 414141ee 414141f6 414141ff +39393900 39393908 39393910 39393918 39393920 39393929 39393931 39393939 39393941 3939394a 39393952 3939395a 39393962 3939396a 39393973 3939397b 39393983 3939398b 39393994 3939399c 393939a4 393939ac 393939b4 393939bd 393939c5 393939cd 393939d5 393939de 393939e6 393939ee 393939f6 393939ff +31313100 31313108 31313110 31313118 31313120 31313129 31313131 31313139 31313141 3131314a 31313152 3131315a 31313162 3131316a 31313173 3131317b 31313183 3131318b 31313194 3131319c 313131a4 313131ac 313131b4 313131bd 313131c5 313131cd 313131d5 313131de 313131e6 313131ee 313131f6 313131ff +29292900 29292908 29292910 29292918 29292920 29292929 29292931 29292939 29292941 2929294a 29292952 2929295a 29292962 2929296a 29292973 2929297b 29292983 2929298b 29292994 2929299c 292929a4 292929ac 292929b4 292929bd 292929c5 292929cd 292929d5 292929de 292929e6 292929ee 292929f6 292929ff +20202000 20202008 20202010 20202018 20202020 20202029 20202031 20202039 20202041 2020204a 20202052 2020205a 20202062 2020206a 20202073 2020207b 20202083 2020208b 20202094 2020209c 202020a4 202020ac 202020b4 202020bd 202020c5 202020cd 202020d5 202020de 202020e6 202020ee 202020f6 202020ff +18181800 18181808 18181810 18181818 18181820 18181829 18181831 18181839 18181841 1818184a 18181852 1818185a 18181862 1818186a 18181873 1818187b 18181883 1818188b 18181894 1818189c 181818a4 181818ac 181818b4 181818bd 181818c5 181818cd 181818d5 181818de 181818e6 181818ee 181818f6 181818ff +10101000 10101008 10101010 10101018 10101020 10101029 10101031 10101039 10101041 1010104a 10101052 1010105a 10101062 1010106a 10101073 1010107b 10101083 1010108b 10101094 1010109c 101010a4 101010ac 101010b4 101010bd 101010c5 101010cd 101010d5 101010de 101010e6 101010ee 101010f6 101010ff +08080800 08080808 08080810 08080818 08080820 08080829 08080831 08080839 08080841 0808084a 08080852 0808085a 08080862 0808086a 08080873 0808087b 08080883 0808088b 08080894 0808089c 080808a4 080808ac 080808b4 080808bd 080808c5 080808cd 080808d5 080808de 080808e6 080808ee 080808f6 080808ff +00000000 00000008 00000010 00000018 00000020 00000029 00000031 00000039 00000041 0000004a 00000052 0000005a 00000062 0000006a 00000073 0000007b 00000083 0000008b 00000094 0000009c 000000a4 000000ac 000000b4 000000bd 000000c5 000000cd 000000d5 000000de 000000e6 000000ee 000000f6 000000ff } diff --git a/src/pkg/image/testdata/video-001.bmp b/src/pkg/image/testdata/video-001.bmp Binary files differnew file mode 100644 index 000000000..ca3dd42a7 --- /dev/null +++ b/src/pkg/image/testdata/video-001.bmp diff --git a/src/pkg/image/testdata/video-001.gif b/src/pkg/image/testdata/video-001.gif Binary files differnew file mode 100644 index 000000000..ca06af61b --- /dev/null +++ b/src/pkg/image/testdata/video-001.gif diff --git a/src/pkg/image/testdata/video-001.jpeg b/src/pkg/image/testdata/video-001.jpeg Binary files differnew file mode 100644 index 000000000..1b87c933b --- /dev/null +++ b/src/pkg/image/testdata/video-001.jpeg diff --git a/src/pkg/image/testdata/video-001.png b/src/pkg/image/testdata/video-001.png Binary files differnew file mode 100644 index 000000000..d3468bbe8 --- /dev/null +++ b/src/pkg/image/testdata/video-001.png diff --git a/src/pkg/image/testdata/video-001.tiff b/src/pkg/image/testdata/video-001.tiff Binary files differnew file mode 100644 index 000000000..0dd6cd931 --- /dev/null +++ b/src/pkg/image/testdata/video-001.tiff diff --git a/src/pkg/io/ioutil/tempfile.go b/src/pkg/io/ioutil/tempfile.go index 114eca2b5..62f8849c0 100644 --- a/src/pkg/io/ioutil/tempfile.go +++ b/src/pkg/io/ioutil/tempfile.go @@ -6,6 +6,7 @@ package ioutil import ( "os" + "path/filepath" "strconv" ) @@ -46,7 +47,7 @@ func TempFile(dir, prefix string) (f *os.File, err os.Error) { nconflict := 0 for i := 0; i < 10000; i++ { - name := dir + "/" + prefix + nextSuffix() + name := filepath.Join(dir, prefix+nextSuffix()) f, err = os.Open(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST { if nconflict++; nconflict > 10 { @@ -58,3 +59,33 @@ func TempFile(dir, prefix string) (f *os.File, err os.Error) { } return } + +// TempDir creates a new temporary directory in the directory dir +// with a name beginning with prefix and returns the path of the +// new directory. If dir is the empty string, TempDir uses the +// default directory for temporary files (see os.TempDir). +// Multiple programs calling TempDir simultaneously +// will not choose the same directory. It is the caller's responsibility +// to remove the directory when no longer needed. +func TempDir(dir, prefix string) (name string, err os.Error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + try := filepath.Join(dir, prefix+nextSuffix()) + err = os.Mkdir(try, 0700) + if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST { + if nconflict++; nconflict > 10 { + rand = reseed() + } + continue + } + if err == nil { + name = try + } + break + } + return +} diff --git a/src/pkg/io/ioutil/tempfile_test.go b/src/pkg/io/ioutil/tempfile_test.go index d949a86cf..80c62f672 100644 --- a/src/pkg/io/ioutil/tempfile_test.go +++ b/src/pkg/io/ioutil/tempfile_test.go @@ -7,6 +7,7 @@ package ioutil_test import ( . "io/ioutil" "os" + "path/filepath" "regexp" "testing" ) @@ -23,11 +24,31 @@ func TestTempFile(t *testing.T) { t.Errorf("TempFile(dir, `ioutil_test`) = %v, %v", f, err) } if f != nil { - re := regexp.MustCompile("^" + regexp.QuoteMeta(dir) + "/ioutil_test[0-9]+$") + f.Close() + os.Remove(f.Name()) + re := regexp.MustCompile("^" + regexp.QuoteMeta(filepath.Join(dir, "ioutil_test")) + "[0-9]+$") if !re.MatchString(f.Name()) { t.Errorf("TempFile(`"+dir+"`, `ioutil_test`) created bad name %s", f.Name()) } - os.Remove(f.Name()) } - f.Close() +} + +func TestTempDir(t *testing.T) { + name, err := TempDir("/_not_exists_", "foo") + if name != "" || err == nil { + t.Errorf("TempDir(`/_not_exists_`, `foo`) = %v, %v", name, err) + } + + dir := os.TempDir() + name, err = TempDir(dir, "ioutil_test") + if name == "" || err != nil { + t.Errorf("TempDir(dir, `ioutil_test`) = %v, %v", name, err) + } + if name != "" { + os.Remove(name) + re := regexp.MustCompile("^" + regexp.QuoteMeta(filepath.Join(dir, "ioutil_test")) + "[0-9]+$") + if !re.MatchString(name) { + t.Errorf("TempDir(`"+dir+"`, `ioutil_test`) created bad name %s", name) + } + } } diff --git a/src/pkg/json/decode.go b/src/pkg/json/decode.go index ff91dd83c..501230c0c 100644 --- a/src/pkg/json/decode.go +++ b/src/pkg/json/decode.go @@ -9,6 +9,7 @@ package json import ( "container/vector" + "encoding/base64" "os" "reflect" "runtime" @@ -466,13 +467,15 @@ func (d *decodeState) object(v reflect.Value) { } else { var f reflect.StructField var ok bool - // First try for field with that tag. st := sv.Type().(*reflect.StructType) - for i := 0; i < sv.NumField(); i++ { - f = st.Field(i) - if f.Tag == key { - ok = true - break + // First try for field with that tag. + if isValidTag(key) { + for i := 0; i < sv.NumField(); i++ { + f = st.Field(i) + if f.Tag == key { + ok = true + break + } } } if !ok { @@ -568,17 +571,29 @@ func (d *decodeState) literal(v reflect.Value) { } case '"': // string - s, ok := unquote(item) + s, ok := unquoteBytes(item) if !ok { d.error(errPhase) } switch v := v.(type) { default: d.saveError(&UnmarshalTypeError{"string", v.Type()}) + case *reflect.SliceValue: + if v.Type() != byteSliceType { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.NewValue(b[0:n]).(*reflect.SliceValue)) case *reflect.StringValue: - v.Set(s) + v.Set(string(s)) case *reflect.InterfaceValue: - v.Set(reflect.NewValue(s)) + v.Set(reflect.NewValue(string(s))) } default: // number @@ -772,12 +787,43 @@ func getu4(s []byte) int { // unquote converts a quoted JSON string literal s into an actual string t. // The rules are different than for Go, so cannot use strconv.Unquote. func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { return } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rune, size := utf8.DecodeRune(s[r:]) + if rune == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + b := make([]byte, len(s)+2*utf8.UTFMax) - w := 0 - for r := 1; r < len(s)-1; { + w := copy(b, s[0:r]) + for r < len(s) { // Out of room? Can only happen if s is full of // malformed UTF-8 and we're replacing each // byte with RuneError. @@ -789,7 +835,7 @@ func unquote(s []byte) (t string, ok bool) { switch c := s[r]; { case c == '\\': r++ - if r >= len(s)-1 { + if r >= len(s) { return } switch s[r] { @@ -857,5 +903,5 @@ func unquote(s []byte) (t string, ok bool) { w += utf8.EncodeRune(b[w:], rune) } } - return string(b[0:w]), true + return b[0:w], true } diff --git a/src/pkg/json/decode_test.go b/src/pkg/json/decode_test.go index 9cb27af41..ad6026363 100644 --- a/src/pkg/json/decode_test.go +++ b/src/pkg/json/decode_test.go @@ -40,6 +40,11 @@ var ( umtrue = unmarshaler{true} ) +type badTag struct { + X string + Y string "y" + Z string "@#*%(#@" +} type unmarshalTest struct { in string @@ -62,6 +67,9 @@ var unmarshalTests = []unmarshalTest{ {`{"X": [1,2,3], "Y": 4}`, new(T), T{Y: 4}, &UnmarshalTypeError{"array", reflect.Typeof("")}}, {`{"x": 1}`, new(tx), tx{}, &UnmarshalFieldError{"x", txType, txType.Field(0)}}, + // skip invalid tags + {`{"X":"a", "y":"b", "Z":"c"}`, new(badTag), badTag{"a", "b", "c"}, nil}, + // syntax errors {`{"X": "foo", "Y"}`, nil, nil, SyntaxError("invalid character '}' after object key")}, @@ -164,6 +172,25 @@ func TestUnmarshalMarshal(t *testing.T) { } } +func TestLargeByteSlice(t *testing.T) { + s0 := make([]byte, 2000) + for i := range s0 { + s0[i] = byte(i) + } + b, err := Marshal(s0) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + var s1 []byte + if err := Unmarshal(b, &s1); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if bytes.Compare(s0, s1) != 0 { + t.Errorf("Marshal large byte slice") + diff(t, s0, s1) + } +} + type Xint struct { X int } @@ -412,11 +439,7 @@ var allValueIndent = `{ "str25", "str26" ], - "ByteSlice": [ - 27, - 28, - 29 - ], + "ByteSlice": "Gxwd", "Small": { "Tag": "tag30" }, @@ -502,7 +525,7 @@ var pallValueIndent = `{ "EmptySlice": [], "NilSlice": [], "StringSlice": [], - "ByteSlice": [], + "ByteSlice": "", "Small": { "Tag": "" }, diff --git a/src/pkg/json/encode.go b/src/pkg/json/encode.go index 0fcc78aa8..26ce47039 100644 --- a/src/pkg/json/encode.go +++ b/src/pkg/json/encode.go @@ -7,12 +7,14 @@ package json import ( - "os" "bytes" + "encoding/base64" + "os" "reflect" "runtime" "sort" "strconv" + "unicode" "utf8" ) @@ -31,12 +33,14 @@ import ( // String values encode as JSON strings, with each invalid UTF-8 sequence // replaced by the encoding of the Unicode replacement character U+FFFD. // -// Array and slice values encode as JSON arrays. +// Array and slice values encode as JSON arrays, except that +// []byte encodes as a base64-encoded string. // // Struct values encode as JSON objects. Each struct field becomes // a member of the object. By default the object's key name is the -// struct field name. If the struct field has a tag, that tag will -// be used as the name instead. Only exported fields will be encoded. +// struct field name. If the struct field has a non-empty tag consisting +// of only Unicode letters, digits, and underscores, that tag will be used +// as the name instead. Only exported fields will be encoded. // // Map values encode as JSON objects. // The map's key type must be string; the object keys are used directly @@ -176,6 +180,8 @@ func (e *encodeState) error(err os.Error) { panic(err) } +var byteSliceType = reflect.Typeof([]byte(nil)) + func (e *encodeState) reflectValue(v reflect.Value) { if v == nil { e.WriteString("null") @@ -230,7 +236,7 @@ func (e *encodeState) reflectValue(v reflect.Value) { } else { e.WriteByte(',') } - if f.Tag != "" { + if isValidTag(f.Tag) { e.string(f.Tag) } else { e.string(f.Name) @@ -262,6 +268,24 @@ func (e *encodeState) reflectValue(v reflect.Value) { e.WriteByte('}') case reflect.ArrayOrSliceValue: + if v.Type() == byteSliceType { + e.WriteByte('"') + s := v.Interface().([]byte) + if len(s) < 1024 { + // for small buffers, using Encode directly is much faster. + dst := make([]byte, base64.StdEncoding.EncodedLen(len(s))) + base64.StdEncoding.Encode(dst, s) + e.Write(dst) + } else { + // for large buffers, avoid unnecessary extra temporary + // buffer space. + enc := base64.NewEncoder(base64.StdEncoding, e) + enc.Write(s) + enc.Close() + } + e.WriteByte('"') + break + } e.WriteByte('[') n := v.Len() for i := 0; i < n; i++ { @@ -285,6 +309,18 @@ func (e *encodeState) reflectValue(v reflect.Value) { return } +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + return true +} + // stringValues is a slice of reflect.Value holding *reflect.StringValue. // It implements the methods to sort by string. type stringValues []reflect.Value diff --git a/src/pkg/mime/multipart/multipart.go b/src/pkg/mime/multipart/multipart.go index 1d855c74c..0a65a447d 100644 --- a/src/pkg/mime/multipart/multipart.go +++ b/src/pkg/mime/multipart/multipart.go @@ -17,6 +17,7 @@ import ( "bytes" "io" "mime" + "net/textproto" "os" "regexp" "strings" @@ -40,7 +41,7 @@ type Part struct { // The headers of the body, if any, with the keys canonicalized // in the same fashion that the Go http.Request headers are. // i.e. "foo-bar" changes case to "Foo-Bar" - Header map[string]string + Header textproto.MIMEHeader buffer *bytes.Buffer mr *multiReader @@ -51,8 +52,8 @@ type Part struct { func (p *Part) FormName() string { // See http://tools.ietf.org/html/rfc2183 section 2 for EBNF // of Content-Disposition value format. - v, ok := p.Header["Content-Disposition"] - if !ok { + v := p.Header.Get("Content-Disposition") + if v == "" { return "" } d, params := mime.ParseMediaType(v) @@ -85,7 +86,7 @@ var devNull = devNullWriter(false) func newPart(mr *multiReader) (bp *Part, err os.Error) { bp = new(Part) - bp.Header = make(map[string]string) + bp.Header = make(map[string][]string) bp.mr = mr bp.buffer = new(bytes.Buffer) if err = bp.populateHeaders(); err != nil { @@ -104,10 +105,7 @@ func (bp *Part) populateHeaders() os.Error { return nil } if matches := headerRegexp.FindStringSubmatch(line); len(matches) == 3 { - key := matches[1] - value := matches[2] - // TODO: canonicalize headers ala http.Request.Header? - bp.Header[key] = value + bp.Header.Add(matches[1], matches[2]) continue } return os.NewError("Unexpected header line found parsing multipart body") diff --git a/src/pkg/mime/multipart/multipart_test.go b/src/pkg/mime/multipart/multipart_test.go index 7e1ed133e..1f3d32d7e 100644 --- a/src/pkg/mime/multipart/multipart_test.go +++ b/src/pkg/mime/multipart/multipart_test.go @@ -58,7 +58,7 @@ func expectEq(t *testing.T, expected, actual, what string) { func TestFormName(t *testing.T) { p := new(Part) - p.Header = make(map[string]string) + p.Header = make(map[string][]string) tests := [...][2]string{ {`form-data; name="foo"`, "foo"}, {` form-data ; name=foo`, "foo"}, @@ -69,7 +69,7 @@ func TestFormName(t *testing.T) { {` FORM-DATA ; filename="foo.txt"; name=foo; baz=quux`, "foo"}, } for _, test := range tests { - p.Header["Content-Disposition"] = test[0] + p.Header.Set("Content-Disposition", test[0]) expected := test[1] actual := p.FormName() if actual != expected { @@ -114,12 +114,15 @@ never read data t.Error("Expected part1") return } - if part.Header["Header1"] != "value1" { + if part.Header.Get("Header1") != "value1" { t.Error("Expected Header1: value") } - if part.Header["foo-bar"] != "baz" { + if part.Header.Get("foo-bar") != "baz" { t.Error("Expected foo-bar: baz") } + if part.Header.Get("Foo-Bar") != "baz" { + t.Error("Expected Foo-Bar: baz") + } buf.Reset() io.Copy(buf, part) expectEq(t, "My value\r\nThe end.", @@ -131,7 +134,7 @@ never read data t.Error("Expected part2") return } - if part.Header["foo-bar"] != "bazb" { + if part.Header.Get("foo-bar") != "bazb" { t.Error("Expected foo-bar: bazb") } buf.Reset() diff --git a/src/pkg/net/dial.go b/src/pkg/net/dial.go index 03b9d87be..1cf8e7915 100644 --- a/src/pkg/net/dial.go +++ b/src/pkg/net/dial.go @@ -24,7 +24,7 @@ import "os" // Dial("tcp", "127.0.0.1:123", "127.0.0.1:88") // func Dial(net, laddr, raddr string) (c Conn, err os.Error) { - switch prefixBefore(net, ':') { + switch net { case "tcp", "tcp4", "tcp6": var la, ra *TCPAddr if laddr != "" { @@ -137,7 +137,7 @@ func Listen(net, laddr string) (l Listener, err os.Error) { // The network string net must be a packet-oriented network: // "udp", "udp4", "udp6", or "unixgram". func ListenPacket(net, laddr string) (c PacketConn, err os.Error) { - switch prefixBefore(net, ':') { + switch net { case "udp", "udp4", "udp6": var la *UDPAddr if laddr != "" { @@ -162,18 +162,24 @@ func ListenPacket(net, laddr string) (c PacketConn, err os.Error) { return nil, err } return c, nil - case "ip", "ip4", "ip6": - var la *IPAddr - if laddr != "" { - if la, err = ResolveIPAddr(laddr); err != nil { + } + + if i := last(net, ':'); i > 0 { + switch net[0:i] { + case "ip", "ip4", "ip6": + var la *IPAddr + if laddr != "" { + if la, err = ResolveIPAddr(laddr); err != nil { + return nil, err + } + } + c, err := ListenIP(net, la) + if err != nil { return nil, err } + return c, nil } - c, err := ListenIP(net, la) - if err != nil { - return nil, err - } - return c, nil } + return nil, UnknownNetworkError(net) } diff --git a/src/pkg/net/fd_windows.go b/src/pkg/net/fd_windows.go index d9c83831d..63a8fbc44 100644 --- a/src/pkg/net/fd_windows.go +++ b/src/pkg/net/fd_windows.go @@ -13,147 +13,241 @@ import ( "unsafe" ) +type InvalidConnError struct{} + +func (e *InvalidConnError) String() string { return "invalid net.Conn" } +func (e *InvalidConnError) Temporary() bool { return false } +func (e *InvalidConnError) Timeout() bool { return false } + +var initErr os.Error + +func init() { + var d syscall.WSAData + e := syscall.WSAStartup(uint32(0x101), &d) + if e != 0 { + initErr = os.NewSyscallError("WSAStartup", e) + } +} + +func closesocket(s int) (errno int) { + return syscall.Closesocket(int32(s)) +} + +// Interface for all io operations. +type anOpIface interface { + Op() *anOp + Name() string + Submit() (errno int) +} + // IO completion result parameters. type ioResult struct { - key uint32 - qty uint32 - errno int + qty uint32 + err int } -// Network file descriptor. -type netFD struct { - // locking/lifetime of sysfd - sysmu sync.Mutex - sysref int - closing bool +// anOp implements functionality common to all io operations. +type anOp struct { + // Used by IOCP interface, it must be first field + // of the struct, as our code rely on it. + o syscall.Overlapped - // immutable until Close - sysfd int - family int - proto int - cr chan *ioResult - cw chan *ioResult - net string - laddr Addr - raddr Addr + resultc chan ioResult // io completion results + errnoc chan int // io submit / cancel operation errors + fd *netFD +} - // owned by client - rdeadline_delta int64 - rdeadline int64 - rio sync.Mutex - wdeadline_delta int64 - wdeadline int64 - wio sync.Mutex +func (o *anOp) Init(fd *netFD) { + o.fd = fd + o.resultc = make(chan ioResult, 1) + o.errnoc = make(chan int) } -type InvalidConnError struct{} +func (o *anOp) Op() *anOp { + return o +} -func (e *InvalidConnError) String() string { return "invalid net.Conn" } -func (e *InvalidConnError) Temporary() bool { return false } -func (e *InvalidConnError) Timeout() bool { return false } +// bufOp is used by io operations that read / write +// data from / to client buffer. +type bufOp struct { + anOp + buf syscall.WSABuf +} -// pollServer will run around waiting for io completion request -// to arrive. Every request received will contain channel to signal -// io owner about the completion. +func (o *bufOp) Init(fd *netFD, buf []byte) { + o.anOp.Init(fd) + o.buf.Len = uint32(len(buf)) + if len(buf) == 0 { + o.buf.Buf = nil + } else { + o.buf.Buf = (*byte)(unsafe.Pointer(&buf[0])) + } +} -type pollServer struct { +// resultSrv will retreive all io completion results from +// iocp and send them to the correspondent waiting client +// goroutine via channel supplied in the request. +type resultSrv struct { iocp int32 } -func newPollServer() (s *pollServer, err os.Error) { - s = new(pollServer) - var e int - if s.iocp, e = syscall.CreateIoCompletionPort(-1, 0, 0, 1); e != 0 { - return nil, os.NewSyscallError("CreateIoCompletionPort", e) +func (s *resultSrv) Run() { + var o *syscall.Overlapped + var key uint32 + var r ioResult + for { + r.err = syscall.GetQueuedCompletionStatus(s.iocp, &(r.qty), &key, &o, syscall.INFINITE) + switch { + case r.err == 0: + // Dequeued successfully completed io packet. + case r.err == syscall.WAIT_TIMEOUT && o == nil: + // Wait has timed out (should not happen now, but might be used in the future). + panic("GetQueuedCompletionStatus timed out") + case o == nil: + // Failed to dequeue anything -> report the error. + panic("GetQueuedCompletionStatus failed " + syscall.Errstr(r.err)) + default: + // Dequeued failed io packet. + } + (*anOp)(unsafe.Pointer(o)).resultc <- r } - go s.Run() - return s, nil } -type ioPacket struct { - // Used by IOCP interface, - // it must be first field of the struct, - // as our code rely on it. - o syscall.Overlapped - // Link to the io owner. - c chan *ioResult - - w *syscall.WSABuf +// ioSrv executes net io requests. +type ioSrv struct { + submchan chan anOpIface // submit io requests + canchan chan anOpIface // cancel io requests } -func (s *pollServer) getCompletedIO() (ov *syscall.Overlapped, result *ioResult, err os.Error) { - var r ioResult - var o *syscall.Overlapped - e := syscall.GetQueuedCompletionStatus(s.iocp, &r.qty, &r.key, &o, syscall.INFINITE) - switch { - case e == 0: - // Dequeued successfully completed io packet. - return o, &r, nil - case e == syscall.WAIT_TIMEOUT && o == nil: - // Wait has timed out (should not happen now, but might be used in the future). - return nil, &r, os.NewSyscallError("GetQueuedCompletionStatus", e) - case o == nil: - // Failed to dequeue anything -> report the error. - return nil, &r, os.NewSyscallError("GetQueuedCompletionStatus", e) - default: - // Dequeued failed io packet. - r.errno = e - return o, &r, nil +// ProcessRemoteIO will execute submit io requests on behalf +// of other goroutines, all on a single os thread, so it can +// cancel them later. Results of all operations will be sent +// back to their requesters via channel supplied in request. +func (s *ioSrv) ProcessRemoteIO() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + for { + select { + case o := <-s.submchan: + o.Op().errnoc <- o.Submit() + case o := <-s.canchan: + o.Op().errnoc <- syscall.CancelIo(uint32(o.Op().fd.sysfd)) + } } - return } -func (s *pollServer) Run() { - for { - o, r, err := s.getCompletedIO() - if err != nil { - panic("Run pollServer: " + err.String() + "\n") +// ExecIO executes a single io operation. It either executes it +// inline, or, if timeouts are employed, passes the request onto +// a special goroutine and waits for completion or cancels request. +func (s *ioSrv) ExecIO(oi anOpIface, deadline_delta int64) (n int, err os.Error) { + var e int + o := oi.Op() + if deadline_delta > 0 { + // Send request to a special dedicated thread, + // so it can stop the io with CancelIO later. + s.submchan <- oi + e = <-o.errnoc + } else { + e = oi.Submit() + } + switch e { + case 0: + // IO completed immediately, but we need to get our completion message anyway. + case syscall.ERROR_IO_PENDING: + // IO started, and we have to wait for it's completion. + default: + return 0, &OpError{oi.Name(), o.fd.net, o.fd.laddr, os.Errno(e)} + } + // Wait for our request to complete. + var r ioResult + if deadline_delta > 0 { + select { + case r = <-o.resultc: + case <-time.After(deadline_delta): + s.canchan <- oi + <-o.errnoc + r = <-o.resultc + if r.err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled + r.err = syscall.EWOULDBLOCK + } } - p := (*ioPacket)(unsafe.Pointer(o)) - p.c <- r + } else { + r = <-o.resultc } + if r.err != 0 { + err = &OpError{oi.Name(), o.fd.net, o.fd.laddr, os.Errno(r.err)} + } + return int(r.qty), err } -// Network FD methods. -// All the network FDs use a single pollServer. - -var pollserver *pollServer +// Start helper goroutines. +var resultsrv *resultSrv +var iosrv *ioSrv var onceStartServer sync.Once func startServer() { - p, err := newPollServer() - if err != nil { - panic("Start pollServer: " + err.String() + "\n") - } - pollserver = p - - go timeoutIO() + resultsrv = new(resultSrv) + var errno int + resultsrv.iocp, errno = syscall.CreateIoCompletionPort(-1, 0, 0, 1) + if errno != 0 { + panic("CreateIoCompletionPort failed " + syscall.Errstr(errno)) + } + go resultsrv.Run() + + iosrv = new(ioSrv) + iosrv.submchan = make(chan anOpIface) + iosrv.canchan = make(chan anOpIface) + go iosrv.ProcessRemoteIO() } -var initErr os.Error +// Network file descriptor. +type netFD struct { + // locking/lifetime of sysfd + sysmu sync.Mutex + sysref int + closing bool -func newFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD, err os.Error) { - if initErr != nil { - return nil, initErr - } - onceStartServer.Do(startServer) - // Associate our socket with pollserver.iocp. - if _, e := syscall.CreateIoCompletionPort(int32(fd), pollserver.iocp, 0, 0); e != 0 { - return nil, &OpError{"CreateIoCompletionPort", net, laddr, os.Errno(e)} - } + // immutable until Close + sysfd int + family int + proto int + net string + laddr Addr + raddr Addr + + // owned by client + rdeadline_delta int64 + rdeadline int64 + rio sync.Mutex + wdeadline_delta int64 + wdeadline int64 + wio sync.Mutex +} + +func allocFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD) { f = &netFD{ sysfd: fd, family: family, proto: proto, - cr: make(chan *ioResult, 1), - cw: make(chan *ioResult, 1), net: net, laddr: laddr, raddr: raddr, } runtime.SetFinalizer(f, (*netFD).Close) - return f, nil + return f +} + +func newFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD, err os.Error) { + if initErr != nil { + return nil, initErr + } + onceStartServer.Do(startServer) + // Associate our socket with resultsrv.iocp. + if _, e := syscall.CreateIoCompletionPort(int32(fd), resultsrv.iocp, 0, 0); e != 0 { + return nil, &OpError{"CreateIoCompletionPort", net, laddr, os.Errno(e)} + } + return allocFD(fd, family, proto, net, laddr, raddr), nil } // Add a reference to this fd. @@ -172,7 +266,7 @@ func (fd *netFD) decref() { // In case the user has set linger, switch to blocking mode so // the close blocks. As long as this doesn't happen often, we // can handle the extra OS processes. Otherwise we'll need to - // use the pollserver for Close too. Sigh. + // use the resultsrv for Close too. Sigh. syscall.SetNonblock(fd.sysfd, false) closesocket(fd.sysfd) fd.sysfd = -1 @@ -194,89 +288,22 @@ func (fd *netFD) Close() os.Error { return nil } -func newWSABuf(p []byte) *syscall.WSABuf { - var p0 *byte - if len(p) > 0 { - p0 = (*byte)(unsafe.Pointer(&p[0])) - } - return &syscall.WSABuf{uint32(len(p)), p0} -} - -func waitPacket(fd *netFD, pckt *ioPacket, mode int) (r *ioResult) { - var delta int64 - if mode == 'r' { - delta = fd.rdeadline_delta - } - if mode == 'w' { - delta = fd.wdeadline_delta - } - if delta <= 0 { - return <-pckt.c - } +// Read from network. - select { - case r = <-pckt.c: - case <-time.After(delta): - a := &arg{f: cancel, fd: fd, pckt: pckt, c: make(chan int)} - ioChan <- a - <-a.c - r = <-pckt.c - if r.errno == 995 { // IO Canceled - r.errno = syscall.EWOULDBLOCK - } - } - return r +type readOp struct { + bufOp } -const ( - read = iota - readfrom - write - writeto - cancel -) +func (o *readOp) Submit() (errno int) { + var d, f uint32 + return syscall.WSARecv(uint32(o.fd.sysfd), &o.buf, 1, &d, &f, &o.o, nil) +} -type arg struct { - f int - fd *netFD - pckt *ioPacket - done *uint32 - flags *uint32 - rsa *syscall.RawSockaddrAny - size *int32 - sa *syscall.Sockaddr - c chan int -} - -var ioChan chan *arg = make(chan *arg) - -func timeoutIO() { - // CancelIO only cancels all pending input and output (I/O) operations that are - // issued by the calling thread for the specified file, does not cancel I/O - // operations that other threads issue for a file handle. So we need do all timeout - // I/O in single OS thread. - runtime.LockOSThread() - defer runtime.UnlockOSThread() - for { - o := <-ioChan - var e int - switch o.f { - case read: - e = syscall.WSARecv(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, o.flags, &o.pckt.o, nil) - case readfrom: - e = syscall.WSARecvFrom(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, o.flags, o.rsa, o.size, &o.pckt.o, nil) - case write: - e = syscall.WSASend(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, uint32(0), &o.pckt.o, nil) - case writeto: - e = syscall.WSASendto(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, 0, *o.sa, &o.pckt.o, nil) - case cancel: - e = syscall.CancelIo(uint32(o.fd.sysfd)) - } - o.c <- e - } +func (o *readOp) Name() string { + return "WSARecv" } -func (fd *netFD) Read(p []byte) (n int, err os.Error) { +func (fd *netFD) Read(buf []byte) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL } @@ -287,45 +314,37 @@ func (fd *netFD) Read(p []byte) (n int, err os.Error) { if fd.sysfd == -1 { return 0, os.EINVAL } - // Submit receive request. - var pckt ioPacket - pckt.c = fd.cr - pckt.w = newWSABuf(p) - var done uint32 - flags := uint32(0) - var e int - if fd.rdeadline_delta > 0 { - a := &arg{f: read, fd: fd, pckt: &pckt, done: &done, flags: &flags, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSARecv(uint32(fd.sysfd), pckt.w, 1, &done, &flags, &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, &OpError{"WSARecv", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'r') - if r.errno != 0 { - err = &OpError{"WSARecv", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) + var o readOp + o.Init(fd, buf) + n, err = iosrv.ExecIO(&o, fd.rdeadline_delta) if err == nil && n == 0 { err = os.EOF } return } -func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err os.Error) { +// ReadFrom from network. + +type readFromOp struct { + bufOp + rsa syscall.RawSockaddrAny +} + +func (o *readFromOp) Submit() (errno int) { + var d, f uint32 + l := int32(unsafe.Sizeof(o.rsa)) + return syscall.WSARecvFrom(uint32(o.fd.sysfd), &o.buf, 1, &d, &f, &o.rsa, &l, &o.o, nil) +} + +func (o *readFromOp) Name() string { + return "WSARecvFrom" +} + +func (fd *netFD) ReadFrom(buf []byte) (n int, sa syscall.Sockaddr, err os.Error) { if fd == nil { return 0, nil, os.EINVAL } - if len(p) == 0 { + if len(buf) == 0 { return 0, nil, nil } fd.rio.Lock() @@ -335,41 +354,29 @@ func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err os.Error) { if fd.sysfd == -1 { return 0, nil, os.EINVAL } - // Submit receive request. - var pckt ioPacket - pckt.c = fd.cr - pckt.w = newWSABuf(p) - var done uint32 - flags := uint32(0) - var rsa syscall.RawSockaddrAny - l := int32(unsafe.Sizeof(rsa)) - var e int - if fd.rdeadline_delta > 0 { - a := &arg{f: readfrom, fd: fd, pckt: &pckt, done: &done, flags: &flags, rsa: &rsa, size: &l, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSARecvFrom(uint32(fd.sysfd), pckt.w, 1, &done, &flags, &rsa, &l, &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, nil, &OpError{"WSARecvFrom", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'r') - if r.errno != 0 { - err = &OpError{"WSARecvFrom", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) - sa, _ = rsa.Sockaddr() + var o readFromOp + o.Init(fd, buf) + n, err = iosrv.ExecIO(&o, fd.rdeadline_delta) + sa, _ = o.rsa.Sockaddr() return } -func (fd *netFD) Write(p []byte) (n int, err os.Error) { +// Write to network. + +type writeOp struct { + bufOp +} + +func (o *writeOp) Submit() (errno int) { + var d uint32 + return syscall.WSASend(uint32(o.fd.sysfd), &o.buf, 1, &d, 0, &o.o, nil) +} + +func (o *writeOp) Name() string { + return "WSASend" +} + +func (fd *netFD) Write(buf []byte) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL } @@ -380,41 +387,32 @@ func (fd *netFD) Write(p []byte) (n int, err os.Error) { if fd.sysfd == -1 { return 0, os.EINVAL } - // Submit send request. - var pckt ioPacket - pckt.c = fd.cw - pckt.w = newWSABuf(p) - var done uint32 - var e int - if fd.wdeadline_delta > 0 { - a := &arg{f: write, fd: fd, pckt: &pckt, done: &done, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSASend(uint32(fd.sysfd), pckt.w, 1, &done, uint32(0), &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, &OpError{"WSASend", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'w') - if r.errno != 0 { - err = &OpError{"WSASend", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) - return + var o writeOp + o.Init(fd, buf) + return iosrv.ExecIO(&o, fd.wdeadline_delta) +} + +// WriteTo to network. + +type writeToOp struct { + bufOp + sa syscall.Sockaddr +} + +func (o *writeToOp) Submit() (errno int) { + var d uint32 + return syscall.WSASendto(uint32(o.fd.sysfd), &o.buf, 1, &d, 0, o.sa, &o.o, nil) +} + +func (o *writeToOp) Name() string { + return "WSASendto" } -func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err os.Error) { +func (fd *netFD) WriteTo(buf []byte, sa syscall.Sockaddr) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL } - if len(p) == 0 { + if len(buf) == 0 { return 0, nil } fd.wio.Lock() @@ -424,34 +422,29 @@ func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err os.Error) { if fd.sysfd == -1 { return 0, os.EINVAL } - // Submit send request. - var pckt ioPacket - pckt.c = fd.cw - pckt.w = newWSABuf(p) - var done uint32 - var e int - if fd.wdeadline_delta > 0 { - a := &arg{f: writeto, fd: fd, pckt: &pckt, done: &done, sa: &sa, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSASendto(uint32(fd.sysfd), pckt.w, 1, &done, 0, sa, &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, &OpError{"WSASendTo", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'w') - if r.errno != 0 { - err = &OpError{"WSASendTo", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) - return + var o writeToOp + o.Init(fd, buf) + o.sa = sa + return iosrv.ExecIO(&o, fd.wdeadline_delta) +} + +// Accept new network connections. + +type acceptOp struct { + anOp + newsock int + attrs [2]syscall.RawSockaddrAny // space for local and remote address only +} + +func (o *acceptOp) Submit() (errno int) { + var d uint32 + l := uint32(unsafe.Sizeof(o.attrs[0])) + return syscall.AcceptEx(uint32(o.fd.sysfd), uint32(o.newsock), + (*byte)(unsafe.Pointer(&o.attrs[0])), 0, l, l, &d, &o.o) +} + +func (o *acceptOp) Name() string { + return "AcceptEx" } func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os.Error) { @@ -474,72 +467,40 @@ func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os. // Associate our new socket with IOCP. onceStartServer.Do(startServer) - if _, e = syscall.CreateIoCompletionPort(int32(s), pollserver.iocp, 0, 0); e != 0 { + if _, e = syscall.CreateIoCompletionPort(int32(s), resultsrv.iocp, 0, 0); e != 0 { return nil, &OpError{"CreateIoCompletionPort", fd.net, fd.laddr, os.Errno(e)} } // Submit accept request. - // Will use new unique channel here, because, unlike Read or Write, - // Accept is expected to be executed by many goroutines simultaniously. - var pckt ioPacket - pckt.c = make(chan *ioResult) - attrs, e := syscall.AcceptIOCP(fd.sysfd, s, &pckt.o) - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - closesocket(s) - return nil, &OpError{"AcceptEx", fd.net, fd.laddr, os.Errno(e)} - } - - // Wait for peer connection. - r := <-pckt.c - if r.errno != 0 { + var o acceptOp + o.Init(fd) + o.newsock = s + _, err = iosrv.ExecIO(&o, 0) + if err != nil { closesocket(s) - return nil, &OpError{"AcceptEx", fd.net, fd.laddr, os.Errno(r.errno)} + return nil, err } // Inherit properties of the listening socket. e = syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_UPDATE_ACCEPT_CONTEXT, fd.sysfd) if e != 0 { closesocket(s) - return nil, &OpError{"Setsockopt", fd.net, fd.laddr, os.Errno(r.errno)} + return nil, err } // Get local and peer addr out of AcceptEx buffer. - lsa, rsa := syscall.GetAcceptIOCPSockaddrs(attrs) - - // Create our netFD and return it for further use. - laddr := toAddr(lsa) - raddr := toAddr(rsa) - - f := &netFD{ - sysfd: s, - family: fd.family, - proto: fd.proto, - cr: make(chan *ioResult, 1), - cw: make(chan *ioResult, 1), - net: fd.net, - laddr: laddr, - raddr: raddr, - } - runtime.SetFinalizer(f, (*netFD).Close) - return f, nil -} - -func closesocket(s int) (errno int) { - return syscall.Closesocket(int32(s)) + var lrsa, rrsa *syscall.RawSockaddrAny + var llen, rlen int32 + l := uint32(unsafe.Sizeof(*lrsa)) + syscall.GetAcceptExSockaddrs((*byte)(unsafe.Pointer(&o.attrs[0])), + 0, l, l, &lrsa, &llen, &rrsa, &rlen) + lsa, _ := lrsa.Sockaddr() + rsa, _ := rrsa.Sockaddr() + + return allocFD(s, fd.family, fd.proto, fd.net, toAddr(lsa), toAddr(rsa)), nil } -func init() { - var d syscall.WSAData - e := syscall.WSAStartup(uint32(0x101), &d) - if e != 0 { - initErr = os.NewSyscallError("WSAStartup", e) - } -} +// Not implemeted functions. func (fd *netFD) dup() (f *os.File, err os.Error) { // TODO: Implement this diff --git a/src/pkg/net/iprawsock.go b/src/pkg/net/iprawsock.go index 241be1509..81a918ce5 100644 --- a/src/pkg/net/iprawsock.go +++ b/src/pkg/net/iprawsock.go @@ -245,7 +245,7 @@ func hostToIP(host string) (ip IP, err os.Error) { err = err1 goto Error } - addr = ParseIP(addrs[0]) + addr = firstSupportedAddr(addrs) if addr == nil { // should not happen err = &AddrError{"LookupHost returned invalid address", addrs[0]} @@ -311,7 +311,7 @@ func DialIP(netProto string, laddr, raddr *IPAddr) (c *IPConn, err os.Error) { if err != nil { return } - switch prefixBefore(net, ':') { + switch net { case "ip", "ip4", "ip6": default: return nil, UnknownNetworkError(net) @@ -335,7 +335,7 @@ func ListenIP(netProto string, laddr *IPAddr) (c *IPConn, err os.Error) { if err != nil { return } - switch prefixBefore(net, ':') { + switch net { case "ip", "ip4", "ip6": default: return nil, UnknownNetworkError(net) diff --git a/src/pkg/net/ipsock.go b/src/pkg/net/ipsock.go index 4ba6a55b9..ae4204b48 100644 --- a/src/pkg/net/ipsock.go +++ b/src/pkg/net/ipsock.go @@ -18,19 +18,34 @@ import ( // Unfortunately, we need to run on kernels built without IPv6 support too. // So probe the kernel to figure it out. func kernelSupportsIPv6() bool { - // FreeBSD does not support this sort of interface. - if syscall.OS == "freebsd" { + s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != 0 { return false } - fd, e := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - if fd >= 0 { - closesocket(fd) + defer closesocket(s) + + la := &TCPAddr{IP: IPv4(127, 0, 0, 1)} + sa, oserr := la.toAddr().sockaddr(syscall.AF_INET6) + if oserr != nil { + return false } - return e == 0 + + return syscall.Bind(s, sa) == 0 } var preferIPv4 = !kernelSupportsIPv6() +func firstSupportedAddr(addrs []string) (addr IP) { + for _, s := range addrs { + addr = ParseIP(s) + if !preferIPv4 || addr.To4() != nil { + break + } + addr = nil + } + return addr +} + // TODO(rsc): if syscall.OS == "linux", we're supposd to read // /proc/sys/net/core/somaxconn, // to take advantage of kernels that have raised the limit. @@ -208,7 +223,7 @@ func hostPortToIP(net, hostport string) (ip IP, iport int, err os.Error) { err = err1 goto Error } - addr = ParseIP(addrs[0]) + addr = firstSupportedAddr(addrs) if addr == nil { // should not happen err = &AddrError{"LookupHost returned invalid address", addrs[0]} diff --git a/src/pkg/net/multicast_test.go b/src/pkg/net/multicast_test.go new file mode 100644 index 000000000..32fdec85b --- /dev/null +++ b/src/pkg/net/multicast_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "runtime" + "testing" +) + +func TestMulticastJoinAndLeave(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + + addr := &UDPAddr{ + IP: IPv4zero, + Port: 0, + } + // open a UDPConn + conn, err := ListenUDP("udp4", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // try to join group + mcast := IPv4(224, 0, 0, 254) + err = conn.JoinGroup(mcast) + if err != nil { + t.Fatal(err) + } + + // try to leave group + err = conn.LeaveGroup(mcast) + if err != nil { + t.Fatal(err) + } +} + +func TestJoinFailureWithIPv6Address(t *testing.T) { + addr := &UDPAddr{ + IP: IPv4zero, + Port: 0, + } + + // open a UDPConn + conn, err := ListenUDP("udp4", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // try to join group + mcast := ParseIP("ff02::1") + err = conn.JoinGroup(mcast) + if err == nil { + t.Fatal("JoinGroup succeeded, should fail") + } + t.Logf("%s", err) +} diff --git a/src/pkg/net/net.go b/src/pkg/net/net.go index c0c1c3b8a..04a898a9a 100644 --- a/src/pkg/net/net.go +++ b/src/pkg/net/net.go @@ -31,7 +31,6 @@ type Conn interface { Write(b []byte) (n int, err os.Error) // Close closes the connection. - // The error returned is an os.Error to satisfy io.Closer; Close() os.Error // LocalAddr returns the local network address. @@ -83,7 +82,6 @@ type PacketConn interface { WriteTo(b []byte, addr Addr) (n int, err os.Error) // Close closes the connection. - // The error returned is an os.Error to satisfy io.Closer; Close() os.Error // LocalAddr returns the local network address. @@ -112,7 +110,6 @@ type Listener interface { Accept() (c Conn, err os.Error) // Close closes the listener. - // The error returned is an os.Error to satisfy io.Closer; Close() os.Error // Addr returns the listener's network address. diff --git a/src/pkg/net/parse.go b/src/pkg/net/parse.go index 605f3110b..2bc0db465 100644 --- a/src/pkg/net/parse.go +++ b/src/pkg/net/parse.go @@ -192,16 +192,6 @@ func count(s string, b byte) int { return n } -// Returns the prefix of s up to but not including the character c -func prefixBefore(s string, c byte) string { - for i, v := range s { - if v == int(c) { - return s[0:i] - } - } - return s -} - // Index of rightmost occurrence of b in s. func last(s string, b byte) int { i := len(s) diff --git a/src/pkg/net/textproto/Makefile b/src/pkg/net/textproto/Makefile index 7897fa711..cadf3ab69 100644 --- a/src/pkg/net/textproto/Makefile +++ b/src/pkg/net/textproto/Makefile @@ -6,6 +6,7 @@ include ../../../Make.inc TARG=net/textproto GOFILES=\ + header.go\ pipeline.go\ reader.go\ textproto.go\ diff --git a/src/pkg/net/textproto/header.go b/src/pkg/net/textproto/header.go new file mode 100644 index 000000000..288deb2ce --- /dev/null +++ b/src/pkg/net/textproto/header.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +// A MIMEHeader represents a MIME-style header mapping +// keys to sets of values. +type MIMEHeader map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +func (h MIMEHeader) Add(key, value string) { + key = CanonicalMIMEHeaderKey(key) + h[key] = append(h[key], value) +} + +// Set sets the header entries associated with key to +// the single element value. It replaces any existing +// values associated with key. +func (h MIMEHeader) Set(key, value string) { + h[CanonicalMIMEHeaderKey(key)] = []string{value} +} + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +// Get is a convenience method. For more complex queries, +// access the map directly. +func (h MIMEHeader) Get(key string) string { + if h == nil { + return "" + } + v := h[CanonicalMIMEHeaderKey(key)] + if len(v) == 0 { + return "" + } + return v[0] +} + +// Del deletes the values associated with key. +func (h MIMEHeader) Del(key string) { + h[CanonicalMIMEHeaderKey(key)] = nil, false +} diff --git a/src/pkg/net/textproto/reader.go b/src/pkg/net/textproto/reader.go index c8e34b758..ac1278689 100644 --- a/src/pkg/net/textproto/reader.go +++ b/src/pkg/net/textproto/reader.go @@ -402,7 +402,7 @@ func (r *Reader) ReadDotLines() ([]string, os.Error) { // ReadMIMEHeader reads a MIME-style header from r. // The header is a sequence of possibly continued Key: Value lines // ending in a blank line. -// The returned map m maps CanonicalHeaderKey(key) to a +// The returned map m maps CanonicalMIMEHeaderKey(key) to a // sequence of values in the same order encountered in the input. // // For example, consider this input: @@ -415,12 +415,12 @@ func (r *Reader) ReadDotLines() ([]string, os.Error) { // Given that input, ReadMIMEHeader returns the map: // // map[string][]string{ -// "My-Key": []string{"Value 1", "Value 2"}, -// "Long-Key": []string{"Even Longer Value"}, +// "My-Key": {"Value 1", "Value 2"}, +// "Long-Key": {"Even Longer Value"}, // } // -func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) { - m := make(map[string][]string) +func (r *Reader) ReadMIMEHeader() (MIMEHeader, os.Error) { + m := make(MIMEHeader) for { kv, err := r.ReadContinuedLineBytes() if len(kv) == 0 { @@ -432,7 +432,7 @@ func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) { if i < 0 || bytes.IndexByte(kv[0:i], ' ') >= 0 { return m, ProtocolError("malformed MIME header line: " + string(kv)) } - key := CanonicalHeaderKey(string(kv[0:i])) + key := CanonicalMIMEHeaderKey(string(kv[0:i])) // Skip initial spaces in value. i++ // skip colon @@ -452,12 +452,12 @@ func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) { panic("unreachable") } -// CanonicalHeaderKey returns the canonical format of the +// CanonicalMIMEHeaderKey returns the canonical format of the // MIME header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; // the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". -func CanonicalHeaderKey(s string) string { +func CanonicalMIMEHeaderKey(s string) string { // Quick check for canonical encoding. needUpper := true for i := 0; i < len(s); i++ { diff --git a/src/pkg/net/textproto/reader_test.go b/src/pkg/net/textproto/reader_test.go index 2cecbc75f..0658e58b8 100644 --- a/src/pkg/net/textproto/reader_test.go +++ b/src/pkg/net/textproto/reader_test.go @@ -26,10 +26,10 @@ var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ {"USER-AGENT", "User-Agent"}, } -func TestCanonicalHeaderKey(t *testing.T) { +func TestCanonicalMIMEHeaderKey(t *testing.T) { for _, tt := range canonicalHeaderKeyTests { - if s := CanonicalHeaderKey(tt.in); s != tt.out { - t.Errorf("CanonicalHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) + if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { + t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) } } } @@ -130,7 +130,7 @@ func TestReadDotBytes(t *testing.T) { func TestReadMIMEHeader(t *testing.T) { r := reader("my-key: Value 1 \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n") m, err := r.ReadMIMEHeader() - want := map[string][]string{ + want := MIMEHeader{ "My-Key": {"Value 1", "Value 2"}, "Long-Key": {"Even Longer Value"}, } diff --git a/src/pkg/net/udpsock.go b/src/pkg/net/udpsock.go index 0270954c1..f9274493e 100644 --- a/src/pkg/net/udpsock.go +++ b/src/pkg/net/udpsock.go @@ -279,3 +279,44 @@ func (c *UDPConn) BindToDevice(device string) os.Error { // It is the caller's responsibility to close f when finished. // Closing c does not affect f, and closing f does not affect c. func (c *UDPConn) File() (f *os.File, err os.Error) { return c.fd.dup() } + +var errInvalidMulticast = os.ErrorString("invalid IPv4 multicast address") + +// JoinGroup joins the IPv4 multicast group named by addr. +// The UDPConn must use the "udp4" network. +func (c *UDPConn) JoinGroup(addr IP) os.Error { + if !c.ok() { + return os.EINVAL + } + ip := addr.To4() + if ip == nil { + return &OpError{"joingroup", "udp", &IPAddr{ip}, errInvalidMulticast} + } + mreq := &syscall.IpMreq{ + Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, + } + err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) + if err != nil { + return &OpError{"joingroup", "udp", &IPAddr{ip}, err} + } + return nil +} + +// LeaveGroup exits the IPv4 multicast group named by addr. +func (c *UDPConn) LeaveGroup(addr IP) os.Error { + if !c.ok() { + return os.EINVAL + } + ip := addr.To4() + if ip == nil { + return &OpError{"leavegroup", "udp", &IPAddr{ip}, errInvalidMulticast} + } + mreq := &syscall.IpMreq{ + Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, + } + err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq)) + if err != nil { + return &OpError{"leavegroup", "udp", &IPAddr{ip}, err} + } + return nil +} diff --git a/src/pkg/netchan/common.go b/src/pkg/netchan/common.go index 6c085484e..dd06050ee 100644 --- a/src/pkg/netchan/common.go +++ b/src/pkg/netchan/common.go @@ -6,7 +6,7 @@ package netchan import ( "gob" - "net" + "io" "os" "reflect" "sync" @@ -93,7 +93,7 @@ type encDec struct { enc *gob.Encoder } -func newEncDec(conn net.Conn) *encDec { +func newEncDec(conn io.ReadWriter) *encDec { return &encDec{ dec: gob.NewDecoder(conn), enc: gob.NewEncoder(conn), @@ -199,9 +199,10 @@ func (cs *clientSet) sync(timeout int64) os.Error { // are delivered into the local channel. type netChan struct { *chanDir - name string - id int - size int // buffer size of channel. + name string + id int + size int // buffer size of channel. + closed bool // sender-specific state ackCh chan bool // buffered with space for all the acks we need @@ -227,6 +228,9 @@ func newNetChan(name string, id int, ch *chanDir, ed *encDec, size int, count in // Close the channel. func (nch *netChan) close() { + if nch.closed { + return + } if nch.dir == Recv { if nch.sendCh != nil { // If the sender goroutine is active, close the channel to it. @@ -239,6 +243,7 @@ func (nch *netChan) close() { nch.ch.Close() close(nch.ackCh) } + nch.closed = true } // Send message from remote side to local receiver. diff --git a/src/pkg/netchan/export.go b/src/pkg/netchan/export.go index 675e252d5..55eba0e2e 100644 --- a/src/pkg/netchan/export.go +++ b/src/pkg/netchan/export.go @@ -23,6 +23,7 @@ package netchan import ( "log" + "io" "net" "os" "reflect" @@ -43,7 +44,6 @@ func expLog(args ...interface{}) { // but they must use different ports. type Exporter struct { *clientSet - listener net.Listener } type expClient struct { @@ -57,7 +57,7 @@ type expClient struct { seqLock sync.Mutex // guarantees messages are in sequence, only locked under mu } -func newClient(exp *Exporter, conn net.Conn) *expClient { +func newClient(exp *Exporter, conn io.ReadWriter) *expClient { client := new(expClient) client.exp = exp client.encDec = newEncDec(conn) @@ -260,39 +260,50 @@ func (client *expClient) ack() int64 { return n } -// Wait for incoming connections, start a new runner for each -func (exp *Exporter) listen() { +// Serve waits for incoming connections on the listener +// and serves the Exporter's channels on each. +// It blocks until the listener is closed. +func (exp *Exporter) Serve(listener net.Listener) { for { - conn, err := exp.listener.Accept() + conn, err := listener.Accept() if err != nil { expLog("listen:", err) break } - client := exp.addClient(conn) - go client.run() + go exp.ServeConn(conn) } } -// NewExporter creates a new Exporter to export channels -// on the network and local address defined as in net.Listen. -func NewExporter(network, localaddr string) (*Exporter, os.Error) { - listener, err := net.Listen(network, localaddr) - if err != nil { - return nil, err - } +// ServeConn exports the Exporter's channels on conn. +// It blocks until the connection is terminated. +func (exp *Exporter) ServeConn(conn io.ReadWriter) { + exp.addClient(conn).run() +} + +// NewExporter creates a new Exporter that exports a set of channels. +func NewExporter() *Exporter { e := &Exporter{ - listener: listener, clientSet: &clientSet{ names: make(map[string]*chanDir), clients: make(map[unackedCounter]bool), }, } - go e.listen() - return e, nil + return e +} + +// ListenAndServe exports the exporter's channels through the +// given network and local address defined as in net.Listen. +func (exp *Exporter) ListenAndServe(network, localaddr string) os.Error { + listener, err := net.Listen(network, localaddr) + if err != nil { + return err + } + go exp.Serve(listener) + return nil } // addClient creates a new expClient and records its existence -func (exp *Exporter) addClient(conn net.Conn) *expClient { +func (exp *Exporter) addClient(conn io.ReadWriter) *expClient { client := newClient(exp, conn) exp.mu.Lock() exp.clients[client] = true @@ -329,9 +340,6 @@ func (exp *Exporter) Sync(timeout int64) os.Error { return exp.clientSet.sync(timeout) } -// Addr returns the Exporter's local network address. -func (exp *Exporter) Addr() net.Addr { return exp.listener.Addr() } - func checkChan(chT interface{}, dir Dir) (*reflect.ChanValue, os.Error) { chanType, ok := reflect.Typeof(chT).(*reflect.ChanType) if !ok { diff --git a/src/pkg/netchan/import.go b/src/pkg/netchan/import.go index d220d9a66..30edcd812 100644 --- a/src/pkg/netchan/import.go +++ b/src/pkg/netchan/import.go @@ -5,6 +5,7 @@ package netchan import ( + "io" "log" "net" "os" @@ -25,7 +26,6 @@ func impLog(args ...interface{}) { // importers, even from the same machine/network port. type Importer struct { *encDec - conn net.Conn chanLock sync.Mutex // protects access to channel map names map[string]*netChan chans map[int]*netChan @@ -33,23 +33,26 @@ type Importer struct { maxId int } -// NewImporter creates a new Importer object to import channels -// from an Exporter at the network and remote address as defined in net.Dial. -// The Exporter must be available and serving when the Importer is -// created. -func NewImporter(network, remoteaddr string) (*Importer, os.Error) { - conn, err := net.Dial(network, "", remoteaddr) - if err != nil { - return nil, err - } +// NewImporter creates a new Importer object to import a set of channels +// from the given connection. The Exporter must be available and serving when +// the Importer is created. +func NewImporter(conn io.ReadWriter) *Importer { imp := new(Importer) imp.encDec = newEncDec(conn) - imp.conn = conn imp.chans = make(map[int]*netChan) imp.names = make(map[string]*netChan) imp.errors = make(chan os.Error, 10) go imp.run() - return imp, nil + return imp +} + +// Import imports a set of channels from the given network and address. +func Import(network, remoteaddr string) (*Importer, os.Error) { + conn, err := net.Dial(network, "", remoteaddr) + if err != nil { + return nil, err + } + return NewImporter(conn), nil } // shutdown closes all channels for which we are receiving data from the remote side. @@ -231,15 +234,13 @@ func (imp *Importer) ImportNValues(name string, chT interface{}, dir Dir, size, // the channel. Messages in flight for the channel may be dropped. func (imp *Importer) Hangup(name string) os.Error { imp.chanLock.Lock() - nc, ok := imp.names[name] - if ok { - imp.names[name] = nil, false - imp.chans[nc.id] = nil, false - } - imp.chanLock.Unlock() - if !ok { + defer imp.chanLock.Unlock() + nc := imp.names[name] + if nc == nil { return os.ErrorString("netchan import: hangup: no such channel: " + name) } + imp.names[name] = nil, false + imp.chans[nc.id] = nil, false nc.close() return nil } diff --git a/src/pkg/netchan/netchan_test.go b/src/pkg/netchan/netchan_test.go index 4076aefeb..1c84a9d14 100644 --- a/src/pkg/netchan/netchan_test.go +++ b/src/pkg/netchan/netchan_test.go @@ -5,6 +5,7 @@ package netchan import ( + "net" "strings" "testing" "time" @@ -94,27 +95,13 @@ func importReceive(imp *Importer, t *testing.T, done chan bool) { } func TestExportSendImportReceive(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) exportSend(exp, count, t, nil) importReceive(imp, t, nil) } func TestExportReceiveImportSend(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) expDone := make(chan bool) done := make(chan bool) go func() { @@ -127,27 +114,13 @@ func TestExportReceiveImportSend(t *testing.T) { } func TestClosingExportSendImportReceive(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) exportSend(exp, closeCount, t, nil) importReceive(imp, t, nil) } func TestClosingImportSendExportReceive(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) expDone := make(chan bool) done := make(chan bool) go func() { @@ -160,17 +133,10 @@ func TestClosingImportSendExportReceive(t *testing.T) { } func TestErrorForIllegalChannel(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) // Now export a channel. ch := make(chan int, 1) - err = exp.Export("aChannel", ch, Send) + err := exp.Export("aChannel", ch, Send) if err != nil { t.Fatal("export:", err) } @@ -200,14 +166,7 @@ func TestErrorForIllegalChannel(t *testing.T) { // Not a great test but it does at least invoke Drain. func TestExportDrain(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) done := make(chan bool) go func() { exportSend(exp, closeCount, t, nil) @@ -221,14 +180,7 @@ func TestExportDrain(t *testing.T) { // Not a great test but it does at least invoke Sync. func TestExportSync(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) done := make(chan bool) exportSend(exp, closeCount, t, nil) go importReceive(imp, t, done) @@ -239,16 +191,9 @@ func TestExportSync(t *testing.T) { // Test hanging up the send side of an export. // TODO: test hanging up the receive side of an export. func TestExportHangup(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) ech := make(chan int) - err = exp.Export("exportedSend", ech, Send) + err := exp.Export("exportedSend", ech, Send) if err != nil { t.Fatal("export:", err) } @@ -276,16 +221,9 @@ func TestExportHangup(t *testing.T) { // Test hanging up the send side of an import. // TODO: test hanging up the receive side of an import. func TestImportHangup(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) ech := make(chan int) - err = exp.Export("exportedRecv", ech, Recv) + err := exp.Export("exportedRecv", ech, Recv) if err != nil { t.Fatal("export:", err) } @@ -343,14 +281,7 @@ func exportLoopback(exp *Exporter, t *testing.T) { // This test checks that channel operations can proceed // even when other concurrent operations are blocked. func TestIndependentSends(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) exportLoopback(exp, t) @@ -377,23 +308,8 @@ type value struct { } func TestCrossConnect(t *testing.T) { - e1, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - i1, err := NewImporter("tcp", e1.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } - - e2, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - i2, err := NewImporter("tcp", e2.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + e1, i1 := pair(t) + e2, i2 := pair(t) crossExport(e1, e2, t) crossImport(i1, i2, t) @@ -452,20 +368,13 @@ const flowCount = 100 // test flow control from exporter to importer. func TestExportFlowControl(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) sendDone := make(chan bool, 1) exportSend(exp, flowCount, t, sendDone) ch := make(chan int) - err = imp.ImportNValues("exportedSend", ch, Recv, 20, -1) + err := imp.ImportNValues("exportedSend", ch, Recv, 20, -1) if err != nil { t.Fatal("importReceive:", err) } @@ -475,17 +384,10 @@ func TestExportFlowControl(t *testing.T) { // test flow control from importer to exporter. func TestImportFlowControl(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) ch := make(chan int) - err = exp.Export("exportedRecv", ch, Recv) + err := exp.Export("exportedRecv", ch, Recv) if err != nil { t.Fatal("importReceive:", err) } @@ -513,3 +415,11 @@ func testFlow(sendDone chan bool, ch <-chan int, N int, t *testing.T) { t.Fatalf("expected %d values; got %d", N, n) } } + +func pair(t *testing.T) (*Exporter, *Importer) { + c0, c1 := net.Pipe() + exp := NewExporter() + go exp.ServeConn(c0) + imp := NewImporter(c1) + return exp, imp +} diff --git a/src/pkg/os/error.go b/src/pkg/os/error.go index 4738d1a42..635a3fe50 100644 --- a/src/pkg/os/error.go +++ b/src/pkg/os/error.go @@ -80,6 +80,7 @@ var ( ENAMETOOLONG Error = Errno(syscall.ENAMETOOLONG) EAFNOSUPPORT Error = Errno(syscall.EAFNOSUPPORT) ETIMEDOUT Error = Errno(syscall.ETIMEDOUT) + ENOTCONN Error = Errno(syscall.ENOTCONN) ) // PathError records an error and the operation and file path that caused it. diff --git a/src/pkg/path/Makefile b/src/pkg/path/Makefile index 4371913e8..fc3e2519c 100644 --- a/src/pkg/path/Makefile +++ b/src/pkg/path/Makefile @@ -9,18 +9,6 @@ GOFILES=\ match.go\ path.go\ -GOFILES_freebsd=\ - path_unix.go - -GOFILES_darwin=\ - path_unix.go - -GOFILES_linux=\ - path_unix.go - -GOFILES_windows=\ - path_windows.go - GOFILES+=$(GOFILES_$(GOOS)) include ../../Make.pkg diff --git a/src/pkg/path/filepath/Makefile b/src/pkg/path/filepath/Makefile new file mode 100644 index 000000000..2330fc09d --- /dev/null +++ b/src/pkg/path/filepath/Makefile @@ -0,0 +1,26 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=path/filepath +GOFILES=\ + match.go\ + path.go\ + +GOFILES_freebsd=\ + path_unix.go + +GOFILES_darwin=\ + path_unix.go + +GOFILES_linux=\ + path_unix.go + +GOFILES_windows=\ + path_unix.go + +GOFILES+=$(GOFILES_$(GOOS)) + +include ../../../Make.pkg diff --git a/src/pkg/path/filepath/match.go b/src/pkg/path/filepath/match.go new file mode 100644 index 000000000..ad4053fa2 --- /dev/null +++ b/src/pkg/path/filepath/match.go @@ -0,0 +1,282 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package filepath + +import ( + "os" + "sort" + "strings" + "utf8" +) + +var ErrBadPattern = os.NewError("syntax error in pattern") + +// Match returns true if name matches the shell file name pattern. +// The pattern syntax is: +// +// pattern: +// { term } +// term: +// '*' matches any sequence of non-Separator characters +// '?' matches any single non-Separator character +// '[' [ '^' ] { character-range } ']' +// character class (must be non-empty) +// c matches character c (c != '*', '?', '\\', '[') +// '\\' c matches character c +// +// character-range: +// c matches character c (c != '\\', '-', ']') +// '\\' c matches character c +// lo '-' hi matches character c for lo <= c <= hi +// +// Match requires pattern to match all of name, not just a substring. +// The only possible error return is when pattern is malformed. +// +func Match(pattern, name string) (matched bool, err os.Error) { +Pattern: + for len(pattern) > 0 { + var star bool + var chunk string + star, chunk, pattern = scanChunk(pattern) + if star && chunk == "" { + // Trailing * matches rest of string unless it has a /. + return strings.Index(name, string(Separator)) < 0, nil + } + // Look for match at current position. + t, ok, err := matchChunk(chunk, name) + // if we're the last chunk, make sure we've exhausted the name + // otherwise we'll give a false result even if we could still match + // using the star + if ok && (len(t) == 0 || len(pattern) > 0) { + name = t + continue + } + if err != nil { + return false, err + } + if star { + // Look for match skipping i+1 bytes. + // Cannot skip /. + for i := 0; i < len(name) && name[i] != Separator; i++ { + t, ok, err := matchChunk(chunk, name[i+1:]) + if ok { + // if we're the last chunk, make sure we exhausted the name + if len(pattern) == 0 && len(t) > 0 { + continue + } + name = t + continue Pattern + } + if err != nil { + return false, err + } + } + } + return false, nil + } + return len(name) == 0, nil +} + +// scanChunk gets the next segment of pattern, which is a non-star string +// possibly preceded by a star. +func scanChunk(pattern string) (star bool, chunk, rest string) { + for len(pattern) > 0 && pattern[0] == '*' { + pattern = pattern[1:] + star = true + } + inrange := false + var i int +Scan: + for i = 0; i < len(pattern); i++ { + switch pattern[i] { + case '\\': + // error check handled in matchChunk: bad pattern. + if i+1 < len(pattern) { + i++ + } + case '[': + inrange = true + case ']': + inrange = false + case '*': + if !inrange { + break Scan + } + } + } + return star, pattern[0:i], pattern[i:] +} + +// matchChunk checks whether chunk matches the beginning of s. +// If so, it returns the remainder of s (after the match). +// Chunk is all single-character operators: literals, char classes, and ?. +func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) { + for len(chunk) > 0 { + if len(s) == 0 { + return + } + switch chunk[0] { + case '[': + // character class + r, n := utf8.DecodeRuneInString(s) + s = s[n:] + chunk = chunk[1:] + // possibly negated + notNegated := true + if len(chunk) > 0 && chunk[0] == '^' { + notNegated = false + chunk = chunk[1:] + } + // parse all ranges + match := false + nrange := 0 + for { + if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 { + chunk = chunk[1:] + break + } + var lo, hi int + if lo, chunk, err = getEsc(chunk); err != nil { + return + } + hi = lo + if chunk[0] == '-' { + if hi, chunk, err = getEsc(chunk[1:]); err != nil { + return + } + } + if lo <= r && r <= hi { + match = true + } + nrange++ + } + if match != notNegated { + return + } + + case '?': + if s[0] == Separator { + return + } + _, n := utf8.DecodeRuneInString(s) + s = s[n:] + chunk = chunk[1:] + + case '\\': + chunk = chunk[1:] + if len(chunk) == 0 { + err = ErrBadPattern + return + } + fallthrough + + default: + if chunk[0] != s[0] { + return + } + s = s[1:] + chunk = chunk[1:] + } + } + return s, true, nil +} + +// getEsc gets a possibly-escaped character from chunk, for a character class. +func getEsc(chunk string) (r int, nchunk string, err os.Error) { + if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' { + err = ErrBadPattern + return + } + if chunk[0] == '\\' { + chunk = chunk[1:] + if len(chunk) == 0 { + err = ErrBadPattern + return + } + } + r, n := utf8.DecodeRuneInString(chunk) + if r == utf8.RuneError && n == 1 { + err = ErrBadPattern + } + nchunk = chunk[n:] + if len(nchunk) == 0 { + err = ErrBadPattern + } + return +} + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +func Glob(pattern string) (matches []string) { + if !hasMeta(pattern) { + if _, err := os.Stat(pattern); err == nil { + return []string{pattern} + } + return nil + } + + dir, file := Split(pattern) + switch dir { + case "": + dir = "." + case string(Separator): + // nothing + default: + dir = dir[0 : len(dir)-1] // chop off trailing separator + } + + if hasMeta(dir) { + for _, d := range Glob(dir) { + matches = glob(d, file, matches) + } + } else { + return glob(dir, file, nil) + } + return matches +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. +func glob(dir, pattern string, matches []string) []string { + fi, err := os.Stat(dir) + if err != nil { + return nil + } + if !fi.IsDirectory() { + return matches + } + d, err := os.Open(dir, os.O_RDONLY, 0666) + if err != nil { + return nil + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return nil + } + sort.SortStrings(names) + + for _, n := range names { + matched, err := Match(pattern, n) + if err != nil { + return matches + } + if matched { + matches = append(matches, Join(dir, n)) + } + } + return matches +} + +// hasMeta returns true if path contains any of the magic characters +// recognized by Match. +func hasMeta(path string) bool { + // TODO(niemeyer): Should other magic characters be added here? + return strings.IndexAny(path, "*?[") >= 0 +} diff --git a/src/pkg/path/filepath/match_test.go b/src/pkg/path/filepath/match_test.go new file mode 100644 index 000000000..ad0c90b75 --- /dev/null +++ b/src/pkg/path/filepath/match_test.go @@ -0,0 +1,106 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package filepath_test + +import ( + "os" + "path/filepath" + "testing" +) + +type MatchTest struct { + pattern, s string + match bool + err os.Error +} + +var matchTests = []MatchTest{ + {"abc", "abc", true, nil}, + {"*", "abc", true, nil}, + {"*c", "abc", true, nil}, + {"a*", "a", true, nil}, + {"a*", "abc", true, nil}, + {"a*", "ab/c", false, nil}, + {"a*/b", "abc/b", true, nil}, + {"a*/b", "a/c/b", false, nil}, + {"a*b*c*d*e*/f", "axbxcxdxe/f", true, nil}, + {"a*b*c*d*e*/f", "axbxcxdxexxx/f", true, nil}, + {"a*b*c*d*e*/f", "axbxcxdxe/xxx/f", false, nil}, + {"a*b*c*d*e*/f", "axbxcxdxexxx/fff", false, nil}, + {"a*b?c*x", "abxbbxdbxebxczzx", true, nil}, + {"a*b?c*x", "abxbbxdbxebxczzy", false, nil}, + {"ab[c]", "abc", true, nil}, + {"ab[b-d]", "abc", true, nil}, + {"ab[e-g]", "abc", false, nil}, + {"ab[^c]", "abc", false, nil}, + {"ab[^b-d]", "abc", false, nil}, + {"ab[^e-g]", "abc", true, nil}, + {"a\\*b", "a*b", true, nil}, + {"a\\*b", "ab", false, nil}, + {"a?b", "a☺b", true, nil}, + {"a[^a]b", "a☺b", true, nil}, + {"a???b", "a☺b", false, nil}, + {"a[^a][^a][^a]b", "a☺b", false, nil}, + {"[a-ζ]*", "α", true, nil}, + {"*[a-ζ]", "A", false, nil}, + {"a?b", "a/b", false, nil}, + {"a*b", "a/b", false, nil}, + {"[\\]a]", "]", true, nil}, + {"[\\-]", "-", true, nil}, + {"[x\\-]", "x", true, nil}, + {"[x\\-]", "-", true, nil}, + {"[x\\-]", "z", false, nil}, + {"[\\-x]", "x", true, nil}, + {"[\\-x]", "-", true, nil}, + {"[\\-x]", "a", false, nil}, + {"[]a]", "]", false, filepath.ErrBadPattern}, + {"[-]", "-", false, filepath.ErrBadPattern}, + {"[x-]", "x", false, filepath.ErrBadPattern}, + {"[x-]", "-", false, filepath.ErrBadPattern}, + {"[x-]", "z", false, filepath.ErrBadPattern}, + {"[-x]", "x", false, filepath.ErrBadPattern}, + {"[-x]", "-", false, filepath.ErrBadPattern}, + {"[-x]", "a", false, filepath.ErrBadPattern}, + {"\\", "a", false, filepath.ErrBadPattern}, + {"[a-b-c]", "a", false, filepath.ErrBadPattern}, + {"*x", "xxx", true, nil}, +} + +func TestMatch(t *testing.T) { + for _, tt := range matchTests { + ok, err := filepath.Match(tt.pattern, tt.s) + if ok != tt.match || err != tt.err { + t.Errorf("Match(%#q, %#q) = %v, %v want %v, nil", tt.pattern, tt.s, ok, err, tt.match) + } + } +} + +// contains returns true if vector contains the string s. +func contains(vector []string, s string) bool { + for _, elem := range vector { + if elem == s { + return true + } + } + return false +} + +var globTests = []struct { + pattern, result string +}{ + {"match.go", "match.go"}, + {"mat?h.go", "match.go"}, + {"*", "match.go"}, + {"../*/match.go", "../filepath/match.go"}, +} + +func TestGlob(t *testing.T) { + for _, tt := range globTests { + matches := filepath.Glob(tt.pattern) + if !contains(matches, tt.result) { + t.Errorf("Glob(%#q) = %#v want %v", tt.pattern, matches, tt.result) + } + } +} diff --git a/src/pkg/path/filepath/path.go b/src/pkg/path/filepath/path.go new file mode 100644 index 000000000..414df7d20 --- /dev/null +++ b/src/pkg/path/filepath/path.go @@ -0,0 +1,270 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The filepath package implements utility routines for manipulating +// filename paths in a way compatible with the target operating +// system-defined file paths. +package filepath + +import ( + "os" + "sort" + "strings" +) + +// BUG(niemeyer): Package filepath does not yet work on Windows. + +// Clean returns the shortest path name equivalent to path +// by purely lexical processing. It applies the following rules +// iteratively until no further processing can be done: +// +// 1. Replace multiple Separator elements with a single one. +// 2. Eliminate each . path name element (the current directory). +// 3. Eliminate each inner .. path name element (the parent directory) +// along with the non-.. element that precedes it. +// 4. Eliminate .. elements that begin a rooted path: +// that is, replace "/.." by "/" at the beginning of a path, +// assuming Separator is '/'. +// +// If the result of this process is an empty string, Clean +// returns the string ".". +// +// See also Rob Pike, ``Lexical File Names in Plan 9 or +// Getting Dot-Dot right,'' +// http://plan9.bell-labs.com/sys/doc/lexnames.html +func Clean(path string) string { + if path == "" { + return "." + } + + rooted := path[0] == Separator + n := len(path) + + // Invariants: + // reading from path; r is index of next byte to process. + // writing to buf; w is index of next byte to write. + // dotdot is index in buf where .. must stop, either because + // it is the leading slash or it is a leading ../../.. prefix. + buf := []byte(path) + r, w, dotdot := 0, 0, 0 + if rooted { + r, w, dotdot = 1, 1, 1 + } + + for r < n { + switch { + case path[r] == Separator: + // empty path element + r++ + case path[r] == '.' && (r+1 == n || path[r+1] == Separator): + // . element + r++ + case path[r] == '.' && path[r+1] == '.' && (r+2 == n || path[r+2] == Separator): + // .. element: remove to last separator + r += 2 + switch { + case w > dotdot: + // can backtrack + w-- + for w > dotdot && buf[w] != Separator { + w-- + } + case !rooted: + // cannot backtrack, but not rooted, so append .. element. + if w > 0 { + buf[w] = Separator + w++ + } + buf[w] = '.' + w++ + buf[w] = '.' + w++ + dotdot = w + } + default: + // real path element. + // add slash if needed + if rooted && w != 1 || !rooted && w != 0 { + buf[w] = Separator + w++ + } + // copy element + for ; r < n && path[r] != Separator; r++ { + buf[w] = path[r] + w++ + } + } + } + + // Turn empty string into "." + if w == 0 { + buf[w] = '.' + w++ + } + + return string(buf[0:w]) +} + +// ToSlash returns the result of replacing each separator character +// in path with a slash ('/') character. +func ToSlash(path string) string { + if Separator == '/' { + return path + } + return strings.Replace(path, string(Separator), "/", -1) +} + +// FromSlash returns the result of replacing each slash ('/') character +// in path with a separator character. +func FromSlash(path string) string { + if Separator == '/' { + return path + } + return strings.Replace(path, "/", string(Separator), -1) +} + +// SplitList splits a list of paths joined by the OS-specific ListSeparator. +func SplitList(path string) []string { + if path == "" { + return []string{} + } + return strings.Split(path, string(ListSeparator), -1) +} + +// Split splits path immediately following the final Separator, +// partitioning it into a directory and a file name components. +// If there are no separators in path, Split returns an empty base +// and file set to path. +func Split(path string) (dir, file string) { + i := strings.LastIndex(path, string(Separator)) + return path[:i+1], path[i+1:] +} + +// Join joins any number of path elements into a single path, adding +// a Separator if necessary. All empty strings are ignored. +func Join(elem ...string) string { + for i, e := range elem { + if e != "" { + return Clean(strings.Join(elem[i:], string(Separator))) + } + } + return "" +} + +// Ext returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +func Ext(path string) string { + for i := len(path) - 1; i >= 0 && path[i] != Separator; i-- { + if path[i] == '.' { + return path[i:] + } + } + return "" +} + +// Visitor methods are invoked for corresponding file tree entries +// visited by Walk. The parameter path is the full path of f relative +// to root. +type Visitor interface { + VisitDir(path string, f *os.FileInfo) bool + VisitFile(path string, f *os.FileInfo) +} + +func walk(path string, f *os.FileInfo, v Visitor, errors chan<- os.Error) { + if !f.IsDirectory() { + v.VisitFile(path, f) + return + } + + if !v.VisitDir(path, f) { + return // skip directory entries + } + + list, err := readDir(path) + if err != nil { + if errors != nil { + errors <- err + } + } + + for _, e := range list { + walk(Join(path, e.Name), e, v, errors) + } +} + +// readDir reads the directory named by dirname and returns +// a list of sorted directory entries. +// Copied from io/ioutil to avoid the circular import. +func readDir(dirname string) ([]*os.FileInfo, os.Error) { + f, err := os.Open(dirname, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + fi := make(fileInfoList, len(list)) + for i := range list { + fi[i] = &list[i] + } + sort.Sort(fi) + return fi, nil +} + +// A dirList implements sort.Interface. +type fileInfoList []*os.FileInfo + +func (f fileInfoList) Len() int { return len(f) } +func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name } +func (f fileInfoList) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +// Walk walks the file tree rooted at root, calling v.VisitDir or +// v.VisitFile for each directory or file in the tree, including root. +// If v.VisitDir returns false, Walk skips the directory's entries; +// otherwise it invokes itself for each directory entry in sorted order. +// An error reading a directory does not abort the Walk. +// If errors != nil, Walk sends each directory read error +// to the channel. Otherwise Walk discards the error. +func Walk(root string, v Visitor, errors chan<- os.Error) { + f, err := os.Lstat(root) + if err != nil { + if errors != nil { + errors <- err + } + return // can't progress + } + walk(root, f, v, errors) +} + +// Base returns the last element of path. +// Trailing path separators are removed before extracting the last element. +// If the path is empty, Base returns ".". +// If the path consists entirely of separators, Base returns a single separator. +func Base(path string) string { + if path == "" { + return "." + } + // Strip trailing slashes. + for len(path) > 0 && path[len(path)-1] == Separator { + path = path[0 : len(path)-1] + } + // Find the last element + if i := strings.LastIndex(path, string(Separator)); i >= 0 { + path = path[i+1:] + } + // If empty now, it had only slashes. + if path == "" { + return string(Separator) + } + return path +} + +// IsAbs returns true if the path is absolute. +func IsAbs(path string) bool { + return len(path) > 0 && path[0] == Separator +} diff --git a/src/pkg/path/filepath/path_test.go b/src/pkg/path/filepath/path_test.go new file mode 100644 index 000000000..8f887f00b --- /dev/null +++ b/src/pkg/path/filepath/path_test.go @@ -0,0 +1,392 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package filepath_test + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +type PathTest struct { + path, result string +} + +var cleantests = []PathTest{ + // Already clean + {"", "."}, + {"abc", "abc"}, + {"abc/def", "abc/def"}, + {"a/b/c", "a/b/c"}, + {".", "."}, + {"..", ".."}, + {"../..", "../.."}, + {"../../abc", "../../abc"}, + {"/abc", "/abc"}, + {"/", "/"}, + + // Remove trailing slash + {"abc/", "abc"}, + {"abc/def/", "abc/def"}, + {"a/b/c/", "a/b/c"}, + {"./", "."}, + {"../", ".."}, + {"../../", "../.."}, + {"/abc/", "/abc"}, + + // Remove doubled slash + {"abc//def//ghi", "abc/def/ghi"}, + {"//abc", "/abc"}, + {"///abc", "/abc"}, + {"//abc//", "/abc"}, + {"abc//", "abc"}, + + // Remove . elements + {"abc/./def", "abc/def"}, + {"/./abc/def", "/abc/def"}, + {"abc/.", "abc"}, + + // Remove .. elements + {"abc/def/ghi/../jkl", "abc/def/jkl"}, + {"abc/def/../ghi/../jkl", "abc/jkl"}, + {"abc/def/..", "abc"}, + {"abc/def/../..", "."}, + {"/abc/def/../..", "/"}, + {"abc/def/../../..", ".."}, + {"/abc/def/../../..", "/"}, + {"abc/def/../../../ghi/jkl/../../../mno", "../../mno"}, + + // Combinations + {"abc/./../def", "def"}, + {"abc//./../def", "def"}, + {"abc/../../././../def", "../../def"}, +} + +func TestClean(t *testing.T) { + for _, test := range cleantests { + if s := filepath.Clean(test.path); s != test.result { + t.Errorf("Clean(%q) = %q, want %q", test.path, s, test.result) + } + } +} + +const sep = filepath.Separator + +var slashtests = []PathTest{ + {"", ""}, + {"/", string(sep)}, + {"/a/b", string([]byte{sep, 'a', sep, 'b'})}, + {"a//b", string([]byte{'a', sep, sep, 'b'})}, +} + +func TestFromAndToSlash(t *testing.T) { + for _, test := range slashtests { + if s := filepath.FromSlash(test.path); s != test.result { + t.Errorf("FromSlash(%q) = %q, want %q", test.path, s, test.result) + } + if s := filepath.ToSlash(test.result); s != test.path { + t.Errorf("ToSlash(%q) = %q, want %q", test.result, s, test.path) + } + } +} + +type SplitListTest struct { + list string + result []string +} + +const lsep = filepath.ListSeparator + +var splitlisttests = []SplitListTest{ + {"", []string{}}, + {string([]byte{'a', lsep, 'b'}), []string{"a", "b"}}, + {string([]byte{lsep, 'a', lsep, 'b'}), []string{"", "a", "b"}}, +} + +func TestSplitList(t *testing.T) { + for _, test := range splitlisttests { + if l := filepath.SplitList(test.list); !reflect.DeepEqual(l, test.result) { + t.Errorf("SplitList(%q) = %s, want %s", test.list, l, test.result) + } + } +} + +type SplitTest struct { + path, dir, file string +} + +var unixsplittests = []SplitTest{ + {"a/b", "a/", "b"}, + {"a/b/", "a/b/", ""}, + {"a/", "a/", ""}, + {"a", "", "a"}, + {"/", "/", ""}, +} + +func TestSplit(t *testing.T) { + var splittests []SplitTest + splittests = unixsplittests + for _, test := range splittests { + if d, f := filepath.Split(test.path); d != test.dir || f != test.file { + t.Errorf("Split(%q) = %q, %q, want %q, %q", test.path, d, f, test.dir, test.file) + } + } +} + +type JoinTest struct { + elem []string + path string +} + +var jointests = []JoinTest{ + // zero parameters + {[]string{}, ""}, + + // one parameter + {[]string{""}, ""}, + {[]string{"a"}, "a"}, + + // two parameters + {[]string{"a", "b"}, "a/b"}, + {[]string{"a", ""}, "a"}, + {[]string{"", "b"}, "b"}, + {[]string{"/", "a"}, "/a"}, + {[]string{"/", ""}, "/"}, + {[]string{"a/", "b"}, "a/b"}, + {[]string{"a/", ""}, "a"}, + {[]string{"", ""}, ""}, +} + +// join takes a []string and passes it to Join. +func join(elem []string, args ...string) string { + args = elem + return filepath.Join(args...) +} + +func TestJoin(t *testing.T) { + for _, test := range jointests { + if p := join(test.elem); p != test.path { + t.Errorf("join(%q) = %q, want %q", test.elem, p, test.path) + } + } +} + +type ExtTest struct { + path, ext string +} + +var exttests = []ExtTest{ + {"path.go", ".go"}, + {"path.pb.go", ".go"}, + {"a.dir/b", ""}, + {"a.dir/b.go", ".go"}, + {"a.dir/", ""}, +} + +func TestExt(t *testing.T) { + for _, test := range exttests { + if x := filepath.Ext(test.path); x != test.ext { + t.Errorf("Ext(%q) = %q, want %q", test.path, x, test.ext) + } + } +} + +type Node struct { + name string + entries []*Node // nil if the entry is a file + mark int +} + +var tree = &Node{ + "testdata", + []*Node{ + &Node{"a", nil, 0}, + &Node{"b", []*Node{}, 0}, + &Node{"c", nil, 0}, + &Node{ + "d", + []*Node{ + &Node{"x", nil, 0}, + &Node{"y", []*Node{}, 0}, + &Node{ + "z", + []*Node{ + &Node{"u", nil, 0}, + &Node{"v", nil, 0}, + }, + 0, + }, + }, + 0, + }, + }, + 0, +} + +func walkTree(n *Node, path string, f func(path string, n *Node)) { + f(path, n) + for _, e := range n.entries { + walkTree(e, filepath.Join(path, e.name), f) + } +} + +func makeTree(t *testing.T) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.entries == nil { + fd, err := os.Open(path, os.O_CREAT, 0660) + if err != nil { + t.Errorf("makeTree: %v", err) + } + fd.Close() + } else { + os.Mkdir(path, 0770) + } + }) +} + +func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } + +func checkMarks(t *testing.T) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.mark != 1 { + t.Errorf("node %s mark = %d; expected 1", path, n.mark) + } + n.mark = 0 + }) +} + +// Assumes that each node name is unique. Good enough for a test. +func mark(name string) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.name == name { + n.mark++ + } + }) +} + +type TestVisitor struct{} + +func (v *TestVisitor) VisitDir(path string, f *os.FileInfo) bool { + mark(f.Name) + return true +} + +func (v *TestVisitor) VisitFile(path string, f *os.FileInfo) { + mark(f.Name) +} + +func TestWalk(t *testing.T) { + // TODO(brainman): enable test once Windows version is implemented. + if runtime.GOOS == "windows" { + return + } + makeTree(t) + + // 1) ignore error handling, expect none + v := &TestVisitor{} + filepath.Walk(tree.name, v, nil) + checkMarks(t) + + // 2) handle errors, expect none + errors := make(chan os.Error, 64) + filepath.Walk(tree.name, v, errors) + select { + case err := <-errors: + t.Errorf("no error expected, found: %s", err) + default: + // ok + } + checkMarks(t) + + if os.Getuid() != 0 { + // introduce 2 errors: chmod top-level directories to 0 + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + + // 3) handle errors, expect two + errors = make(chan os.Error, 64) + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) + filepath.Walk(tree.name, v, errors) + Loop: + for i := 1; i <= 2; i++ { + select { + case <-errors: + // ok + default: + t.Errorf("%d. error expected, none found", i) + break Loop + } + } + select { + case err := <-errors: + t.Errorf("only two errors expected, found 3rd: %v", err) + default: + // ok + } + // the inaccessible subtrees were marked manually + checkMarks(t) + } + + // cleanup + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) + if err := os.RemoveAll(tree.name); err != nil { + t.Errorf("removeTree: %v", err) + } +} + +var basetests = []PathTest{ + {"", "."}, + {".", "."}, + {"/.", "."}, + {"/", "/"}, + {"////", "/"}, + {"x/", "x"}, + {"abc", "abc"}, + {"abc/def", "def"}, + {"a/b/.x", ".x"}, + {"a/b/c.", "c."}, + {"a/b/c.x", "c.x"}, +} + +func TestBase(t *testing.T) { + for _, test := range basetests { + if s := filepath.Base(test.path); s != test.result { + t.Errorf("Base(%q) = %q, want %q", test.path, s, test.result) + } + } +} + +type IsAbsTest struct { + path string + isAbs bool +} + +var isAbsTests = []IsAbsTest{ + {"", false}, + {"/", true}, + {"/usr/bin/gcc", true}, + {"..", false}, + {"/a/../bb", true}, + {".", false}, + {"./", false}, + {"lala", false}, +} + +func TestIsAbs(t *testing.T) { + for _, test := range isAbsTests { + if r := filepath.IsAbs(test.path); r != test.isAbs { + t.Errorf("IsAbs(%q) = %v, want %v", test.path, r, test.isAbs) + } + } +} diff --git a/src/pkg/path/filepath/path_unix.go b/src/pkg/path/filepath/path_unix.go new file mode 100644 index 000000000..7d07794e3 --- /dev/null +++ b/src/pkg/path/filepath/path_unix.go @@ -0,0 +1,10 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package filepath + +const ( + Separator = '/' // OS-specific path separator + ListSeparator = ':' // OS-specific path list separator +) diff --git a/src/pkg/path/match.go b/src/pkg/path/match.go index dd3422c42..efb8c5ce7 100644 --- a/src/pkg/path/match.go +++ b/src/pkg/path/match.go @@ -1,8 +1,11 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package path import ( "os" - "sort" "strings" "utf8" ) @@ -10,7 +13,7 @@ import ( var ErrBadPattern = os.NewError("syntax error in pattern") // Match returns true if name matches the shell file name pattern. -// The syntax used by pattern is: +// The pattern syntax is: // // pattern: // { term } @@ -75,7 +78,7 @@ Pattern: return len(name) == 0, nil } -// scanChunk gets the next section of pattern, which is a non-star string +// scanChunk gets the next segment of pattern, which is a non-star string // possibly preceded by a star. func scanChunk(pattern string) (star bool, chunk, rest string) { for len(pattern) > 0 && pattern[0] == '*' { @@ -92,7 +95,6 @@ Scan: if i+1 < len(pattern) { i++ } - continue case '[': inrange = true case ']': @@ -203,76 +205,3 @@ func getEsc(chunk string) (r int, nchunk string, err os.Error) { } return } - -// Glob returns the names of all files matching pattern or nil -// if there is no matching file. The syntax of patterns is the same -// as in Match. The pattern may describe hierarchical names such as -// /usr/*/bin/ed. -// -func Glob(pattern string) (matches []string) { - if !hasMeta(pattern) { - if _, err := os.Stat(pattern); err == nil { - return []string{pattern} - } - return nil - } - - dir, file := Split(pattern) - switch dir { - case "": - dir = "." - case "/": - // nothing - default: - dir = dir[0 : len(dir)-1] // chop off trailing '/' - } - - if hasMeta(dir) { - for _, d := range Glob(dir) { - matches = glob(d, file, matches) - } - } else { - return glob(dir, file, nil) - } - return matches -} - -// glob searches for files matching pattern in the directory dir -// and appends them to matches. -func glob(dir, pattern string, matches []string) []string { - fi, err := os.Stat(dir) - if err != nil { - return nil - } - if !fi.IsDirectory() { - return matches - } - d, err := os.Open(dir, os.O_RDONLY, 0666) - if err != nil { - return nil - } - defer d.Close() - - names, err := d.Readdirnames(-1) - if err != nil { - return nil - } - sort.SortStrings(names) - - for _, n := range names { - matched, err := Match(pattern, n) - if err != nil { - return matches - } - if matched { - matches = append(matches, Join(dir, n)) - } - } - return matches -} - -// hasMeta returns true if path contains any of the magic characters -// recognized by Match. -func hasMeta(path string) bool { - return strings.IndexAny(path, "*?[") != -1 -} diff --git a/src/pkg/path/match_test.go b/src/pkg/path/match_test.go index a1bf508e3..f377f1083 100644 --- a/src/pkg/path/match_test.go +++ b/src/pkg/path/match_test.go @@ -75,31 +75,3 @@ func TestMatch(t *testing.T) { } } } - -// contains returns true if vector contains the string s. -func contains(vector []string, s string) bool { - for _, elem := range vector { - if elem == s { - return true - } - } - return false -} - -var globTests = []struct { - pattern, result string -}{ - {"match.go", "match.go"}, - {"mat?h.go", "match.go"}, - {"*", "match.go"}, - {"../*/match.go", "../path/match.go"}, -} - -func TestGlob(t *testing.T) { - for _, tt := range globTests { - matches := Glob(tt.pattern) - if !contains(matches, tt.result) { - t.Errorf("Glob(%#q) = %#v want %v", tt.pattern, matches, tt.result) - } - } -} diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index 61eea8858..658eec093 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -7,8 +7,6 @@ package path import ( - "io/ioutil" - "os" "strings" ) @@ -107,7 +105,7 @@ func Clean(path string) string { // If there is no separator in path, Split returns an empty dir and // file set to path. func Split(path string) (dir, file string) { - i := strings.LastIndexAny(path, PathSeps) + i := strings.LastIndex(path, "/") return path[:i+1], path[i+1:] } @@ -135,78 +133,30 @@ func Ext(path string) string { return "" } -// Visitor methods are invoked for corresponding file tree entries -// visited by Walk. The parameter path is the full path of f relative -// to root. -type Visitor interface { - VisitDir(path string, f *os.FileInfo) bool - VisitFile(path string, f *os.FileInfo) -} - -func walk(path string, f *os.FileInfo, v Visitor, errors chan<- os.Error) { - if !f.IsDirectory() { - v.VisitFile(path, f) - return - } - - if !v.VisitDir(path, f) { - return // skip directory entries - } - - list, err := ioutil.ReadDir(path) - if err != nil { - if errors != nil { - errors <- err - } - } - - for _, e := range list { - walk(Join(path, e.Name), e, v, errors) - } -} - -// Walk walks the file tree rooted at root, calling v.VisitDir or -// v.VisitFile for each directory or file in the tree, including root. -// If v.VisitDir returns false, Walk skips the directory's entries; -// otherwise it invokes itself for each directory entry in sorted order. -// An error reading a directory does not abort the Walk. -// If errors != nil, Walk sends each directory read error -// to the channel. Otherwise Walk discards the error. -func Walk(root string, v Visitor, errors chan<- os.Error) { - f, err := os.Lstat(root) - if err != nil { - if errors != nil { - errors <- err - } - return // can't progress - } - walk(root, f, v, errors) -} - -// Base returns the last path element of the slash-separated name. -// Trailing slashes are removed before extracting the last element. If the name is -// empty, "." is returned. If it consists entirely of slashes, "/" is returned. -func Base(name string) string { - if name == "" { +// Base returns the last element of path. +// Trailing slashes are removed before extracting the last element. +// If the path is empty, Base returns ".". +// If the path consists entirely of slashes, Base returns "/". +func Base(path string) string { + if path == "" { return "." } // Strip trailing slashes. - for len(name) > 0 && name[len(name)-1] == '/' { - name = name[0 : len(name)-1] + for len(path) > 0 && path[len(path)-1] == '/' { + path = path[0 : len(path)-1] } // Find the last element - if i := strings.LastIndex(name, "/"); i >= 0 { - name = name[i+1:] + if i := strings.LastIndex(path, "/"); i >= 0 { + path = path[i+1:] } // If empty now, it had only slashes. - if name == "" { + if path == "" { return "/" } - return name + return path } // IsAbs returns true if the path is absolute. func IsAbs(path string) bool { - // TODO: Add Windows support - return strings.HasPrefix(path, "/") + return len(path) > 0 && path[0] == '/' } diff --git a/src/pkg/path/path_test.go b/src/pkg/path/path_test.go index ab0b48ad6..1fd57cc80 100644 --- a/src/pkg/path/path_test.go +++ b/src/pkg/path/path_test.go @@ -5,8 +5,6 @@ package path import ( - "os" - "runtime" "testing" ) @@ -84,18 +82,7 @@ var splittests = []SplitTest{ {"/", "/", ""}, } -var winsplittests = []SplitTest{ - {`C:\Windows\System32`, `C:\Windows\`, `System32`}, - {`C:\Windows\`, `C:\Windows\`, ``}, - {`C:\Windows`, `C:\`, `Windows`}, - {`C:Windows`, `C:`, `Windows`}, - {`\\?\c:\`, `\\?\c:\`, ``}, -} - func TestSplit(t *testing.T) { - if runtime.GOOS == "windows" { - splittests = append(splittests, winsplittests...) - } for _, test := range splittests { if d, f := Split(test.path); d != test.dir || f != test.file { t.Errorf("Split(%q) = %q, %q, want %q, %q", test.path, d, f, test.dir, test.file) @@ -161,152 +148,6 @@ func TestExt(t *testing.T) { } } -type Node struct { - name string - entries []*Node // nil if the entry is a file - mark int -} - -var tree = &Node{ - "testdata", - []*Node{ - &Node{"a", nil, 0}, - &Node{"b", []*Node{}, 0}, - &Node{"c", nil, 0}, - &Node{ - "d", - []*Node{ - &Node{"x", nil, 0}, - &Node{"y", []*Node{}, 0}, - &Node{ - "z", - []*Node{ - &Node{"u", nil, 0}, - &Node{"v", nil, 0}, - }, - 0, - }, - }, - 0, - }, - }, - 0, -} - -func walkTree(n *Node, path string, f func(path string, n *Node)) { - f(path, n) - for _, e := range n.entries { - walkTree(e, Join(path, e.name), f) - } -} - -func makeTree(t *testing.T) { - walkTree(tree, tree.name, func(path string, n *Node) { - if n.entries == nil { - fd, err := os.Open(path, os.O_CREAT, 0660) - if err != nil { - t.Errorf("makeTree: %v", err) - } - fd.Close() - } else { - os.Mkdir(path, 0770) - } - }) -} - -func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } - -func checkMarks(t *testing.T) { - walkTree(tree, tree.name, func(path string, n *Node) { - if n.mark != 1 { - t.Errorf("node %s mark = %d; expected 1", path, n.mark) - } - n.mark = 0 - }) -} - -// Assumes that each node name is unique. Good enough for a test. -func mark(name string) { - walkTree(tree, tree.name, func(path string, n *Node) { - if n.name == name { - n.mark++ - } - }) -} - -type TestVisitor struct{} - -func (v *TestVisitor) VisitDir(path string, f *os.FileInfo) bool { - mark(f.Name) - return true -} - -func (v *TestVisitor) VisitFile(path string, f *os.FileInfo) { - mark(f.Name) -} - -func TestWalk(t *testing.T) { - makeTree(t) - - // 1) ignore error handling, expect none - v := &TestVisitor{} - Walk(tree.name, v, nil) - checkMarks(t) - - // 2) handle errors, expect none - errors := make(chan os.Error, 64) - Walk(tree.name, v, errors) - select { - case err := <-errors: - t.Errorf("no error expected, found: %s", err) - default: - // ok - } - checkMarks(t) - - if os.Getuid() != 0 { - // introduce 2 errors: chmod top-level directories to 0 - os.Chmod(Join(tree.name, tree.entries[1].name), 0) - os.Chmod(Join(tree.name, tree.entries[3].name), 0) - // mark respective subtrees manually - markTree(tree.entries[1]) - markTree(tree.entries[3]) - // correct double-marking of directory itself - tree.entries[1].mark-- - tree.entries[3].mark-- - - // 3) handle errors, expect two - errors = make(chan os.Error, 64) - os.Chmod(Join(tree.name, tree.entries[1].name), 0) - Walk(tree.name, v, errors) - Loop: - for i := 1; i <= 2; i++ { - select { - case <-errors: - // ok - default: - t.Errorf("%d. error expected, none found", i) - break Loop - } - } - select { - case err := <-errors: - t.Errorf("only two errors expected, found 3rd: %v", err) - default: - // ok - } - // the inaccessible subtrees were marked manually - checkMarks(t) - } - - // cleanup - os.Chmod(Join(tree.name, tree.entries[1].name), 0770) - os.Chmod(Join(tree.name, tree.entries[3].name), 0770) - if err := os.RemoveAll(tree.name); err != nil { - t.Errorf("removeTree: %v", err) - } -} - var basetests = []CleanTest{ // Already clean {"", "."}, diff --git a/src/pkg/path/path_unix.go b/src/pkg/path/path_unix.go deleted file mode 100644 index 7e8c5eb8b..000000000 --- a/src/pkg/path/path_unix.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package path - -const ( - DirSeps = `/` // directory separators - VolumeSeps = `` // volume separators - PathSeps = DirSeps + VolumeSeps // all path separators -) diff --git a/src/pkg/path/path_windows.go b/src/pkg/path/path_windows.go deleted file mode 100644 index 966eb49fb..000000000 --- a/src/pkg/path/path_windows.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package path - -const ( - DirSeps = `\/` // directory separators - VolumeSeps = `:` // volume separators - PathSeps = DirSeps + VolumeSeps // all path separators -) diff --git a/src/pkg/reflect/all_test.go b/src/pkg/reflect/all_test.go index 3675be6f1..7a97ea173 100644 --- a/src/pkg/reflect/all_test.go +++ b/src/pkg/reflect/all_test.go @@ -1093,6 +1093,18 @@ func TestMethod(t *testing.T) { t.Errorf("Value Method returned %d; want 250", i) } + // Curried method of pointer. + i = NewValue(&p).Method(0).Call([]Value{NewValue(10)})[0].(*IntValue).Get() + if i != 250 { + t.Errorf("Value Method returned %d; want 250", i) + } + + // Curried method of pointer to value. + i = NewValue(p).Addr().Method(0).Call([]Value{NewValue(10)})[0].(*IntValue).Get() + if i != 250 { + t.Errorf("Value Method returned %d; want 250", i) + } + // Curried method of interface value. // Have to wrap interface value in a struct to get at it. // Passing it to NewValue directly would @@ -1390,3 +1402,66 @@ func TestEmbeddedMethods(t *testing.T) { t.Errorf("f(o) = %d, want 2", v) } } + +func TestPtrTo(t *testing.T) { + var i int + + typ := Typeof(i) + for i = 0; i < 100; i++ { + typ = PtrTo(typ) + } + for i = 0; i < 100; i++ { + typ = typ.(*PtrType).Elem() + } + if typ != Typeof(i) { + t.Errorf("after 100 PtrTo and Elem, have %s, want %s", typ, Typeof(i)) + } +} + +func TestAddr(t *testing.T) { + var p struct { + X, Y int + } + + v := NewValue(&p) + v = v.(*PtrValue).Elem() + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*StructValue).Field(0) + v.(*IntValue).Set(2) + if p.X != 2 { + t.Errorf("Addr.Elem.Set failed to set value") + } + + // Again but take address of the NewValue value. + // Exercises generation of PtrTypes not present in the binary. + v = NewValue(&p) + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*PtrValue).Elem() + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*StructValue).Field(0) + v.(*IntValue).Set(3) + if p.X != 3 { + t.Errorf("Addr.Elem.Set failed to set value") + } + + // Starting without pointer we should get changed value + // in interface. + v = NewValue(p) + v0 := v + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*StructValue).Field(0) + v.(*IntValue).Set(4) + if p.X != 3 { // should be unchanged from last time + t.Errorf("somehow value Set changed original p") + } + p = v0.Interface().(struct { + X, Y int + }) + if p.X != 4 { + t.Errorf("Addr.Elem.Set valued to set value in top value") + } +} diff --git a/src/pkg/reflect/deepequal.go b/src/pkg/reflect/deepequal.go index a50925e51..c9beec506 100644 --- a/src/pkg/reflect/deepequal.go +++ b/src/pkg/reflect/deepequal.go @@ -31,8 +31,8 @@ func deepValueEqual(v1, v2 Value, visited map[uintptr]*visit, depth int) bool { // if depth > 10 { panic("deepValueEqual") } // for debugging - addr1 := v1.Addr() - addr2 := v2.Addr() + addr1 := v1.UnsafeAddr() + addr2 := v2.UnsafeAddr() if addr1 > addr2 { // Canonicalize order to reduce number of entries in visited. addr1, addr2 = addr2, addr1 diff --git a/src/pkg/reflect/type.go b/src/pkg/reflect/type.go index 8dcbb2413..2cc1f576a 100644 --- a/src/pkg/reflect/type.go +++ b/src/pkg/reflect/type.go @@ -18,6 +18,7 @@ package reflect import ( "runtime" "strconv" + "sync" "unsafe" ) @@ -48,6 +49,7 @@ type commonType struct { kind uint8 string *string *uncommonType + ptrToThis *runtime.Type } type method struct { @@ -163,6 +165,10 @@ type SliceType struct { elem *runtime.Type } +// arrayOrSliceType is an unexported method that guarantees only +// arrays and slices implement ArrayOrSliceType. +func (*SliceType) arrayOrSliceType() {} + // Struct field type structField struct { name *string @@ -235,11 +241,19 @@ type Type interface { // Kind returns the specific kind of this type. Kind() Kind - // For non-interface types, Method returns the i'th method with receiver T. - // For interface types, Method returns the i'th method in the interface. - // NumMethod returns the number of such methods. + // Method returns the i'th method in the type's method set. + // + // For a non-interface type T or *T, the returned Method's Type and Func + // fields describe a function whose first argument is the receiver. + // + // For an interface type, the returned Method's Type field gives the + // method signature, without a receiver, and the Func field is nil. Method(int) Method + + // NumMethods returns the number of methods in the type's method set. NumMethod() int + + common() *commonType uncommon() *uncommonType } @@ -350,6 +364,8 @@ func (t *commonType) FieldAlign() int { return int(t.fieldAlign) } func (t *commonType) Kind() Kind { return Kind(t.kind & kindMask) } +func (t *commonType) common() *commonType { return t } + func (t *uncommonType) Method(i int) (m Method) { if t == nil || i < 0 || i >= len(t.methods) { return @@ -363,7 +379,7 @@ func (t *uncommonType) Method(i int) (m Method) { } m.Type = toType(*p.typ).(*FuncType) fn := p.tfn - m.Func = &FuncValue{value: value{m.Type, addr(&fn), true}} + m.Func = &FuncValue{value: value{m.Type, addr(&fn), canSet}} return } @@ -391,6 +407,10 @@ func (t *ArrayType) Len() int { return int(t.len) } // Elem returns the type of the array's elements. func (t *ArrayType) Elem() Type { return toType(*t.elem) } +// arrayOrSliceType is an unexported method that guarantees only +// arrays and slices implement ArrayOrSliceType. +func (*ArrayType) arrayOrSliceType() {} + // Dir returns the channel direction. func (t *ChanType) Dir() ChanDir { return ChanDir(t.dir) } @@ -444,7 +464,7 @@ func (t *FuncType) Out(i int) Type { // NumOut returns the number of function output parameters. func (t *FuncType) NumOut() int { return len(t.out) } -// Method returns the i'th interface method. +// Method returns the i'th method in the type's method set. func (t *InterfaceType) Method(i int) (m Method) { if i < 0 || i >= len(t.methods) { return @@ -458,7 +478,7 @@ func (t *InterfaceType) Method(i int) (m Method) { return } -// NumMethod returns the number of interface methods. +// NumMethod returns the number of interface methods in the type's method set. func (t *InterfaceType) NumMethod() int { return len(t.methods) } // Key returns the map key type. @@ -669,7 +689,89 @@ func toType(i interface{}) Type { type ArrayOrSliceType interface { Type Elem() Type + arrayOrSliceType() // Guarantees only Array and Slice implement this interface. } // Typeof returns the reflection Type of the value in the interface{}. func Typeof(i interface{}) Type { return toType(unsafe.Typeof(i)) } + +// ptrMap is the cache for PtrTo. +var ptrMap struct { + sync.RWMutex + m map[Type]*PtrType +} + +// runtimePtrType is the runtime layout for a *PtrType. +// The memory immediately before the *PtrType is always +// the canonical runtime.Type to be used for a *runtime.Type +// describing this PtrType. +type runtimePtrType struct { + runtime.Type + runtime.PtrType +} + +// PtrTo returns the pointer type with element t. +// For example, if t represents type Foo, PtrTo(t) represents *Foo. +func PtrTo(t Type) *PtrType { + // If t records its pointer-to type, use it. + ct := t.common() + if p := ct.ptrToThis; p != nil { + return toType(*p).(*PtrType) + } + + // Otherwise, synthesize one. + // This only happens for pointers with no methods. + // We keep the mapping in a map on the side, because + // this operation is rare and a separate map lets us keep + // the type structures in read-only memory. + ptrMap.RLock() + if m := ptrMap.m; m != nil { + if p := m[t]; p != nil { + ptrMap.RUnlock() + return p + } + } + ptrMap.RUnlock() + ptrMap.Lock() + if ptrMap.m == nil { + ptrMap.m = make(map[Type]*PtrType) + } + p := ptrMap.m[t] + if p != nil { + // some other goroutine won the race and created it + ptrMap.Unlock() + return p + } + + // runtime.Type value is always right before type structure. + // 2*ptrSize is size of interface header + rt := (*runtime.Type)(unsafe.Pointer(uintptr(unsafe.Pointer(ct)) - uintptr(unsafe.Sizeof(runtime.Type(nil))))) + + rp := new(runtimePtrType) + rp.Type = &rp.PtrType + + // initialize rp.PtrType using *byte's PtrType as a prototype. + // have to do assignment as PtrType, not runtime.PtrType, + // in order to write to unexported fields. + p = (*PtrType)(unsafe.Pointer(&rp.PtrType)) + bp := (*PtrType)(unsafe.Pointer(unsafe.Typeof((*byte)(nil)).(*runtime.PtrType))) + *p = *bp + + s := "*" + *ct.string + p.string = &s + + // For the type structures linked into the binary, the + // compiler provides a good hash of the string. + // Create a good hash for the new string by using + // the FNV-1 hash's mixing function to combine the + // old hash and the new "*". + p.hash = ct.hash*16777619 ^ '*' + + p.uncommonType = nil + p.ptrToThis = nil + p.elem = rt + + ptrMap.m[t] = (*PtrType)(unsafe.Pointer(&rp.PtrType)) + ptrMap.Unlock() + return p +} diff --git a/src/pkg/reflect/value.go b/src/pkg/reflect/value.go index 4d7d87237..0b70b17f8 100644 --- a/src/pkg/reflect/value.go +++ b/src/pkg/reflect/value.go @@ -11,7 +11,7 @@ import ( ) const ptrSize = uintptr(unsafe.Sizeof((*byte)(nil))) -const cannotSet = "cannot set value obtained via unexported struct field" +const cannotSet = "cannot set value obtained from unexported struct field" type addr unsafe.Pointer @@ -51,20 +51,32 @@ type Value interface { // Interface returns the value as an interface{}. Interface() interface{} - // CanSet returns whether the value can be changed. + // CanSet returns true if the value can be changed. // Values obtained by the use of non-exported struct fields // can be used in Get but not Set. - // If CanSet() returns false, calling the type-specific Set - // will cause a crash. + // If CanSet returns false, calling the type-specific Set will panic. CanSet() bool // SetValue assigns v to the value; v must have the same type as the value. SetValue(v Value) - // Addr returns a pointer to the underlying data. - // It is for advanced clients that also - // import the "unsafe" package. - Addr() uintptr + // CanAddr returns true if the value's address can be obtained with Addr. + // Such values are called addressable. A value is addressable if it is + // an element of a slice, an element of an addressable array, + // a field of an addressable struct, the result of dereferencing a pointer, + // or the result of a call to NewValue, MakeChan, MakeMap, or MakeZero. + // If CanAddr returns false, calling Addr will panic. + CanAddr() bool + + // Addr returns the address of the value. + // If the value is not addressable, Addr panics. + // Addr is typically used to obtain a pointer to a struct field or slice element + // in order to call a method that requires a pointer receiver. + Addr() *PtrValue + + // UnsafeAddr returns a pointer to the underlying data. + // It is for advanced clients that also import the "unsafe" package. + UnsafeAddr() uintptr // Method returns a FuncValue corresponding to the value's i'th method. // The arguments to a Call on the returned FuncValue @@ -75,19 +87,42 @@ type Value interface { getAddr() addr } +// flags for value +const ( + canSet uint32 = 1 << iota // can set value (write to *v.addr) + canAddr // can take address of value + canStore // can store through value (write to **v.addr) +) + // value is the common implementation of most values. // It is embedded in other, public struct types, but always // with a unique tag like "uint" or "float" so that the client cannot // convert from, say, *UintValue to *FloatValue. type value struct { - typ Type - addr addr - canSet bool + typ Type + addr addr + flag uint32 } func (v *value) Type() Type { return v.typ } -func (v *value) Addr() uintptr { return uintptr(v.addr) } +func (v *value) Addr() *PtrValue { + if !v.CanAddr() { + panic("reflect: cannot take address of value") + } + a := v.addr + flag := canSet + if v.CanSet() { + flag |= canStore + } + // We could safely set canAddr here too - + // the caller would get the address of a - + // but it doesn't match the Go model. + // The language doesn't let you say &&v. + return newValue(PtrTo(v.typ), addr(&a), flag).(*PtrValue) +} + +func (v *value) UnsafeAddr() uintptr { return uintptr(v.addr) } func (v *value) getAddr() addr { return v.addr } @@ -109,7 +144,10 @@ func (v *value) Interface() interface{} { return unsafe.Unreflect(v.typ, unsafe.Pointer(v.addr)) } -func (v *value) CanSet() bool { return v.canSet } +func (v *value) CanSet() bool { return v.flag&canSet != 0 } + +func (v *value) CanAddr() bool { return v.flag&canAddr != 0 } + /* * basic types @@ -125,7 +163,7 @@ func (v *BoolValue) Get() bool { return *(*bool)(v.addr) } // Set sets v to the value x. func (v *BoolValue) Set(x bool) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } *(*bool)(v.addr) = x @@ -152,7 +190,7 @@ func (v *FloatValue) Get() float64 { // Set sets v to the value x. func (v *FloatValue) Set(x float64) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -197,7 +235,7 @@ func (v *ComplexValue) Get() complex128 { // Set sets v to the value x. func (v *ComplexValue) Set(x complex128) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -237,7 +275,7 @@ func (v *IntValue) Get() int64 { // Set sets v to the value x. func (v *IntValue) Set(x int64) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -282,7 +320,7 @@ func (v *StringValue) Get() string { return *(*string)(v.addr) } // Set sets v to the value x. func (v *StringValue) Set(x string) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } *(*string)(v.addr) = x @@ -317,7 +355,7 @@ func (v *UintValue) Get() uint64 { // Set sets v to the value x. func (v *UintValue) Set(x uint64) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -361,7 +399,7 @@ func (v *UnsafePointerValue) Get() uintptr { return uintptr(*(*unsafe.Pointer)(v // Set sets v to the value x. func (v *UnsafePointerValue) Set(x unsafe.Pointer) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } *(*unsafe.Pointer)(v.addr) = x @@ -473,7 +511,7 @@ func (v *ArrayValue) addr() addr { return v.value.addr } // Set assigns x to v. // The new value x must have the same type as v. func (v *ArrayValue) Set(x *ArrayValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -491,7 +529,7 @@ func (v *ArrayValue) Elem(i int) Value { panic("array index out of bounds") } p := addr(uintptr(v.addr()) + uintptr(i)*typ.Size()) - return newValue(typ, p, v.canSet) + return newValue(typ, p, v.flag) } /* @@ -537,7 +575,7 @@ func (v *SliceValue) SetLen(n int) { // Set assigns x to v. // The new value x must have the same type as v. func (v *SliceValue) Set(x *SliceValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -566,7 +604,14 @@ func (v *SliceValue) Slice(beg, end int) *SliceValue { s.Data = uintptr(v.addr()) + uintptr(beg)*typ.Elem().Size() s.Len = end - beg s.Cap = cap - beg - return newValue(typ, addr(s), v.canSet).(*SliceValue) + + // Like the result of Addr, we treat Slice as an + // unaddressable temporary, so don't set canAddr. + flag := canSet + if v.flag&canStore != 0 { + flag |= canStore + } + return newValue(typ, addr(s), flag).(*SliceValue) } // Elem returns the i'th element of v. @@ -577,7 +622,11 @@ func (v *SliceValue) Elem(i int) Value { panic("reflect: slice index out of range") } p := addr(uintptr(v.addr()) + uintptr(i)*typ.Size()) - return newValue(typ, p, v.canSet) + flag := canAddr + if v.flag&canStore != 0 { + flag |= canSet | canStore + } + return newValue(typ, p, flag) } // MakeSlice creates a new zero-initialized slice value @@ -588,7 +637,7 @@ func MakeSlice(typ *SliceType, len, cap int) *SliceValue { Len: len, Cap: cap, } - return newValue(typ, addr(s), true).(*SliceValue) + return newValue(typ, addr(s), canAddr|canSet|canStore).(*SliceValue) } /* @@ -606,7 +655,7 @@ func (v *ChanValue) IsNil() bool { return *(*uintptr)(v.addr) == 0 } // Set assigns x to v. // The new value x must have the same type as v. func (v *ChanValue) Set(x *ChanValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -733,7 +782,7 @@ func (v *FuncValue) Get() uintptr { return *(*uintptr)(v.addr) } // Set assigns x to v. // The new value x must have the same type as v. func (v *FuncValue) Set(x *FuncValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -754,7 +803,7 @@ func (v *value) Method(i int) *FuncValue { } p := &t.methods[i] fn := p.tfn - fv := &FuncValue{value: value{toType(*p.typ), addr(&fn), true}, first: v, isInterface: false} + fv := &FuncValue{value: value{toType(*p.typ), addr(&fn), 0}, first: v, isInterface: false} return fv } @@ -765,6 +814,17 @@ type tiny struct { b byte } +// Interface returns the fv as an interface value. +// If fv is a method obtained by invoking Value.Method +// (as opposed to Type.Method), Interface cannot return an +// interface value, so it panics. +func (fv *FuncValue) Interface() interface{} { + if fv.first != nil { + panic("FuncValue: cannot create interface value for method with bound receiver") + } + return fv.value.Interface() +} + // Call calls the function fv with input parameters in. // It returns the function's output parameters as Values. func (fv *FuncValue) Call(in []Value) []Value { @@ -902,7 +962,7 @@ func (v *InterfaceValue) Set(x Value) { if x != nil { i = x.Interface() } - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } // Two different representations; see comment in Get. @@ -933,11 +993,11 @@ func (v *InterfaceValue) Method(i int) *FuncValue { // Interface is two words: itable, data. tab := *(**runtime.Itable)(v.addr) - data := &value{Typeof((*byte)(nil)), addr(uintptr(v.addr) + ptrSize), true} + data := &value{Typeof((*byte)(nil)), addr(uintptr(v.addr) + ptrSize), 0} // Function pointer is at p.perm in the table. fn := tab.Fn[i] - fv := &FuncValue{value: value{toType(*p.typ), addr(&fn), true}, first: data, isInterface: true} + fv := &FuncValue{value: value{toType(*p.typ), addr(&fn), 0}, first: data, isInterface: true} return fv } @@ -956,7 +1016,7 @@ func (v *MapValue) IsNil() bool { return *(*uintptr)(v.addr) == 0 } // Set assigns x to v. // The new value x must have the same type as v. func (v *MapValue) Set(x *MapValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } if x == nil { @@ -1075,15 +1135,18 @@ func (v *PtrValue) IsNil() bool { return *(*uintptr)(v.addr) == 0 } func (v *PtrValue) Get() uintptr { return *(*uintptr)(v.addr) } // Set assigns x to v. -// The new value x must have the same type as v. +// The new value x must have the same type as v, and x.Elem().CanSet() must be true. func (v *PtrValue) Set(x *PtrValue) { if x == nil { *(**uintptr)(v.addr) = nil return } - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } + if x.flag&canStore == 0 { + panic("cannot copy pointer obtained from unexported struct field") + } typesMustMatch(v.typ, x.typ) // TODO: This will have to move into the runtime // once the new gc goes in @@ -1112,7 +1175,7 @@ func (v *PtrValue) PointTo(x Value) { typesMustMatch(v.typ.(*PtrType).Elem(), x.Type()) // TODO: This will have to move into the runtime // once the new gc goes in. - *(*uintptr)(v.addr) = x.Addr() + *(*uintptr)(v.addr) = x.UnsafeAddr() } // Elem returns the value that v points to. @@ -1121,7 +1184,11 @@ func (v *PtrValue) Elem() Value { if v.IsNil() { return nil } - return newValue(v.typ.(*PtrType).Elem(), *(*addr)(v.addr), v.canSet) + flag := canAddr + if v.flag&canStore != 0 { + flag |= canSet | canStore + } + return newValue(v.typ.(*PtrType).Elem(), *(*addr)(v.addr), flag) } // Indirect returns the value that v points to. @@ -1148,7 +1215,7 @@ type StructValue struct { func (v *StructValue) Set(x *StructValue) { // TODO: This will have to move into the runtime // once the gc goes in. - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -1165,7 +1232,12 @@ func (v *StructValue) Field(i int) Value { return nil } f := t.Field(i) - return newValue(f.Type, addr(uintptr(v.addr)+f.Offset), v.canSet && f.PkgPath == "") + flag := v.flag + if f.PkgPath != "" { + // unexported field + flag &^= canSet | canStore + } + return newValue(f.Type, addr(uintptr(v.addr)+f.Offset), flag) } // FieldByIndex returns the nested field corresponding to index. @@ -1221,11 +1293,11 @@ func NewValue(i interface{}) Value { return nil } t, a := unsafe.Reflect(i) - return newValue(toType(t), addr(a), true) + return newValue(toType(t), addr(a), canSet|canAddr|canStore) } -func newValue(typ Type, addr addr, canSet bool) Value { - v := value{typ, addr, canSet} +func newValue(typ Type, addr addr, flag uint32) Value { + v := value{typ, addr, flag} switch typ.(type) { case *ArrayType: return &ArrayValue{v} @@ -1266,5 +1338,5 @@ func MakeZero(typ Type) Value { if typ == nil { return nil } - return newValue(typ, addr(unsafe.New(typ)), true) + return newValue(typ, addr(unsafe.New(typ)), canSet|canAddr|canStore) } diff --git a/src/pkg/rpc/server.go b/src/pkg/rpc/server.go index 9dcda4148..f185cd16e 100644 --- a/src/pkg/rpc/server.go +++ b/src/pkg/rpc/server.go @@ -527,7 +527,7 @@ func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { io.WriteString(w, "405 must CONNECT\n") return } - conn, _, err := w.Hijack() + conn, _, err := w.(http.Hijacker).Hijack() if err != nil { log.Print("rpc hijacking ", w.RemoteAddr(), ": ", err.String()) return diff --git a/src/pkg/runtime/Makefile b/src/pkg/runtime/Makefile index 521c095b9..e4cc08175 100644 --- a/src/pkg/runtime/Makefile +++ b/src/pkg/runtime/Makefile @@ -40,11 +40,8 @@ OFILES_386=\ vlop.$O\ vlrt.$O\ -GOARM?=6 - # arm-specific object files OFILES_arm=\ - cas$(GOARM).$O\ memset.$O\ softfloat.$O\ vlop.$O\ @@ -94,6 +91,7 @@ HFILES=\ runtime.h\ hashmap.h\ malloc.h\ + stack.h\ $(GOARCH)/asm.h\ $(GOOS)/os.h\ $(GOOS)/signals.h\ @@ -129,7 +127,7 @@ mkversion: mkversion.c quietgcc -o $@ -I "$(GOROOT)/include" $< "$(GOROOT)/lib/lib9.a" version.go: mkversion - ./mkversion >version.go + GOROOT="$(GOROOT_FINAL)" ./mkversion >version.go version_$(GOARCH).go: (echo 'package runtime'; echo 'const theGoarch = "$(GOARCH)"') >$@ @@ -141,13 +139,13 @@ version_$(GOOS).go: ./goc2c "`pwd`/$<" > $@.tmp mv -f $@.tmp $@ -%.$O: $(GOARCH)/%.c +%.$O: $(GOARCH)/%.c $(HFILES) $(CC) $(CFLAGS) $< -%.$O: $(GOOS)/%.c +%.$O: $(GOOS)/%.c $(HFILES) $(CC) $(CFLAGS) $< -%.$O: $(GOOS)/$(GOARCH)/%.c +%.$O: $(GOOS)/$(GOARCH)/%.c $(HFILES) $(CC) $(CFLAGS) $< %.$O: $(GOARCH)/%.s $(GOARCH)/asm.h diff --git a/src/pkg/runtime/amd64/asm.s b/src/pkg/runtime/amd64/asm.s index b6642c13c..cc05435f7 100644 --- a/src/pkg/runtime/amd64/asm.s +++ b/src/pkg/runtime/amd64/asm.s @@ -317,6 +317,26 @@ TEXT runtime·cas(SB), 7, $0 MOVL $1, AX RET +// bool casp(void **val, void *old, void *new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime·casp(SB), 7, $0 + MOVQ 8(SP), BX + MOVQ 16(SP), AX + MOVQ 24(SP), CX + LOCK + CMPXCHGQ CX, 0(BX) + JZ 3(PC) + MOVL $0, AX + RET + MOVL $1, AX + RET + + // void jmpdefer(fn, sp); // called from deferreturn. // 1. pop the caller diff --git a/src/pkg/runtime/amd64/traceback.c b/src/pkg/runtime/amd64/traceback.c index d3aae0db9..0f6733c36 100644 --- a/src/pkg/runtime/amd64/traceback.c +++ b/src/pkg/runtime/amd64/traceback.c @@ -164,12 +164,35 @@ gentraceback(byte *pc0, byte *sp, G *g, int32 skip, uintptr *pcbuf, int32 max) continue; } + if(pcbuf == nil && f->entry == (uintptr)runtime·lessstack && g == m->g0) { + // Lessstack is running on scheduler stack. Switch to original goroutine. + runtime·printf("----- lessstack called from goroutine %d -----\n", m->curg->goid); + g = m->curg; + stk = (Stktop*)g->stackbase; + sp = stk->gobuf.sp; + pc = (uintptr)stk->gobuf.pc; + fp = nil; + lr = 0; + continue; + } + // Unwind to next frame. pc = lr; lr = 0; sp = fp; fp = nil; } + + if(pcbuf == nil && (pc = g->gopc) != 0 && (f = runtime·findfunc(pc)) != nil) { + runtime·printf("----- goroutine created by -----\n%S", f->name); + if(pc > f->entry) + runtime·printf("+%p", (uintptr)(pc - f->entry)); + tracepc = pc; // back up to CALL instruction for funcline. + if(n > 0 && pc > f->entry) + tracepc--; + runtime·printf(" %S:%d\n", f->src, runtime·funcline(f, tracepc)); + } + return n; } diff --git a/src/pkg/runtime/arm/asm.s b/src/pkg/runtime/arm/asm.s index a4e4b3283..f9fe7e628 100644 --- a/src/pkg/runtime/arm/asm.s +++ b/src/pkg/runtime/arm/asm.s @@ -12,10 +12,10 @@ TEXT _rt0_arm(SB),7,$-4 // use R13 instead of SP to avoid linker rewriting the offsets MOVW 0(R13), R0 // argc MOVW $4(R13), R1 // argv - SUB $128, R13 // plenty of scratch + SUB $64, R13 // plenty of scratch AND $~7, R13 - MOVW R0, 120(R13) // save argc, argv away - MOVW R1, 124(R13) + MOVW R0, 60(R13) // save argc, argv away + MOVW R1, 64(R13) // set up m and g registers // g is R10, m is R9 @@ -34,9 +34,9 @@ TEXT _rt0_arm(SB),7,$-4 BL runtime·check(SB) // saved argc, argv - MOVW 120(R13), R0 + MOVW 60(R13), R0 MOVW R0, 4(R13) - MOVW 124(R13), R1 + MOVW 64(R13), R1 MOVW R1, 8(R13) BL runtime·args(SB) BL runtime·osinit(SB) @@ -274,3 +274,34 @@ TEXT runtime·abort(SB),7,$-4 TEXT runtime·runcgocallback(SB),7,$0 MOVW $0, R0 MOVW (R0), R1 + +// bool armcas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// }else +// return 0; +// +// To implement runtime·cas in ../$GOOS/arm/sys.s +// using the native instructions, use: +// +// TEXT runtime·cas(SB),7,$0 +// B runtime·armcas(SB) +// +TEXT runtime·armcas(SB),7,$0 + MOVW valptr+0(FP), R1 + MOVW old+4(FP), R2 + MOVW new+8(FP), R3 +casl: + LDREX (R1), R0 + CMP R0, R2 + BNE casfail + STREX R3, (R1), R0 + CMP $0, R0 + BNE casl + MOVW $1, R0 + RET +casfail: + MOVW $0, R0 + RET diff --git a/src/pkg/runtime/arm/cas5.s b/src/pkg/runtime/arm/cas5.s deleted file mode 100644 index 20bd3c3e2..000000000 --- a/src/pkg/runtime/arm/cas5.s +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "arm/asm.h" - -// This version works on pre v6 architectures - -// bool cas(int32 *val, int32 old, int32 new) -// Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// }else -// return 0; - -TEXT runtime·cas(SB),7,$0 - MOVW 0(FP), R0 // *val - MOVW 4(FP), R1 // old - MOVW 8(FP), R2 // new - MOVW $1, R3 - MOVW $runtime·cas_mutex(SB), R4 -l: - SWPW (R4), R3 // acquire mutex - CMP $0, R3 - BNE fail0 - - MOVW (R0), R5 - CMP R1, R5 - BNE fail1 - - MOVW R2, (R0) - MOVW R3, (R4) // release mutex - MOVW $1, R0 - RET -fail1: - MOVW R3, (R4) // release mutex -fail0: - MOVW $0, R0 - RET - -DATA runtime·cas_mutex(SB)/4, $0 -GLOBL runtime·cas_mutex(SB), $4 diff --git a/src/pkg/runtime/arm/cas6.s b/src/pkg/runtime/arm/cas6.s deleted file mode 100644 index 43788b28a..000000000 --- a/src/pkg/runtime/arm/cas6.s +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// bool cas(int32 *val, int32 old, int32 new) -// Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// }else -// return 0; - -TEXT runtime·cas(SB),7,$0 - MOVW 0(FP), R1 // *val - MOVW 4(FP), R2 // old - MOVW 8(FP), R3 // new -l: - LDREX (R1), R0 - CMP R0, R2 - BNE fail - STREX R3, (R1), R0 - CMP $0, R0 - BNE l - MOVW $1, R0 - RET -fail: - MOVW $0, R0 - RET - diff --git a/src/pkg/runtime/arm/traceback.c b/src/pkg/runtime/arm/traceback.c index 2307e98e8..ad3096823 100644 --- a/src/pkg/runtime/arm/traceback.c +++ b/src/pkg/runtime/arm/traceback.c @@ -9,6 +9,10 @@ void runtime·deferproc(void); void runtime·newproc(void); void runtime·newstack(void); void runtime·morestack(void); +void _div(void); +void _mod(void); +void _divu(void); +void _modu(void); static int32 gentraceback(byte *pc0, byte *sp, byte *lr0, G *g, int32 skip, uintptr *pcbuf, int32 max) @@ -113,7 +117,7 @@ gentraceback(byte *pc0, byte *sp, byte *lr0, G *g, int32 skip, uintptr *pcbuf, i // Print during crash. // main+0xf /home/rsc/go/src/runtime/x.go:23 // main(0x1, 0x2, 0x3) - runtime·printf("%S", f->name); + runtime·printf("[%p] %S", fp, f->name); if(pc > f->entry) runtime·printf("+%p", (uintptr)(pc - f->entry)); tracepc = pc; // back up to CALL instruction for funcline. @@ -145,12 +149,43 @@ gentraceback(byte *pc0, byte *sp, byte *lr0, G *g, int32 skip, uintptr *pcbuf, i continue; } + if(pcbuf == nil && f->entry == (uintptr)runtime·lessstack && g == m->g0) { + runtime·printf("----- lessstack called from goroutine %d -----\n", m->curg->goid); + g = m->curg; + stk = (Stktop*)g->stackbase; + sp = stk->gobuf.sp; + pc = (uintptr)stk->gobuf.pc; + fp = nil; + lr = 0; + continue; + } + // Unwind to next frame. pc = lr; lr = 0; sp = fp; fp = nil; + + // If this was div or divu or mod or modu, the caller had + // an extra 8 bytes on its stack. Adjust sp. + if(f->entry == (uintptr)_div || f->entry == (uintptr)_divu || f->entry == (uintptr)_mod || f->entry == (uintptr)_modu) + sp += 8; + + // If this was deferproc or newproc, the caller had an extra 12. + if(f->entry == (uintptr)runtime·deferproc || f->entry == (uintptr)runtime·newproc) + sp += 12; } + + if(pcbuf == nil && (pc = g->gopc) != 0 && (f = runtime·findfunc(pc)) != nil) { + runtime·printf("----- goroutine created by -----\n%S", f->name); + if(pc > f->entry) + runtime·printf("+%p", (uintptr)(pc - f->entry)); + tracepc = pc; // back up to CALL instruction for funcline. + if(n > 0 && pc > f->entry) + tracepc -= sizeof(uintptr); + runtime·printf(" %S:%d\n", f->src, runtime·funcline(f, tracepc)); + } + return n; } diff --git a/src/pkg/runtime/cgocall.c b/src/pkg/runtime/cgocall.c index 74e5a3085..741e8f0b8 100644 --- a/src/pkg/runtime/cgocall.c +++ b/src/pkg/runtime/cgocall.c @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "stack.h" #include "cgocall.h" void *initcgo; /* filled in by dynamic linker when Cgo is available */ @@ -70,7 +71,7 @@ runtime·cgocallback(void (*fn)(void), void *arg, int32 argsize) runtime·startcgocallback(g1); sp = g1->sched.sp - argsize; - if(sp < g1->stackguard - StackGuard + 8) // +8 for return address + if(sp < g1->stackguard - StackGuard - StackSystem + 8) // +8 for return address runtime·throw("g stack overflow in cgocallback"); runtime·mcpy(sp, arg, argsize); diff --git a/src/pkg/runtime/chan.c b/src/pkg/runtime/chan.c index 8d3ac2ca4..3177c2295 100644 --- a/src/pkg/runtime/chan.c +++ b/src/pkg/runtime/chan.c @@ -495,17 +495,27 @@ runtime·selectnbrecv(byte *v, Hchan *c, bool ok) runtime·chanrecv(c, v, &ok, nil); } +static void newselect(int32, Select**); + // newselect(size uint32) (sel *byte); #pragma textflag 7 void runtime·newselect(int32 size, ...) { - int32 n, o; + int32 o; Select **selp; - Select *sel; o = runtime·rnd(sizeof(size), Structrnd); selp = (Select**)((byte*)&size + o); + newselect(size, selp); +} + +static void +newselect(int32 size, Select **selp) +{ + int32 n; + Select *sel; + n = 0; if(size > 1) n = size-1; @@ -589,11 +599,19 @@ runtime·selectrecv(Select *sel, Hchan *c, ...) } -// selectdefaul(sel *byte) (selected bool); +static void selectdefault(Select*, void*); + +// selectdefault(sel *byte) (selected bool); #pragma textflag 7 void runtime·selectdefault(Select *sel, ...) { + selectdefault(sel, runtime·getcallerpc(&sel)); +} + +static void +selectdefault(Select *sel, void *callerpc) +{ int32 i; Scase *cas; @@ -603,7 +621,7 @@ runtime·selectdefault(Select *sel, ...) sel->ncase = i+1; cas = runtime·mal(sizeof *cas); sel->scase[i] = cas; - cas->pc = runtime·getcallerpc(&sel); + cas->pc = callerpc; cas->chan = nil; cas->so = runtime·rnd(sizeof(sel), Structrnd); @@ -662,23 +680,32 @@ runtime·block(void) runtime·gosched(); } +static void* selectgo(Select**); + // selectgo(sel *byte); // // overwrites return pc on stack to signal which case of the select // to run, so cannot appear at the top of a split stack. -// frame has 6 pointers and 4 int32 so 64 bytes max. -// that's less than StackGuard-StackSmall, so okay. #pragma textflag 7 void runtime·selectgo(Select *sel) { + runtime·setcallerpc(&sel, selectgo(&sel)); +} + +static void* +selectgo(Select **selp) +{ + Select *sel; uint32 o, i, j; Scase *cas, *dfl; Hchan *c; SudoG *sg; G *gp; byte *as; + void *pc; + sel = *selp; if(runtime·gcwaiting) runtime·gosched(); @@ -889,16 +916,18 @@ retc: selunlock(sel); // return to pc corresponding to chosen case - runtime·setcallerpc(&sel, cas->pc); - as = (byte*)&sel + cas->so; + + pc = cas->pc; + as = (byte*)selp + cas->so; freesel(sel); *as = true; - return; + return pc; sclose: // send on closed channel selunlock(sel); runtime·panicstring("send on closed channel"); + return nil; // not reached } // closechan(sel *byte); diff --git a/src/pkg/runtime/darwin/386/signal.c b/src/pkg/runtime/darwin/386/signal.c index 33f47d44f..aeef5de3f 100644 --- a/src/pkg/runtime/darwin/386/signal.c +++ b/src/pkg/runtime/darwin/386/signal.c @@ -34,20 +34,19 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo *info, void *context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Mcontext *mc; Regs *r; uintptr *sp; - G *gp; byte *pc; uc = context; mc = uc->uc_mcontext; r = &mc->ss; - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Work around Leopard bug that doesn't set FPE_INTDIV. // Look at instruction to see if it is a divide. // Not necessary in Snow Leopard (si_code will be != 0). @@ -103,12 +102,11 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->eip, (void*)r->esp, 0, m->curg); - runtime·tracebackothers(m->curg); + runtime·traceback((void*)r->eip, (void*)r->esp, 0, gp); + runtime·tracebackothers(gp); runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); } diff --git a/src/pkg/runtime/darwin/386/sys.s b/src/pkg/runtime/darwin/386/sys.s index 7961e369c..9d2caca0a 100644 --- a/src/pkg/runtime/darwin/386/sys.s +++ b/src/pkg/runtime/darwin/386/sys.s @@ -80,33 +80,34 @@ TEXT runtime·sigtramp(SB),7,$40 get_tls(CX) // save g - MOVL g(CX), BP - MOVL BP, 20(SP) + MOVL g(CX), DI + MOVL DI, 20(SP) // g = m->gsignal MOVL m(CX), BP MOVL m_gsignal(BP), BP MOVL BP, g(CX) - MOVL handler+0(FP), DI - // 4(FP) is sigstyle - MOVL signo+8(FP), AX - MOVL siginfo+12(FP), BX - MOVL context+16(FP), CX - - MOVL AX, 0(SP) + // copy arguments to sighandler + MOVL sig+8(FP), BX + MOVL BX, 0(SP) + MOVL info+12(FP), BX MOVL BX, 4(SP) - MOVL CX, 8(SP) - CALL DI + MOVL context+16(FP), BX + MOVL BX, 8(SP) + MOVL DI, 12(SP) + + MOVL handler+0(FP), BX + CALL BX // restore g get_tls(CX) - MOVL 20(SP), BP - MOVL BP, g(CX) + MOVL 20(SP), DI + MOVL DI, g(CX) + // call sigreturn MOVL context+16(FP), CX MOVL style+4(FP), BX - MOVL $0, 0(SP) // "caller PC" - ignored MOVL CX, 4(SP) MOVL BX, 8(SP) diff --git a/src/pkg/runtime/darwin/amd64/signal.c b/src/pkg/runtime/darwin/amd64/signal.c index 948b6c9c2..402ab33ca 100644 --- a/src/pkg/runtime/darwin/amd64/signal.c +++ b/src/pkg/runtime/darwin/amd64/signal.c @@ -42,12 +42,11 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo *info, void *context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Mcontext *mc; Regs *r; - G *gp; uintptr *sp; byte *pc; @@ -55,7 +54,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) mc = uc->uc_mcontext; r = &mc->ss; - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Work around Leopard bug that doesn't set FPE_INTDIV. // Look at instruction to see if it is a divide. // Not necessary in Snow Leopard (si_code will be != 0). @@ -113,12 +112,11 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->rip, (void*)r->rsp, 0, g); - runtime·tracebackothers(g); + runtime·traceback((void*)r->rip, (void*)r->rsp, 0, gp); + runtime·tracebackothers(gp); runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); } diff --git a/src/pkg/runtime/darwin/amd64/sys.s b/src/pkg/runtime/darwin/amd64/sys.s index bc970156a..4f9e0d77a 100644 --- a/src/pkg/runtime/darwin/amd64/sys.s +++ b/src/pkg/runtime/darwin/amd64/sys.s @@ -66,8 +66,8 @@ TEXT runtime·sigtramp(SB),7,$64 get_tls(BX) // save g - MOVQ g(BX), BP - MOVQ BP, 40(SP) + MOVQ g(BX), R10 + MOVQ R10, 48(SP) // g = m->gsignal MOVQ m(BX), BP @@ -77,18 +77,21 @@ TEXT runtime·sigtramp(SB),7,$64 MOVL DX, 0(SP) MOVQ CX, 8(SP) MOVQ R8, 16(SP) - MOVQ R8, 24(SP) // save ucontext - MOVQ SI, 32(SP) // save infostyle + MOVQ R10, 24(SP) + + MOVQ R8, 32(SP) // save ucontext + MOVQ SI, 40(SP) // save infostyle CALL DI // restore g get_tls(BX) - MOVQ 40(SP), BP - MOVQ BP, g(BX) + MOVQ 48(SP), R10 + MOVQ R10, g(BX) + // call sigreturn MOVL $(0x2000000+184), AX // sigreturn(ucontext, infostyle) - MOVQ 24(SP), DI // saved ucontext - MOVQ 32(SP), SI // saved infostyle + MOVQ 32(SP), DI // saved ucontext + MOVQ 40(SP), SI // saved infostyle SYSCALL INT $3 // not reached diff --git a/src/pkg/runtime/darwin/thread.c b/src/pkg/runtime/darwin/thread.c index 57e813109..235d69abf 100644 --- a/src/pkg/runtime/darwin/thread.c +++ b/src/pkg/runtime/darwin/thread.c @@ -5,6 +5,7 @@ #include "runtime.h" #include "defs.h" #include "os.h" +#include "stack.h" extern SigTab runtime·sigtab[]; @@ -176,7 +177,7 @@ runtime·minit(void) { // Initialize signal handling. m->gsignal = runtime·malg(32*1024); // OS X wants >=8K, Linux >=2K - runtime·signalstack(m->gsignal->stackguard, 32*1024); + runtime·signalstack(m->gsignal->stackguard - StackGuard, 32*1024); } // Mach IPC, to get at semaphores diff --git a/src/pkg/runtime/freebsd/386/signal.c b/src/pkg/runtime/freebsd/386/signal.c index ddb11fc3b..8e9d74256 100644 --- a/src/pkg/runtime/freebsd/386/signal.c +++ b/src/pkg/runtime/freebsd/386/signal.c @@ -45,17 +45,16 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo* info, void* context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Mcontext *r; - G *gp; uintptr *sp; uc = context; r = &uc->uc_mcontext; - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Make it look like a call to the signal func. // Have to pass arguments out of band since // augmenting the stack frame would break @@ -99,12 +98,11 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->mc_eip, (void*)r->mc_esp, 0, m->curg); - runtime·tracebackothers(m->curg); + runtime·traceback((void*)r->mc_eip, (void*)r->mc_esp, 0, gp); + runtime·tracebackothers(gp); runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); } diff --git a/src/pkg/runtime/freebsd/386/sys.s b/src/pkg/runtime/freebsd/386/sys.s index 7110e6924..60c189bf8 100644 --- a/src/pkg/runtime/freebsd/386/sys.s +++ b/src/pkg/runtime/freebsd/386/sys.s @@ -111,30 +111,36 @@ TEXT runtime·sigaction(SB),7,$-4 CALL runtime·notok(SB) RET -TEXT runtime·sigtramp(SB),7,$40 - // g = m->gsignal - get_tls(DX) - MOVL m(DX), BP - MOVL m_gsignal(BP), BP - MOVL BP, g(DX) +TEXT runtime·sigtramp(SB),7,$44 + get_tls(CX) - MOVL signo+0(FP), AX - MOVL siginfo+4(FP), BX - MOVL context+8(FP), CX + // save g + MOVL g(CX), DI + MOVL DI, 20(SP) + + // g = m->gsignal + MOVL m(CX), BX + MOVL m_gsignal(BX), BX + MOVL BX, g(CX) - MOVL AX, 0(SP) + // copy arguments for call to sighandler + MOVL signo+0(FP), BX + MOVL BX, 0(SP) + MOVL info+4(FP), BX MOVL BX, 4(SP) - MOVL CX, 8(SP) - CALL runtime·sighandler(SB) + MOVL context+8(FP), BX + MOVL BX, 8(SP) + MOVL DI, 12(SP) - // g = m->curg - get_tls(DX) - MOVL m(DX), BP - MOVL m_curg(BP), BP - MOVL BP, g(DX) + CALL runtime·sighandler(SB) + // restore g + get_tls(CX) + MOVL 20(SP), BX + MOVL BX, g(CX) + + // call sigreturn MOVL context+8(FP), AX - MOVL $0, 0(SP) // syscall gap MOVL AX, 4(SP) MOVL $417, AX // sigreturn(ucontext) diff --git a/src/pkg/runtime/freebsd/amd64/signal.c b/src/pkg/runtime/freebsd/amd64/signal.c index 9f873d276..f145371b4 100644 --- a/src/pkg/runtime/freebsd/amd64/signal.c +++ b/src/pkg/runtime/freebsd/amd64/signal.c @@ -53,17 +53,16 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo* info, void* context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Mcontext *r; - G *gp; uintptr *sp; uc = context; r = &uc->uc_mcontext; - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Make it look like a call to the signal func. // Have to pass arguments out of band since // augmenting the stack frame would break @@ -107,12 +106,11 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->mc_rip, (void*)r->mc_rsp, 0, g); - runtime·tracebackothers(g); + runtime·traceback((void*)r->mc_rip, (void*)r->mc_rsp, 0, gp); + runtime·tracebackothers(gp); runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); } diff --git a/src/pkg/runtime/freebsd/amd64/sys.s b/src/pkg/runtime/freebsd/amd64/sys.s index b9cf3832d..d986e9ac0 100644 --- a/src/pkg/runtime/freebsd/amd64/sys.s +++ b/src/pkg/runtime/freebsd/amd64/sys.s @@ -90,15 +90,29 @@ TEXT runtime·sigaction(SB),7,$-8 CALL runtime·notok(SB) RET -TEXT runtime·sigtramp(SB),7,$24-16 - get_tls(CX) - MOVQ m(CX), AX - MOVQ m_gsignal(AX), AX - MOVQ AX, g(CX) +TEXT runtime·sigtramp(SB),7,$64 + get_tls(BX) + + // save g + MOVQ g(BX), R10 + MOVQ R10, 40(SP) + + // g = m->signal + MOVQ m(BX), BP + MOVQ m_gsignal(BP), BP + MOVQ BP, g(BX) + MOVQ DI, 0(SP) MOVQ SI, 8(SP) MOVQ DX, 16(SP) + MOVQ R10, 24(SP) + CALL runtime·sighandler(SB) + + // restore g + get_tls(BX) + MOVQ 40(SP), R10 + MOVQ R10, g(BX) RET TEXT runtime·mmap(SB),7,$0 diff --git a/src/pkg/runtime/freebsd/thread.c b/src/pkg/runtime/freebsd/thread.c index 9bd883833..569098aa2 100644 --- a/src/pkg/runtime/freebsd/thread.c +++ b/src/pkg/runtime/freebsd/thread.c @@ -4,6 +4,7 @@ #include "runtime.h" #include "defs.h" #include "os.h" +#include "stack.h" extern SigTab runtime·sigtab[]; extern int32 runtime·sys_umtx_op(uint32*, int32, uint32, void*, void*); @@ -175,7 +176,7 @@ runtime·minit(void) { // Initialize signal handling m->gsignal = runtime·malg(32*1024); - runtime·signalstack(m->gsignal->stackguard, 32*1024); + runtime·signalstack(m->gsignal->stackguard - StackGuard, 32*1024); } void diff --git a/src/pkg/runtime/hashmap.c b/src/pkg/runtime/hashmap.c index f0d5ce90a..e50cefd9a 100644 --- a/src/pkg/runtime/hashmap.c +++ b/src/pkg/runtime/hashmap.c @@ -781,6 +781,9 @@ runtime·mapaccess(Hmap *h, byte *ak, byte *av, bool *pres) { byte *res; + if(h == nil) + runtime·panicstring("lookup in nil map"); + if(runtime·gcwaiting) runtime·gosched(); @@ -802,6 +805,9 @@ runtime·mapaccess1(Hmap *h, ...) byte *ak, *av; bool pres; + if(h == nil) + runtime·panicstring("lookup in nil map"); + ak = (byte*)&h + h->ko1; av = (byte*)&h + h->vo1; @@ -827,6 +833,9 @@ runtime·mapaccess2(Hmap *h, ...) { byte *ak, *av, *ap; + if(h == nil) + runtime·panicstring("lookup in nil map"); + ak = (byte*)&h + h->ko1; av = (byte*)&h + h->vo1; ap = (byte*)&h + h->po1; @@ -852,6 +861,9 @@ runtime·mapassign(Hmap *h, byte *ak, byte *av) byte *res; int32 hit; + if(h == nil) + runtime·panicstring("assignment to entry in nil map"); + if(runtime·gcwaiting) runtime·gosched(); @@ -889,6 +901,9 @@ runtime·mapassign1(Hmap *h, ...) { byte *ak, *av; + if(h == nil) + runtime·panicstring("assignment to entry in nil map"); + ak = (byte*)&h + h->ko2; av = (byte*)&h + h->vo2; @@ -902,6 +917,9 @@ runtime·mapassign2(Hmap *h, ...) { byte *ak, *av, *ap; + if(h == nil) + runtime·panicstring("assignment to entry in nil map"); + ak = (byte*)&h + h->ko2; av = (byte*)&h + h->vo2; ap = (byte*)&h + h->po2; diff --git a/src/pkg/runtime/iface.c b/src/pkg/runtime/iface.c index 3dec45e2b..698aead3d 100644 --- a/src/pkg/runtime/iface.c +++ b/src/pkg/runtime/iface.c @@ -209,16 +209,25 @@ runtime·convT2E(Type *t, ...) copyin(t, elem, &ret->data); } +static void assertI2Tret(Type *t, Iface i, byte *ret); + // func ifaceI2T(typ *byte, iface any) (ret any) #pragma textflag 7 void runtime·assertI2T(Type *t, Iface i, ...) { - Itab *tab; byte *ret; - Eface err; ret = (byte*)(&i+1); + assertI2Tret(t, i, ret); +} + +static void +assertI2Tret(Type *t, Iface i, byte *ret) +{ + Itab *tab; + Eface err; + tab = i.tab; if(tab == nil) { runtime·newTypeAssertionError(nil, nil, t, @@ -258,15 +267,23 @@ runtime·assertI2T2(Type *t, Iface i, ...) copyout(t, &i.data, ret); } +static void assertE2Tret(Type *t, Eface e, byte *ret); + // func ifaceE2T(typ *byte, iface any) (ret any) #pragma textflag 7 void runtime·assertE2T(Type *t, Eface e, ...) { byte *ret; - Eface err; ret = (byte*)(&e+1); + assertE2Tret(t, e, ret); +} + +static void +assertE2Tret(Type *t, Eface e, byte *ret) +{ + Eface err; if(e.type == nil) { runtime·newTypeAssertionError(nil, nil, t, @@ -307,7 +324,6 @@ runtime·assertE2T2(Type *t, Eface e, ...) } // func convI2E(elem any) (ret any) -#pragma textflag 7 void runtime·convI2E(Iface i, Eface ret) { @@ -322,7 +338,6 @@ runtime·convI2E(Iface i, Eface ret) } // func ifaceI2E(typ *byte, iface any) (ret any) -#pragma textflag 7 void runtime·assertI2E(InterfaceType* inter, Iface i, Eface ret) { @@ -343,7 +358,6 @@ runtime·assertI2E(InterfaceType* inter, Iface i, Eface ret) } // func ifaceI2E2(typ *byte, iface any) (ret any, ok bool) -#pragma textflag 7 void runtime·assertI2E2(InterfaceType* inter, Iface i, Eface ret, bool ok) { @@ -364,7 +378,6 @@ runtime·assertI2E2(InterfaceType* inter, Iface i, Eface ret, bool ok) } // func convI2I(typ *byte, elem any) (ret any) -#pragma textflag 7 void runtime·convI2I(InterfaceType* inter, Iface i, Iface ret) { @@ -399,7 +412,6 @@ runtime·ifaceI2I(InterfaceType *inter, Iface i, Iface *ret) } // func ifaceI2I(sigi *byte, iface any) (ret any) -#pragma textflag 7 void runtime·assertI2I(InterfaceType* inter, Iface i, Iface ret) { @@ -407,7 +419,6 @@ runtime·assertI2I(InterfaceType* inter, Iface i, Iface ret) } // func ifaceI2I2(sigi *byte, iface any) (ret any, ok bool) -#pragma textflag 7 void runtime·assertI2I2(InterfaceType *inter, Iface i, Iface ret, bool ok) { @@ -446,7 +457,6 @@ runtime·ifaceE2I(InterfaceType *inter, Eface e, Iface *ret) } // func ifaceE2I(sigi *byte, iface any) (ret any) -#pragma textflag 7 void runtime·assertE2I(InterfaceType* inter, Eface e, Iface ret) { @@ -454,7 +464,6 @@ runtime·assertE2I(InterfaceType* inter, Eface e, Iface ret) } // ifaceE2I2(sigi *byte, iface any) (ret any, ok bool) -#pragma textflag 7 void runtime·assertE2I2(InterfaceType *inter, Eface e, Iface ret, bool ok) { @@ -474,7 +483,6 @@ runtime·assertE2I2(InterfaceType *inter, Eface e, Iface ret, bool ok) } // func ifaceE2E(typ *byte, iface any) (ret any) -#pragma textflag 7 void runtime·assertE2E(InterfaceType* inter, Eface e, Eface ret) { @@ -494,7 +502,6 @@ runtime·assertE2E(InterfaceType* inter, Eface e, Eface ret) } // func ifaceE2E2(iface any) (ret any, ok bool) -#pragma textflag 7 void runtime·assertE2E2(InterfaceType* inter, Eface e, Eface ret, bool ok) { diff --git a/src/pkg/runtime/linux/386/signal.c b/src/pkg/runtime/linux/386/signal.c index 9651a6f28..bd918c7ea 100644 --- a/src/pkg/runtime/linux/386/signal.c +++ b/src/pkg/runtime/linux/386/signal.c @@ -42,17 +42,16 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo* info, void* context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Sigcontext *r; uintptr *sp; - G *gp; uc = context; r = &uc->uc_mcontext; - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Make it look like a call to the signal func. // Have to pass arguments out of band since // augmenting the stack frame would break @@ -96,12 +95,11 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->eip, (void*)r->esp, 0, m->curg); - runtime·tracebackothers(m->curg); + runtime·traceback((void*)r->eip, (void*)r->esp, 0, gp); + runtime·tracebackothers(gp); runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); } diff --git a/src/pkg/runtime/linux/386/sys.s b/src/pkg/runtime/linux/386/sys.s index a1505b0b0..a684371be 100644 --- a/src/pkg/runtime/linux/386/sys.s +++ b/src/pkg/runtime/linux/386/sys.s @@ -56,12 +56,12 @@ TEXT runtime·rt_sigaction(SB),7,$0 INT $0x80 RET -TEXT runtime·sigtramp(SB),7,$40 +TEXT runtime·sigtramp(SB),7,$44 get_tls(CX) // save g - MOVL g(CX), BX - MOVL BX, 20(SP) + MOVL g(CX), DI + MOVL DI, 20(SP) // g = m->gsignal MOVL m(CX), BX @@ -75,6 +75,7 @@ TEXT runtime·sigtramp(SB),7,$40 MOVL BX, 4(SP) MOVL context+8(FP), BX MOVL BX, 8(SP) + MOVL DI, 12(SP) CALL runtime·sighandler(SB) diff --git a/src/pkg/runtime/linux/amd64/signal.c b/src/pkg/runtime/linux/amd64/signal.c index 9e501c96d..ea0932523 100644 --- a/src/pkg/runtime/linux/amd64/signal.c +++ b/src/pkg/runtime/linux/amd64/signal.c @@ -50,19 +50,18 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo* info, void* context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Mcontext *mc; Sigcontext *r; uintptr *sp; - G *gp; uc = context; mc = &uc->uc_mcontext; r = (Sigcontext*)mc; // same layout, more conveient names - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Make it look like a call to the signal func. // Have to pass arguments out of band since // augmenting the stack frame would break @@ -106,12 +105,11 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->rip, (void*)r->rsp, 0, g); - runtime·tracebackothers(g); + runtime·traceback((void*)r->rip, (void*)r->rsp, 0, gp); + runtime·tracebackothers(gp); runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); } diff --git a/src/pkg/runtime/linux/amd64/sys.s b/src/pkg/runtime/linux/amd64/sys.s index 170b659fc..1bf734dc0 100644 --- a/src/pkg/runtime/linux/amd64/sys.s +++ b/src/pkg/runtime/linux/amd64/sys.s @@ -64,8 +64,8 @@ TEXT runtime·sigtramp(SB),7,$64 get_tls(BX) // save g - MOVQ g(BX), BP - MOVQ BP, 40(SP) + MOVQ g(BX), R10 + MOVQ R10, 40(SP) // g = m->gsignal MOVQ m(BX), BP @@ -75,12 +75,14 @@ TEXT runtime·sigtramp(SB),7,$64 MOVQ DI, 0(SP) MOVQ SI, 8(SP) MOVQ DX, 16(SP) + MOVQ R10, 24(SP) + CALL runtime·sighandler(SB) // restore g get_tls(BX) - MOVQ 40(SP), BP - MOVQ BP, g(BX) + MOVQ 40(SP), R10 + MOVQ R10, g(BX) RET TEXT runtime·sigignore(SB),7,$0 diff --git a/src/pkg/runtime/linux/arm/signal.c b/src/pkg/runtime/linux/arm/signal.c index 481bd13c6..843c40b68 100644 --- a/src/pkg/runtime/linux/arm/signal.c +++ b/src/pkg/runtime/linux/arm/signal.c @@ -50,16 +50,15 @@ runtime·signame(int32 sig) } void -runtime·sighandler(int32 sig, Siginfo *info, void *context) +runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) { Ucontext *uc; Sigcontext *r; - G *gp; uc = context; r = &uc->uc_mcontext; - if((gp = m->curg) != nil && (runtime·sigtab[sig].flags & SigPanic)) { + if(gp != nil && (runtime·sigtab[sig].flags & SigPanic)) { // Make it look like a call to the signal func. // Have to pass arguments out of band since // augmenting the stack frame would break @@ -99,8 +98,8 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) runtime·printf("\n"); if(runtime·gotraceback()){ - runtime·traceback((void*)r->arm_pc, (void*)r->arm_sp, (void*)r->arm_lr, m->curg); - runtime·tracebackothers(m->curg); + runtime·traceback((void*)r->arm_pc, (void*)r->arm_sp, (void*)r->arm_lr, gp); + runtime·tracebackothers(gp); runtime·printf("\n"); runtime·dumpregs(r); } diff --git a/src/pkg/runtime/linux/arm/sys.s b/src/pkg/runtime/linux/arm/sys.s index b25cf81aa..9daf9c2e4 100644 --- a/src/pkg/runtime/linux/arm/sys.s +++ b/src/pkg/runtime/linux/arm/sys.s @@ -197,11 +197,24 @@ TEXT runtime·sigignore(SB),7,$0 RET TEXT runtime·sigtramp(SB),7,$24 + // save g + MOVW g, R3 + MOVW g, 20(R13) + + // g = m->gsignal MOVW m_gsignal(m), g + + // copy arguments for call to sighandler MOVW R0, 4(R13) MOVW R1, 8(R13) MOVW R2, 12(R13) + MOVW R3, 16(R13) + BL runtime·sighandler(SB) + + // restore g + MOVW 20(R13), g + RET TEXT runtime·rt_sigaction(SB),7,$0 @@ -217,3 +230,21 @@ TEXT runtime·sigreturn(SB),7,$0 MOVW $SYS_rt_sigreturn, R7 SWI $0 RET + +// Use kernel version instead of native armcas in ../../arm.s. +// See ../../../sync/atomic/asm_linux_arm.s for details. +TEXT cas<>(SB),7,$0 + MOVW $0xffff0fc0, PC + +TEXT runtime·cas(SB),7,$0 + MOVW valptr+0(FP), R2 + MOVW old+4(FP), R0 + MOVW new+8(FP), R1 + BL cas<>(SB) + MOVW $0, R0 + MOVW.CS $1, R0 + RET + +TEXT runtime·casp(SB),7,$0 + B runtime·cas(SB) + diff --git a/src/pkg/runtime/linux/thread.c b/src/pkg/runtime/linux/thread.c index d5f9a8fb0..7166b0ef2 100644 --- a/src/pkg/runtime/linux/thread.c +++ b/src/pkg/runtime/linux/thread.c @@ -5,6 +5,7 @@ #include "runtime.h" #include "defs.h" #include "os.h" +#include "stack.h" extern SigTab runtime·sigtab[]; @@ -274,7 +275,7 @@ runtime·minit(void) { // Initialize signal handling. m->gsignal = runtime·malg(32*1024); // OS X wants >=8K, Linux >=2K - runtime·signalstack(m->gsignal->stackguard, 32*1024); + runtime·signalstack(m->gsignal->stackguard - StackGuard, 32*1024); } void diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc index 70b85d68d..41060682e 100644 --- a/src/pkg/runtime/malloc.goc +++ b/src/pkg/runtime/malloc.goc @@ -8,6 +8,7 @@ package runtime #include "runtime.h" +#include "stack.h" #include "malloc.h" #include "defs.h" #include "type.h" @@ -146,6 +147,9 @@ runtime·free(void *v) // Large object. size = s->npages<<PageShift; *(uintptr*)(s->start<<PageShift) = 1; // mark as "needs to be zeroed" + // Must mark v freed before calling unmarkspan and MHeap_Free: + // they might coalesce v into other spans and change the bitmap further. + runtime·markfreed(v, size); runtime·unmarkspan(v, 1<<PageShift); runtime·MHeap_Free(&runtime·mheap, s, 1); } else { @@ -154,10 +158,13 @@ runtime·free(void *v) size = runtime·class_to_size[sizeclass]; if(size > sizeof(uintptr)) ((uintptr*)v)[1] = 1; // mark as "needs to be zeroed" + // Must mark v freed before calling MCache_Free: + // it might coalesce v and other blocks into a bigger span + // and change the bitmap further. + runtime·markfreed(v, size); mstats.by_size[sizeclass].nfree++; runtime·MCache_Free(c, v, sizeclass, size); } - runtime·markfreed(v, size); mstats.alloc -= size; if(prof) runtime·MProf_Free(v, size); @@ -379,7 +386,7 @@ static struct { } stacks; enum { - FixedStack = StackBig + StackExtra + FixedStack = StackMin, }; void* @@ -387,6 +394,12 @@ runtime·stackalloc(uint32 n) { void *v; + // Stackalloc must be called on scheduler stack, so that we + // never try to grow the stack during the code that stackalloc runs. + // Doing so would cause a deadlock (issue 1547). + if(g != m->g0) + runtime·throw("stackalloc not on scheduler stack"); + if(m->mallocing || m->gcing || n == FixedStack) { runtime·lock(&stacks); if(stacks.size == 0) diff --git a/src/pkg/runtime/mfinal.c b/src/pkg/runtime/mfinal.c index 03ee777c0..f3138145b 100644 --- a/src/pkg/runtime/mfinal.c +++ b/src/pkg/runtime/mfinal.c @@ -5,7 +5,9 @@ #include "runtime.h" #include "malloc.h" -// TODO(rsc): Why not just use mheap.Lock? +// Lock to protect finalizer data structures. +// Cannot reuse mheap.Lock because the finalizer +// maintenance requires allocation. static Lock finlock; // Finalizer hash table. Direct hash, linear scan, at most 3/4 full. @@ -90,7 +92,6 @@ runtime·addfinalizer(void *p, void (*f)(void*), int32 nret) { Fintab newtab; int32 i; - uint32 *ref; byte *base; Finalizer *e; diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index 232c6cdcd..7c175b308 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -380,6 +380,7 @@ mark(void) break; case Grunning: case Grecovery: + case Gstackalloc: if(gp != g) runtime·throw("mark - world not stopped"); scanstack(gp); @@ -584,7 +585,7 @@ runtime·gc(int32 force) if(fp != nil) { // kick off or wake up goroutine to run queued finalizers if(fing == nil) - fing = runtime·newproc1((byte*)runfinq, nil, 0, 0); + fing = runtime·newproc1((byte*)runfinq, nil, 0, 0, runtime·gc); else if(fingwait) { fingwait = 0; runtime·ready(fing); @@ -663,7 +664,7 @@ runfinq(void) void runtime·markallocated(void *v, uintptr n, bool noptr) { - uintptr *b, bits, off, shift; + uintptr *b, obits, bits, off, shift; if(0) runtime·printf("markallocated %p+%p\n", v, n); @@ -675,17 +676,27 @@ runtime·markallocated(void *v, uintptr n, bool noptr) b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; - bits = (*b & ~(bitMask<<shift)) | (bitAllocated<<shift); - if(noptr) - bits |= bitNoPointers<<shift; - *b = bits; + for(;;) { + obits = *b; + bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); + if(noptr) + bits |= bitNoPointers<<shift; + if(runtime·gomaxprocs == 1) { + *b = bits; + break; + } else { + // gomaxprocs > 1: use atomic op + if(runtime·casp((void**)b, (void*)obits, (void*)bits)) + break; + } + } } // mark the block at v of size n as freed. void runtime·markfreed(void *v, uintptr n) { - uintptr *b, off, shift; + uintptr *b, obits, bits, off, shift; if(0) runtime·printf("markallocated %p+%p\n", v, n); @@ -697,7 +708,18 @@ runtime·markfreed(void *v, uintptr n) b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; - *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); + for(;;) { + obits = *b; + bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); + if(runtime·gomaxprocs == 1) { + *b = bits; + break; + } else { + // gomaxprocs > 1: use atomic op + if(runtime·casp((void**)b, (void*)obits, (void*)bits)) + break; + } + } } // check that the block at v of size n is marked freed. @@ -739,6 +761,10 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) if(leftover) // mark a boundary just past end of last block too n++; for(; n-- > 0; p += size) { + // Okay to use non-atomic ops here, because we control + // the entire span, and each bitmap word has bits for only + // one span, so no other goroutines are changing these + // bitmap words. off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // word offset b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; @@ -763,6 +789,10 @@ runtime·unmarkspan(void *v, uintptr n) n /= PtrSize; if(n%wordsPerBitmapWord != 0) runtime·throw("unmarkspan: unaligned length"); + // Okay to use non-atomic ops here, because we control + // the entire span, and each bitmap word has bits for only + // one span, so no other goroutines are changing these + // bitmap words. n /= wordsPerBitmapWord; while(n-- > 0) *b-- = 0; @@ -783,13 +813,24 @@ runtime·blockspecial(void *v) void runtime·setblockspecial(void *v) { - uintptr *b, off, shift; + uintptr *b, off, shift, bits, obits; off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; - *b |= bitSpecial<<shift; + for(;;) { + obits = *b; + bits = obits | (bitSpecial<<shift); + if(runtime·gomaxprocs == 1) { + *b = bits; + break; + } else { + // gomaxprocs > 1: use atomic op + if(runtime·casp((void**)b, (void*)obits, (void*)bits)) + break; + } + } } void diff --git a/src/pkg/runtime/print.c b/src/pkg/runtime/print.c index 3b4bb103d..b8069aa39 100644 --- a/src/pkg/runtime/print.c +++ b/src/pkg/runtime/print.c @@ -42,113 +42,108 @@ runtime·printf(int8 *s, ...) vprintf(s, arg); } -static byte* -vrnd(byte *p, int32 x) -{ - if((uint32)(uintptr)p&(x-1)) - p += x - ((uint32)(uintptr)p&(x-1)); - return p; -} - // Very simple printf. Only for debugging prints. // Do not add to this without checking with Rob. static void -vprintf(int8 *s, byte *arg) +vprintf(int8 *s, byte *base) { int8 *p, *lp; - byte *narg; + uintptr arg, narg; + byte *v; // lock(&debuglock); lp = p = s; + arg = 0; for(; *p; p++) { if(*p != '%') continue; if(p > lp) runtime·write(2, lp, p-lp); p++; - narg = nil; + narg = 0; switch(*p) { case 't': narg = arg + 1; break; case 'd': // 32-bit case 'x': - arg = vrnd(arg, 4); + arg = runtime·rnd(arg, 4); narg = arg + 4; break; case 'D': // 64-bit case 'U': case 'X': case 'f': - arg = vrnd(arg, sizeof(uintptr)); + arg = runtime·rnd(arg, sizeof(uintptr)); narg = arg + 8; break; case 'C': - arg = vrnd(arg, sizeof(uintptr)); + arg = runtime·rnd(arg, sizeof(uintptr)); narg = arg + 16; break; case 'p': // pointer-sized case 's': - arg = vrnd(arg, sizeof(uintptr)); + arg = runtime·rnd(arg, sizeof(uintptr)); narg = arg + sizeof(uintptr); break; case 'S': // pointer-aligned but bigger - arg = vrnd(arg, sizeof(uintptr)); + arg = runtime·rnd(arg, sizeof(uintptr)); narg = arg + sizeof(String); break; case 'a': // pointer-aligned but bigger - arg = vrnd(arg, sizeof(uintptr)); + arg = runtime·rnd(arg, sizeof(uintptr)); narg = arg + sizeof(Slice); break; case 'i': // pointer-aligned but bigger case 'e': - arg = vrnd(arg, sizeof(uintptr)); + arg = runtime·rnd(arg, sizeof(uintptr)); narg = arg + sizeof(Eface); break; } + v = base+arg; switch(*p) { case 'a': - runtime·printslice(*(Slice*)arg); + runtime·printslice(*(Slice*)v); break; case 'd': - runtime·printint(*(int32*)arg); + runtime·printint(*(int32*)v); break; case 'D': - runtime·printint(*(int64*)arg); + runtime·printint(*(int64*)v); break; case 'e': - runtime·printeface(*(Eface*)arg); + runtime·printeface(*(Eface*)v); break; case 'f': - runtime·printfloat(*(float64*)arg); + runtime·printfloat(*(float64*)v); break; case 'C': - runtime·printcomplex(*(Complex128*)arg); + runtime·printcomplex(*(Complex128*)v); break; case 'i': - runtime·printiface(*(Iface*)arg); + runtime·printiface(*(Iface*)v); break; case 'p': - runtime·printpointer(*(void**)arg); + runtime·printpointer(*(void**)v); break; case 's': - runtime·prints(*(int8**)arg); + runtime·prints(*(int8**)v); break; case 'S': - runtime·printstring(*(String*)arg); + runtime·printstring(*(String*)v); break; case 't': - runtime·printbool(*(bool*)arg); + runtime·printbool(*(bool*)v); break; case 'U': - runtime·printuint(*(uint64*)arg); + runtime·printuint(*(uint64*)v); break; case 'x': - runtime·printhex(*(uint32*)arg); + runtime·printhex(*(uint32*)v); break; case 'X': - runtime·printhex(*(uint64*)arg); + runtime·printhex(*(uint64*)v); break; } arg = narg; diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index 26c1f13a4..db6072b5c 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -7,6 +7,7 @@ #include "defs.h" #include "malloc.h" #include "os.h" +#include "stack.h" bool runtime·iscgo; @@ -63,7 +64,6 @@ struct Sched { int32 mcount; // number of ms that have been created int32 mcpu; // number of ms executing on cpu int32 mcpumax; // max number of ms allowed on cpu - int32 gomaxprocs; int32 msyscall; // number of ms in system calls int32 predawn; // running initialization, don't run new gs. @@ -73,6 +73,7 @@ struct Sched { }; Sched runtime·sched; +int32 gomaxprocs; // Scheduling helpers. Sched must be locked. static void gput(G*); // put/get on ghead/gtail @@ -116,13 +117,13 @@ runtime·schedinit(void) // For debugging: // Allocate internal symbol table representation now, // so that we don't need to call malloc when we crash. - // findfunc(0); + // runtime·findfunc(0); - runtime·sched.gomaxprocs = 1; + runtime·gomaxprocs = 1; p = runtime·getenv("GOMAXPROCS"); if(p != nil && (n = runtime·atoi(p)) != 0) - runtime·sched.gomaxprocs = n; - runtime·sched.mcpumax = runtime·sched.gomaxprocs; + runtime·gomaxprocs = n; + runtime·sched.mcpumax = runtime·gomaxprocs; runtime·sched.mcount = 1; runtime·sched.predawn = 1; @@ -165,6 +166,18 @@ runtime·tracebackothers(G *me) } } +// Mark this g as m's idle goroutine. +// This functionality might be used in environments where programs +// are limited to a single thread, to simulate a select-driven +// network server. It is not exposed via the standard runtime API. +void +runtime·idlegoroutine(void) +{ + if(g->idlem != nil) + runtime·throw("g is already an idle goroutine"); + g->idlem = m; +} + // Put on `g' queue. Sched must be locked. static void gput(G *g) @@ -176,6 +189,18 @@ gput(G *g) mnextg(m, g); return; } + + // If g is the idle goroutine for an m, hand it off. + if(g->idlem != nil) { + if(g->idlem->idleg != nil) { + runtime·printf("m%d idle out of sync: g%d g%d\n", + g->idlem->id, + g->idlem->idleg->goid, g->goid); + runtime·throw("runtime: double idle"); + } + g->idlem->idleg = g; + return; + } g->schedlink = nil; if(runtime·sched.ghead == nil) @@ -198,6 +223,9 @@ gget(void) if(runtime·sched.ghead == nil) runtime·sched.gtail = nil; runtime·sched.gwait--; + } else if(m->idleg != nil) { + g = m->idleg; + m->idleg = nil; } return g; } @@ -252,8 +280,10 @@ readylocked(G *g) } // Mark runnable. - if(g->status == Grunnable || g->status == Grunning || g->status == Grecovery) + if(g->status == Grunnable || g->status == Grunning || g->status == Grecovery || g->status == Gstackalloc) { + runtime·printf("goroutine %d has status %d\n", g->goid, g->status); runtime·throw("bad g->status in ready"); + } g->status = Grunnable; gput(g); @@ -376,7 +406,7 @@ runtime·starttheworld(void) { runtime·lock(&runtime·sched); runtime·gcwaiting = 0; - runtime·sched.mcpumax = runtime·sched.gomaxprocs; + runtime·sched.mcpumax = runtime·gomaxprocs; matchmg(); runtime·unlock(&runtime·sched); } @@ -491,6 +521,13 @@ scheduler(void) runtime·free(d); runtime·gogo(&gp->sched, 1); } + + if(gp->status == Gstackalloc) { + // switched to scheduler stack to call stackalloc. + gp->param = runtime·stackalloc((uintptr)gp->param); + gp->status = Grunning; + runtime·gogo(&gp->sched, 1); + } // Jumped here via runtime·gosave/gogo, so didn't // execute lock(&runtime·sched) above. @@ -508,6 +545,8 @@ scheduler(void) switch(gp->status){ case Grunnable: case Gdead: + case Grecovery: + case Gstackalloc: // Shouldn't have been running! runtime·throw("bad gp->status in sched"); case Grunning: @@ -520,6 +559,7 @@ scheduler(void) gp->lockedm = nil; m->lockedg = nil; } + gp->idlem = nil; unwindstack(gp, nil); gfput(gp); if(--runtime·sched.gcount == 0) @@ -701,7 +741,7 @@ runtime·oldstack(void) goid = old.gobuf.g->goid; // fault if g is bad, before gogo if(old.free != 0) - runtime·stackfree(g1->stackguard - StackGuard, old.free); + runtime·stackfree(g1->stackguard - StackGuard - StackSystem, old.free); g1->stackbase = old.stackbase; g1->stackguard = old.stackguard; @@ -739,14 +779,15 @@ runtime·newstack(void) // the new Stktop* is necessary to unwind, but // we don't need to create a new segment. top = (Stktop*)(m->morebuf.sp - sizeof(*top)); - stk = g1->stackguard - StackGuard; + stk = g1->stackguard - StackGuard - StackSystem; free = 0; } else { // allocate new segment. framesize += argsize; - if(framesize < StackBig) - framesize = StackBig; framesize += StackExtra; // room for more functions, Stktop. + if(framesize < StackMin) + framesize = StackMin; + framesize += StackSystem; stk = runtime·stackalloc(framesize); top = (Stktop*)(stk+framesize-sizeof(*top)); free = framesize; @@ -767,7 +808,7 @@ runtime·newstack(void) g1->ispanic = false; g1->stackbase = (byte*)top; - g1->stackguard = stk + StackGuard; + g1->stackguard = stk + StackGuard + StackSystem; sp = (byte*)top; if(argsize > 0) { @@ -793,18 +834,35 @@ runtime·newstack(void) G* runtime·malg(int32 stacksize) { - G *g; + G *newg; byte *stk; + int32 oldstatus; - g = runtime·malloc(sizeof(G)); + newg = runtime·malloc(sizeof(G)); if(stacksize >= 0) { - stk = runtime·stackalloc(stacksize + StackGuard); - g->stack0 = stk; - g->stackguard = stk + StackGuard; - g->stackbase = stk + StackGuard + stacksize - sizeof(Stktop); - runtime·memclr(g->stackbase, sizeof(Stktop)); + if(g == m->g0) { + // running on scheduler stack already. + stk = runtime·stackalloc(StackSystem + stacksize); + } else { + // have to call stackalloc on scheduler stack. + oldstatus = g->status; + g->param = (void*)(StackSystem + stacksize); + g->status = Gstackalloc; + // next two lines are runtime·gosched without the check + // of m->locks. we're almost certainly holding a lock, + // but this is not a real rescheduling so it's okay. + if(runtime·gosave(&g->sched) == 0) + runtime·gogo(&m->sched, 1); + stk = g->param; + g->param = nil; + g->status = oldstatus; + } + newg->stack0 = stk; + newg->stackguard = stk + StackSystem + StackGuard; + newg->stackbase = stk + StackSystem + stacksize - sizeof(Stktop); + runtime·memclr(newg->stackbase, sizeof(Stktop)); } - return g; + return newg; } /* @@ -826,11 +884,11 @@ runtime·newproc(int32 siz, byte* fn, ...) argp = (byte*)(&fn+2); // skip caller's saved LR else argp = (byte*)(&fn+1); - runtime·newproc1(fn, argp, siz, 0); + runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz)); } G* -runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret) +runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc) { byte *sp; G *newg; @@ -846,10 +904,10 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret) if((newg = gfget()) != nil){ newg->status = Gwaiting; - if(newg->stackguard - StackGuard != newg->stack0) + if(newg->stackguard - StackGuard - StackSystem != newg->stack0) runtime·throw("invalid stack in newg"); } else { - newg = runtime·malg(StackBig); + newg = runtime·malg(StackMin); newg->status = Gwaiting; newg->alllink = runtime·allg; runtime·allg = newg; @@ -868,6 +926,7 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret) newg->sched.pc = (byte*)runtime·goexit; newg->sched.g = newg; newg->entry = fn; + newg->gopc = (uintptr)callerpc; runtime·sched.gcount++; runtime·goidgen++; @@ -1019,6 +1078,7 @@ runtime·panic(Eface e) } // ran out of deferred calls - old-school panic now + runtime·startpanic(); printpanics(g->panic); runtime·dopanic(0); } @@ -1098,7 +1158,7 @@ nomatch: static void gfput(G *g) { - if(g->stackguard - StackGuard != g->stack0) + if(g->stackguard - StackGuard - StackSystem != g->stack0) runtime·throw("invalid stack in gfput"); g->schedlink = runtime·sched.gfree; runtime·sched.gfree = g; @@ -1151,10 +1211,10 @@ runtime·gomaxprocsfunc(int32 n) int32 ret; runtime·lock(&runtime·sched); - ret = runtime·sched.gomaxprocs; + ret = runtime·gomaxprocs; if (n <= 0) n = ret; - runtime·sched.gomaxprocs = n; + runtime·gomaxprocs = n; runtime·sched.mcpumax = n; // handle fewer procs? if(runtime·sched.mcpu > runtime·sched.mcpumax) { diff --git a/src/pkg/runtime/runtime-gdb.py b/src/pkg/runtime/runtime-gdb.py index 677e9bde4..68dd8abdc 100644 --- a/src/pkg/runtime/runtime-gdb.py +++ b/src/pkg/runtime/runtime-gdb.py @@ -13,7 +13,7 @@ path to this file based on the path to the runtime package. # - pretty printing only works for the 'native' strings. E.g. 'type # foo string' will make foo a plain struct in the eyes of gdb, # circumventing the pretty print triggering. -# - + import sys, re @@ -39,7 +39,8 @@ class StringTypePrinter: return 'string' def to_string(self): - return self.val['str'] + l = int(self.val['len']) + return self.val['str'].string("utf-8", "ignore", l) class SliceTypePrinter: diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c index e3a20d48a..1a3653f10 100644 --- a/src/pkg/runtime/runtime.c +++ b/src/pkg/runtime/runtime.c @@ -3,12 +3,13 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "stack.h" enum { maxround = sizeof(uintptr), }; -int32 runtime·panicking = 0; +uint32 runtime·panicking; int32 runtime·gotraceback(void) @@ -21,14 +22,24 @@ runtime·gotraceback(void) return runtime·atoi(p); } +static Lock paniclk; + void -runtime·dopanic(int32 unused) +runtime·startpanic(void) { - if(runtime·panicking) { - runtime·printf("double panic\n"); + if(m->dying) { + runtime·printf("panic during panic\n"); runtime·exit(3); } - runtime·panicking++; + m->dying = 1; + runtime·xadd(&runtime·panicking, 1); + runtime·lock(&paniclk); +} + +void +runtime·dopanic(int32 unused) +{ + static bool didothers; if(g->sig != 0) runtime·printf("\n[signal %x code=%p addr=%p pc=%p]\n", @@ -37,10 +48,22 @@ runtime·dopanic(int32 unused) runtime·printf("\n"); if(runtime·gotraceback()){ runtime·traceback(runtime·getcallerpc(&unused), runtime·getcallersp(&unused), 0, g); - runtime·tracebackothers(g); + if(!didothers) { + didothers = true; + runtime·tracebackothers(g); + } } - - runtime·breakpoint(); // so we can grab it in a debugger + runtime·unlock(&paniclk); + if(runtime·xadd(&runtime·panicking, -1) != 0) { + // Some other m is panicking too. + // Let it print what it needs to print. + // Wait forever without chewing up cpu. + // It will exit when it's done. + static Lock deadlock; + runtime·lock(&deadlock); + runtime·lock(&deadlock); + } + runtime·exit(2); } @@ -73,6 +96,7 @@ runtime·throwinit(void) void runtime·throw(int8 *s) { + runtime·startpanic(); runtime·printf("throw: %s\n", s); runtime·dopanic(0); *(int32*)0 = 0; // not reached diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index cea07e4a7..85dca54f7 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -104,6 +104,7 @@ enum Gmoribund, Gdead, Grecovery, + Gstackalloc, }; enum { @@ -196,10 +197,12 @@ struct G bool ispanic; M* m; // for debuggers, but offset not hard-coded M* lockedm; + M* idlem; int32 sig; uintptr sigcode0; uintptr sigcode1; uintptr sigpc; + uintptr gopc; // pc of go statement that created this goroutine }; struct M { @@ -224,6 +227,7 @@ struct M int32 locks; int32 nomemprof; int32 waitnextg; + int32 dying; Note havenextg; G* nextg; M* alllink; // on allm @@ -231,6 +235,7 @@ struct M uint32 machport; // Return address for Mach IPC (OS X) MCache *mcache; G* lockedg; + G* idleg; uint32 freglo[16]; // D[i] lsb and F[i] uint32 freghi[16]; // D[i] msb and F[i+16] uint32 fflag; // floating point compare flags @@ -358,7 +363,7 @@ G* runtime·allg; M* runtime·allm; int32 runtime·goidgen; extern int32 runtime·gomaxprocs; -extern int32 runtime·panicking; +extern uint32 runtime·panicking; extern int32 runtime·gcwaiting; // gc is waiting to run int8* runtime·goos; extern bool runtime·iscgo; @@ -448,13 +453,14 @@ void runtime·entersyscall(void); void runtime·exitsyscall(void); void runtime·startcgocallback(G*); void runtime·endcgocallback(G*); -G* runtime·newproc1(byte*, byte*, int32, int32); +G* runtime·newproc1(byte*, byte*, int32, int32, void*); void runtime·siginit(void); bool runtime·sigsend(int32 sig); void runtime·gettime(int64*, int32*); int32 runtime·callers(int32, uintptr*, int32); int64 runtime·nanotime(void); void runtime·dopanic(int32); +void runtime·startpanic(void); #pragma varargck argpos runtime·printf 1 #pragma varargck type "d" int32 @@ -590,83 +596,17 @@ int32 runtime·chancap(Hchan*); void runtime·ifaceE2I(struct InterfaceType*, Eface, Iface*); -/* - * Stack layout parameters. - * Known to linkers. - * - * The per-goroutine g->stackguard is set to point - * StackGuard bytes above the bottom of the stack. - * Each function compares its stack pointer against - * g->stackguard to check for overflow. To cut one - * instruction from the check sequence for functions - * with tiny frames, the stack is allowed to protrude - * StackSmall bytes below the stack guard. Functions - * with large frames don't bother with the check and - * always call morestack. The sequences are - * (for amd64, others are similar): - * - * guard = g->stackguard - * frame = function's stack frame size - * argsize = size of function arguments (call + return) - * - * stack frame size <= StackSmall: - * CMPQ guard, SP - * JHI 3(PC) - * MOVQ m->morearg, $(argsize << 32) - * CALL morestack(SB) - * - * stack frame size > StackSmall but < StackBig - * LEAQ (frame-StackSmall)(SP), R0 - * CMPQ guard, R0 - * JHI 3(PC) - * MOVQ m->morearg, $(argsize << 32) - * CALL morestack(SB) - * - * stack frame size >= StackBig: - * MOVQ m->morearg, $((argsize << 32) | frame) - * CALL morestack(SB) - * - * The bottom StackGuard - StackSmall bytes are important: - * there has to be enough room to execute functions that - * refuse to check for stack overflow, either because they - * need to be adjacent to the actual caller's frame (deferproc) - * or because they handle the imminent stack overflow (morestack). - * - * For example, deferproc might call malloc, which does one - * of the above checks (without allocating a full frame), - * which might trigger a call to morestack. This sequence - * needs to fit in the bottom section of the stack. On amd64, - * morestack's frame is 40 bytes, and deferproc's frame is 56 bytes. - * That fits well within the StackGuard - StackSmall = 128 bytes - * at the bottom. There may be other sequences lurking or yet to - * be written that require more stack. Morestack checks to make - * sure the stack has not completely overflowed and should catch - * such sequences. - */ enum { + // StackSystem is a number of additional bytes to add + // to each stack below the usual guard area for OS-specific + // purposes like signal handling. + // TODO(rsc): This is only for Windows. Can't Windows use + // a separate exception stack like every other operating system? #ifdef __WINDOWS__ - // need enough room in guard area for exception handler. - // use larger stacks to compensate for larger stack guard. - StackSmall = 256, - StackGuard = 2048, - StackBig = 8192, - StackExtra = StackGuard, + StackSystem = 2048, #else - // byte offset of stack guard (g->stackguard) above bottom of stack. - StackGuard = 256, - - // checked frames are allowed to protrude below the guard by - // this many bytes. this saves an instruction in the checking - // sequence when the stack frame is tiny. - StackSmall = 128, - - // extra space in the frame (beyond the function for which - // the frame is allocated) is assumed not to be much bigger - // than this amount. it may not be used efficiently if it is. - StackBig = 4096, - - // extra room over frame size when allocating a stack. - StackExtra = 1024, + StackSystem = 0, #endif }; + diff --git a/src/pkg/runtime/stack.h b/src/pkg/runtime/stack.h new file mode 100644 index 000000000..ebf0462b5 --- /dev/null +++ b/src/pkg/runtime/stack.h @@ -0,0 +1,86 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Stack layout parameters. +Included both by runtime (compiled via 6c) and linkers (compiled via gcc). + +The per-goroutine g->stackguard is set to point StackGuard bytes +above the bottom of the stack. Each function compares its stack +pointer against g->stackguard to check for overflow. To cut one +instruction from the check sequence for functions with tiny frames, +the stack is allowed to protrude StackSmall bytes below the stack +guard. Functions with large frames don't bother with the check and +always call morestack. The sequences are (for amd64, others are +similar): + + guard = g->stackguard + frame = function's stack frame size + argsize = size of function arguments (call + return) + + stack frame size <= StackSmall: + CMPQ guard, SP + JHI 3(PC) + MOVQ m->morearg, $(argsize << 32) + CALL morestack(SB) + + stack frame size > StackSmall but < StackBig + LEAQ (frame-StackSmall)(SP), R0 + CMPQ guard, R0 + JHI 3(PC) + MOVQ m->morearg, $(argsize << 32) + CALL morestack(SB) + + stack frame size >= StackBig: + MOVQ m->morearg, $((argsize << 32) | frame) + CALL morestack(SB) + +The bottom StackGuard - StackSmall bytes are important: there has +to be enough room to execute functions that refuse to check for +stack overflow, either because they need to be adjacent to the +actual caller's frame (deferproc) or because they handle the imminent +stack overflow (morestack). + +For example, deferproc might call malloc, which does one of the +above checks (without allocating a full frame), which might trigger +a call to morestack. This sequence needs to fit in the bottom +section of the stack. On amd64, morestack's frame is 40 bytes, and +deferproc's frame is 56 bytes. That fits well within the +StackGuard - StackSmall = 128 bytes at the bottom. +The linkers explore all possible call traces involving non-splitting +functions to make sure that this limit cannot be violated. + */ + +enum { + // The amount of extra stack to allocate beyond the size + // needed for the single frame that triggered the split. + StackExtra = 1024, + + // The minimum stack segment size to allocate. + // If the amount needed for the splitting frame + StackExtra + // is less than this number, the stack will have this size instead. + StackMin = 4096, + + // Functions that need frames bigger than this call morestack + // unconditionally. That is, on entry to a function it is assumed + // that the amount of space available in the current stack segment + // couldn't possibly be bigger than StackBig. If stack segments + // do run with more space than StackBig, the space may not be + // used efficiently. As a result, StackBig should not be significantly + // smaller than StackMin or StackExtra. + StackBig = 4096, + + // The stack guard is a pointer this many bytes above the + // bottom of the stack. + StackGuard = 256, + + // After a stack split check the SP is allowed to be this + // many bytes below the stack guard. This saves an instruction + // in the checking sequence for tiny frames. + StackSmall = 128, + + // The maximum number of bytes that a chain of NOSPLIT + // functions can use. + StackLimit = StackGuard - StackSmall, +}; diff --git a/src/pkg/runtime/type.go b/src/pkg/runtime/type.go index 87268db4c..71ad4e7a5 100644 --- a/src/pkg/runtime/type.go +++ b/src/pkg/runtime/type.go @@ -9,7 +9,7 @@ * data structures and must be kept in sync with this file: * * ../../cmd/gc/reflect.c - * ../../cmd/ld/dwarf.c + * ../../cmd/ld/dwarf.c decodetype_* * ../reflect/type.go * type.h */ @@ -35,6 +35,7 @@ type commonType struct { kind uint8 // enumeration for C string *string // string form; unnecessary but undeniably useful *uncommonType // (relatively) uncommon fields + ptrToThis *Type // pointer to this type, if used in binary or has methods } // Values for commonType.kind. diff --git a/src/pkg/runtime/type.h b/src/pkg/runtime/type.h index c7d9dace2..1adb6dc2e 100644 --- a/src/pkg/runtime/type.h +++ b/src/pkg/runtime/type.h @@ -31,6 +31,7 @@ struct CommonType uint8 kind; String *string; UncommonType *x; + Type *ptrto; }; enum { diff --git a/src/pkg/runtime/windows/386/signal.c b/src/pkg/runtime/windows/386/signal.c index 903636910..08b32a137 100644 --- a/src/pkg/runtime/windows/386/signal.c +++ b/src/pkg/runtime/windows/386/signal.c @@ -85,7 +85,6 @@ runtime·sighandler(ExceptionRecord *info, void *frame, Context *r) runtime·dumpregs(r); } - runtime·breakpoint(); runtime·exit(2); return 0; } diff --git a/src/pkg/runtime/windows/signals.h b/src/pkg/runtime/windows/signals.h new file mode 100644 index 000000000..6943714b0 --- /dev/null +++ b/src/pkg/runtime/windows/signals.h @@ -0,0 +1,3 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. diff --git a/src/pkg/sync/Makefile b/src/pkg/sync/Makefile index fd8e5d998..e8a766226 100644 --- a/src/pkg/sync/Makefile +++ b/src/pkg/sync/Makefile @@ -6,26 +6,10 @@ include ../../Make.inc TARG=sync GOFILES=\ + cond.go\ mutex.go\ once.go \ rwmutex.go\ waitgroup.go\ -# 386-specific object files -OFILES_386=\ - asm_386.$O\ - -# amd64-specific object files -OFILES_amd64=\ - asm_amd64.$O\ - -GOARM?=6 - -# arm-specific object files -OFILES_arm=\ - asm_arm$(GOARM).$O\ - -OFILES=\ - $(OFILES_$(GOARCH))\ - include ../../Make.pkg diff --git a/src/pkg/sync/asm_386.s b/src/pkg/sync/asm_386.s deleted file mode 100644 index 228bad044..000000000 --- a/src/pkg/sync/asm_386.s +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// func cas(val *int32, old, new int32) bool -// Atomically: -// if *val == old { -// *val = new; -// return true; -// }else -// return false; -TEXT ·cas(SB), 7, $0 - MOVL 4(SP), BX - MOVL 8(SP), AX - MOVL 12(SP), CX - LOCK - CMPXCHGL CX, 0(BX) - JZ ok - MOVL $0, 16(SP) - RET -ok: - MOVL $1, 16(SP) - RET diff --git a/src/pkg/sync/asm_amd64.s b/src/pkg/sync/asm_amd64.s deleted file mode 100644 index 870236482..000000000 --- a/src/pkg/sync/asm_amd64.s +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// func cas(val *int32, old, new int32) bool -// Atomically: -// if *val == old { -// *val = new; -// return true; -// }else -// return false; -TEXT ·cas(SB), 7, $0 - MOVQ 8(SP), BX - MOVL 16(SP), AX - MOVL 20(SP), CX - LOCK - CMPXCHGL CX, 0(BX) - JZ ok - MOVL $0, 24(SP) - RET -ok: - MOVL $1, 24(SP) - RET diff --git a/src/pkg/sync/asm_arm5.s b/src/pkg/sync/asm_arm5.s deleted file mode 100644 index 2cb496887..000000000 --- a/src/pkg/sync/asm_arm5.s +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This version works on pre v6 architectures -// func cas(val *int32, old, new int32) bool -// Atomically: -// if *val == old { -// *val = new; -// return true; -// }else -// return false; - -TEXT ·cas(SB),7,$0 - MOVW 0(FP), R0 // *val - MOVW 4(FP), R1 // old - MOVW 8(FP), R2 // new - MOVW $1, R3 - MOVW $runtime·cas_mutex(SB), R4 -l: - SWPW (R4), R3 // acquire mutex - CMP $0, R3 - BNE fail0 - - MOVW (R0), R5 - CMP R1, R5 - BNE fail1 - - MOVW R2, (R0) - MOVW R3, (R4) // release mutex - MOVW $1, R0 - MOVW R0, 16(SP) - RET -fail1: - MOVW R3, (R4) // release mutex -fail0: - MOVW $0, R0 - MOVW R0, 16(SP) - RET - diff --git a/src/pkg/sync/asm_arm6.s b/src/pkg/sync/asm_arm6.s deleted file mode 100644 index d1e0851d0..000000000 --- a/src/pkg/sync/asm_arm6.s +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// func cas(val *int32, old, new int32) bool -// Atomically: -// if *val == old { -// *val = new; -// return true; -// }else -// return false; - -TEXT ·cas(SB),7,$0 - MOVW 0(FP), R1 // *val - MOVW 4(FP), R2 // old - MOVW 8(FP), R3 // new -l: - LDREX (R1), R0 - CMP R0, R2 - BNE fail - STREX R3, (R1), R0 - CMP $0, R0 - BNE l - MOVW $1, R0 - MOVW R0, 16(SP) - RET -fail: - MOVW $0, R0 - MOVW R0, 16(SP) - RET diff --git a/src/pkg/sync/atomic/Makefile b/src/pkg/sync/atomic/Makefile new file mode 100644 index 000000000..38d8998c0 --- /dev/null +++ b/src/pkg/sync/atomic/Makefile @@ -0,0 +1,18 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=sync/atomic +GOFILES=\ + doc.go\ + +OFILES=\ + asm_$(GOARCH).$O\ + +ifeq ($(GOARCH),arm) +OFILES+=asm_$(GOOS)_$(GOARCH).$O +endif + +include ../../../Make.pkg diff --git a/src/pkg/sync/atomic/asm_386.s b/src/pkg/sync/atomic/asm_386.s new file mode 100644 index 000000000..a9360efae --- /dev/null +++ b/src/pkg/sync/atomic/asm_386.s @@ -0,0 +1,87 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·CompareAndSwapInt32(SB),7,$0 + JMP ·CompareAndSwapUint32(SB) + +TEXT ·CompareAndSwapUint32(SB),7,$0 + MOVL valptr+0(FP), BP + MOVL old+4(FP), AX + MOVL new+8(FP), CX + // CMPXCHGL was introduced on the 486. + LOCK + CMPXCHGL CX, 0(BP) + SETEQ ret+12(FP) + RET + +TEXT ·CompareAndSwapUintptr(SB),7,$0 + JMP ·CompareAndSwapUint32(SB) + +TEXT ·CompareAndSwapInt64(SB),7,$0 + JMP ·CompareAndSwapUint64(SB) + +TEXT ·CompareAndSwapUint64(SB),7,$0 + MOVL valptr+0(FP), BP + MOVL oldlo+4(FP), AX + MOVL oldhi+8(FP), DX + MOVL newlo+12(FP), BX + MOVL newhi+16(FP), CX + // CMPXCHG8B was introduced on the Pentium. + LOCK + CMPXCHG8B 0(BP) + SETEQ ret+20(FP) + RET + +TEXT ·AddInt32(SB),7,$0 + JMP ·AddUint32(SB) + +TEXT ·AddUint32(SB),7,$0 + MOVL valptr+0(FP), BP + MOVL delta+4(FP), AX + MOVL AX, CX + // XADD was introduced on the 486. + LOCK + XADDL AX, 0(BP) + ADDL AX, CX + MOVL CX, ret+8(FP) + RET + +TEXT ·AddUintptr(SB),7,$0 + JMP ·AddUint32(SB) + +TEXT ·AddInt64(SB),7,$0 + JMP ·AddUint64(SB) + +TEXT ·AddUint64(SB),7,$0 + // no XADDQ so use CMPXCHG8B loop + MOVL valptr+0(FP), BP + // DI:SI = delta + MOVL deltalo+4(FP), SI + MOVL deltahi+8(FP), DI + // DX:AX = *valptr + MOVL 0(BP), AX + MOVL 4(BP), DX +addloop: + // CX:BX = DX:AX (*valptr) + DI:SI (delta) + MOVL AX, BX + MOVL DX, CX + ADDL SI, BX + ADCL DI, CX + + // if *valptr == DX:AX { + // *valptr = CX:BX + // } else { + // DX:AX = *valptr + // } + // all in one instruction + LOCK + CMPXCHG8B 0(BP) + + JNZ addloop + + // success + // return CX:BX + MOVL BX, retlo+12(FP) + MOVL CX, rethi+16(FP) + RET diff --git a/src/pkg/sync/atomic/asm_amd64.s b/src/pkg/sync/atomic/asm_amd64.s new file mode 100644 index 000000000..a260902a7 --- /dev/null +++ b/src/pkg/sync/atomic/asm_amd64.s @@ -0,0 +1,59 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·CompareAndSwapInt32(SB),7,$0 + JMP ·CompareAndSwapUint32(SB) + +TEXT ·CompareAndSwapUint32(SB),7,$0 + MOVQ valptr+0(FP), BP + MOVL old+8(FP), AX + MOVL new+12(FP), CX + LOCK + CMPXCHGL CX, 0(BP) + SETEQ ret+16(FP) + RET + +TEXT ·CompareAndSwapUintptr(SB),7,$0 + JMP ·CompareAndSwapUint64(SB) + +TEXT ·CompareAndSwapInt64(SB),7,$0 + JMP ·CompareAndSwapUint64(SB) + +TEXT ·CompareAndSwapUint64(SB),7,$0 + MOVQ valptr+0(FP), BP + MOVQ old+8(FP), AX + MOVQ new+16(FP), CX + LOCK + CMPXCHGQ CX, 0(BP) + SETEQ ret+24(FP) + RET + +TEXT ·AddInt32(SB),7,$0 + JMP ·AddUint32(SB) + +TEXT ·AddUint32(SB),7,$0 + MOVQ valptr+0(FP), BP + MOVL delta+8(FP), AX + MOVL AX, CX + LOCK + XADDL AX, 0(BP) + ADDL AX, CX + MOVL CX, ret+16(FP) + RET + +TEXT ·AddUintptr(SB),7,$0 + JMP ·AddUint64(SB) + +TEXT ·AddInt64(SB),7,$0 + JMP ·AddUint64(SB) + +TEXT ·AddUint64(SB),7,$0 + MOVQ valptr+0(FP), BP + MOVQ delta+8(FP), AX + MOVQ AX, CX + LOCK + XADDQ AX, 0(BP) + ADDQ AX, CX + MOVQ CX, ret+16(FP) + RET diff --git a/src/pkg/sync/atomic/asm_arm.s b/src/pkg/sync/atomic/asm_arm.s new file mode 100644 index 000000000..1ae0a995e --- /dev/null +++ b/src/pkg/sync/atomic/asm_arm.s @@ -0,0 +1,78 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// ARM atomic operations, for use by asm_$(GOOS)_arm.s. + +TEXT ·armCompareAndSwapUint32(SB),7,$0 + MOVW valptr+0(FP), R1 + MOVW old+4(FP), R2 + MOVW new+8(FP), R3 +casloop: + // LDREX and STREX were introduced in ARM 6. + LDREX (R1), R0 + CMP R0, R2 + BNE casfail + STREX R3, (R1), R0 + CMP $0, R0 + BNE casloop + MOVW $1, R0 + MOVBU R0, ret+12(FP) + RET +casfail: + MOVW $0, R0 + MOVBU R0, ret+12(FP) + RET + +TEXT ·armCompareAndSwapUint64(SB),7,$0 + MOVW valptr+0(FP), R1 + MOVW oldlo+4(FP), R2 + MOVW oldhi+8(FP), R3 + MOVW newlo+12(FP), R4 + MOVW newhi+16(FP), R5 +cas64loop: + // LDREXD and STREXD were introduced in ARM 11. + LDREXD (R1), R6 // loads R6 and R7 + CMP R2, R6 + BNE cas64fail + CMP R3, R7 + BNE cas64fail + STREXD R4, (R1), R0 // stores R4 and R5 + CMP $0, R0 + BNE cas64loop + MOVW $1, R0 + MOVBU R0, ret+20(FP) + RET +cas64fail: + MOVW $0, R0 + MOVBU R0, ret+20(FP) + RET + +TEXT ·armAddUint32(SB),7,$0 + MOVW valptr+0(FP), R1 + MOVW delta+4(FP), R2 +addloop: + // LDREX and STREX were introduced in ARM 6. + LDREX (R1), R3 + ADD R2, R3 + STREX R3, (R1), R0 + CMP $0, R0 + BNE addloop + MOVW R3, ret+8(FP) + RET + +TEXT ·armAddUint64(SB),7,$0 + MOVW valptr+0(FP), R1 + MOVW deltalo+4(FP), R2 + MOVW deltahi+8(FP), R3 +add64loop: + // LDREXD and STREXD were introduced in ARM 11. + LDREXD (R1), R4 // loads R4 and R5 + ADD.S R2, R4 + ADC R3, R5 + STREXD R4, (R1), R0 // stores R4 and R5 + CMP $0, R0 + BNE add64loop + MOVW R4, retlo+12(FP) + MOVW R5, rethi+16(FP) + RET diff --git a/src/pkg/sync/atomic/asm_linux_arm.s b/src/pkg/sync/atomic/asm_linux_arm.s new file mode 100644 index 000000000..5e7aea292 --- /dev/null +++ b/src/pkg/sync/atomic/asm_linux_arm.s @@ -0,0 +1,68 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Linux/ARM atomic operations. + +// Because there is so much variation in ARM devices, +// the Linux kernel provides an appropriate compare-and-swap +// implementation at address 0xffff0fc0. Caller sets: +// R0 = old value +// R1 = new value +// R2 = valptr +// LR = return address +// The function returns with CS true if the swap happened. +// http://lxr.linux.no/linux+v2.6.37.2/arch/arm/kernel/entry-armv.S#L850 +TEXT cas<>(SB),7,$0 + MOVW $0xffff0fc0, PC + +TEXT ·CompareAndSwapInt32(SB),7,$0 + B ·CompareAndSwapUint32(SB) + +// Implement using kernel cas for portability. +TEXT ·CompareAndSwapUint32(SB),7,$0 + MOVW valptr+0(FP), R2 + MOVW old+4(FP), R0 + MOVW new+8(FP), R1 + BL cas<>(SB) + MOVW $0, R0 + MOVW.CS $1, R0 + MOVW R0, ret+12(FP) + RET + +TEXT ·CompareAndSwapUintptr(SB),7,$0 + B ·CompareAndSwapUint32(SB) + +TEXT ·AddInt32(SB),7,$0 + B ·AddUint32(SB) + +// Implement using kernel cas for portability. +TEXT ·AddUint32(SB),7,$0 + MOVW valptr+0(FP), R2 + MOVW delta+4(FP), R4 +addloop1: + MOVW 0(R2), R0 + MOVW R0, R1 + ADD R4, R1 + BL cas<>(SB) + BCC addloop1 + MOVW R1, ret+8(FP) + RET + +TEXT ·AddUintptr(SB),7,$0 + B ·AddUint32(SB) + +// The kernel provides no 64-bit compare-and-swap, +// so use native ARM instructions, which will only work on +// ARM 11 and later devices. +TEXT ·CompareAndSwapInt64(SB),7,$0 + B ·armCompareAndSwapUint64(SB) + +TEXT ·CompareAndSwapUint64(SB),7,$0 + B ·armCompareAndSwapUint64(SB) + +TEXT ·AddInt64(SB),7,$0 + B ·armAddUint64(SB) + +TEXT ·AddUint64(SB),7,$0 + B ·armAddUint64(SB) diff --git a/src/pkg/sync/atomic/atomic_test.go b/src/pkg/sync/atomic/atomic_test.go new file mode 100644 index 000000000..7b204b1d9 --- /dev/null +++ b/src/pkg/sync/atomic/atomic_test.go @@ -0,0 +1,506 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +import ( + "runtime" + "testing" + "unsafe" +) + +// Tests of correct behavior, without contention. +// (Does the function work as advertised?) +// +// Test that the Add functions add correctly. +// Test that the CompareAndSwap functions actually +// do the comparison and the swap correctly. +// +// The loop over power-of-two values is meant to +// ensure that the operations apply to the full word size. +// The struct fields x.before and x.after check that the +// operations do not extend past the full word size. + +const ( + magic32 = 0xdedbeef + magic64 = 0xdeddeadbeefbeef +) + +func TestAddInt32(t *testing.T) { + var x struct { + before int32 + i int32 + after int32 + } + x.before = magic32 + x.after = magic32 + var j int32 + for delta := int32(1); delta+delta > delta; delta += delta { + k := AddInt32(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestAddUint32(t *testing.T) { + var x struct { + before uint32 + i uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + var j uint32 + for delta := uint32(1); delta+delta > delta; delta += delta { + k := AddUint32(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestAddInt64(t *testing.T) { + var x struct { + before int64 + i int64 + after int64 + } + x.before = magic64 + x.after = magic64 + var j int64 + for delta := int64(1); delta+delta > delta; delta += delta { + k := AddInt64(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, int64(magic64), int64(magic64)) + } +} + +func TestAddUint64(t *testing.T) { + var x struct { + before uint64 + i uint64 + after uint64 + } + x.before = magic64 + x.after = magic64 + var j uint64 + for delta := uint64(1); delta+delta > delta; delta += delta { + k := AddUint64(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestAddUintptr(t *testing.T) { + var x struct { + before uintptr + i uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + var j uintptr + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := AddUintptr(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +func TestCompareAndSwapInt32(t *testing.T) { + var x struct { + before int32 + i int32 + after int32 + } + x.before = magic32 + x.after = magic32 + for val := int32(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapInt32(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapInt32(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestCompareAndSwapUint32(t *testing.T) { + var x struct { + before uint32 + i uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + for val := uint32(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapUint32(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapUint32(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestCompareAndSwapInt64(t *testing.T) { + var x struct { + before int64 + i int64 + after int64 + } + x.before = magic64 + x.after = magic64 + for val := int64(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapInt64(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapInt64(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestCompareAndSwapUint64(t *testing.T) { + var x struct { + before uint64 + i uint64 + after uint64 + } + x.before = magic64 + x.after = magic64 + for val := uint64(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapUint64(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapUint64(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestCompareAndSwapUintptr(t *testing.T) { + var x struct { + before uintptr + i uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for val := uintptr(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapUintptr(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapUintptr(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +// Tests of correct behavior, with contention. +// (Is the function atomic?) +// +// For each function, we write a "hammer" function that repeatedly +// uses the atomic operation to add 1 to a value. After running +// multiple hammers in parallel, check that we end with the correct +// total. + +var hammer32 = []struct { + name string + f func(*uint32, int) +}{ + {"AddInt32", hammerAddInt32}, + {"AddUint32", hammerAddUint32}, + {"AddUintptr", hammerAddUintptr32}, + {"CompareAndSwapInt32", hammerCompareAndSwapInt32}, + {"CompareAndSwapUint32", hammerCompareAndSwapUint32}, + {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr32}, +} + +func init() { + var v uint64 = 1 << 50 + if uintptr(v) != 0 { + // 64-bit system; clear uintptr tests + hammer32[2].f = nil + hammer32[5].f = nil + } +} + +func hammerAddInt32(uval *uint32, count int) { + val := (*int32)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddInt32(val, 1) + } +} + +func hammerAddUint32(val *uint32, count int) { + for i := 0; i < count; i++ { + AddUint32(val, 1) + } +} + +func hammerAddUintptr32(uval *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddUintptr(val, 1) + } +} + +func hammerCompareAndSwapInt32(uval *uint32, count int) { + val := (*int32)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapInt32(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUint32(val *uint32, count int) { + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUint32(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUintptr32(uval *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUintptr(val, v, v+1) { + break + } + } + } +} + +func TestHammer32(t *testing.T) { + const ( + n = 100000 + p = 4 + ) + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(p)) + + for _, tt := range hammer32 { + if tt.f == nil { + continue + } + c := make(chan int) + var val uint32 + for i := 0; i < p; i++ { + go func() { + tt.f(&val, n) + c <- 1 + }() + } + for i := 0; i < p; i++ { + <-c + } + if val != n*p { + t.Errorf("%s: val=%d want %d", tt.name, val, n*p) + } + } +} + +var hammer64 = []struct { + name string + f func(*uint64, int) +}{ + {"AddInt64", hammerAddInt64}, + {"AddUint64", hammerAddUint64}, + {"AddUintptr", hammerAddUintptr64}, + {"CompareAndSwapInt64", hammerCompareAndSwapInt64}, + {"CompareAndSwapUint64", hammerCompareAndSwapUint64}, + {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr64}, +} + +func init() { + var v uint64 = 1 << 50 + if uintptr(v) == 0 { + // 32-bit system; clear uintptr tests + hammer64[2].f = nil + hammer64[5].f = nil + } +} + +func hammerAddInt64(uval *uint64, count int) { + val := (*int64)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddInt64(val, 1) + } +} + +func hammerAddUint64(val *uint64, count int) { + for i := 0; i < count; i++ { + AddUint64(val, 1) + } +} + +func hammerAddUintptr64(uval *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddUintptr(val, 1) + } +} + +func hammerCompareAndSwapInt64(uval *uint64, count int) { + val := (*int64)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapInt64(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUint64(val *uint64, count int) { + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUint64(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUintptr64(uval *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUintptr(val, v, v+1) { + break + } + } + } +} + +func TestHammer64(t *testing.T) { + const ( + n = 100000 + p = 4 + ) + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(p)) + + for _, tt := range hammer64 { + if tt.f == nil { + continue + } + c := make(chan int) + var val uint64 + for i := 0; i < p; i++ { + go func() { + tt.f(&val, n) + c <- 1 + }() + } + for i := 0; i < p; i++ { + <-c + } + if val != n*p { + t.Errorf("%s: val=%d want %d", tt.name, val, n*p) + } + } +} diff --git a/src/pkg/sync/atomic/doc.go b/src/pkg/sync/atomic/doc.go new file mode 100644 index 000000000..1335def59 --- /dev/null +++ b/src/pkg/sync/atomic/doc.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package atomic provides low-level atomic memory primitives +// useful for implementing synchronization algorithms. +// +// These functions require great care to be used correctly. +// Except for special, low-level applications, synchronization is better +// done with channels or the facilities of the sync package. +// Share memory by communicating; +// don't communicate by sharing memory. +// +// The compare-and-swap operation, implemented by the CompareAndSwapT +// functions, is the atomic equivalent of: +// +// if *val == old { +// *val = new +// return true +// } +// return false +// +package atomic + +// BUG(rsc): On ARM, the 64-bit functions use instructions unavailable before ARM 11. +// +// On x86-32, the 64-bit functions use instructions unavailable before the Pentium. + +// CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value. +func CompareAndSwapInt32(val *int32, old, new int32) (swapped bool) + +// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. +func CompareAndSwapInt64(val *int64, old, new int64) (swapped bool) + +// CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value. +func CompareAndSwapUint32(val *uint32, old, new uint32) (swapped bool) + +// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. +func CompareAndSwapUint64(val *uint64, old, new uint64) (swapped bool) + +// CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value. +func CompareAndSwapUintptr(val *uintptr, old, new uintptr) (swapped bool) + +// AddInt32 atomically adds delta to *val and returns the new value. +func AddInt32(val *int32, delta int32) (new int32) + +// AddUint32 atomically adds delta to *val and returns the new value. +func AddUint32(val *uint32, delta uint32) (new uint32) + +// AddInt64 atomically adds delta to *val and returns the new value. +func AddInt64(val *int64, delta int64) (new int64) + +// AddUint64 atomically adds delta to *val and returns the new value. +func AddUint64(val *uint64, delta uint64) (new uint64) + +// AddUintptr atomically adds delta to *val and returns the new value. +func AddUintptr(val *uintptr, delta uintptr) (new uintptr) diff --git a/src/pkg/sync/cond.go b/src/pkg/sync/cond.go new file mode 100644 index 000000000..ea48f2e7a --- /dev/null +++ b/src/pkg/sync/cond.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import "runtime" + +// Cond implements a condition variable, a rendezvous point +// for goroutines waiting for or announcing the occurrence +// of an event. +// +// Each Cond has an associated Locker L (often a *Mutex or *RWMutex), +// which must be held when changing the condition and +// when calling the Wait method. +type Cond struct { + L Locker // held while observing or changing the condition + m Mutex // held to avoid internal races + waiters int // number of goroutines blocked on Wait + sema *uint32 +} + +// NewCond returns a new Cond with Locker l. +func NewCond(l Locker) *Cond { + return &Cond{L: l} +} + +// Wait atomically unlocks c.L and suspends execution +// of the calling goroutine. After later resuming execution, +// Wait locks c.L before returning. +// +// Because L is not locked when Wait first resumes, the caller +// typically cannot assume that the condition is true when +// Wait returns. Instead, the caller should Wait in a loop: +// +// c.L.Lock() +// for !condition() { +// c.Wait() +// } +// ... make use of condition ... +// c.L.Unlock() +// +func (c *Cond) Wait() { + c.m.Lock() + if c.sema == nil { + c.sema = new(uint32) + } + s := c.sema + c.waiters++ + c.m.Unlock() + c.L.Unlock() + runtime.Semacquire(s) + c.L.Lock() +} + +// Signal wakes one goroutine waiting on c, if there is any. +// +// It is allowed but not required for the caller to hold c.L +// during the call. +func (c *Cond) Signal() { + c.m.Lock() + if c.waiters > 0 { + c.waiters-- + runtime.Semrelease(c.sema) + } + c.m.Unlock() +} + +// Broadcast wakes all goroutines waiting on c. +// +// It is allowed but not required for the caller to hold c.L +// during the call. +func (c *Cond) Broadcast() { + c.m.Lock() + if c.waiters > 0 { + s := c.sema + n := c.waiters + for i := 0; i < n; i++ { + runtime.Semrelease(s) + } + // We just issued n wakeups via the semaphore s. + // To ensure that they wake up the existing waiters + // and not waiters that arrive after Broadcast returns, + // clear c.sema. The next operation will allocate + // a new one. + c.sema = nil + c.waiters = 0 + } + c.m.Unlock() +} diff --git a/src/pkg/sync/cond_test.go b/src/pkg/sync/cond_test.go new file mode 100644 index 000000000..846f98bf3 --- /dev/null +++ b/src/pkg/sync/cond_test.go @@ -0,0 +1,99 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package sync_test + +import ( + . "sync" + "testing" +) + +func TestCondSignal(t *testing.T) { + var m Mutex + c := NewCond(&m) + n := 2 + running := make(chan bool, n) + awake := make(chan bool, n) + for i := 0; i < n; i++ { + go func() { + m.Lock() + running <- true + c.Wait() + awake <- true + m.Unlock() + }() + } + for i := 0; i < n; i++ { + <-running // Wait for everyone to run. + } + for n > 0 { + select { + case <-awake: + t.Fatal("goroutine not asleep") + default: + } + m.Lock() + c.Signal() + m.Unlock() + <-awake // Will deadlock if no goroutine wakes up + select { + case <-awake: + t.Fatal("too many goroutines awake") + default: + } + n-- + } + c.Signal() +} + +func TestCondBroadcast(t *testing.T) { + var m Mutex + c := NewCond(&m) + n := 200 + running := make(chan int, n) + awake := make(chan int, n) + exit := false + for i := 0; i < n; i++ { + go func(g int) { + m.Lock() + for !exit { + running <- g + c.Wait() + awake <- g + } + m.Unlock() + }(i) + } + for i := 0; i < n; i++ { + for i := 0; i < n; i++ { + <-running // Will deadlock unless n are running. + } + if i == n-1 { + m.Lock() + exit = true + m.Unlock() + } + select { + case <-awake: + t.Fatal("goroutine not asleep") + default: + } + m.Lock() + c.Broadcast() + m.Unlock() + seen := make([]bool, n) + for i := 0; i < n; i++ { + g := <-awake + if seen[g] { + t.Fatal("goroutine woke up twice") + } + seen[g] = true + } + } + select { + case <-running: + t.Fatal("goroutine did not exit") + default: + } + c.Broadcast() +} diff --git a/src/pkg/sync/mutex.go b/src/pkg/sync/mutex.go index 2a1270b9c..da565d38d 100644 --- a/src/pkg/sync/mutex.go +++ b/src/pkg/sync/mutex.go @@ -9,37 +9,30 @@ // done via channels and communication. package sync -import "runtime" - -func cas(val *uint32, old, new uint32) bool +import ( + "runtime" + "sync/atomic" +) // A Mutex is a mutual exclusion lock. // Mutexes can be created as part of other structures; // the zero value for a Mutex is an unlocked mutex. type Mutex struct { - key uint32 + key int32 sema uint32 } -// Add delta to *val, and return the new *val in a thread-safe way. If multiple -// goroutines call xadd on the same val concurrently, the changes will be -// serialized, and all the deltas will be added in an undefined order. -func xadd(val *uint32, delta int32) (new uint32) { - for { - v := *val - nv := v + uint32(delta) - if cas(val, v, nv) { - return nv - } - } - panic("unreached") +// A Locker represents an object that can be locked and unlocked. +type Locker interface { + Lock() + Unlock() } // Lock locks m. // If the lock is already in use, the calling goroutine // blocks until the mutex is available. func (m *Mutex) Lock() { - if xadd(&m.key, 1) == 1 { + if atomic.AddInt32(&m.key, 1) == 1 { // changed from 0 to 1; we hold lock return } @@ -53,11 +46,11 @@ func (m *Mutex) Lock() { // It is allowed for one goroutine to lock a Mutex and then // arrange for another goroutine to unlock it. func (m *Mutex) Unlock() { - switch v := xadd(&m.key, -1); { + switch v := atomic.AddInt32(&m.key, -1); { case v == 0: // changed from 1 to 0; no contention return - case int32(v) == -1: + case v == -1: // changed from 0 to -1: wasn't locked // (or there are 4 billion goroutines waiting) panic("sync: unlock of unlocked mutex") diff --git a/src/pkg/sync/rwmutex.go b/src/pkg/sync/rwmutex.go index 25696aca2..9248b4b03 100644 --- a/src/pkg/sync/rwmutex.go +++ b/src/pkg/sync/rwmutex.go @@ -4,6 +4,8 @@ package sync +import "sync/atomic" + // An RWMutex is a reader/writer mutual exclusion lock. // The lock can be held by an arbitrary number of readers // or a single writer. @@ -14,9 +16,9 @@ package sync // Writers take priority over Readers: no new RLocks // are granted while a blocked Lock call is waiting. type RWMutex struct { - w Mutex // held if there are pending readers or writers - r Mutex // held if the w is being rd - readerCount uint32 // number of pending readers + w Mutex // held if there are pending readers or writers + r Mutex // held if the w is being rd + readerCount int32 // number of pending readers } // RLock locks rw for reading. @@ -33,7 +35,7 @@ func (rw *RWMutex) RLock() { // B: rw.RUnlock() // ... (new readers come and go indefinitely, W is starving) rw.r.Lock() - if xadd(&rw.readerCount, 1) == 1 { + if atomic.AddInt32(&rw.readerCount, 1) == 1 { // The first reader locks rw.w, so writers will be blocked // while the readers have the RLock. rw.w.Lock() @@ -46,7 +48,7 @@ func (rw *RWMutex) RLock() { // It is a run-time error if rw is not locked for reading // on entry to RUnlock. func (rw *RWMutex) RUnlock() { - if xadd(&rw.readerCount, -1) == 0 { + if atomic.AddInt32(&rw.readerCount, -1) == 0 { // last reader finished, enable writers rw.w.Unlock() } @@ -71,3 +73,14 @@ func (rw *RWMutex) Lock() { // goroutine. One goroutine may RLock (Lock) an RWMutex and then // arrange for another goroutine to RUnlock (Unlock) it. func (rw *RWMutex) Unlock() { rw.w.Unlock() } + +// RLocker returns a Locker interface that implements +// the Lock and Unlock methods by calling rw.RLock and rw.RUnlock. +func (rw *RWMutex) RLocker() Locker { + return (*rlocker)(rw) +} + +type rlocker RWMutex + +func (r *rlocker) Lock() { (*RWMutex)(r).RLock() } +func (r *rlocker) Unlock() { (*RWMutex)(r).RUnlock() } diff --git a/src/pkg/sync/rwmutex_test.go b/src/pkg/sync/rwmutex_test.go index 111bca1e3..405079270 100644 --- a/src/pkg/sync/rwmutex_test.go +++ b/src/pkg/sync/rwmutex_test.go @@ -10,6 +10,7 @@ import ( "fmt" "runtime" . "sync" + "sync/atomic" "testing" ) @@ -49,31 +50,31 @@ func TestParallelReaders(t *testing.T) { doTestParallelReaders(4, 2) } -func reader(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) { +func reader(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) { for i := 0; i < num_iterations; i++ { rwm.RLock() - n := Xadd(activity, 1) + n := atomic.AddInt32(activity, 1) if n < 1 || n >= 10000 { panic(fmt.Sprintf("wlock(%d)\n", n)) } for i := 0; i < 100; i++ { } - Xadd(activity, -1) + atomic.AddInt32(activity, -1) rwm.RUnlock() } cdone <- true } -func writer(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) { +func writer(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) { for i := 0; i < num_iterations; i++ { rwm.Lock() - n := Xadd(activity, 10000) + n := atomic.AddInt32(activity, 10000) if n != 10000 { panic(fmt.Sprintf("wlock(%d)\n", n)) } for i := 0; i < 100; i++ { } - Xadd(activity, -10000) + atomic.AddInt32(activity, -10000) rwm.Unlock() } cdone <- true @@ -82,7 +83,7 @@ func writer(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) { runtime.GOMAXPROCS(gomaxprocs) // Number of active readers + 10000 * number of active writers. - var activity uint32 + var activity int32 var rwm RWMutex cdone := make(chan bool) go writer(&rwm, num_iterations, &activity, cdone) @@ -112,3 +113,38 @@ func TestRWMutex(t *testing.T) { HammerRWMutex(10, 10, 1000) HammerRWMutex(10, 5, 10000) } + +func TestRLocker(t *testing.T) { + var wl RWMutex + var rl Locker + wlocked := make(chan bool, 1) + rlocked := make(chan bool, 1) + rl = wl.RLocker() + n := 10 + go func() { + for i := 0; i < n; i++ { + rl.Lock() + rl.Lock() + rlocked <- true + wl.Lock() + wlocked <- true + } + }() + for i := 0; i < n; i++ { + <-rlocked + rl.Unlock() + select { + case <-wlocked: + t.Fatal("RLocker() didn't read-lock it") + default: + } + rl.Unlock() + <-wlocked + select { + case <-rlocked: + t.Fatal("RLocker() didn't respect the write lock") + default: + } + wl.Unlock() + } +} diff --git a/src/pkg/sync/xadd_test.go b/src/pkg/sync/xadd_test.go deleted file mode 100644 index 8b2ef76e6..000000000 --- a/src/pkg/sync/xadd_test.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sync - -func Xadd(val *uint32, delta int32) (new uint32) { - return xadd(val, delta) -} diff --git a/src/pkg/syscall/exec_unix.go b/src/pkg/syscall/exec_unix.go index 04c066918..2e09539ee 100644 --- a/src/pkg/syscall/exec_unix.go +++ b/src/pkg/syscall/exec_unix.go @@ -238,6 +238,10 @@ func forkExec(argv0 string, argv []string, envv []string, traceme bool, dir stri dirp = StringBytePtr(dir) } + if OS == "freebsd" && len(argv[0]) > len(argv0) { + argvp[0] = argv0p + } + // Acquire the fork lock so that no other threads // create new fds that are not yet close-on-exec // before we fork. diff --git a/src/pkg/syscall/exec_windows.go b/src/pkg/syscall/exec_windows.go index 7256c3ae3..73c3c8624 100644 --- a/src/pkg/syscall/exec_windows.go +++ b/src/pkg/syscall/exec_windows.go @@ -185,7 +185,7 @@ func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []i startupInfo, processInfo) - if err != 0 { + if err == 0 { pid = int(processInfo.ProcessId) handle = int(processInfo.Process) CloseHandle(processInfo.Thread) diff --git a/src/pkg/syscall/mkerrors.sh b/src/pkg/syscall/mkerrors.sh index a13c0e91b..d40d1f6b3 100755 --- a/src/pkg/syscall/mkerrors.sh +++ b/src/pkg/syscall/mkerrors.sh @@ -116,7 +116,7 @@ done $2 ~ /^E[A-Z0-9_]+$/ || $2 ~ /^SIG[^_]/ || $2 ~ /^IN_/ || - $2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|EVFILT|EV|SHUT|PROT|MAP|PACKET|MSG|SCM|IFF|NET_RT|RTM|RTF|RTV|RTA|RTAX)_/ || + $2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|EVFILT|EV|SHUT|PROT|MAP|PACKET|MSG|SCM|IFF|NET_RT|RTM|RTF|RTV|RTA|RTAX|MCL)_/ || $2 == "SOMAXCONN" || $2 == "NAME_MAX" || $2 == "IFNAMSIZ" || diff --git a/src/pkg/syscall/syscall_windows.go b/src/pkg/syscall/syscall_windows.go index 0cd89d426..394e06442 100644 --- a/src/pkg/syscall/syscall_windows.go +++ b/src/pkg/syscall/syscall_windows.go @@ -154,6 +154,7 @@ func NewCallback(fn interface{}) uintptr //sys SetEnvironmentVariable(name *uint16, value *uint16) (errno int) = kernel32.SetEnvironmentVariableW //sys SetFileTime(handle int32, ctime *Filetime, atime *Filetime, wtime *Filetime) (errno int) //sys GetFileAttributes(name *uint16) (attrs uint32, errno int) [failretval==INVALID_FILE_ATTRIBUTES] = kernel32.GetFileAttributesW +//sys SetFileAttributes(name *uint16, attrs uint32) (errno int) = kernel32.SetFileAttributesW //sys GetCommandLine() (cmd *uint16) = kernel32.GetCommandLineW //sys CommandLineToArgv(cmd *uint16, argc *int32) (argv *[8192]*[8192]uint16, errno int) [failretval==nil] = shell32.CommandLineToArgvW //sys LocalFree(hmem uint32) (handle uint32, errno int) [failretval!=0] @@ -470,6 +471,23 @@ func Fsync(fd int) (errno int) { return FlushFileBuffers(int32(fd)) } +func Chmod(path string, mode uint32) (errno int) { + if mode == 0 { + return EINVAL + } + p := StringToUTF16Ptr(path) + attrs, e := GetFileAttributes(p) + if e != 0 { + return e + } + if mode&S_IWRITE != 0 { + attrs &^= FILE_ATTRIBUTE_READONLY + } else { + attrs |= FILE_ATTRIBUTE_READONLY + } + return SetFileAttributes(p, attrs) +} + // net api calls //sys WSAStartup(verreq uint32, data *WSAData) (sockerrno int) = wsock32.WSAStartup @@ -635,26 +653,6 @@ func Shutdown(fd, how int) (errno int) { return int(shutdown(int32(fd), int32(how))) } -func AcceptIOCP(iocpfd, fd int, o *Overlapped) (attrs *byte, errno int) { - // Will ask for local and remote address only. - rsa := make([]RawSockaddrAny, 2) - attrs = (*byte)(unsafe.Pointer(&rsa[0])) - alen := uint32(unsafe.Sizeof(rsa[0])) - var done uint32 - errno = AcceptEx(uint32(iocpfd), uint32(fd), attrs, 0, alen, alen, &done, o) - return -} - -func GetAcceptIOCPSockaddrs(attrs *byte) (lsa, rsa Sockaddr) { - var lrsa, rrsa *RawSockaddrAny - var llen, rlen int32 - alen := uint32(unsafe.Sizeof(*lrsa)) - GetAcceptExSockaddrs(attrs, 0, alen, alen, &lrsa, &llen, &rrsa, &rlen) - lsa, _ = lrsa.Sockaddr() - rsa, _ = rrsa.Sockaddr() - return -} - func WSASendto(s uint32, bufs *WSABuf, bufcnt uint32, sent *uint32, flags uint32, to Sockaddr, overlapped *Overlapped, croutine *byte) (errno int) { rsa, l, err := to.sockaddr() if err != 0 { @@ -702,8 +700,19 @@ type Linger struct { Linger int32 } -func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) { return EWINDOWS } -func BindToDevice(fd int, device string) (errno int) { return EWINDOWS } +const ( + IP_ADD_MEMBERSHIP = iota + IP_DROP_MEMBERSHIP +) + +type IpMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) { return EWINDOWS } +func SetsockoptIpMreq(fd, level, opt int, mreq *IpMreq) (errno int) { return EWINDOWS } +func BindToDevice(fd int, device string) (errno int) { return EWINDOWS } // TODO(brainman): fix all needed for os @@ -718,11 +727,11 @@ func Fchdir(fd int) (errno int) { return EWINDOWS } func Link(oldpath, newpath string) (errno int) { return EWINDOWS } func Symlink(path, link string) (errno int) { return EWINDOWS } func Readlink(path string, buf []byte) (n int, errno int) { return 0, EWINDOWS } -func Chmod(path string, mode uint32) (errno int) { return EWINDOWS } -func Fchmod(fd int, mode uint32) (errno int) { return EWINDOWS } -func Chown(path string, uid int, gid int) (errno int) { return EWINDOWS } -func Lchown(path string, uid int, gid int) (errno int) { return EWINDOWS } -func Fchown(fd int, uid int, gid int) (errno int) { return EWINDOWS } + +func Fchmod(fd int, mode uint32) (errno int) { return EWINDOWS } +func Chown(path string, uid int, gid int) (errno int) { return EWINDOWS } +func Lchown(path string, uid int, gid int) (errno int) { return EWINDOWS } +func Fchown(fd int, uid int, gid int) (errno int) { return EWINDOWS } func Getuid() (uid int) { return -1 } func Geteuid() (euid int) { return -1 } diff --git a/src/pkg/syscall/zerrors_freebsd_amd64.go b/src/pkg/syscall/zerrors_freebsd_amd64.go index ce3ff5402..770d293a2 100644 --- a/src/pkg/syscall/zerrors_freebsd_amd64.go +++ b/src/pkg/syscall/zerrors_freebsd_amd64.go @@ -698,7 +698,6 @@ const ( SIOCGIFBRDADDR = 0xc0206923 SIOCGIFCAP = 0xc020691f SIOCGIFCONF = 0xc0106924 - SIOCGIFCONF32 = 0xc0086924 SIOCGIFDESCR = 0xc020692a SIOCGIFDSTADDR = 0xc0206922 SIOCGIFFLAGS = 0xc0206911 diff --git a/src/pkg/syscall/zerrors_linux_386.go b/src/pkg/syscall/zerrors_linux_386.go index 43d27827c..a73a94961 100644 --- a/src/pkg/syscall/zerrors_linux_386.go +++ b/src/pkg/syscall/zerrors_linux_386.go @@ -424,6 +424,8 @@ const ( MAP_SHARED = 0x1 MAP_STACK = 0x20000 MAP_TYPE = 0xf + MCL_CURRENT = 0x1 + MCL_FUTURE = 0x2 MSG_CMSG_CLOEXEC = 0x40000000 MSG_CONFIRM = 0x800 MSG_CTRUNC = 0x8 diff --git a/src/pkg/syscall/zerrors_linux_amd64.go b/src/pkg/syscall/zerrors_linux_amd64.go index 04f4dad2c..eea55a275 100644 --- a/src/pkg/syscall/zerrors_linux_amd64.go +++ b/src/pkg/syscall/zerrors_linux_amd64.go @@ -424,6 +424,8 @@ const ( MAP_SHARED = 0x1 MAP_STACK = 0x20000 MAP_TYPE = 0xf + MCL_CURRENT = 0x1 + MCL_FUTURE = 0x2 MSG_CMSG_CLOEXEC = 0x40000000 MSG_CONFIRM = 0x800 MSG_CTRUNC = 0x8 diff --git a/src/pkg/syscall/zsyscall_windows_386.go b/src/pkg/syscall/zsyscall_windows_386.go index 46e16f43c..543992ea6 100644 --- a/src/pkg/syscall/zsyscall_windows_386.go +++ b/src/pkg/syscall/zsyscall_windows_386.go @@ -63,6 +63,7 @@ var ( procSetEnvironmentVariableW = getSysProcAddr(modkernel32, "SetEnvironmentVariableW") procSetFileTime = getSysProcAddr(modkernel32, "SetFileTime") procGetFileAttributesW = getSysProcAddr(modkernel32, "GetFileAttributesW") + procSetFileAttributesW = getSysProcAddr(modkernel32, "SetFileAttributesW") procGetCommandLineW = getSysProcAddr(modkernel32, "GetCommandLineW") procCommandLineToArgvW = getSysProcAddr(modshell32, "CommandLineToArgvW") procLocalFree = getSysProcAddr(modkernel32, "LocalFree") @@ -806,6 +807,20 @@ func GetFileAttributes(name *uint16) (attrs uint32, errno int) { return } +func SetFileAttributes(name *uint16, attrs uint32) (errno int) { + r1, _, e1 := Syscall(procSetFileAttributesW, 2, uintptr(unsafe.Pointer(name)), uintptr(attrs), 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = EINVAL + } + } else { + errno = 0 + } + return +} + func GetCommandLine() (cmd *uint16) { r0, _, _ := Syscall(procGetCommandLineW, 0, 0, 0, 0) cmd = (*uint16)(unsafe.Pointer(r0)) diff --git a/src/pkg/syscall/zsysnum_freebsd_amd64.go b/src/pkg/syscall/zsysnum_freebsd_amd64.go index 4286b487d..a5b7b664f 100644 --- a/src/pkg/syscall/zsysnum_freebsd_amd64.go +++ b/src/pkg/syscall/zsysnum_freebsd_amd64.go @@ -1,4 +1,4 @@ -// mksysnum_freebsd.sh +// mksysnum_freebsd.sh // MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT package syscall diff --git a/src/pkg/syscall/ztypes_windows_386.go b/src/pkg/syscall/ztypes_windows_386.go index e9ab35461..ff367a858 100644 --- a/src/pkg/syscall/ztypes_windows_386.go +++ b/src/pkg/syscall/ztypes_windows_386.go @@ -28,6 +28,7 @@ const ( ERROR_PROC_NOT_FOUND = 127 ERROR_ENVVAR_NOT_FOUND = 203 ERROR_DIRECTORY = 267 + ERROR_OPERATION_ABORTED = 995 ERROR_IO_PENDING = 997 ) @@ -380,6 +381,7 @@ const ( S_ISGID = 0x400 S_ISVTX = 0x200 S_IRUSR = 0x100 + S_IWRITE = 0x80 S_IWUSR = 0x80 S_IXUSR = 0x40 ) diff --git a/src/pkg/template/template.go b/src/pkg/template/template.go index 36fd06dc2..c3cb8901a 100644 --- a/src/pkg/template/template.go +++ b/src/pkg/template/template.go @@ -24,7 +24,8 @@ - The result of invoking a niladic single-valued method with that name (result = data.field()) - Major constructs ({} are metacharacters; [] marks optional elements): + Major constructs ({} are the default delimiters for template actions; + [] are the notation in this comment for optional elements): {# comment } @@ -74,6 +75,11 @@ Multiple formatters separated by the pipeline character | are executed sequentially, with each formatter receiving the bytes emitted by the one to its left. + + The delimiter strings get their default value, "{" and "}", from + JSON-template. They may be set to any non-empty, space-free + string using the SetDelims method. Their value can be printed + in the output using {.meta-left} and {.meta-right}. */ package template diff --git a/src/pkg/testing/benchmark.go b/src/pkg/testing/benchmark.go index ad938027d..cf73e2b48 100644 --- a/src/pkg/testing/benchmark.go +++ b/src/pkg/testing/benchmark.go @@ -8,10 +8,11 @@ import ( "flag" "fmt" "os" + "runtime" "time" ) -var matchBenchmarks = flag.String("benchmarks", "", "regular expression to select benchmarks to run") +var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run") // An internal type but exported because it is cross-package; part of the implementation // of gotest. @@ -64,6 +65,9 @@ func (b *B) nsPerOp() int64 { // runN runs a single benchmark for the specified number of iterations. func (b *B) runN(n int) { + // Try to get a comparable environment for each run + // by clearing garbage from previous runs. + runtime.GC() b.N = n b.ResetTimer() b.StartTimer() @@ -175,7 +179,7 @@ func RunBenchmarks(matchString func(pat, str string) (bool, os.Error), benchmark for _, Benchmark := range benchmarks { matched, err := matchString(*matchBenchmarks, Benchmark.Name) if err != nil { - println("invalid regexp for -benchmarks:", err.String()) + println("invalid regexp for -test.bench:", err.String()) os.Exit(1) } if !matched { diff --git a/src/pkg/testing/testing.go b/src/pkg/testing/testing.go index edbf0847c..324b5a70e 100644 --- a/src/pkg/testing/testing.go +++ b/src/pkg/testing/testing.go @@ -12,7 +12,7 @@ // // Functions of the form // func BenchmarkXxx(*testing.B) -// are considered benchmarks, and are executed by gotest when the -benchmarks +// are considered benchmarks, and are executed by gotest when the -test.bench // flag is provided. // // A sample benchmark function looks like this: @@ -47,8 +47,8 @@ import ( ) // Report as tests are run; default is silent for success. -var chatty = flag.Bool("v", false, "verbose: print additional output") -var match = flag.String("match", "", "regular expression to select tests to run") +var chatty = flag.Bool("test.v", false, "verbose: print additional output") +var match = flag.String("test.run", "", "regular expression to select tests to run") // Insert final newline if needed and tabs after internal newlines. @@ -92,7 +92,7 @@ func (t *T) FailNow() { // and records the text in the error log. func (t *T) Log(args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintln(args...)) } -// Log formats its arguments according to the format, analogous to Printf(), +// Logf formats its arguments according to the format, analogous to Printf(), // and records the text in the error log. func (t *T) Logf(format string, args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintf(format, args...)) @@ -145,7 +145,7 @@ func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTe for i := 0; i < len(tests); i++ { matched, err := matchString(*match, tests[i].Name) if err != nil { - println("invalid regexp for -match:", err.String()) + println("invalid regexp for -test.run:", err.String()) os.Exit(1) } if !matched { diff --git a/src/pkg/time/Makefile b/src/pkg/time/Makefile index 5213e4457..3fa96065e 100644 --- a/src/pkg/time/Makefile +++ b/src/pkg/time/Makefile @@ -8,6 +8,7 @@ TARG=time GOFILES=\ format.go\ sleep.go\ + sys.go\ tick.go\ time.go\ diff --git a/src/pkg/time/sleep.go b/src/pkg/time/sleep.go index 833552d68..7b3f01f01 100644 --- a/src/pkg/time/sleep.go +++ b/src/pkg/time/sleep.go @@ -5,7 +5,6 @@ package time import ( - "os" "syscall" "sync" "container/heap" @@ -47,30 +46,6 @@ func init() { timers.Push(&Timer{t: forever}) // sentinel } -// Sleep pauses the current goroutine for at least ns nanoseconds. -// Higher resolution sleeping may be provided by syscall.Nanosleep -// on some operating systems. -func Sleep(ns int64) os.Error { - _, err := sleep(Nanoseconds(), ns) - return err -} - -// sleep takes the current time and a duration, -// pauses for at least ns nanoseconds, and -// returns the current time and an error. -func sleep(t, ns int64) (int64, os.Error) { - // TODO(cw): use monotonic-time once it's available - end := t + ns - for t < end { - errno := syscall.Sleep(end - t) - if errno != 0 && errno != syscall.EINTR { - return 0, os.NewSyscallError("sleep", errno) - } - t = Nanoseconds() - } - return t, nil -} - // NewTimer creates a new Timer that will send // the current time on its channel after at least ns nanoseconds. func NewTimer(ns int64) *Timer { diff --git a/src/pkg/time/sys.go b/src/pkg/time/sys.go new file mode 100644 index 000000000..8a2e6fadc --- /dev/null +++ b/src/pkg/time/sys.go @@ -0,0 +1,54 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time + +import ( + "os" + "syscall" +) + +// Seconds reports the number of seconds since the Unix epoch, +// January 1, 1970 00:00:00 UTC. +func Seconds() int64 { + sec, _, err := os.Time() + if err != nil { + panic(err) + } + return sec +} + +// Nanoseconds reports the number of nanoseconds since the Unix epoch, +// January 1, 1970 00:00:00 UTC. +func Nanoseconds() int64 { + sec, nsec, err := os.Time() + if err != nil { + panic(err) + } + return sec*1e9 + nsec +} + +// Sleep pauses the current goroutine for at least ns nanoseconds. +// Higher resolution sleeping may be provided by syscall.Nanosleep +// on some operating systems. +func Sleep(ns int64) os.Error { + _, err := sleep(Nanoseconds(), ns) + return err +} + +// sleep takes the current time and a duration, +// pauses for at least ns nanoseconds, and +// returns the current time and an error. +func sleep(t, ns int64) (int64, os.Error) { + // TODO(cw): use monotonic-time once it's available + end := t + ns + for t < end { + errno := syscall.Sleep(end - t) + if errno != 0 && errno != syscall.EINTR { + return 0, os.NewSyscallError("sleep", errno) + } + t = Nanoseconds() + } + return t, nil +} diff --git a/src/pkg/time/time.go b/src/pkg/time/time.go index 4abd11230..432b3523a 100644 --- a/src/pkg/time/time.go +++ b/src/pkg/time/time.go @@ -6,30 +6,6 @@ // displaying time. package time -import ( - "os" -) - -// Seconds reports the number of seconds since the Unix epoch, -// January 1, 1970 00:00:00 UTC. -func Seconds() int64 { - sec, _, err := os.Time() - if err != nil { - panic(err) - } - return sec -} - -// Nanoseconds reports the number of nanoseconds since the Unix epoch, -// January 1, 1970 00:00:00 UTC. -func Nanoseconds() int64 { - sec, nsec, err := os.Time() - if err != nil { - panic(err) - } - return sec*1e9 + nsec -} - // Days of the week. const ( Sunday = iota diff --git a/src/pkg/unsafe/unsafe.go b/src/pkg/unsafe/unsafe.go index 3a4e30c00..3cd4cff6e 100644 --- a/src/pkg/unsafe/unsafe.go +++ b/src/pkg/unsafe/unsafe.go @@ -41,13 +41,14 @@ func Typeof(i interface{}) (typ interface{}) // Reflect unpacks an interface value into its type and the address of a copy of the // internal value. -func Reflect(i interface{}) (typ interface{}, addr uintptr) +func Reflect(i interface{}) (typ interface{}, addr Pointer) -// Unreflect inverts Reflect: Given a type and a pointer, it returns an empty interface value -// with those contents. The typ is assumed to contain a pointer to a runtime type; -// the type information in the interface{} is ignored, so that, for example, both +// Unreflect inverts Reflect: Given a type and a pointer to a value, it returns an +// empty interface value with contents the type and the value (not the pointer to +// the value). The typ is assumed to contain a pointer to a runtime type; the type +// information in the interface{} is ignored, so that, for example, both // *reflect.StructType and *runtime.StructType can be passed for typ. -func Unreflect(typ interface{}, addr uintptr) (ret interface{}) +func Unreflect(typ interface{}, addr Pointer) (ret interface{}) // New allocates and returns a pointer to memory for a new value of the given type. // The typ is assumed to hold a pointer to a runtime type. diff --git a/src/pkg/websocket/client.go b/src/pkg/websocket/client.go index 091345944..d8a7aa0a2 100644 --- a/src/pkg/websocket/client.go +++ b/src/pkg/websocket/client.go @@ -245,20 +245,20 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. } // Step 41. check websocket headers. - if resp.Header["Upgrade"] != "WebSocket" || - strings.ToLower(resp.Header["Connection"]) != "upgrade" { + if resp.Header.Get("Upgrade") != "WebSocket" || + strings.ToLower(resp.Header.Get("Connection")) != "upgrade" { return ErrBadUpgrade } - if resp.Header["Sec-Websocket-Origin"] != origin { + if resp.Header.Get("Sec-Websocket-Origin") != origin { return ErrBadWebSocketOrigin } - if resp.Header["Sec-Websocket-Location"] != location { + if resp.Header.Get("Sec-Websocket-Location") != location { return ErrBadWebSocketLocation } - if protocol != "" && resp.Header["Sec-Websocket-Protocol"] != protocol { + if protocol != "" && resp.Header.Get("Sec-Websocket-Protocol") != protocol { return ErrBadWebSocketProtocol } @@ -304,17 +304,17 @@ func draft75handshake(resourceName, host, origin, location, protocol string, br if resp.Status != "101 Web Socket Protocol Handshake" { return ErrBadStatus } - if resp.Header["Upgrade"] != "WebSocket" || - resp.Header["Connection"] != "Upgrade" { + if resp.Header.Get("Upgrade") != "WebSocket" || + resp.Header.Get("Connection") != "Upgrade" { return ErrBadUpgrade } - if resp.Header["Websocket-Origin"] != origin { + if resp.Header.Get("Websocket-Origin") != origin { return ErrBadWebSocketOrigin } - if resp.Header["Websocket-Location"] != location { + if resp.Header.Get("Websocket-Location") != location { return ErrBadWebSocketLocation } - if protocol != "" && resp.Header["Websocket-Protocol"] != protocol { + if protocol != "" && resp.Header.Get("Websocket-Protocol") != protocol { return ErrBadWebSocketProtocol } return diff --git a/src/pkg/websocket/server.go b/src/pkg/websocket/server.go index dd797f24e..37149f044 100644 --- a/src/pkg/websocket/server.go +++ b/src/pkg/websocket/server.go @@ -58,7 +58,7 @@ func getKeyNumber(s string) (r uint32) { // ServeHTTP implements the http.Handler interface for a Web Socket func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - rwc, buf, err := w.Hijack() + rwc, buf, err := w.(http.Hijacker).Hijack() if err != nil { panic("Hijack failed: " + err.String()) return @@ -73,23 +73,23 @@ func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // HTTP version can be safely ignored. - if strings.ToLower(req.Header["Upgrade"]) != "websocket" || - strings.ToLower(req.Header["Connection"]) != "upgrade" { + if strings.ToLower(req.Header.Get("Upgrade")) != "websocket" || + strings.ToLower(req.Header.Get("Connection")) != "upgrade" { return } // TODO(ukai): check Host - origin, found := req.Header["Origin"] - if !found { + origin := req.Header.Get("Origin") + if origin == "" { return } - key1, found := req.Header["Sec-Websocket-Key1"] - if !found { + key1 := req.Header.Get("Sec-Websocket-Key1") + if key1 == "" { return } - key2, found := req.Header["Sec-Websocket-Key2"] - if !found { + key2 := req.Header.Get("Sec-Websocket-Key2") + if key2 == "" { return } key3 := make([]byte, 8) @@ -138,8 +138,8 @@ func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { buf.WriteString("Connection: Upgrade\r\n") buf.WriteString("Sec-WebSocket-Location: " + location + "\r\n") buf.WriteString("Sec-WebSocket-Origin: " + origin + "\r\n") - protocol, found := req.Header["Sec-Websocket-Protocol"] - if found { + protocol := strings.TrimSpace(req.Header.Get("Sec-Websocket-Protocol")) + if protocol != "" { buf.WriteString("Sec-WebSocket-Protocol: " + protocol + "\r\n") } // Step 12. send CRLF. @@ -167,24 +167,24 @@ func (f Draft75Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { io.WriteString(w, "Unexpected request") return } - if req.Header["Upgrade"] != "WebSocket" { + if req.Header.Get("Upgrade") != "WebSocket" { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, "missing Upgrade: WebSocket header") return } - if req.Header["Connection"] != "Upgrade" { + if req.Header.Get("Connection") != "Upgrade" { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, "missing Connection: Upgrade header") return } - origin, found := req.Header["Origin"] - if !found { + origin := strings.TrimSpace(req.Header.Get("Origin")) + if origin == "" { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, "missing Origin header") return } - rwc, buf, err := w.Hijack() + rwc, buf, err := w.(http.Hijacker).Hijack() if err != nil { panic("Hijack failed: " + err.String()) return @@ -205,9 +205,9 @@ func (f Draft75Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { buf.WriteString("Connection: Upgrade\r\n") buf.WriteString("WebSocket-Origin: " + origin + "\r\n") buf.WriteString("WebSocket-Location: " + location + "\r\n") - protocol, found := req.Header["Websocket-Protocol"] + protocol := strings.TrimSpace(req.Header.Get("Websocket-Protocol")) // canonical header key of WebSocket-Protocol. - if found { + if protocol != "" { buf.WriteString("WebSocket-Protocol: " + protocol + "\r\n") } buf.WriteString("\r\n") diff --git a/src/pkg/websocket/websocket_test.go b/src/pkg/websocket/websocket_test.go index 204a9de1e..14d708a3b 100644 --- a/src/pkg/websocket/websocket_test.go +++ b/src/pkg/websocket/websocket_test.go @@ -9,6 +9,7 @@ import ( "bytes" "fmt" "http" + "http/httptest" "io" "log" "net" @@ -22,15 +23,11 @@ var once sync.Once func echoServer(ws *Conn) { io.Copy(ws, ws) } func startServer() { - l, e := net.Listen("tcp", "127.0.0.1:0") // any available address - if e != nil { - log.Fatalf("net.Listen tcp :0 %v", e) - } - serverAddr = l.Addr().String() - log.Print("Test WebSocket server listening on ", serverAddr) http.Handle("/echo", Handler(echoServer)) http.Handle("/echoDraft75", Draft75Handler(echoServer)) - go http.Serve(l, nil) + server := httptest.NewServer(nil) + serverAddr = server.Listener.Addr().String() + log.Print("Test WebSocket server listening on ", serverAddr) } // Test the getChallengeResponse function with values from section diff --git a/src/pkg/xml/read_test.go b/src/pkg/xml/read_test.go index 71ceddce4..a6b9a8ed1 100644 --- a/src/pkg/xml/read_test.go +++ b/src/pkg/xml/read_test.go @@ -13,16 +13,16 @@ import ( func TestUnmarshalFeed(t *testing.T) { var f Feed - if err := Unmarshal(StringReader(rssFeedString), &f); err != nil { + if err := Unmarshal(StringReader(atomFeedString), &f); err != nil { t.Fatalf("Unmarshal: %s", err) } - if !reflect.DeepEqual(f, rssFeed) { - t.Fatalf("have %#v\nwant %#v", f, rssFeed) + if !reflect.DeepEqual(f, atomFeed) { + t.Fatalf("have %#v\nwant %#v", f, atomFeed) } } // hget http://codereview.appspot.com/rss/mine/rsc -const rssFeedString = ` +const atomFeedString = ` <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-us"><title>Code Review - My issues</title><link href="http://codereview.appspot.com/" rel="alternate"></link><li-nk href="http://codereview.appspot.com/rss/mine/rsc" rel="self"></li-nk><id>http://codereview.appspot.com/</id><updated>2009-10-04T01:35:58+00:00</updated><author><name>rietveld<></name></author><entry><title>rietveld: an attempt at pubsubhubbub </title><link hre-f="http://codereview.appspot.com/126085" rel="alternate"></link><updated>2009-10-04T01:35:58+00:00</updated><author><name>email-address-removed</name></author><id>urn:md5:134d9179c41f806be79b3a5f7877d19a</id><summary type="html"> @@ -115,7 +115,7 @@ type Text struct { type Time string -var rssFeed = Feed{ +var atomFeed = Feed{ XMLName: Name{"http://www.w3.org/2005/Atom", "feed"}, Title: "Code Review - My issues", Link: []Link{ diff --git a/src/pkg/xml/xml.go b/src/pkg/xml/xml.go index 417b4edfd..691c13a11 100644 --- a/src/pkg/xml/xml.go +++ b/src/pkg/xml/xml.go @@ -541,17 +541,36 @@ func (p *Parser) RawToken() (Token, os.Error) { } // Probably a directive: <!DOCTYPE ...>, <!ENTITY ...>, etc. - // We don't care, but accumulate for caller. + // We don't care, but accumulate for caller. Quoted angle + // brackets do not count for nesting. p.buf.Reset() p.buf.WriteByte(b) + inquote := uint8(0) + depth := 0 for { if b, ok = p.mustgetc(); !ok { return nil, p.err } - if b == '>' { + if inquote == 0 && b == '>' && depth == 0 { break } p.buf.WriteByte(b) + switch { + case b == inquote: + inquote = 0 + + case inquote != 0: + // in quotes, no special action + + case b == '\'' || b == '"': + inquote = b + + case b == '>' && inquote == 0: + depth-- + + case b == '<' && inquote == 0: + depth++ + } } return Directive(p.buf.Bytes()), nil } diff --git a/src/pkg/xml/xml_test.go b/src/pkg/xml/xml_test.go index 317ecabd9..887bc3d14 100644 --- a/src/pkg/xml/xml_test.go +++ b/src/pkg/xml/xml_test.go @@ -185,6 +185,52 @@ func TestRawToken(t *testing.T) { } } +// Ensure that directives (specifically !DOCTYPE) include the complete +// text of any nested directives, noting that < and > do not change +// nesting depth if they are in single or double quotes. + +var nestedDirectivesInput = ` +<!DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]> +<!DOCTYPE [<!ENTITY xlt ">">]> +<!DOCTYPE [<!ENTITY xlt "<">]> +<!DOCTYPE [<!ENTITY xlt '>'>]> +<!DOCTYPE [<!ENTITY xlt '<'>]> +<!DOCTYPE [<!ENTITY xlt '">'>]> +<!DOCTYPE [<!ENTITY xlt "'<">]> +` + +var nestedDirectivesTokens = []Token{ + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt ">">]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt "<">]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt '>'>]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt '<'>]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt '">'>]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt "'<">]`)), + CharData([]byte("\n")), +} + +func TestNestedDirectives(t *testing.T) { + p := NewParser(StringReader(nestedDirectivesInput)) + + for i, want := range nestedDirectivesTokens { + have, err := p.Token() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + t.Errorf("token %d = %#v want %#v", i, have, want) + } + } +} + func TestToken(t *testing.T) { p := NewParser(StringReader(testInput)) diff --git a/src/run.bash b/src/run.bash index a3e90cc4f..aec490109 100755 --- a/src/run.bash +++ b/src/run.bash @@ -91,7 +91,6 @@ time gomake ogle time ./run ) || exit $? -[ "$GOHOSTOS" == windows ] || (xcd ../doc/codelab/wiki gomake clean gomake diff --git a/test/bench/Makefile b/test/bench/Makefile new file mode 100644 index 000000000..145fe0cea --- /dev/null +++ b/test/bench/Makefile @@ -0,0 +1,14 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../src/Make.inc + +all: + @echo "make clean or timing" + +timing: + ./timing.sh + +clean: + rm -f [568].out *.[568] diff --git a/test/bench/clean.bash b/test/bench/clean.bash deleted file mode 100755 index d56c0e394..000000000 --- a/test/bench/clean.bash +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -OS=568 -rm -f [$OS].out *.[$OS] diff --git a/test/bench/timing.log b/test/bench/timing.log index f2b6a1f40..a967f27d2 100644 --- a/test/bench/timing.log +++ b/test/bench/timing.log @@ -592,3 +592,85 @@ regex-dna 100000 gc regex-dna 6.80u 0.00s 6.81r gc regex-dna-parallel 6.82u 0.01s 2.75r gc_B regex-dna 6.69u 0.02s 6.70r + +Feb 15, 2011 + +# Improved GC, still single-threaded but more efficient + +fasta -n 25000000 + gcc -O2 fasta.c 3.40u 0.00s 3.40r + gccgo -O2 fasta.go 3.51u 0.00s 3.50r + gc fasta 3.66u 0.01s 3.66r + gc_B fasta 3.66u 0.00s 3.66r + +reverse-complement < output-of-fasta-25000000 + gcc -O2 reverse-complement.c 1.86u 1.29s 4.93r + gccgo -O2 reverse-complement.go 2.18u 0.41s 2.60r + gc reverse-complement 1.67u 0.48s 2.15r + gc_B reverse-complement 1.71u 0.45s 2.15r + +nbody -n 50000000 + gcc -O2 -lm nbody.c 21.64u 0.00s 21.64r + gccgo -O2 nbody.go 21.46u 0.00s 21.45r + gc nbody 29.07u 0.00s 29.06r + gc_B nbody 31.61u 0.00s 31.61r + +binary-tree 15 # too slow to use 20 + gcc -O2 binary-tree.c -lm 0.88u 0.00s 0.87r + gccgo -O2 binary-tree.go 2.74u 0.07s 2.81r + gccgo -O2 binary-tree-freelist.go 0.01u 0.00s 0.00r + gc binary-tree 4.22u 0.02s 4.24r + gc binary-tree-freelist 0.54u 0.02s 0.55r + +fannkuch 12 + gcc -O2 fannkuch.c 57.64u 0.00s 57.64r + gccgo -O2 fannkuch.go 65.79u 0.00s 65.82r + gccgo -O2 fannkuch-parallel.go 160.91u 0.02s 43.90r + gc fannkuch 126.36u 0.03s 126.53r + gc fannkuch-parallel 175.23u 0.04s 45.49r + gc_B fannkuch 89.23u 0.00s 89.24r + +regex-dna 100000 + gcc -O2 regex-dna.c -lpcre 0.77u 0.01s 0.80r + gccgo -O2 regex-dna.go 12.38u 0.10s 12.52r + gccgo -O2 regex-dna-parallel.go 43.96u 4.64s 15.11r + gc regex-dna 7.03u 0.01s 7.05r + gc regex-dna-parallel 6.85u 0.05s 2.70r + gc_B regex-dna 6.87u 0.02s 6.89r + +spectral-norm 5500 + gcc -O2 spectral-norm.c -lm 12.29u 0.00s 12.28r + gccgo -O2 spectral-norm.go 11.79u 0.00s 11.79r + gc spectral-norm 24.00u 0.02s 24.05r + gc_B spectral-norm 24.59u 0.01s 24.59r + +k-nucleotide 1000000 + gcc -O2 k-nucleotide.c 9.75u 0.07s 9.82r + gccgo -O2 k-nucleotide.go 8.92u 0.06s 8.98r + gccgo -O2 k-nucleotide-parallel.go 8.40u 0.04s 2.76r + gc k-nucleotide 17.01u 0.03s 17.04r + gc k-nucleotide-parallel 16.51u 0.08s 6.21r + gc_B k-nucleotide 16.94u 0.08s 17.02r + +mandelbrot 16000 + gcc -O2 mandelbrot.c 54.60u 0.00s 54.66r + gccgo -O2 mandelbrot.go 59.38u 0.00s 59.41r + gc mandelbrot 64.93u 0.04s 65.08r + gc_B mandelbrot 64.85u 0.03s 64.92r + +meteor 2098 + gcc -O2 meteor-contest.c 0.10u 0.01s 0.10r + gccgo -O2 meteor-contest.go 0.11u 0.00s 0.11r + gc meteor-contest 0.18u 0.00s 0.17r + gc_B meteor-contest 0.17u 0.00s 0.16r + +pidigits 10000 + gcc -O2 pidigits.c -lgmp 2.24u 0.00s 2.23r + gccgo -O2 pidigits.go 14.05u 0.00s 14.06r + gc pidigits 6.34u 0.05s 6.38r + gc_B pidigits 6.37u 0.02s 6.38r + +threadring 50000000 + gcc -O2 threadring.c 30.50u 258.05s 325.72r + gccgo -O2 threadring.go 92.87u 748.39s 728.46r + gc threadring 38.03u 0.01s 38.04r diff --git a/test/bench/timing.sh b/test/bench/timing.sh index fec39182c..473c9b312 100755 --- a/test/bench/timing.sh +++ b/test/bench/timing.sh @@ -123,7 +123,8 @@ regexdna() { runonly a.out 100000 > x runonly echo 'regex-dna 100000' run 'gcc -O2 regex-dna.c -lpcre' a.out <x -# run 'gccgo -O2 regex-dna.go' a.out <x # restore after regexp.FindIndex is in library + run 'gccgo -O2 regex-dna.go' a.out <x + run 'gccgo -O2 regex-dna-parallel.go' a.out <x run 'gc regex-dna' $O.out <x run 'gc regex-dna-parallel' $O.out <x run 'gc_B regex-dna' $O.out <x diff --git a/test/bugs/bug324.dir/main.go b/test/bugs/bug324.dir/main.go new file mode 100644 index 000000000..37f2a59e4 --- /dev/null +++ b/test/bugs/bug324.dir/main.go @@ -0,0 +1,48 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "./p" +) + +type Exported interface { + private() +} + +type Implementation struct{} + +func (p *Implementation) private() { println("main.Implementation.private()") } + + +func main() { + // nothing unusual here + var x Exported + x = new(Implementation) + x.private() // main.Implementation.private() + + // same here - should be and is legal + var px p.Exported + px = p.X + + // this assignment is correctly illegal: + // px.private undefined (cannot refer to unexported field or method private) + // px.private() + + // this assignment is correctly illegal: + // *Implementation does not implement p.Exported (missing p.private method) + // px = new(Implementation) + + // this assignment is correctly illegal: + // p.Exported does not implement Exported (missing private method) + // x = px + + // this assignment unexpectedly compiles and then executes + x = px.(Exported) // ERROR "does not implement" + + // this is a legitimate call, but because of the previous assignment, + // it invokes the method private in p! + x.private() // p.Implementation.private() +} diff --git a/test/bugs/bug324.dir/p.go b/test/bugs/bug324.dir/p.go new file mode 100644 index 000000000..d1e3b991a --- /dev/null +++ b/test/bugs/bug324.dir/p.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type Exported interface { + private() +} + +type Implementation struct{} + +func (p *Implementation) private() { println("p.Implementation.private()") } + +var X = new(Implementation) diff --git a/test/bugs/bug324.go b/test/bugs/bug324.go new file mode 100644 index 000000000..8b4e29200 --- /dev/null +++ b/test/bugs/bug324.go @@ -0,0 +1,8 @@ +// $G $D/$F.dir/p.go && errchk $G $D/$F.dir/main.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 1550 +ignored diff --git a/test/fixedbugs/bug001.go b/test/fixedbugs/bug001.go deleted file mode 100644 index 2df8791ff..000000000 --- a/test/fixedbugs/bug001.go +++ /dev/null @@ -1,11 +0,0 @@ -// $G $D/$F.go && $L $F.$A && ./$A.out - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -func main() { - if {} // compiles; should be an error (must be an expression) -} diff --git a/test/fixedbugs/bug140.go b/test/fixedbugs/bug140.go index 33d1deb8a..298081663 100644 --- a/test/fixedbugs/bug140.go +++ b/test/fixedbugs/bug140.go @@ -7,8 +7,8 @@ package main func main() { - if {} else L1: ; - if {} else L2: main() ; + if true {} else L1: ; + if true {} else L2: main() ; } /* diff --git a/test/fixedbugs/bug1515.go b/test/fixedbugs/bug1515.go new file mode 100644 index 000000000..740252516 --- /dev/null +++ b/test/fixedbugs/bug1515.go @@ -0,0 +1,20 @@ +// $G $D/$F.go && $L $F.$A && ./$A.out + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +const ( + joao = "João" + jose = "José" +) + +func main() { + s1 := joao + s2 := jose + if (s1 < s2) != (joao < jose) { + panic("unequal") + } +} diff --git a/test/fixedbugs/bug219.go b/test/fixedbugs/bug219.go index 21361a2aa..966d3fcf3 100644 --- a/test/fixedbugs/bug219.go +++ b/test/fixedbugs/bug219.go @@ -12,8 +12,8 @@ func f(func()) int { return 0 } // bug219.go:16: syntax error near if func g1() { if x := f(func() { - if {} - }); { + if true {} + }); true { _ = x; } } @@ -21,8 +21,8 @@ func g1() { // this works func g2() { if x := f(func() { - //if {} - }); { + //if true {} + }); true { _ = x; } } @@ -30,9 +30,9 @@ func g2() { // this works func g3() { x := f(func() { - if {} + if true {} }); - if { + if true { _ = x; } } diff --git a/test/fixedbugs/bug322.go b/test/fixedbugs/bug323.go index bfb528318..bfb528318 100644 --- a/test/fixedbugs/bug322.go +++ b/test/fixedbugs/bug323.go diff --git a/test/fixedbugs/bug325.go b/test/fixedbugs/bug325.go new file mode 100644 index 000000000..23dbc8b3c --- /dev/null +++ b/test/fixedbugs/bug325.go @@ -0,0 +1,14 @@ +// errchk $G $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "unsafe" + +func main() { + var x unsafe.Pointer + println(*x) // ERROR "invalid indirect.*unsafe.Pointer" +} diff --git a/test/golden.out b/test/golden.out index 7883973e0..cc699d450 100644 --- a/test/golden.out +++ b/test/golden.out @@ -164,3 +164,6 @@ bugs/bug322.dir/main.go:19: implicit assignment of unexported field 'x' of lib.T bugs/bug322.dir/main.go:22: implicit assignment of unexported field 'x' of lib.T in assignment bugs/bug322.dir/main.go:31: implicit assignment of unexported field 'x' of lib.T in method receiver BUG: fails incorrectly + +=========== bugs/bug324.go +BUG: errchk: command succeeded unexpectedly diff --git a/test/if.go b/test/if.go index db1fe8b79..c1bb69d27 100644 --- a/test/if.go +++ b/test/if.go @@ -45,18 +45,6 @@ func main() { assertequal(count, 0, "if false one") count = 0 - if { - count = count + 1 - } - assertequal(count, 1, "if empty") - - count = 0 - if one := 1; true { - count = count + one - } - assertequal(count, 1, "if empty one") - - count = 0 if i5 < i7 { count = count + 1 } diff --git a/test/if1.go b/test/if1.go deleted file mode 100644 index 061c36411..000000000 --- a/test/if1.go +++ /dev/null @@ -1,20 +0,0 @@ -// $G $F.go && $L $F.$A && ./$A.out - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "os" - -func main() { - count := 7 - if one := 1; { - count = count + one - } - if count != 8 { - print(count, " should be 8\n") - os.Exit(1) - } -} diff --git a/test/init.go b/test/init.go new file mode 100644 index 000000000..b6c8c9706 --- /dev/null +++ b/test/init.go @@ -0,0 +1,18 @@ +// errchk $G -e $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "runtime" + +func init() { +} + +func main() { + init() // ERROR "undefined: init" + runtime.init() // ERROR "unexported.*runtime\.init" + var _ = init // ERROR "undefined: init" +} diff --git a/test/ken/robif.go b/test/ken/robif.go deleted file mode 100644 index b6fe4e433..000000000 --- a/test/ken/robif.go +++ /dev/null @@ -1,97 +0,0 @@ -// $G $D/$F.go && $L $F.$A && ./$A.out - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -func assertequal(is, shouldbe int, msg string) { - if is != shouldbe { - print("assertion fail" + msg + "\n"); - panic(1); - } -} - -func main() { - i5 := 5; - i7 := 7; - - var count int; - - count = 0; - if true { - count = count + 1; - } - assertequal(count, 1, "if true"); - - count = 0; - if false { - count = count + 1; - } - assertequal(count, 0, "if false"); - - count = 0; - if one := 1; true { - count = count + one; - } - assertequal(count, 1, "if true one"); - - count = 0; - if one := 1; false { - _ = one; - count = count + 1; - } - assertequal(count, 0, "if false one"); - - count = 0; - if { - count = count + 1; - } - assertequal(count, 1, "if empty"); - - count = 0; - if one := 1; { - count = count + one; - } - assertequal(count, 1, "if empty one"); - - count = 0; - if i5 < i7 { - count = count + 1; - } - assertequal(count, 1, "if cond"); - - count = 0; - if true { - count = count + 1; - } else - count = count - 1; - assertequal(count, 1, "if else true"); - - count = 0; - if false { - count = count + 1; - } else - count = count - 1; - assertequal(count, -1, "if else false"); - - count = 0; - if t:=1; false { - count = count + 1; - t := 7; - _ = t; - } else - count = count - t; - assertequal(count, -1, "if else false var"); - - count = 0; - t := 1; - if false { - count = count + 1; - t := 7; - _ = t; - } else - count = count - t; - assertequal(count, -1, "if else false var outside"); -} diff --git a/test/syntax/if.go b/test/syntax/if.go new file mode 100644 index 000000000..913d41885 --- /dev/null +++ b/test/syntax/if.go @@ -0,0 +1,15 @@ +// errchk $G $D/$F.go + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() { + if { // ERROR "missing condition" + } + + if x(); { // ERROR "missing condition" + } +} |
