Basic infrastructure and process documentation

- Updated source code layout to use standard Go project structure. - Makefile for build and test execution. - Glide for dependency management. - Integrated spf13/cobra for command line entry point. - Added docs on release and development process.
2026-08-12 19:32:48 -06:00 · 2016-03-29 08:26:07 -07:00
parent 50cc2891cb
commit e2297a7833
2472 changed files with 889185 additions and 22 deletions
@@ -1 +1,2 @@
 .vscode
+opa
@@ -1,4 +1,4 @@
 language: go
-
-install: ./install-deps-gen-code.sh
-
+go:
+  - 1.5
+  - 1.6
@@ -0,0 +1,6 @@
+# Change Log
+
+All notable changes to this project will be documented in this file. This
+project adheres to [Semantic Versioning](http://semver.org/).
+
+## Unreleased
@@ -0,0 +1,35 @@
+# Copyright 2015 The OPA Authors.  All rights reserved.
+# Use of this source code is governed by an Apache2
+# license that can be found in the LICENSE file.
+
+PACKAGES := github.com/open-policy-agent/opa/jsonlog/.../ \
+	github.com/open-policy-agent/opa/cmd/.../
+
+BUILD_COMMIT := $(shell ./build/get-build-commit.sh)
+BUILD_TIMESTAMP := $(shell ./build/get-build-timestamp.sh)
+BUILD_HOSTNAME := $(shell ./build/get-build-hostname.sh)
+
+LDFLAGS := -ldflags "-X github.com/open-policy-agent/opa/version.Vcs=$(BUILD_COMMIT) \
+	-X github.com/open-policy-agent/opa/version.Timestamp=$(BUILD_TIMESTAMP) \
+	-X github.com/open-policy-agent/opa/version.Hostname=$(BUILD_HOSTNAME)"
+
+GO := go
+
+GO15VENDOREXPERIMENT := 1
+export GO15VENDOREXPERIMENT
+
+.PHONY: all generate build test clean
+
+all: build test
+
+generate:
+	$(GO) generate
+
+build:
+	$(GO) build -o opa $(LDFLAGS)
+
+test:
+	$(GO) test -v $(PACKAGES)
+
+clean:
+	rm -f ./opa
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+GIT_SHA=$(git rev-parse --short HEAD)
+
+if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
+    echo $GIT_SHA
+else
+    echo "$GIT_SHA-dirty"
+fi
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hostname -f
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+date -u +"%Y-%m-%dT%H:%M:%SZ"
@@ -0,0 +1,15 @@
+// Copyright 2016 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import "github.com/spf13/cobra"
+import "path"
+import "os"
+
+var RootCommand = &cobra.Command{
+	Use:   path.Base(os.Args[0]),
+	Short: "Open Policy Agent (OPA)",
+	Long:  "An open source project to policy enable any application.",
+}
@@ -0,0 +1,25 @@
+// Copyright 2016 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import "fmt"
+import "github.com/spf13/cobra"
+import "github.com/open-policy-agent/opa/version"
+
+var versionCommand = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version of OPA",
+	Long:  "Show version and build information for OPA.",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Version: " + version.Version)
+		fmt.Println("Build Commit: " + version.Vcs)
+		fmt.Println("Build Timestamp: " + version.Timestamp)
+		fmt.Println("Build Hostname: " + version.Hostname)
+	},
+}
+
+func init() {
+	RootCommand.AddCommand(versionCommand)
+}
@@ -0,0 +1,112 @@
+# Development
+
+## Environment
+
+OPA is written in the [Go](https://golang.org) programming language.
+
+If you are not familiar with Go we recommend you read through the [How to Write Go
+Code](https://golang.org/doc/code.html) article to familiarize yourself with the standard Go development environment.
+
+Requirements:
+
+- Git
+- GitHub account (if you are contributing)
+- Go (version 1.5.x and 1.6.x are supported)
+- GNU Make
+
+## Getting Started
+
+After cloning the repository, you can run `make all` to build the project and
+execute all of the tests. If this succeeds, there should be a binary
+in the top directory (opa).
+
+Verify the build was successful by running `opa version`.
+
+You can re-build the project with `make build` and execute all of the tests
+with `make test`.
+
+## Workflow
+
+1. Go to [https://github.com/open-policy-agent/opa](https://github.com/open-policy-agent/opa) and fork the repository
+   into your account by clicking the "Fork" button.
+
+1. Clone the fork to your local machine.
+
+    ```
+    cd $GOPATH
+    mkdir -p src/github.com/open-policy-agent
+    cd src/github.com/open-policy-agent
+    git clone git@github.com/<GITHUB USERNAME>/opa.git opa
+    cd opa
+    git remote add upstream https://github.com/open-policy-agent/opa.git
+    ```
+
+1. Create a branch for your changes.
+
+    ```
+    git checkout -b somefeature
+    ```
+
+1. Update your local branch with upstream.
+
+    ```
+    git fetch upstream
+    git rebase upstream/master
+    ```
+
+1. Develop your changes and regularly update your local branch against upstream.
+
+	- Make sure you run `go fmt` on your code before submitting a Pull Request.
+
+1. Commit changes and push to your fork.
+
+    ```
+    git commit
+    git push origin somefeature
+    ```
+
+1. Submit a Pull Request via https://github.com/\<GITHUB USERNAME>/opa. You
+   should be prompted to with a "Compare and Pull Request" button that
+   mentions your branch.
+
+1. Once your Pull Request has been reviewed and signed off please squash your
+   commits. If you have a specific reason to leave multiple commits in the
+   Pull Request, please mention it in the discussion.
+
+   > If you are not familiar with squashing commits, see [the following blog post for a good overview](http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html).
+
+## Dependencies
+
+[Glide](https://github.com/Masterminds/glide) is a command line tool used for
+dependency management. You must have Glide installed in order to add new
+dependencies or update existing dependencies. If you are not changing
+dependencies you do not have to install Glide, all of the dependencies are
+contained in the vendor directory.
+
+If you need to add a dependency to the project:
+
+1. Run `glide get <package>` to download the package.
+    - This command should be used instead of `go get <package>`.
+	- The package will be stored under the vendor directory.
+	- The glide.yaml file will be updated.
+1. Manually remove the VCS directories (e.g., .git, .hg, etc.) from the new
+   vendor directories.
+1. Commit the changes in glide.yaml, glide.lock, and new vendor directories.
+
+If you need to update the dependencies:
+
+1. Run `glide update --update-vendored`.
+1. Commit the changes to the glide.lock file and any files under the vendor
+   directory.
+
+## Opalog
+
+If you need to modify the Opalog syntax you must update jsonlog/parser.peg
+and run `make generate` to re-generate the parser code.
+
+> If you encounter an error because "pigeon" is not installed, run `glide
+> rebuild` to build and install the vendored dependencies (which include the
+> parser generator). Note, you will need to have [Glide](https://github.com/Masterminds/glide)
+> installed for this.
+
+Commit the changes to the parser.peg and parser.go files.
@@ -0,0 +1,80 @@
+# Release Process
+
+## Overview
+
+The release process consists of three phases: versioning, building, and
+publishing.
+
+Versioning involves maintaining the CHANGELOG.md and version.go files inside
+the repository and tagging the repository to identify specific releases.
+
+Building involves obtaining a copy of the repository, checking out the release
+tag, and building the packages.
+
+Publishing involves creating a new *Release* on GitHub with the relevant
+CHANGELOG.md snippet and uploading the packages from the build phase.
+
+## Versioning
+
+1. Obtain copy of remote repository.
+
+	```
+	git clone git@github.com/open-policy-agent/opa.git
+	```
+
+1. Edit CHANGELOG.md to update the Unreleased header (e.g., s/Unreleased/0.12.8/) and add any missing items to prepare for release.
+
+1. Edit version/version.go to set Version variable to prepare for release (e.g., s/Version = “0.12.8-dev”/Version = "0.12.8”/).
+
+1. Commit the changes and push to remote repository.
+
+	```
+	git commit -a -m “Prepare v<version> release”
+	git push origin master
+	```
+
+1. Tag repository with release version and push tags to remote repository.
+
+	```
+	git tag v<semver>
+	git push origin --tags
+	```
+
+1. Edit CHANGELOG.md to add back the Unreleased header to prepare for development.
+
+1. Edit version/version.go to set Version variable to prepare for development (e.g., s/Version = “0.12.8”/Version = “0.12.9-dev”/).
+
+1. Commit the changes and push to remote repository.
+
+	```
+	git commit -a -m “Prepare v<next_semvar> development”
+	git push origin master
+	```
+
+## Building
+
+1. Obtain copy of remote repository.
+
+	```
+	git clone git@github.com/open-policy-agent/opa.git
+	```
+
+1. Checkout release tag.
+
+	```
+	git checkout v<semver>
+	```
+
+1. Run command to build packages. This will produce a bunch of binaries (e.g., amd64/linux, i386/linux, amd64/darwin, etc.) that can be published (“distributions”).
+
+	```
+	make dist
+	```
+
+## Publishing
+
+1. Open browser and go to https://github.com/open-policy-agent/opa/releases
+
+1. Create a new release for the version.
+	- Copy the changelog content into the message.
+	- Upload the distributions packages.
@@ -0,0 +1,77 @@
+hash: 60660dbeea966624ef47c09512b10812b4fd5e9e82876d915ca69d72dbc3157c
+updated: 2016-03-29T17:05:39.985980976-07:00
+imports:
+- name: github.com/armon/consul-api
+  version: dcfedd50ed5334f96adee43fc88518a4f095e15c
+  repo: https://github.com/armon/consul-api
+- name: github.com/BurntSushi/toml
+  version: bbd5bb678321a0d6e58f1099321dfa73391c1b6f
+  repo: https://github.com/BurntSushi/toml
+- name: github.com/coreos/go-etcd
+  version: 003851be7bb0694fe3cc457a49529a19388ee7cf
+  repo: https://github.com/coreos/go-etcd
+- name: github.com/cpuguy83/go-md2man
+  version: 2724a9c9051aa62e9cca11304e7dd518e9e41599
+  repo: https://github.com/cpuguy83/go-md2man
+- name: github.com/hashicorp/hcl
+  version: 2604f3bda7e8960c1be1063709e7d7f0765048d0
+  repo: https://github.com/hashicorp/hcl
+- name: github.com/kr/pretty
+  version: add1dbc86daf0f983cd4a48ceb39deb95c729b67
+  repo: https://github.com/kr/pretty
+- name: github.com/kr/pty
+  version: f7ee69f31298ecbe5d2b349c711e2547a617d398
+  repo: https://github.com/kr/pty
+- name: github.com/kr/text
+  version: bb797dc4fb8320488f47bf11de07a733d7233e1f
+  repo: https://github.com/kr/text
+- name: github.com/magiconair/properties
+  version: c265cfa48dda6474e208715ca93e987829f572f8
+  repo: https://github.com/magiconair/properties
+- name: github.com/mitchellh/mapstructure
+  version: d2dd0262208475919e1a362f675cfc0e7c10e905
+  repo: https://github.com/mitchellh/mapstructure
+- name: github.com/PuerkitoBio/pigeon
+  version: a5221784523de14130c00a8c389148a1b2ad260c
+- name: github.com/russross/blackfriday
+  version: b43df972fb5fdf3af8d2e90f38a69d374fe26dd0
+  repo: https://github.com/russross/blackfriday
+- name: github.com/shurcooL/sanitized_anchor_name
+  version: 10ef21a441db47d8b13ebcc5fd2310f636973c77
+  repo: https://github.com/shurcooL/sanitized_anchor_name
+- name: github.com/spf13/cast
+  version: 27b586b42e29bec072fe7379259cc719e1289da6
+  repo: https://github.com/spf13/cast
+- name: github.com/spf13/cobra
+  version: c678ff029ee250b65714e518f4f5c5cb934955de
+- name: github.com/spf13/jwalterweatherman
+  version: 33c24e77fb80341fe7130ee7c594256ff08ccc46
+  repo: https://github.com/spf13/jwalterweatherman
+- name: github.com/spf13/pflag
+  version: 7f60f83a2c81bc3c3c0d5297f61ddfa68da9d3b7
+- name: github.com/spf13/viper
+  version: c975dc1b4eacf4ec7fdbf0873638de5d090ba323
+  repo: https://github.com/spf13/viper
+- name: github.com/ugorji/go
+  version: a396ed22fc049df733440d90efe17475e3929ccb
+  repo: https://github.com/ugorji/go
+- name: github.com/xordataexchange/crypt
+  version: 749e360c8f236773f28fc6d3ddfce4a470795227
+  repo: https://github.com/xordataexchange/crypt
+- name: golang.org/x/crypto
+  version: 9e7f5dc375abeb9619ea3c5c58502c428f457aa2
+- name: golang.org/x/net
+  version: 31df19d69da8728e9220def59b80ee577c3e48bf
+- name: golang.org/x/text
+  version: 1b466db55e0ba5d56ef5315c728216b42f796491
+- name: golang.org/x/tools
+  version: 84e7bc0dd39bab24b696dde4d714641fa738f945
+  subpackages:
+  - cmd/goimports
+- name: gopkg.in/fsnotify.v1
+  version: 875cf421b32f8f1b31bd43776297876d01542279
+  repo: https://gopkg.in/fsnotify.v1
+- name: gopkg.in/yaml.v2
+  version: a83829b6f1293c91addabc89d0571c246397bbf4
+  repo: https://gopkg.in/yaml.v2
+devImports: []
@@ -0,0 +1,7 @@
+package: github.com/open-policy-agent/opa
+import:
+  - package: github.com/PuerkitoBio/pigeon
+  - package: golang.org/x/tools
+    subpackages:
+      - cmd/goimports
+  - package: github.com/spf13/cobra
@@ -1,11 +0,0 @@
-#!/usr/bin/env sh
-
-# install dependencies
-go get -u github.com/PuerkitoBio/pigeon
-go get golang.org/x/tools/cmd/goimports
-
-# generate source code for parser.  Delete first so no silent errors.
-rm src/jsonlog/parser.go
-pigeon src/jsonlog/jsonlog.peg | goimports > src/jsonlog/parser.go
-
-
@@ -0,0 +1,21 @@
+// Copyright 2016 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+
+package main
+
+import "fmt"
+import "os"
+import "github.com/open-policy-agent/opa/cmd"
+
+func main() {
+	if err := cmd.RootCommand.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+// Opalog parser generation:
+//
+//go:generate pigeon -o jsonlog/parser.go jsonlog/jsonlog.peg
+//go:generate goimports -w jsonlog/parser.go
@@ -1,8 +0,0 @@
-package main
-
-import "fmt"
-
-func main() {
-	fmt.Println("Hello world")
-}
-
@@ -0,0 +1,5 @@
+TAGS
+tags
+.*.swp
+tomlcheck/tomlcheck
+toml.test
@@ -0,0 +1,12 @@
+language: go
+go:
+  - 1.1
+  - 1.2
+  - tip
+install:
+  - go install ./...
+  - go get github.com/BurntSushi/toml-test
+script:
+  - export PATH="$PATH:$HOME/gopath/bin"
+  - make test
+
@@ -0,0 +1,3 @@
+Compatible with TOML version
+[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md)
+
@@ -0,0 +1,14 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.
+
@@ -0,0 +1,19 @@
+install:
+	go install ./...
+
+test: install
+	go test -v
+	toml-test toml-test-decoder
+	toml-test -encoder toml-test-encoder
+
+fmt:
+	gofmt -w *.go */*.go
+	colcheck *.go */*.go
+
+tags:
+	find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS
+
+push:
+	git push origin master
+	git push github master
+
@@ -0,0 +1,220 @@
+## TOML parser and encoder for Go with reflection
+
+TOML stands for Tom's Obvious, Minimal Language. This Go package provides a
+reflection interface similar to Go's standard library `json` and `xml` 
+packages. This package also supports the `encoding.TextUnmarshaler` and
+`encoding.TextMarshaler` interfaces so that you can define custom data 
+representations. (There is an example of this below.)
+
+Spec: https://github.com/mojombo/toml
+
+Compatible with TOML version
+[v0.2.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.2.0.md)
+
+Documentation: http://godoc.org/github.com/BurntSushi/toml
+
+Installation:
+
+```bash
+go get github.com/BurntSushi/toml
+```
+
+Try the toml validator:
+
+```bash
+go get github.com/BurntSushi/toml/cmd/tomlv
+tomlv some-toml-file.toml
+```
+
+[![Build status](https://api.travis-ci.org/BurntSushi/toml.png)](https://travis-ci.org/BurntSushi/toml)
+
+
+### Testing
+
+This package passes all tests in
+[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder
+and the encoder.
+
+### Examples
+
+This package works similarly to how the Go standard library handles `XML`
+and `JSON`. Namely, data is loaded into Go values via reflection.
+
+For the simplest example, consider some TOML file as just a list of keys
+and values:
+
+```toml
+Age = 25
+Cats = [ "Cauchy", "Plato" ]
+Pi = 3.14
+Perfection = [ 6, 28, 496, 8128 ]
+DOB = 1987-07-05T05:45:00Z
+```
+
+Which could be defined in Go as:
+
+```go
+type Config struct {
+  Age int
+  Cats []string
+  Pi float64
+  Perfection []int
+  DOB time.Time // requires `import time`
+}
+```
+
+And then decoded with:
+
+```go
+var conf Config
+if _, err := toml.Decode(tomlData, &conf); err != nil {
+  // handle error
+}
+```
+
+You can also use struct tags if your struct field name doesn't map to a TOML
+key value directly:
+
+```toml
+some_key_NAME = "wat"
+```
+
+```go
+type TOML struct {
+  ObscureKey string `toml:"some_key_NAME"`
+}
+```
+
+### Using the `encoding.TextUnmarshaler` interface
+
+Here's an example that automatically parses duration strings into 
+`time.Duration` values:
+
+```toml
+[[song]]
+name = "Thunder Road"
+duration = "4m49s"
+
+[[song]]
+name = "Stairway to Heaven"
+duration = "8m03s"
+```
+
+Which can be decoded with:
+
+```go
+type song struct {
+  Name     string
+  Duration duration
+}
+type songs struct {
+  Song []song
+}
+var favorites songs
+if _, err := toml.Decode(blob, &favorites); err != nil {
+  log.Fatal(err)
+}
+
+for _, s := range favorites.Song {
+  fmt.Printf("%s (%s)\n", s.Name, s.Duration)
+}
+```
+
+And you'll also need a `duration` type that satisfies the 
+`encoding.TextUnmarshaler` interface:
+
+```go
+type duration struct {
+	time.Duration
+}
+
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}
+```
+
+### More complex usage
+
+Here's an example of how to load the example from the official spec page:
+
+```toml
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
+
+# Line breaks are OK when inside arrays
+hosts = [
+  "alpha",
+  "omega"
+]
+```
+
+And the corresponding Go types are:
+
+```go
+type tomlConfig struct {
+	Title string
+	Owner ownerInfo
+	DB database `toml:"database"`
+	Servers map[string]server
+	Clients clients
+}
+
+type ownerInfo struct {
+	Name string
+	Org string `toml:"organization"`
+	Bio string
+	DOB time.Time
+}
+
+type database struct {
+	Server string
+	Ports []int
+	ConnMax int `toml:"connection_max"`
+	Enabled bool
+}
+
+type server struct {
+	IP string
+	DC string
+}
+
+type clients struct {
+	Data [][]interface{}
+	Hosts []string
+}
+```
+
+Note that a case insensitive match will be tried if an exact match can't be
+found.
+
+A working example of the above can be found in `_examples/example.{go,toml}`.
+
@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+type tomlConfig struct {
+	Title   string
+	Owner   ownerInfo
+	DB      database `toml:"database"`
+	Servers map[string]server
+	Clients clients
+}
+
+type ownerInfo struct {
+	Name string
+	Org  string `toml:"organization"`
+	Bio  string
+	DOB  time.Time
+}
+
+type database struct {
+	Server  string
+	Ports   []int
+	ConnMax int `toml:"connection_max"`
+	Enabled bool
+}
+
+type server struct {
+	IP string
+	DC string
+}
+
+type clients struct {
+	Data  [][]interface{}
+	Hosts []string
+}
+
+func main() {
+	var config tomlConfig
+	if _, err := toml.DecodeFile("example.toml", &config); err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	fmt.Printf("Title: %s\n", config.Title)
+	fmt.Printf("Owner: %s (%s, %s), Born: %s\n",
+		config.Owner.Name, config.Owner.Org, config.Owner.Bio,
+		config.Owner.DOB)
+	fmt.Printf("Database: %s %v (Max conn. %d), Enabled? %v\n",
+		config.DB.Server, config.DB.Ports, config.DB.ConnMax,
+		config.DB.Enabled)
+	for serverName, server := range config.Servers {
+		fmt.Printf("Server: %s (%s, %s)\n", serverName, server.IP, server.DC)
+	}
+	fmt.Printf("Client data: %v\n", config.Clients.Data)
+	fmt.Printf("Client hosts: %v\n", config.Clients.Hosts)
+}
@@ -0,0 +1,35 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
+
+# Line breaks are OK when inside arrays
+hosts = [
+  "alpha",
+  "omega"
+]
@@ -0,0 +1,22 @@
+# Test file for TOML
+# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate
+# This part you'll really hate
+
+[the]
+test_string = "You'll hate me after this - #"          # " Annoying, isn't it?
+
+    [the.hard]
+    test_array = [ "] ", " # "]      # ] There you go, parse this!
+    test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ]
+    # You didn't think it'd as easy as chucking out the last #, did you?
+    another_test_string = " Same thing, but with a string #"
+    harder_test_string = " And when \"'s are in the string, along with # \""   # "and comments are there too"
+    # Things will get harder
+    
+        [the.hard.bit#]
+        what? = "You don't think some user won't do that?"
+        multi_line_array = [
+            "]",
+            # ] Oh yes I did
+            ]
+
@@ -0,0 +1,4 @@
+# [x] you
+# [x.y] don't
+# [x.y.z] need these
+[x.y.z.w] # for this to work
@@ -0,0 +1,6 @@
+# DO NOT WANT
+[fruit]
+type = "apple"
+
+[fruit.type]
+apple = "yes"
@@ -0,0 +1,35 @@
+# This is an INVALID TOML document. Boom.
+# Can you spot the error without help?
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T7:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
+
+# Line breaks are OK when inside arrays
+hosts = [
+  "alpha",
+  "omega"
+]
@@ -0,0 +1,5 @@
+Age = 25
+Cats = [ "Cauchy", "Plato" ]
+Pi = 3.14
+Perfection = [ 6, 28, 496, 8128 ]
+DOB = 1987-07-05T05:45:00Z
@@ -0,0 +1 @@
+some_key_NAME = "wat"
@@ -0,0 +1,14 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.
+
@@ -0,0 +1,14 @@
+# Implements the TOML test suite interface
+
+This is an implementation of the interface expected by
+[toml-test](https://github.com/BurntSushi/toml-test) for my
+[toml parser written in Go](https://github.com/BurntSushi/toml).
+In particular, it maps TOML data on `stdin` to a JSON format on `stdout`.
+
+
+Compatible with TOML version
+[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md)
+
+Compatible with `toml-test` version
+[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0)
+
@@ -0,0 +1,90 @@
+// Command toml-test-decoder satisfies the toml-test interface for testing
+// TOML decoders. Namely, it accepts TOML on stdin and outputs JSON on stdout.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+func init() {
+	log.SetFlags(0)
+
+	flag.Usage = usage
+	flag.Parse()
+}
+
+func usage() {
+	log.Printf("Usage: %s < toml-file\n", path.Base(os.Args[0]))
+	flag.PrintDefaults()
+
+	os.Exit(1)
+}
+
+func main() {
+	if flag.NArg() != 0 {
+		flag.Usage()
+	}
+
+	var tmp interface{}
+	if _, err := toml.DecodeReader(os.Stdin, &tmp); err != nil {
+		log.Fatalf("Error decoding TOML: %s", err)
+	}
+
+	typedTmp := translate(tmp)
+	if err := json.NewEncoder(os.Stdout).Encode(typedTmp); err != nil {
+		log.Fatalf("Error encoding JSON: %s", err)
+	}
+}
+
+func translate(tomlData interface{}) interface{} {
+	switch orig := tomlData.(type) {
+	case map[string]interface{}:
+		typed := make(map[string]interface{}, len(orig))
+		for k, v := range orig {
+			typed[k] = translate(v)
+		}
+		return typed
+	case []map[string]interface{}:
+		typed := make([]map[string]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v).(map[string]interface{})
+		}
+		return typed
+	case []interface{}:
+		typed := make([]interface{}, len(orig))
+		for i, v := range orig {
+			typed[i] = translate(v)
+		}
+
+		// We don't really need to tag arrays, but let's be future proof.
+		// (If TOML ever supports tuples, we'll need this.)
+		return tag("array", typed)
+	case time.Time:
+		return tag("datetime", orig.Format("2006-01-02T15:04:05Z"))
+	case bool:
+		return tag("bool", fmt.Sprintf("%v", orig))
+	case int64:
+		return tag("integer", fmt.Sprintf("%d", orig))
+	case float64:
+		return tag("float", fmt.Sprintf("%v", orig))
+	case string:
+		return tag("string", orig)
+	}
+
+	panic(fmt.Sprintf("Unknown type: %T", tomlData))
+}
+
+func tag(typeName string, data interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"type":  typeName,
+		"value": data,
+	}
+}
@@ -0,0 +1,14 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.
+
@@ -0,0 +1,14 @@
+# Implements the TOML test suite interface for TOML encoders
+
+This is an implementation of the interface expected by
+[toml-test](https://github.com/BurntSushi/toml-test) for the
+[TOML encoder](https://github.com/BurntSushi/toml).
+In particular, it maps JSON data on `stdin` to a TOML format on `stdout`.
+
+
+Compatible with TOML version
+[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md)
+
+Compatible with `toml-test` version
+[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0)
+
@@ -0,0 +1,131 @@
+// Command toml-test-encoder satisfies the toml-test interface for testing
+// TOML encoders. Namely, it accepts JSON on stdin and outputs TOML on stdout.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	"os"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+func init() {
+	log.SetFlags(0)
+
+	flag.Usage = usage
+	flag.Parse()
+}
+
+func usage() {
+	log.Printf("Usage: %s < json-file\n", path.Base(os.Args[0]))
+	flag.PrintDefaults()
+
+	os.Exit(1)
+}
+
+func main() {
+	if flag.NArg() != 0 {
+		flag.Usage()
+	}
+
+	var tmp interface{}
+	if err := json.NewDecoder(os.Stdin).Decode(&tmp); err != nil {
+		log.Fatalf("Error decoding JSON: %s", err)
+	}
+
+	tomlData := translate(tmp)
+	if err := toml.NewEncoder(os.Stdout).Encode(tomlData); err != nil {
+		log.Fatalf("Error encoding TOML: %s", err)
+	}
+}
+
+func translate(typedJson interface{}) interface{} {
+	switch v := typedJson.(type) {
+	case map[string]interface{}:
+		if len(v) == 2 && in("type", v) && in("value", v) {
+			return untag(v)
+		}
+		m := make(map[string]interface{}, len(v))
+		for k, v2 := range v {
+			m[k] = translate(v2)
+		}
+		return m
+	case []interface{}:
+		tabArray := make([]map[string]interface{}, len(v))
+		for i := range v {
+			if m, ok := translate(v[i]).(map[string]interface{}); ok {
+				tabArray[i] = m
+			} else {
+				log.Fatalf("JSON arrays may only contain objects. This " +
+					"corresponds to only tables being allowed in " +
+					"TOML table arrays.")
+			}
+		}
+		return tabArray
+	}
+	log.Fatalf("Unrecognized JSON format '%T'.", typedJson)
+	panic("unreachable")
+}
+
+func untag(typed map[string]interface{}) interface{} {
+	t := typed["type"].(string)
+	v := typed["value"]
+	switch t {
+	case "string":
+		return v.(string)
+	case "integer":
+		v := v.(string)
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			log.Fatalf("Could not parse '%s' as integer: %s", v, err)
+		}
+		return n
+	case "float":
+		v := v.(string)
+		f, err := strconv.ParseFloat(v, 64)
+		if err != nil {
+			log.Fatalf("Could not parse '%s' as float64: %s", v, err)
+		}
+		return f
+	case "datetime":
+		v := v.(string)
+		t, err := time.Parse("2006-01-02T15:04:05Z", v)
+		if err != nil {
+			log.Fatalf("Could not parse '%s' as a datetime: %s", v, err)
+		}
+		return t
+	case "bool":
+		v := v.(string)
+		switch v {
+		case "true":
+			return true
+		case "false":
+			return false
+		}
+		log.Fatalf("Could not parse '%s' as a boolean.", v)
+	case "array":
+		v := v.([]interface{})
+		array := make([]interface{}, len(v))
+		for i := range v {
+			if m, ok := v[i].(map[string]interface{}); ok {
+				array[i] = untag(m)
+			} else {
+				log.Fatalf("Arrays may only contain other arrays or "+
+					"primitive values, but found a '%T'.", m)
+			}
+		}
+		return array
+	}
+	log.Fatalf("Unrecognized tag type '%s'.", t)
+	panic("unreachable")
+}
+
+func in(key string, m map[string]interface{}) bool {
+	_, ok := m[key]
+	return ok
+}
@@ -0,0 +1,14 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.
+
@@ -0,0 +1,22 @@
+# TOML Validator
+
+If Go is installed, it's simple to try it out:
+
+```bash
+go get github.com/BurntSushi/toml/cmd/tomlv
+tomlv some-toml-file.toml
+```
+
+You can see the types of every key in a TOML file with:
+
+```bash
+tomlv -types some-toml-file.toml
+```
+
+At the moment, only one error message is reported at a time. Error messages
+include line numbers. No output means that the files given are valid TOML, or 
+there is a bug in `tomlv`.
+
+Compatible with TOML version
+[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md)
+
@@ -0,0 +1,61 @@
+// Command tomlv validates TOML documents and prints each key's type.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	flagTypes = false
+)
+
+func init() {
+	log.SetFlags(0)
+
+	flag.BoolVar(&flagTypes, "types", flagTypes,
+		"When set, the types of every defined key will be shown.")
+
+	flag.Usage = usage
+	flag.Parse()
+}
+
+func usage() {
+	log.Printf("Usage: %s toml-file [ toml-file ... ]\n",
+		path.Base(os.Args[0]))
+	flag.PrintDefaults()
+
+	os.Exit(1)
+}
+
+func main() {
+	if flag.NArg() < 1 {
+		flag.Usage()
+	}
+	for _, f := range flag.Args() {
+		var tmp interface{}
+		md, err := toml.DecodeFile(f, &tmp)
+		if err != nil {
+			log.Fatalf("Error in '%s': %s", f, err)
+		}
+		if flagTypes {
+			printTypes(md)
+		}
+	}
+}
+
+func printTypes(md toml.MetaData) {
+	tabw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	for _, key := range md.Keys() {
+		fmt.Fprintf(tabw, "%s%s\t%s\n",
+			strings.Repeat("    ", len(key)-1), key, md.Type(key...))
+	}
+	tabw.Flush()
+}
@@ -0,0 +1,505 @@
+package toml
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+)
+
+var e = fmt.Errorf
+
+// Unmarshaler is the interface implemented by objects that can unmarshal a
+// TOML description of themselves.
+type Unmarshaler interface {
+	UnmarshalTOML(interface{}) error
+}
+
+// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`.
+func Unmarshal(p []byte, v interface{}) error {
+	_, err := Decode(string(p), v)
+	return err
+}
+
+// Primitive is a TOML value that hasn't been decoded into a Go value.
+// When using the various `Decode*` functions, the type `Primitive` may
+// be given to any value, and its decoding will be delayed.
+//
+// A `Primitive` value can be decoded using the `PrimitiveDecode` function.
+//
+// The underlying representation of a `Primitive` value is subject to change.
+// Do not rely on it.
+//
+// N.B. Primitive values are still parsed, so using them will only avoid
+// the overhead of reflection. They can be useful when you don't know the
+// exact type of TOML data until run time.
+type Primitive struct {
+	undecoded interface{}
+	context   Key
+}
+
+// DEPRECATED!
+//
+// Use MetaData.PrimitiveDecode instead.
+func PrimitiveDecode(primValue Primitive, v interface{}) error {
+	md := MetaData{decoded: make(map[string]bool)}
+	return md.unify(primValue.undecoded, rvalue(v))
+}
+
+// PrimitiveDecode is just like the other `Decode*` functions, except it
+// decodes a TOML value that has already been parsed. Valid primitive values
+// can *only* be obtained from values filled by the decoder functions,
+// including this method. (i.e., `v` may contain more `Primitive`
+// values.)
+//
+// Meta data for primitive values is included in the meta data returned by
+// the `Decode*` functions with one exception: keys returned by the Undecoded
+// method will only reflect keys that were decoded. Namely, any keys hidden
+// behind a Primitive will be considered undecoded. Executing this method will
+// update the undecoded keys in the meta data. (See the example.)
+func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error {
+	md.context = primValue.context
+	defer func() { md.context = nil }()
+	return md.unify(primValue.undecoded, rvalue(v))
+}
+
+// Decode will decode the contents of `data` in TOML format into a pointer
+// `v`.
+//
+// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be
+// used interchangeably.)
+//
+// TOML arrays of tables correspond to either a slice of structs or a slice
+// of maps.
+//
+// TOML datetimes correspond to Go `time.Time` values.
+//
+// All other TOML types (float, string, int, bool and array) correspond
+// to the obvious Go types.
+//
+// An exception to the above rules is if a type implements the
+// encoding.TextUnmarshaler interface. In this case, any primitive TOML value
+// (floats, strings, integers, booleans and datetimes) will be converted to
+// a byte string and given to the value's UnmarshalText method. See the
+// Unmarshaler example for a demonstration with time duration strings.
+//
+// Key mapping
+//
+// TOML keys can map to either keys in a Go map or field names in a Go
+// struct. The special `toml` struct tag may be used to map TOML keys to
+// struct fields that don't match the key name exactly. (See the example.)
+// A case insensitive match to struct names will be tried if an exact match
+// can't be found.
+//
+// The mapping between TOML values and Go values is loose. That is, there
+// may exist TOML values that cannot be placed into your representation, and
+// there may be parts of your representation that do not correspond to
+// TOML values. This loose mapping can be made stricter by using the IsDefined
+// and/or Undecoded methods on the MetaData returned.
+//
+// This decoder will not handle cyclic types. If a cyclic type is passed,
+// `Decode` will not terminate.
+func Decode(data string, v interface{}) (MetaData, error) {
+	p, err := parse(data)
+	if err != nil {
+		return MetaData{}, err
+	}
+	md := MetaData{
+		p.mapping, p.types, p.ordered,
+		make(map[string]bool, len(p.ordered)), nil,
+	}
+	return md, md.unify(p.mapping, rvalue(v))
+}
+
+// DecodeFile is just like Decode, except it will automatically read the
+// contents of the file at `fpath` and decode it for you.
+func DecodeFile(fpath string, v interface{}) (MetaData, error) {
+	bs, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return MetaData{}, err
+	}
+	return Decode(string(bs), v)
+}
+
+// DecodeReader is just like Decode, except it will consume all bytes
+// from the reader and decode it for you.
+func DecodeReader(r io.Reader, v interface{}) (MetaData, error) {
+	bs, err := ioutil.ReadAll(r)
+	if err != nil {
+		return MetaData{}, err
+	}
+	return Decode(string(bs), v)
+}
+
+// unify performs a sort of type unification based on the structure of `rv`,
+// which is the client representation.
+//
+// Any type mismatch produces an error. Finding a type that we don't know
+// how to handle produces an unsupported type error.
+func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
+
+	// Special case. Look for a `Primitive` value.
+	if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() {
+		// Save the undecoded data and the key context into the primitive
+		// value.
+		context := make(Key, len(md.context))
+		copy(context, md.context)
+		rv.Set(reflect.ValueOf(Primitive{
+			undecoded: data,
+			context:   context,
+		}))
+		return nil
+	}
+
+	// Special case. Unmarshaler Interface support.
+	if rv.CanAddr() {
+		if v, ok := rv.Addr().Interface().(Unmarshaler); ok {
+			return v.UnmarshalTOML(data)
+		}
+	}
+
+	// Special case. Handle time.Time values specifically.
+	// TODO: Remove this code when we decide to drop support for Go 1.1.
+	// This isn't necessary in Go 1.2 because time.Time satisfies the encoding
+	// interfaces.
+	if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) {
+		return md.unifyDatetime(data, rv)
+	}
+
+	// Special case. Look for a value satisfying the TextUnmarshaler interface.
+	if v, ok := rv.Interface().(TextUnmarshaler); ok {
+		return md.unifyText(data, v)
+	}
+	// BUG(burntsushi)
+	// The behavior here is incorrect whenever a Go type satisfies the
+	// encoding.TextUnmarshaler interface but also corresponds to a TOML
+	// hash or array. In particular, the unmarshaler should only be applied
+	// to primitive TOML values. But at this point, it will be applied to
+	// all kinds of values and produce an incorrect error whenever those values
+	// are hashes or arrays (including arrays of tables).
+
+	k := rv.Kind()
+
+	// laziness
+	if k >= reflect.Int && k <= reflect.Uint64 {
+		return md.unifyInt(data, rv)
+	}
+	switch k {
+	case reflect.Ptr:
+		elem := reflect.New(rv.Type().Elem())
+		err := md.unify(data, reflect.Indirect(elem))
+		if err != nil {
+			return err
+		}
+		rv.Set(elem)
+		return nil
+	case reflect.Struct:
+		return md.unifyStruct(data, rv)
+	case reflect.Map:
+		return md.unifyMap(data, rv)
+	case reflect.Array:
+		return md.unifyArray(data, rv)
+	case reflect.Slice:
+		return md.unifySlice(data, rv)
+	case reflect.String:
+		return md.unifyString(data, rv)
+	case reflect.Bool:
+		return md.unifyBool(data, rv)
+	case reflect.Interface:
+		// we only support empty interfaces.
+		if rv.NumMethod() > 0 {
+			return e("Unsupported type '%s'.", rv.Kind())
+		}
+		return md.unifyAnything(data, rv)
+	case reflect.Float32:
+		fallthrough
+	case reflect.Float64:
+		return md.unifyFloat64(data, rv)
+	}
+	return e("Unsupported type '%s'.", rv.Kind())
+}
+
+func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error {
+	tmap, ok := mapping.(map[string]interface{})
+	if !ok {
+		if mapping == nil {
+			return nil
+		}
+		return mismatch(rv, "map", mapping)
+	}
+
+	for key, datum := range tmap {
+		var f *field
+		fields := cachedTypeFields(rv.Type())
+		for i := range fields {
+			ff := &fields[i]
+			if ff.name == key {
+				f = ff
+				break
+			}
+			if f == nil && strings.EqualFold(ff.name, key) {
+				f = ff
+			}
+		}
+		if f != nil {
+			subv := rv
+			for _, i := range f.index {
+				subv = indirect(subv.Field(i))
+			}
+			if isUnifiable(subv) {
+				md.decoded[md.context.add(key).String()] = true
+				md.context = append(md.context, key)
+				if err := md.unify(datum, subv); err != nil {
+					return e("Type mismatch for '%s.%s': %s",
+						rv.Type().String(), f.name, err)
+				}
+				md.context = md.context[0 : len(md.context)-1]
+			} else if f.name != "" {
+				// Bad user! No soup for you!
+				return e("Field '%s.%s' is unexported, and therefore cannot "+
+					"be loaded with reflection.", rv.Type().String(), f.name)
+			}
+		}
+	}
+	return nil
+}
+
+func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error {
+	tmap, ok := mapping.(map[string]interface{})
+	if !ok {
+		if tmap == nil {
+			return nil
+		}
+		return badtype("map", mapping)
+	}
+	if rv.IsNil() {
+		rv.Set(reflect.MakeMap(rv.Type()))
+	}
+	for k, v := range tmap {
+		md.decoded[md.context.add(k).String()] = true
+		md.context = append(md.context, k)
+
+		rvkey := indirect(reflect.New(rv.Type().Key()))
+		rvval := reflect.Indirect(reflect.New(rv.Type().Elem()))
+		if err := md.unify(v, rvval); err != nil {
+			return err
+		}
+		md.context = md.context[0 : len(md.context)-1]
+
+		rvkey.SetString(k)
+		rv.SetMapIndex(rvkey, rvval)
+	}
+	return nil
+}
+
+func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error {
+	datav := reflect.ValueOf(data)
+	if datav.Kind() != reflect.Slice {
+		if !datav.IsValid() {
+			return nil
+		}
+		return badtype("slice", data)
+	}
+	sliceLen := datav.Len()
+	if sliceLen != rv.Len() {
+		return e("expected array length %d; got TOML array of length %d",
+			rv.Len(), sliceLen)
+	}
+	return md.unifySliceArray(datav, rv)
+}
+
+func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error {
+	datav := reflect.ValueOf(data)
+	if datav.Kind() != reflect.Slice {
+		if !datav.IsValid() {
+			return nil
+		}
+		return badtype("slice", data)
+	}
+	n := datav.Len()
+	if rv.IsNil() || rv.Cap() < n {
+		rv.Set(reflect.MakeSlice(rv.Type(), n, n))
+	}
+	rv.SetLen(n)
+	return md.unifySliceArray(datav, rv)
+}
+
+func (md *MetaData) unifySliceArray(data, rv reflect.Value) error {
+	sliceLen := data.Len()
+	for i := 0; i < sliceLen; i++ {
+		v := data.Index(i).Interface()
+		sliceval := indirect(rv.Index(i))
+		if err := md.unify(v, sliceval); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error {
+	if _, ok := data.(time.Time); ok {
+		rv.Set(reflect.ValueOf(data))
+		return nil
+	}
+	return badtype("time.Time", data)
+}
+
+func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error {
+	if s, ok := data.(string); ok {
+		rv.SetString(s)
+		return nil
+	}
+	return badtype("string", data)
+}
+
+func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error {
+	if num, ok := data.(float64); ok {
+		switch rv.Kind() {
+		case reflect.Float32:
+			fallthrough
+		case reflect.Float64:
+			rv.SetFloat(num)
+		default:
+			panic("bug")
+		}
+		return nil
+	}
+	return badtype("float", data)
+}
+
+func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error {
+	if num, ok := data.(int64); ok {
+		if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 {
+			switch rv.Kind() {
+			case reflect.Int, reflect.Int64:
+				// No bounds checking necessary.
+			case reflect.Int8:
+				if num < math.MinInt8 || num > math.MaxInt8 {
+					return e("Value '%d' is out of range for int8.", num)
+				}
+			case reflect.Int16:
+				if num < math.MinInt16 || num > math.MaxInt16 {
+					return e("Value '%d' is out of range for int16.", num)
+				}
+			case reflect.Int32:
+				if num < math.MinInt32 || num > math.MaxInt32 {
+					return e("Value '%d' is out of range for int32.", num)
+				}
+			}
+			rv.SetInt(num)
+		} else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 {
+			unum := uint64(num)
+			switch rv.Kind() {
+			case reflect.Uint, reflect.Uint64:
+				// No bounds checking necessary.
+			case reflect.Uint8:
+				if num < 0 || unum > math.MaxUint8 {
+					return e("Value '%d' is out of range for uint8.", num)
+				}
+			case reflect.Uint16:
+				if num < 0 || unum > math.MaxUint16 {
+					return e("Value '%d' is out of range for uint16.", num)
+				}
+			case reflect.Uint32:
+				if num < 0 || unum > math.MaxUint32 {
+					return e("Value '%d' is out of range for uint32.", num)
+				}
+			}
+			rv.SetUint(unum)
+		} else {
+			panic("unreachable")
+		}
+		return nil
+	}
+	return badtype("integer", data)
+}
+
+func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error {
+	if b, ok := data.(bool); ok {
+		rv.SetBool(b)
+		return nil
+	}
+	return badtype("boolean", data)
+}
+
+func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error {
+	rv.Set(reflect.ValueOf(data))
+	return nil
+}
+
+func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error {
+	var s string
+	switch sdata := data.(type) {
+	case TextMarshaler:
+		text, err := sdata.MarshalText()
+		if err != nil {
+			return err
+		}
+		s = string(text)
+	case fmt.Stringer:
+		s = sdata.String()
+	case string:
+		s = sdata
+	case bool:
+		s = fmt.Sprintf("%v", sdata)
+	case int64:
+		s = fmt.Sprintf("%d", sdata)
+	case float64:
+		s = fmt.Sprintf("%f", sdata)
+	default:
+		return badtype("primitive (string-like)", data)
+	}
+	if err := v.UnmarshalText([]byte(s)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// rvalue returns a reflect.Value of `v`. All pointers are resolved.
+func rvalue(v interface{}) reflect.Value {
+	return indirect(reflect.ValueOf(v))
+}
+
+// indirect returns the value pointed to by a pointer.
+// Pointers are followed until the value is not a pointer.
+// New values are allocated for each nil pointer.
+//
+// An exception to this rule is if the value satisfies an interface of
+// interest to us (like encoding.TextUnmarshaler).
+func indirect(v reflect.Value) reflect.Value {
+	if v.Kind() != reflect.Ptr {
+		if v.CanAddr() {
+			pv := v.Addr()
+			if _, ok := pv.Interface().(TextUnmarshaler); ok {
+				return pv
+			}
+		}
+		return v
+	}
+	if v.IsNil() {
+		v.Set(reflect.New(v.Type().Elem()))
+	}
+	return indirect(reflect.Indirect(v))
+}
+
+func isUnifiable(rv reflect.Value) bool {
+	if rv.CanSet() {
+		return true
+	}
+	if _, ok := rv.Interface().(TextUnmarshaler); ok {
+		return true
+	}
+	return false
+}
+
+func badtype(expected string, data interface{}) error {
+	return e("Expected %s but found '%T'.", expected, data)
+}
+
+func mismatch(user reflect.Value, expected string, data interface{}) error {
+	return e("Type mismatch for %s. Expected %s but found '%T'.",
+		user.Type().String(), expected, data)
+}
@@ -0,0 +1,122 @@
+package toml
+
+import "strings"
+
+// MetaData allows access to meta information about TOML data that may not
+// be inferrable via reflection. In particular, whether a key has been defined
+// and the TOML type of a key.
+type MetaData struct {
+	mapping map[string]interface{}
+	types   map[string]tomlType
+	keys    []Key
+	decoded map[string]bool
+	context Key // Used only during decoding.
+}
+
+// IsDefined returns true if the key given exists in the TOML data. The key
+// should be specified hierarchially. e.g.,
+//
+//	// access the TOML key 'a.b.c'
+//	IsDefined("a", "b", "c")
+//
+// IsDefined will return false if an empty key given. Keys are case sensitive.
+func (md *MetaData) IsDefined(key ...string) bool {
+	if len(key) == 0 {
+		return false
+	}
+
+	var hash map[string]interface{}
+	var ok bool
+	var hashOrVal interface{} = md.mapping
+	for _, k := range key {
+		if hash, ok = hashOrVal.(map[string]interface{}); !ok {
+			return false
+		}
+		if hashOrVal, ok = hash[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// Type returns a string representation of the type of the key specified.
+//
+// Type will return the empty string if given an empty key or a key that
+// does not exist. Keys are case sensitive.
+func (md *MetaData) Type(key ...string) string {
+	fullkey := strings.Join(key, ".")
+	if typ, ok := md.types[fullkey]; ok {
+		return typ.typeString()
+	}
+	return ""
+}
+
+// Key is the type of any TOML key, including key groups. Use (MetaData).Keys
+// to get values of this type.
+type Key []string
+
+func (k Key) String() string {
+	return strings.Join(k, ".")
+}
+
+func (k Key) maybeQuotedAll() string {
+	var ss []string
+	for i := range k {
+		ss = append(ss, k.maybeQuoted(i))
+	}
+	return strings.Join(ss, ".")
+}
+
+func (k Key) maybeQuoted(i int) string {
+	quote := false
+	for _, c := range k[i] {
+		if !isBareKeyChar(c) {
+			quote = true
+			break
+		}
+	}
+	if quote {
+		return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\""
+	} else {
+		return k[i]
+	}
+}
+
+func (k Key) add(piece string) Key {
+	newKey := make(Key, len(k)+1)
+	copy(newKey, k)
+	newKey[len(k)] = piece
+	return newKey
+}
+
+// Keys returns a slice of every key in the TOML data, including key groups.
+// Each key is itself a slice, where the first element is the top of the
+// hierarchy and the last is the most specific.
+//
+// The list will have the same order as the keys appeared in the TOML data.
+//
+// All keys returned are non-empty.
+func (md *MetaData) Keys() []Key {
+	return md.keys
+}
+
+// Undecoded returns all keys that have not been decoded in the order in which
+// they appear in the original TOML document.
+//
+// This includes keys that haven't been decoded because of a Primitive value.
+// Once the Primitive value is decoded, the keys will be considered decoded.
+//
+// Also note that decoding into an empty interface will result in no decoding,
+// and so no keys will be considered decoded.
+//
+// In this sense, the Undecoded keys correspond to keys in the TOML document
+// that do not have a concrete type in your representation.
+func (md *MetaData) Undecoded() []Key {
+	undecoded := make([]Key, 0, len(md.keys))
+	for _, key := range md.keys {
+		if !md.decoded[key.String()] {
+			undecoded = append(undecoded, key)
+		}
+	}
+	return undecoded
+}
@@ -0,0 +1,27 @@
+/*
+Package toml provides facilities for decoding and encoding TOML configuration
+files via reflection. There is also support for delaying decoding with
+the Primitive type, and querying the set of keys in a TOML document with the
+MetaData type.
+
+The specification implemented: https://github.com/mojombo/toml
+
+The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify
+whether a file is a valid TOML document. It can also be used to print the
+type of each key in a TOML document.
+
+Testing
+
+There are two important types of tests used for this package. The first is
+contained inside '*_test.go' files and uses the standard Go unit testing
+framework. These tests are primarily devoted to holistically testing the
+decoder and encoder.
+
+The second type of testing is used to verify the implementation's adherence
+to the TOML specification. These tests have been factored into their own
+project: https://github.com/BurntSushi/toml-test
+
+The reason the tests are in a separate project is so that they can be used by
+any implementation of TOML. Namely, it is language agnostic.
+*/
+package toml
@@ -0,0 +1,549 @@
+package toml
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type tomlEncodeError struct{ error }
+
+var (
+	errArrayMixedElementTypes = errors.New(
+		"can't encode array with mixed element types")
+	errArrayNilElement = errors.New(
+		"can't encode array with nil element")
+	errNonString = errors.New(
+		"can't encode a map with non-string key type")
+	errAnonNonStruct = errors.New(
+		"can't encode an anonymous field that is not a struct")
+	errArrayNoTable = errors.New(
+		"TOML array element can't contain a table")
+	errNoKey = errors.New(
+		"top-level values must be a Go map or struct")
+	errAnything = errors.New("") // used in testing
+)
+
+var quotedReplacer = strings.NewReplacer(
+	"\t", "\\t",
+	"\n", "\\n",
+	"\r", "\\r",
+	"\"", "\\\"",
+	"\\", "\\\\",
+)
+
+// Encoder controls the encoding of Go values to a TOML document to some
+// io.Writer.
+//
+// The indentation level can be controlled with the Indent field.
+type Encoder struct {
+	// A single indentation level. By default it is two spaces.
+	Indent string
+
+	// hasWritten is whether we have written any output to w yet.
+	hasWritten bool
+	w          *bufio.Writer
+}
+
+// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer
+// given. By default, a single indentation level is 2 spaces.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{
+		w:      bufio.NewWriter(w),
+		Indent: "  ",
+	}
+}
+
+// Encode writes a TOML representation of the Go value to the underlying
+// io.Writer. If the value given cannot be encoded to a valid TOML document,
+// then an error is returned.
+//
+// The mapping between Go values and TOML values should be precisely the same
+// as for the Decode* functions. Similarly, the TextMarshaler interface is
+// supported by encoding the resulting bytes as strings. (If you want to write
+// arbitrary binary data then you will need to use something like base64 since
+// TOML does not have any binary types.)
+//
+// When encoding TOML hashes (i.e., Go maps or structs), keys without any
+// sub-hashes are encoded first.
+//
+// If a Go map is encoded, then its keys are sorted alphabetically for
+// deterministic output. More control over this behavior may be provided if
+// there is demand for it.
+//
+// Encoding Go values without a corresponding TOML representation---like map
+// types with non-string keys---will cause an error to be returned. Similarly
+// for mixed arrays/slices, arrays/slices with nil elements, embedded
+// non-struct types and nested slices containing maps or structs.
+// (e.g., [][]map[string]string is not allowed but []map[string]string is OK
+// and so is []map[string][]string.)
+func (enc *Encoder) Encode(v interface{}) error {
+	rv := eindirect(reflect.ValueOf(v))
+	if err := enc.safeEncode(Key([]string{}), rv); err != nil {
+		return err
+	}
+	return enc.w.Flush()
+}
+
+func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			if terr, ok := r.(tomlEncodeError); ok {
+				err = terr.error
+				return
+			}
+			panic(r)
+		}
+	}()
+	enc.encode(key, rv)
+	return nil
+}
+
+func (enc *Encoder) encode(key Key, rv reflect.Value) {
+	// Special case. Time needs to be in ISO8601 format.
+	// Special case. If we can marshal the type to text, then we used that.
+	// Basically, this prevents the encoder for handling these types as
+	// generic structs (or whatever the underlying type of a TextMarshaler is).
+	switch rv.Interface().(type) {
+	case time.Time, TextMarshaler:
+		enc.keyEqElement(key, rv)
+		return
+	}
+
+	k := rv.Kind()
+	switch k {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
+		reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
+		reflect.Uint64,
+		reflect.Float32, reflect.Float64, reflect.String, reflect.Bool:
+		enc.keyEqElement(key, rv)
+	case reflect.Array, reflect.Slice:
+		if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) {
+			enc.eArrayOfTables(key, rv)
+		} else {
+			enc.keyEqElement(key, rv)
+		}
+	case reflect.Interface:
+		if rv.IsNil() {
+			return
+		}
+		enc.encode(key, rv.Elem())
+	case reflect.Map:
+		if rv.IsNil() {
+			return
+		}
+		enc.eTable(key, rv)
+	case reflect.Ptr:
+		if rv.IsNil() {
+			return
+		}
+		enc.encode(key, rv.Elem())
+	case reflect.Struct:
+		enc.eTable(key, rv)
+	default:
+		panic(e("Unsupported type for key '%s': %s", key, k))
+	}
+}
+
+// eElement encodes any value that can be an array element (primitives and
+// arrays).
+func (enc *Encoder) eElement(rv reflect.Value) {
+	switch v := rv.Interface().(type) {
+	case time.Time:
+		// Special case time.Time as a primitive. Has to come before
+		// TextMarshaler below because time.Time implements
+		// encoding.TextMarshaler, but we need to always use UTC.
+		enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z"))
+		return
+	case TextMarshaler:
+		// Special case. Use text marshaler if it's available for this value.
+		if s, err := v.MarshalText(); err != nil {
+			encPanic(err)
+		} else {
+			enc.writeQuoted(string(s))
+		}
+		return
+	}
+	switch rv.Kind() {
+	case reflect.Bool:
+		enc.wf(strconv.FormatBool(rv.Bool()))
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
+		reflect.Int64:
+		enc.wf(strconv.FormatInt(rv.Int(), 10))
+	case reflect.Uint, reflect.Uint8, reflect.Uint16,
+		reflect.Uint32, reflect.Uint64:
+		enc.wf(strconv.FormatUint(rv.Uint(), 10))
+	case reflect.Float32:
+		enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32)))
+	case reflect.Float64:
+		enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64)))
+	case reflect.Array, reflect.Slice:
+		enc.eArrayOrSliceElement(rv)
+	case reflect.Interface:
+		enc.eElement(rv.Elem())
+	case reflect.String:
+		enc.writeQuoted(rv.String())
+	default:
+		panic(e("Unexpected primitive type: %s", rv.Kind()))
+	}
+}
+
+// By the TOML spec, all floats must have a decimal with at least one
+// number on either side.
+func floatAddDecimal(fstr string) string {
+	if !strings.Contains(fstr, ".") {
+		return fstr + ".0"
+	}
+	return fstr
+}
+
+func (enc *Encoder) writeQuoted(s string) {
+	enc.wf("\"%s\"", quotedReplacer.Replace(s))
+}
+
+func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) {
+	length := rv.Len()
+	enc.wf("[")
+	for i := 0; i < length; i++ {
+		elem := rv.Index(i)
+		enc.eElement(elem)
+		if i != length-1 {
+			enc.wf(", ")
+		}
+	}
+	enc.wf("]")
+}
+
+func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) {
+	if len(key) == 0 {
+		encPanic(errNoKey)
+	}
+	for i := 0; i < rv.Len(); i++ {
+		trv := rv.Index(i)
+		if isNil(trv) {
+			continue
+		}
+		panicIfInvalidKey(key)
+		enc.newline()
+		enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll())
+		enc.newline()
+		enc.eMapOrStruct(key, trv)
+	}
+}
+
+func (enc *Encoder) eTable(key Key, rv reflect.Value) {
+	panicIfInvalidKey(key)
+	if len(key) == 1 {
+		// Output an extra new line between top-level tables.
+		// (The newline isn't written if nothing else has been written though.)
+		enc.newline()
+	}
+	if len(key) > 0 {
+		enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll())
+		enc.newline()
+	}
+	enc.eMapOrStruct(key, rv)
+}
+
+func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) {
+	switch rv := eindirect(rv); rv.Kind() {
+	case reflect.Map:
+		enc.eMap(key, rv)
+	case reflect.Struct:
+		enc.eStruct(key, rv)
+	default:
+		panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String())
+	}
+}
+
+func (enc *Encoder) eMap(key Key, rv reflect.Value) {
+	rt := rv.Type()
+	if rt.Key().Kind() != reflect.String {
+		encPanic(errNonString)
+	}
+
+	// Sort keys so that we have deterministic output. And write keys directly
+	// underneath this key first, before writing sub-structs or sub-maps.
+	var mapKeysDirect, mapKeysSub []string
+	for _, mapKey := range rv.MapKeys() {
+		k := mapKey.String()
+		if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) {
+			mapKeysSub = append(mapKeysSub, k)
+		} else {
+			mapKeysDirect = append(mapKeysDirect, k)
+		}
+	}
+
+	var writeMapKeys = func(mapKeys []string) {
+		sort.Strings(mapKeys)
+		for _, mapKey := range mapKeys {
+			mrv := rv.MapIndex(reflect.ValueOf(mapKey))
+			if isNil(mrv) {
+				// Don't write anything for nil fields.
+				continue
+			}
+			enc.encode(key.add(mapKey), mrv)
+		}
+	}
+	writeMapKeys(mapKeysDirect)
+	writeMapKeys(mapKeysSub)
+}
+
+func (enc *Encoder) eStruct(key Key, rv reflect.Value) {
+	// Write keys for fields directly under this key first, because if we write
+	// a field that creates a new table, then all keys under it will be in that
+	// table (not the one we're writing here).
+	rt := rv.Type()
+	var fieldsDirect, fieldsSub [][]int
+	var addFields func(rt reflect.Type, rv reflect.Value, start []int)
+	addFields = func(rt reflect.Type, rv reflect.Value, start []int) {
+		for i := 0; i < rt.NumField(); i++ {
+			f := rt.Field(i)
+			// skip unexported fields
+			if f.PkgPath != "" && !f.Anonymous {
+				continue
+			}
+			frv := rv.Field(i)
+			if f.Anonymous {
+				t := f.Type
+				switch t.Kind() {
+				case reflect.Struct:
+					addFields(t, frv, f.Index)
+					continue
+				case reflect.Ptr:
+					if t.Elem().Kind() == reflect.Struct {
+						if !frv.IsNil() {
+							addFields(t.Elem(), frv.Elem(), f.Index)
+						}
+						continue
+					}
+					// Fall through to the normal field encoding logic below
+					// for non-struct anonymous fields.
+				}
+			}
+
+			if typeIsHash(tomlTypeOfGo(frv)) {
+				fieldsSub = append(fieldsSub, append(start, f.Index...))
+			} else {
+				fieldsDirect = append(fieldsDirect, append(start, f.Index...))
+			}
+		}
+	}
+	addFields(rt, rv, nil)
+
+	var writeFields = func(fields [][]int) {
+		for _, fieldIndex := range fields {
+			sft := rt.FieldByIndex(fieldIndex)
+			sf := rv.FieldByIndex(fieldIndex)
+			if isNil(sf) {
+				// Don't write anything for nil fields.
+				continue
+			}
+
+			tag := sft.Tag.Get("toml")
+			if tag == "-" {
+				continue
+			}
+			keyName, opts := getOptions(tag)
+			if keyName == "" {
+				keyName = sft.Name
+			}
+			if _, ok := opts["omitempty"]; ok && isEmpty(sf) {
+				continue
+			} else if _, ok := opts["omitzero"]; ok && isZero(sf) {
+				continue
+			}
+
+			enc.encode(key.add(keyName), sf)
+		}
+	}
+	writeFields(fieldsDirect)
+	writeFields(fieldsSub)
+}
+
+// tomlTypeName returns the TOML type name of the Go value's type. It is
+// used to determine whether the types of array elements are mixed (which is
+// forbidden). If the Go value is nil, then it is illegal for it to be an array
+// element, and valueIsNil is returned as true.
+
+// Returns the TOML type of a Go value. The type may be `nil`, which means
+// no concrete TOML type could be found.
+func tomlTypeOfGo(rv reflect.Value) tomlType {
+	if isNil(rv) || !rv.IsValid() {
+		return nil
+	}
+	switch rv.Kind() {
+	case reflect.Bool:
+		return tomlBool
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
+		reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
+		reflect.Uint64:
+		return tomlInteger
+	case reflect.Float32, reflect.Float64:
+		return tomlFloat
+	case reflect.Array, reflect.Slice:
+		if typeEqual(tomlHash, tomlArrayType(rv)) {
+			return tomlArrayHash
+		} else {
+			return tomlArray
+		}
+	case reflect.Ptr, reflect.Interface:
+		return tomlTypeOfGo(rv.Elem())
+	case reflect.String:
+		return tomlString
+	case reflect.Map:
+		return tomlHash
+	case reflect.Struct:
+		switch rv.Interface().(type) {
+		case time.Time:
+			return tomlDatetime
+		case TextMarshaler:
+			return tomlString
+		default:
+			return tomlHash
+		}
+	default:
+		panic("unexpected reflect.Kind: " + rv.Kind().String())
+	}
+}
+
+// tomlArrayType returns the element type of a TOML array. The type returned
+// may be nil if it cannot be determined (e.g., a nil slice or a zero length
+// slize). This function may also panic if it finds a type that cannot be
+// expressed in TOML (such as nil elements, heterogeneous arrays or directly
+// nested arrays of tables).
+func tomlArrayType(rv reflect.Value) tomlType {
+	if isNil(rv) || !rv.IsValid() || rv.Len() == 0 {
+		return nil
+	}
+	firstType := tomlTypeOfGo(rv.Index(0))
+	if firstType == nil {
+		encPanic(errArrayNilElement)
+	}
+
+	rvlen := rv.Len()
+	for i := 1; i < rvlen; i++ {
+		elem := rv.Index(i)
+		switch elemType := tomlTypeOfGo(elem); {
+		case elemType == nil:
+			encPanic(errArrayNilElement)
+		case !typeEqual(firstType, elemType):
+			encPanic(errArrayMixedElementTypes)
+		}
+	}
+	// If we have a nested array, then we must make sure that the nested
+	// array contains ONLY primitives.
+	// This checks arbitrarily nested arrays.
+	if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) {
+		nest := tomlArrayType(eindirect(rv.Index(0)))
+		if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) {
+			encPanic(errArrayNoTable)
+		}
+	}
+	return firstType
+}
+
+func getOptions(keyName string) (string, map[string]struct{}) {
+	opts := make(map[string]struct{})
+	ss := strings.Split(keyName, ",")
+	name := ss[0]
+	if len(ss) > 1 {
+		for _, opt := range ss {
+			opts[opt] = struct{}{}
+		}
+	}
+
+	return name, opts
+}
+
+func isZero(rv reflect.Value) bool {
+	switch rv.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return rv.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return rv.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return rv.Float() == 0.0
+	}
+	return false
+}
+
+func isEmpty(rv reflect.Value) bool {
+	switch rv.Kind() {
+	case reflect.Array, reflect.Slice, reflect.Map, reflect.String:
+		return rv.Len() == 0
+	case reflect.Bool:
+		return !rv.Bool()
+	}
+	return false
+}
+
+func (enc *Encoder) newline() {
+	if enc.hasWritten {
+		enc.wf("\n")
+	}
+}
+
+func (enc *Encoder) keyEqElement(key Key, val reflect.Value) {
+	if len(key) == 0 {
+		encPanic(errNoKey)
+	}
+	panicIfInvalidKey(key)
+	enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1))
+	enc.eElement(val)
+	enc.newline()
+}
+
+func (enc *Encoder) wf(format string, v ...interface{}) {
+	if _, err := fmt.Fprintf(enc.w, format, v...); err != nil {
+		encPanic(err)
+	}
+	enc.hasWritten = true
+}
+
+func (enc *Encoder) indentStr(key Key) string {
+	return strings.Repeat(enc.Indent, len(key)-1)
+}
+
+func encPanic(err error) {
+	panic(tomlEncodeError{err})
+}
+
+func eindirect(v reflect.Value) reflect.Value {
+	switch v.Kind() {
+	case reflect.Ptr, reflect.Interface:
+		return eindirect(v.Elem())
+	default:
+		return v
+	}
+}
+
+func isNil(rv reflect.Value) bool {
+	switch rv.Kind() {
+	case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+		return rv.IsNil()
+	default:
+		return false
+	}
+}
+
+func panicIfInvalidKey(key Key) {
+	for _, k := range key {
+		if len(k) == 0 {
+			encPanic(e("Key '%s' is not a valid table name. Key names "+
+				"cannot be empty.", key.maybeQuotedAll()))
+		}
+	}
+}
+
+func isValidKeyName(s string) bool {
+	return len(s) != 0
+}
@@ -0,0 +1,590 @@
+package toml
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestEncodeRoundTrip(t *testing.T) {
+	type Config struct {
+		Age        int
+		Cats       []string
+		Pi         float64
+		Perfection []int
+		DOB        time.Time
+		Ipaddress  net.IP
+	}
+
+	var inputs = Config{
+		13,
+		[]string{"one", "two", "three"},
+		3.145,
+		[]int{11, 2, 3, 4},
+		time.Now(),
+		net.ParseIP("192.168.59.254"),
+	}
+
+	var firstBuffer bytes.Buffer
+	e := NewEncoder(&firstBuffer)
+	err := e.Encode(inputs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var outputs Config
+	if _, err := Decode(firstBuffer.String(), &outputs); err != nil {
+		log.Printf("Could not decode:\n-----\n%s\n-----\n",
+			firstBuffer.String())
+		t.Fatal(err)
+	}
+
+	// could test each value individually, but I'm lazy
+	var secondBuffer bytes.Buffer
+	e2 := NewEncoder(&secondBuffer)
+	err = e2.Encode(outputs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if firstBuffer.String() != secondBuffer.String() {
+		t.Error(
+			firstBuffer.String(),
+			"\n\n is not identical to\n\n",
+			secondBuffer.String())
+	}
+}
+
+// XXX(burntsushi)
+// I think these tests probably should be removed. They are good, but they
+// ought to be obsolete by toml-test.
+func TestEncode(t *testing.T) {
+	type Embedded struct {
+		Int int `toml:"_int"`
+	}
+	type NonStruct int
+
+	date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600))
+	dateStr := "2014-05-11T19:30:40Z"
+
+	tests := map[string]struct {
+		input      interface{}
+		wantOutput string
+		wantError  error
+	}{
+		"bool field": {
+			input: struct {
+				BoolTrue  bool
+				BoolFalse bool
+			}{true, false},
+			wantOutput: "BoolTrue = true\nBoolFalse = false\n",
+		},
+		"int fields": {
+			input: struct {
+				Int   int
+				Int8  int8
+				Int16 int16
+				Int32 int32
+				Int64 int64
+			}{1, 2, 3, 4, 5},
+			wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n",
+		},
+		"uint fields": {
+			input: struct {
+				Uint   uint
+				Uint8  uint8
+				Uint16 uint16
+				Uint32 uint32
+				Uint64 uint64
+			}{1, 2, 3, 4, 5},
+			wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" +
+				"\nUint64 = 5\n",
+		},
+		"float fields": {
+			input: struct {
+				Float32 float32
+				Float64 float64
+			}{1.5, 2.5},
+			wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n",
+		},
+		"string field": {
+			input:      struct{ String string }{"foo"},
+			wantOutput: "String = \"foo\"\n",
+		},
+		"string field and unexported field": {
+			input: struct {
+				String     string
+				unexported int
+			}{"foo", 0},
+			wantOutput: "String = \"foo\"\n",
+		},
+		"datetime field in UTC": {
+			input:      struct{ Date time.Time }{date},
+			wantOutput: fmt.Sprintf("Date = %s\n", dateStr),
+		},
+		"datetime field as primitive": {
+			// Using a map here to fail if isStructOrMap() returns true for
+			// time.Time.
+			input: map[string]interface{}{
+				"Date": date,
+				"Int":  1,
+			},
+			wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr),
+		},
+		"array fields": {
+			input: struct {
+				IntArray0 [0]int
+				IntArray3 [3]int
+			}{[0]int{}, [3]int{1, 2, 3}},
+			wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n",
+		},
+		"slice fields": {
+			input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{
+				nil, []int{}, []int{1, 2, 3},
+			},
+			wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n",
+		},
+		"datetime slices": {
+			input: struct{ DatetimeSlice []time.Time }{
+				[]time.Time{date, date},
+			},
+			wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n",
+				dateStr, dateStr),
+		},
+		"nested arrays and slices": {
+			input: struct {
+				SliceOfArrays         [][2]int
+				ArrayOfSlices         [2][]int
+				SliceOfArraysOfSlices [][2][]int
+				ArrayOfSlicesOfArrays [2][][2]int
+				SliceOfMixedArrays    [][2]interface{}
+				ArrayOfMixedSlices    [2][]interface{}
+			}{
+				[][2]int{{1, 2}, {3, 4}},
+				[2][]int{{1, 2}, {3, 4}},
+				[][2][]int{
+					{
+						{1, 2}, {3, 4},
+					},
+					{
+						{5, 6}, {7, 8},
+					},
+				},
+				[2][][2]int{
+					{
+						{1, 2}, {3, 4},
+					},
+					{
+						{5, 6}, {7, 8},
+					},
+				},
+				[][2]interface{}{
+					{1, 2}, {"a", "b"},
+				},
+				[2][]interface{}{
+					{1, 2}, {"a", "b"},
+				},
+			},
+			wantOutput: `SliceOfArrays = [[1, 2], [3, 4]]
+ArrayOfSlices = [[1, 2], [3, 4]]
+SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+SliceOfMixedArrays = [[1, 2], ["a", "b"]]
+ArrayOfMixedSlices = [[1, 2], ["a", "b"]]
+`,
+		},
+		"empty slice": {
+			input:      struct{ Empty []interface{} }{[]interface{}{}},
+			wantOutput: "Empty = []\n",
+		},
+		"(error) slice with element type mismatch (string and integer)": {
+			input:     struct{ Mixed []interface{} }{[]interface{}{1, "a"}},
+			wantError: errArrayMixedElementTypes,
+		},
+		"(error) slice with element type mismatch (integer and float)": {
+			input:     struct{ Mixed []interface{} }{[]interface{}{1, 2.5}},
+			wantError: errArrayMixedElementTypes,
+		},
+		"slice with elems of differing Go types, same TOML types": {
+			input: struct {
+				MixedInts   []interface{}
+				MixedFloats []interface{}
+			}{
+				[]interface{}{
+					int(1), int8(2), int16(3), int32(4), int64(5),
+					uint(1), uint8(2), uint16(3), uint32(4), uint64(5),
+				},
+				[]interface{}{float32(1.5), float64(2.5)},
+			},
+			wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" +
+				"MixedFloats = [1.5, 2.5]\n",
+		},
+		"(error) slice w/ element type mismatch (one is nested array)": {
+			input: struct{ Mixed []interface{} }{
+				[]interface{}{1, []interface{}{2}},
+			},
+			wantError: errArrayMixedElementTypes,
+		},
+		"(error) slice with 1 nil element": {
+			input:     struct{ NilElement1 []interface{} }{[]interface{}{nil}},
+			wantError: errArrayNilElement,
+		},
+		"(error) slice with 1 nil element (and other non-nil elements)": {
+			input: struct{ NilElement []interface{} }{
+				[]interface{}{1, nil},
+			},
+			wantError: errArrayNilElement,
+		},
+		"simple map": {
+			input:      map[string]int{"a": 1, "b": 2},
+			wantOutput: "a = 1\nb = 2\n",
+		},
+		"map with interface{} value type": {
+			input:      map[string]interface{}{"a": 1, "b": "c"},
+			wantOutput: "a = 1\nb = \"c\"\n",
+		},
+		"map with interface{} value type, some of which are structs": {
+			input: map[string]interface{}{
+				"a": struct{ Int int }{2},
+				"b": 1,
+			},
+			wantOutput: "b = 1\n\n[a]\n  Int = 2\n",
+		},
+		"nested map": {
+			input: map[string]map[string]int{
+				"a": {"b": 1},
+				"c": {"d": 2},
+			},
+			wantOutput: "[a]\n  b = 1\n\n[c]\n  d = 2\n",
+		},
+		"nested struct": {
+			input: struct{ Struct struct{ Int int } }{
+				struct{ Int int }{1},
+			},
+			wantOutput: "[Struct]\n  Int = 1\n",
+		},
+		"nested struct and non-struct field": {
+			input: struct {
+				Struct struct{ Int int }
+				Bool   bool
+			}{struct{ Int int }{1}, true},
+			wantOutput: "Bool = true\n\n[Struct]\n  Int = 1\n",
+		},
+		"2 nested structs": {
+			input: struct{ Struct1, Struct2 struct{ Int int } }{
+				struct{ Int int }{1}, struct{ Int int }{2},
+			},
+			wantOutput: "[Struct1]\n  Int = 1\n\n[Struct2]\n  Int = 2\n",
+		},
+		"deeply nested structs": {
+			input: struct {
+				Struct1, Struct2 struct{ Struct3 *struct{ Int int } }
+			}{
+				struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}},
+				struct{ Struct3 *struct{ Int int } }{nil},
+			},
+			wantOutput: "[Struct1]\n  [Struct1.Struct3]\n    Int = 1" +
+				"\n\n[Struct2]\n",
+		},
+		"nested struct with nil struct elem": {
+			input: struct {
+				Struct struct{ Inner *struct{ Int int } }
+			}{
+				struct{ Inner *struct{ Int int } }{nil},
+			},
+			wantOutput: "[Struct]\n",
+		},
+		"nested struct with no fields": {
+			input: struct {
+				Struct struct{ Inner struct{} }
+			}{
+				struct{ Inner struct{} }{struct{}{}},
+			},
+			wantOutput: "[Struct]\n  [Struct.Inner]\n",
+		},
+		"struct with tags": {
+			input: struct {
+				Struct struct {
+					Int int `toml:"_int"`
+				} `toml:"_struct"`
+				Bool bool `toml:"_bool"`
+			}{
+				struct {
+					Int int `toml:"_int"`
+				}{1}, true,
+			},
+			wantOutput: "_bool = true\n\n[_struct]\n  _int = 1\n",
+		},
+		"embedded struct": {
+			input:      struct{ Embedded }{Embedded{1}},
+			wantOutput: "_int = 1\n",
+		},
+		"embedded *struct": {
+			input:      struct{ *Embedded }{&Embedded{1}},
+			wantOutput: "_int = 1\n",
+		},
+		"nested embedded struct": {
+			input: struct {
+				Struct struct{ Embedded } `toml:"_struct"`
+			}{struct{ Embedded }{Embedded{1}}},
+			wantOutput: "[_struct]\n  _int = 1\n",
+		},
+		"nested embedded *struct": {
+			input: struct {
+				Struct struct{ *Embedded } `toml:"_struct"`
+			}{struct{ *Embedded }{&Embedded{1}}},
+			wantOutput: "[_struct]\n  _int = 1\n",
+		},
+		"embedded non-struct": {
+			input:      struct{ NonStruct }{5},
+			wantOutput: "NonStruct = 5\n",
+		},
+		"array of tables": {
+			input: struct {
+				Structs []*struct{ Int int } `toml:"struct"`
+			}{
+				[]*struct{ Int int }{{1}, {3}},
+			},
+			wantOutput: "[[struct]]\n  Int = 1\n\n[[struct]]\n  Int = 3\n",
+		},
+		"array of tables order": {
+			input: map[string]interface{}{
+				"map": map[string]interface{}{
+					"zero": 5,
+					"arr": []map[string]int{
+						{
+							"friend": 5,
+						},
+					},
+				},
+			},
+			wantOutput: "[map]\n  zero = 5\n\n  [[map.arr]]\n    friend = 5\n",
+		},
+		"(error) top-level slice": {
+			input:     []struct{ Int int }{{1}, {2}, {3}},
+			wantError: errNoKey,
+		},
+		"(error) slice of slice": {
+			input: struct {
+				Slices [][]struct{ Int int }
+			}{
+				[][]struct{ Int int }{{{1}}, {{2}}, {{3}}},
+			},
+			wantError: errArrayNoTable,
+		},
+		"(error) map no string key": {
+			input:     map[int]string{1: ""},
+			wantError: errNonString,
+		},
+		"(error) empty key name": {
+			input:     map[string]int{"": 1},
+			wantError: errAnything,
+		},
+		"(error) empty map name": {
+			input: map[string]interface{}{
+				"": map[string]int{"v": 1},
+			},
+			wantError: errAnything,
+		},
+	}
+	for label, test := range tests {
+		encodeExpected(t, label, test.input, test.wantOutput, test.wantError)
+	}
+}
+
+func TestEncodeNestedTableArrays(t *testing.T) {
+	type song struct {
+		Name string `toml:"name"`
+	}
+	type album struct {
+		Name  string `toml:"name"`
+		Songs []song `toml:"songs"`
+	}
+	type springsteen struct {
+		Albums []album `toml:"albums"`
+	}
+	value := springsteen{
+		[]album{
+			{"Born to Run",
+				[]song{{"Jungleland"}, {"Meeting Across the River"}}},
+			{"Born in the USA",
+				[]song{{"Glory Days"}, {"Dancing in the Dark"}}},
+		},
+	}
+	expected := `[[albums]]
+  name = "Born to Run"
+
+  [[albums.songs]]
+    name = "Jungleland"
+
+  [[albums.songs]]
+    name = "Meeting Across the River"
+
+[[albums]]
+  name = "Born in the USA"
+
+  [[albums.songs]]
+    name = "Glory Days"
+
+  [[albums.songs]]
+    name = "Dancing in the Dark"
+`
+	encodeExpected(t, "nested table arrays", value, expected, nil)
+}
+
+func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) {
+	type Alpha struct {
+		V int
+	}
+	type Beta struct {
+		V int
+	}
+	type Conf struct {
+		V int
+		A Alpha
+		B []Beta
+	}
+
+	val := Conf{
+		V: 1,
+		A: Alpha{2},
+		B: []Beta{{3}},
+	}
+	expected := "V = 1\n\n[A]\n  V = 2\n\n[[B]]\n  V = 3\n"
+	encodeExpected(t, "array hash with normal hash order", val, expected, nil)
+}
+
+func TestEncodeWithOmitEmpty(t *testing.T) {
+	type simple struct {
+		Bool   bool              `toml:"bool,omitempty"`
+		String string            `toml:"string,omitempty"`
+		Array  [0]byte           `toml:"array,omitempty"`
+		Slice  []int             `toml:"slice,omitempty"`
+		Map    map[string]string `toml:"map,omitempty"`
+	}
+
+	var v simple
+	encodeExpected(t, "fields with omitempty are omitted when empty", v, "", nil)
+	v = simple{
+		Bool:   true,
+		String: " ",
+		Slice:  []int{2, 3, 4},
+		Map:    map[string]string{"foo": "bar"},
+	}
+	expected := `bool = true
+string = " "
+slice = [2, 3, 4]
+
+[map]
+  foo = "bar"
+`
+	encodeExpected(t, "fields with omitempty are not omitted when non-empty",
+		v, expected, nil)
+}
+
+func TestEncodeWithOmitZero(t *testing.T) {
+	type simple struct {
+		Number   int     `toml:"number,omitzero"`
+		Real     float64 `toml:"real,omitzero"`
+		Unsigned uint    `toml:"unsigned,omitzero"`
+	}
+
+	value := simple{0, 0.0, uint(0)}
+	expected := ""
+
+	encodeExpected(t, "simple with omitzero, all zero", value, expected, nil)
+
+	value.Number = 10
+	value.Real = 20
+	value.Unsigned = 5
+	expected = `number = 10
+real = 20.0
+unsigned = 5
+`
+	encodeExpected(t, "simple with omitzero, non-zero", value, expected, nil)
+}
+
+func TestEncodeOmitemptyWithEmptyName(t *testing.T) {
+	type simple struct {
+		S []int `toml:",omitempty"`
+	}
+	v := simple{[]int{1, 2, 3}}
+	expected := "S = [1, 2, 3]\n"
+	encodeExpected(t, "simple with omitempty, no name, non-empty field",
+		v, expected, nil)
+}
+
+func TestEncodeAnonymousStructPointerField(t *testing.T) {
+	type Sub struct{}
+	type simple struct {
+		*Sub
+	}
+
+	value := simple{}
+	expected := ""
+	encodeExpected(t, "nil anonymous struct pointer field", value, expected, nil)
+
+	value = simple{Sub: &Sub{}}
+	expected = ""
+	encodeExpected(t, "non-nil anonymous struct pointer field", value, expected, nil)
+}
+
+func TestEncodeIgnoredFields(t *testing.T) {
+	type simple struct {
+		Number int `toml:"-"`
+	}
+	value := simple{}
+	expected := ""
+	encodeExpected(t, "ignored field", value, expected, nil)
+}
+
+func encodeExpected(
+	t *testing.T, label string, val interface{}, wantStr string, wantErr error,
+) {
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	err := enc.Encode(val)
+	if err != wantErr {
+		if wantErr != nil {
+			if wantErr == errAnything && err != nil {
+				return
+			}
+			t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err)
+		} else {
+			t.Errorf("%s: Encode failed: %s", label, err)
+		}
+	}
+	if err != nil {
+		return
+	}
+	if got := buf.String(); wantStr != got {
+		t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n",
+			label, wantStr, got)
+	}
+}
+
+func ExampleEncoder_Encode() {
+	date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC")
+	var config = map[string]interface{}{
+		"date":   date,
+		"counts": []int{1, 1, 2, 3, 5, 8},
+		"hash": map[string]string{
+			"key1": "val1",
+			"key2": "val2",
+		},
+	}
+	buf := new(bytes.Buffer)
+	if err := NewEncoder(buf).Encode(config); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(buf.String())
+
+	// Output:
+	// counts = [1, 1, 2, 3, 5, 8]
+	// date = 2010-03-14T18:00:00Z
+	//
+	// [hash]
+	//   key1 = "val1"
+	//   key2 = "val2"
+}
@@ -0,0 +1,19 @@
+// +build go1.2
+
+package toml
+
+// In order to support Go 1.1, we define our own TextMarshaler and
+// TextUnmarshaler types. For Go 1.2+, we just alias them with the
+// standard library interfaces.
+
+import (
+	"encoding"
+)
+
+// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
+// so that Go 1.1 can be supported.
+type TextMarshaler encoding.TextMarshaler
+
+// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
+// here so that Go 1.1 can be supported.
+type TextUnmarshaler encoding.TextUnmarshaler
@@ -0,0 +1,18 @@
+// +build !go1.2
+
+package toml
+
+// These interfaces were introduced in Go 1.2, so we add them manually when
+// compiling for Go 1.1.
+
+// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
+// so that Go 1.1 can be supported.
+type TextMarshaler interface {
+	MarshalText() (text []byte, err error)
+}
+
+// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
+// here so that Go 1.1 can be supported.
+type TextUnmarshaler interface {
+	UnmarshalText(text []byte) error
+}
@@ -0,0 +1,871 @@
+package toml
+
+import (
+	"fmt"
+	"strings"
+	"unicode/utf8"
+)
+
+type itemType int
+
+const (
+	itemError itemType = iota
+	itemNIL            // used in the parser to indicate no type
+	itemEOF
+	itemText
+	itemString
+	itemRawString
+	itemMultilineString
+	itemRawMultilineString
+	itemBool
+	itemInteger
+	itemFloat
+	itemDatetime
+	itemArray // the start of an array
+	itemArrayEnd
+	itemTableStart
+	itemTableEnd
+	itemArrayTableStart
+	itemArrayTableEnd
+	itemKeyStart
+	itemCommentStart
+)
+
+const (
+	eof             = 0
+	tableStart      = '['
+	tableEnd        = ']'
+	arrayTableStart = '['
+	arrayTableEnd   = ']'
+	tableSep        = '.'
+	keySep          = '='
+	arrayStart      = '['
+	arrayEnd        = ']'
+	arrayValTerm    = ','
+	commentStart    = '#'
+	stringStart     = '"'
+	stringEnd       = '"'
+	rawStringStart  = '\''
+	rawStringEnd    = '\''
+)
+
+type stateFn func(lx *lexer) stateFn
+
+type lexer struct {
+	input string
+	start int
+	pos   int
+	width int
+	line  int
+	state stateFn
+	items chan item
+
+	// A stack of state functions used to maintain context.
+	// The idea is to reuse parts of the state machine in various places.
+	// For example, values can appear at the top level or within arbitrarily
+	// nested arrays. The last state on the stack is used after a value has
+	// been lexed. Similarly for comments.
+	stack []stateFn
+}
+
+type item struct {
+	typ  itemType
+	val  string
+	line int
+}
+
+func (lx *lexer) nextItem() item {
+	for {
+		select {
+		case item := <-lx.items:
+			return item
+		default:
+			lx.state = lx.state(lx)
+		}
+	}
+}
+
+func lex(input string) *lexer {
+	lx := &lexer{
+		input: input + "\n",
+		state: lexTop,
+		line:  1,
+		items: make(chan item, 10),
+		stack: make([]stateFn, 0, 10),
+	}
+	return lx
+}
+
+func (lx *lexer) push(state stateFn) {
+	lx.stack = append(lx.stack, state)
+}
+
+func (lx *lexer) pop() stateFn {
+	if len(lx.stack) == 0 {
+		return lx.errorf("BUG in lexer: no states to pop.")
+	}
+	last := lx.stack[len(lx.stack)-1]
+	lx.stack = lx.stack[0 : len(lx.stack)-1]
+	return last
+}
+
+func (lx *lexer) current() string {
+	return lx.input[lx.start:lx.pos]
+}
+
+func (lx *lexer) emit(typ itemType) {
+	lx.items <- item{typ, lx.current(), lx.line}
+	lx.start = lx.pos
+}
+
+func (lx *lexer) emitTrim(typ itemType) {
+	lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line}
+	lx.start = lx.pos
+}
+
+func (lx *lexer) next() (r rune) {
+	if lx.pos >= len(lx.input) {
+		lx.width = 0
+		return eof
+	}
+
+	if lx.input[lx.pos] == '\n' {
+		lx.line++
+	}
+	r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:])
+	lx.pos += lx.width
+	return r
+}
+
+// ignore skips over the pending input before this point.
+func (lx *lexer) ignore() {
+	lx.start = lx.pos
+}
+
+// backup steps back one rune. Can be called only once per call of next.
+func (lx *lexer) backup() {
+	lx.pos -= lx.width
+	if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' {
+		lx.line--
+	}
+}
+
+// accept consumes the next rune if it's equal to `valid`.
+func (lx *lexer) accept(valid rune) bool {
+	if lx.next() == valid {
+		return true
+	}
+	lx.backup()
+	return false
+}
+
+// peek returns but does not consume the next rune in the input.
+func (lx *lexer) peek() rune {
+	r := lx.next()
+	lx.backup()
+	return r
+}
+
+// errorf stops all lexing by emitting an error and returning `nil`.
+// Note that any value that is a character is escaped if it's a special
+// character (new lines, tabs, etc.).
+func (lx *lexer) errorf(format string, values ...interface{}) stateFn {
+	lx.items <- item{
+		itemError,
+		fmt.Sprintf(format, values...),
+		lx.line,
+	}
+	return nil
+}
+
+// lexTop consumes elements at the top level of TOML data.
+func lexTop(lx *lexer) stateFn {
+	r := lx.next()
+	if isWhitespace(r) || isNL(r) {
+		return lexSkip(lx, lexTop)
+	}
+
+	switch r {
+	case commentStart:
+		lx.push(lexTop)
+		return lexCommentStart
+	case tableStart:
+		return lexTableStart
+	case eof:
+		if lx.pos > lx.start {
+			return lx.errorf("Unexpected EOF.")
+		}
+		lx.emit(itemEOF)
+		return nil
+	}
+
+	// At this point, the only valid item can be a key, so we back up
+	// and let the key lexer do the rest.
+	lx.backup()
+	lx.push(lexTopEnd)
+	return lexKeyStart
+}
+
+// lexTopEnd is entered whenever a top-level item has been consumed. (A value
+// or a table.) It must see only whitespace, and will turn back to lexTop
+// upon a new line. If it sees EOF, it will quit the lexer successfully.
+func lexTopEnd(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == commentStart:
+		// a comment will read to a new line for us.
+		lx.push(lexTop)
+		return lexCommentStart
+	case isWhitespace(r):
+		return lexTopEnd
+	case isNL(r):
+		lx.ignore()
+		return lexTop
+	case r == eof:
+		lx.ignore()
+		return lexTop
+	}
+	return lx.errorf("Expected a top-level item to end with a new line, "+
+		"comment or EOF, but got %q instead.", r)
+}
+
+// lexTable lexes the beginning of a table. Namely, it makes sure that
+// it starts with a character other than '.' and ']'.
+// It assumes that '[' has already been consumed.
+// It also handles the case that this is an item in an array of tables.
+// e.g., '[[name]]'.
+func lexTableStart(lx *lexer) stateFn {
+	if lx.peek() == arrayTableStart {
+		lx.next()
+		lx.emit(itemArrayTableStart)
+		lx.push(lexArrayTableEnd)
+	} else {
+		lx.emit(itemTableStart)
+		lx.push(lexTableEnd)
+	}
+	return lexTableNameStart
+}
+
+func lexTableEnd(lx *lexer) stateFn {
+	lx.emit(itemTableEnd)
+	return lexTopEnd
+}
+
+func lexArrayTableEnd(lx *lexer) stateFn {
+	if r := lx.next(); r != arrayTableEnd {
+		return lx.errorf("Expected end of table array name delimiter %q, "+
+			"but got %q instead.", arrayTableEnd, r)
+	}
+	lx.emit(itemArrayTableEnd)
+	return lexTopEnd
+}
+
+func lexTableNameStart(lx *lexer) stateFn {
+	switch r := lx.peek(); {
+	case r == tableEnd || r == eof:
+		return lx.errorf("Unexpected end of table name. (Table names cannot " +
+			"be empty.)")
+	case r == tableSep:
+		return lx.errorf("Unexpected table separator. (Table names cannot " +
+			"be empty.)")
+	case r == stringStart || r == rawStringStart:
+		lx.ignore()
+		lx.push(lexTableNameEnd)
+		return lexValue // reuse string lexing
+	default:
+		return lexBareTableName
+	}
+}
+
+// lexTableName lexes the name of a table. It assumes that at least one
+// valid character for the table has already been read.
+func lexBareTableName(lx *lexer) stateFn {
+	switch r := lx.next(); {
+	case isBareKeyChar(r):
+		return lexBareTableName
+	case r == tableSep || r == tableEnd:
+		lx.backup()
+		lx.emitTrim(itemText)
+		return lexTableNameEnd
+	default:
+		return lx.errorf("Bare keys cannot contain %q.", r)
+	}
+}
+
+// lexTableNameEnd reads the end of a piece of a table name, optionally
+// consuming whitespace.
+func lexTableNameEnd(lx *lexer) stateFn {
+	switch r := lx.next(); {
+	case isWhitespace(r):
+		return lexTableNameEnd
+	case r == tableSep:
+		lx.ignore()
+		return lexTableNameStart
+	case r == tableEnd:
+		return lx.pop()
+	default:
+		return lx.errorf("Expected '.' or ']' to end table name, but got %q "+
+			"instead.", r)
+	}
+}
+
+// lexKeyStart consumes a key name up until the first non-whitespace character.
+// lexKeyStart will ignore whitespace.
+func lexKeyStart(lx *lexer) stateFn {
+	r := lx.peek()
+	switch {
+	case r == keySep:
+		return lx.errorf("Unexpected key separator %q.", keySep)
+	case isWhitespace(r) || isNL(r):
+		lx.next()
+		return lexSkip(lx, lexKeyStart)
+	case r == stringStart || r == rawStringStart:
+		lx.ignore()
+		lx.emit(itemKeyStart)
+		lx.push(lexKeyEnd)
+		return lexValue // reuse string lexing
+	default:
+		lx.ignore()
+		lx.emit(itemKeyStart)
+		return lexBareKey
+	}
+}
+
+// lexBareKey consumes the text of a bare key. Assumes that the first character
+// (which is not whitespace) has not yet been consumed.
+func lexBareKey(lx *lexer) stateFn {
+	switch r := lx.next(); {
+	case isBareKeyChar(r):
+		return lexBareKey
+	case isWhitespace(r):
+		lx.emitTrim(itemText)
+		return lexKeyEnd
+	case r == keySep:
+		lx.backup()
+		lx.emitTrim(itemText)
+		return lexKeyEnd
+	default:
+		return lx.errorf("Bare keys cannot contain %q.", r)
+	}
+}
+
+// lexKeyEnd consumes the end of a key and trims whitespace (up to the key
+// separator).
+func lexKeyEnd(lx *lexer) stateFn {
+	switch r := lx.next(); {
+	case r == keySep:
+		return lexSkip(lx, lexValue)
+	case isWhitespace(r):
+		return lexSkip(lx, lexKeyEnd)
+	default:
+		return lx.errorf("Expected key separator %q, but got %q instead.",
+			keySep, r)
+	}
+}
+
+// lexValue starts the consumption of a value anywhere a value is expected.
+// lexValue will ignore whitespace.
+// After a value is lexed, the last state on the next is popped and returned.
+func lexValue(lx *lexer) stateFn {
+	// We allow whitespace to precede a value, but NOT new lines.
+	// In array syntax, the array states are responsible for ignoring new
+	// lines.
+	r := lx.next()
+	if isWhitespace(r) {
+		return lexSkip(lx, lexValue)
+	}
+
+	switch {
+	case r == arrayStart:
+		lx.ignore()
+		lx.emit(itemArray)
+		return lexArrayValue
+	case r == stringStart:
+		if lx.accept(stringStart) {
+			if lx.accept(stringStart) {
+				lx.ignore() // Ignore """
+				return lexMultilineString
+			}
+			lx.backup()
+		}
+		lx.ignore() // ignore the '"'
+		return lexString
+	case r == rawStringStart:
+		if lx.accept(rawStringStart) {
+			if lx.accept(rawStringStart) {
+				lx.ignore() // Ignore """
+				return lexMultilineRawString
+			}
+			lx.backup()
+		}
+		lx.ignore() // ignore the "'"
+		return lexRawString
+	case r == 't':
+		return lexTrue
+	case r == 'f':
+		return lexFalse
+	case r == '-':
+		return lexNumberStart
+	case isDigit(r):
+		lx.backup() // avoid an extra state and use the same as above
+		return lexNumberOrDateStart
+	case r == '.': // special error case, be kind to users
+		return lx.errorf("Floats must start with a digit, not '.'.")
+	}
+	return lx.errorf("Expected value but found %q instead.", r)
+}
+
+// lexArrayValue consumes one value in an array. It assumes that '[' or ','
+// have already been consumed. All whitespace and new lines are ignored.
+func lexArrayValue(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isWhitespace(r) || isNL(r):
+		return lexSkip(lx, lexArrayValue)
+	case r == commentStart:
+		lx.push(lexArrayValue)
+		return lexCommentStart
+	case r == arrayValTerm:
+		return lx.errorf("Unexpected array value terminator %q.",
+			arrayValTerm)
+	case r == arrayEnd:
+		return lexArrayEnd
+	}
+
+	lx.backup()
+	lx.push(lexArrayValueEnd)
+	return lexValue
+}
+
+// lexArrayValueEnd consumes the cruft between values of an array. Namely,
+// it ignores whitespace and expects either a ',' or a ']'.
+func lexArrayValueEnd(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isWhitespace(r) || isNL(r):
+		return lexSkip(lx, lexArrayValueEnd)
+	case r == commentStart:
+		lx.push(lexArrayValueEnd)
+		return lexCommentStart
+	case r == arrayValTerm:
+		lx.ignore()
+		return lexArrayValue // move on to the next value
+	case r == arrayEnd:
+		return lexArrayEnd
+	}
+	return lx.errorf("Expected an array value terminator %q or an array "+
+		"terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r)
+}
+
+// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has
+// just been consumed.
+func lexArrayEnd(lx *lexer) stateFn {
+	lx.ignore()
+	lx.emit(itemArrayEnd)
+	return lx.pop()
+}
+
+// lexString consumes the inner contents of a string. It assumes that the
+// beginning '"' has already been consumed and ignored.
+func lexString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isNL(r):
+		return lx.errorf("Strings cannot contain new lines.")
+	case r == '\\':
+		lx.push(lexString)
+		return lexStringEscape
+	case r == stringEnd:
+		lx.backup()
+		lx.emit(itemString)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexString
+}
+
+// lexMultilineString consumes the inner contents of a string. It assumes that
+// the beginning '"""' has already been consumed and ignored.
+func lexMultilineString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == '\\':
+		return lexMultilineStringEscape
+	case r == stringEnd:
+		if lx.accept(stringEnd) {
+			if lx.accept(stringEnd) {
+				lx.backup()
+				lx.backup()
+				lx.backup()
+				lx.emit(itemMultilineString)
+				lx.next()
+				lx.next()
+				lx.next()
+				lx.ignore()
+				return lx.pop()
+			}
+			lx.backup()
+		}
+	}
+	return lexMultilineString
+}
+
+// lexRawString consumes a raw string. Nothing can be escaped in such a string.
+// It assumes that the beginning "'" has already been consumed and ignored.
+func lexRawString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isNL(r):
+		return lx.errorf("Strings cannot contain new lines.")
+	case r == rawStringEnd:
+		lx.backup()
+		lx.emit(itemRawString)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexRawString
+}
+
+// lexMultilineRawString consumes a raw string. Nothing can be escaped in such
+// a string. It assumes that the beginning "'" has already been consumed and
+// ignored.
+func lexMultilineRawString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == rawStringEnd:
+		if lx.accept(rawStringEnd) {
+			if lx.accept(rawStringEnd) {
+				lx.backup()
+				lx.backup()
+				lx.backup()
+				lx.emit(itemRawMultilineString)
+				lx.next()
+				lx.next()
+				lx.next()
+				lx.ignore()
+				return lx.pop()
+			}
+			lx.backup()
+		}
+	}
+	return lexMultilineRawString
+}
+
+// lexMultilineStringEscape consumes an escaped character. It assumes that the
+// preceding '\\' has already been consumed.
+func lexMultilineStringEscape(lx *lexer) stateFn {
+	// Handle the special case first:
+	if isNL(lx.next()) {
+		return lexMultilineString
+	} else {
+		lx.backup()
+		lx.push(lexMultilineString)
+		return lexStringEscape(lx)
+	}
+}
+
+func lexStringEscape(lx *lexer) stateFn {
+	r := lx.next()
+	switch r {
+	case 'b':
+		fallthrough
+	case 't':
+		fallthrough
+	case 'n':
+		fallthrough
+	case 'f':
+		fallthrough
+	case 'r':
+		fallthrough
+	case '"':
+		fallthrough
+	case '\\':
+		return lx.pop()
+	case 'u':
+		return lexShortUnicodeEscape
+	case 'U':
+		return lexLongUnicodeEscape
+	}
+	return lx.errorf("Invalid escape character %q. Only the following "+
+		"escape characters are allowed: "+
+		"\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+
+		"\\uXXXX and \\UXXXXXXXX.", r)
+}
+
+func lexShortUnicodeEscape(lx *lexer) stateFn {
+	var r rune
+	for i := 0; i < 4; i++ {
+		r = lx.next()
+		if !isHexadecimal(r) {
+			return lx.errorf("Expected four hexadecimal digits after '\\u', "+
+				"but got '%s' instead.", lx.current())
+		}
+	}
+	return lx.pop()
+}
+
+func lexLongUnicodeEscape(lx *lexer) stateFn {
+	var r rune
+	for i := 0; i < 8; i++ {
+		r = lx.next()
+		if !isHexadecimal(r) {
+			return lx.errorf("Expected eight hexadecimal digits after '\\U', "+
+				"but got '%s' instead.", lx.current())
+		}
+	}
+	return lx.pop()
+}
+
+// lexNumberOrDateStart consumes either a (positive) integer, float or
+// datetime. It assumes that NO negative sign has been consumed.
+func lexNumberOrDateStart(lx *lexer) stateFn {
+	r := lx.next()
+	if !isDigit(r) {
+		if r == '.' {
+			return lx.errorf("Floats must start with a digit, not '.'.")
+		} else {
+			return lx.errorf("Expected a digit but got %q.", r)
+		}
+	}
+	return lexNumberOrDate
+}
+
+// lexNumberOrDate consumes either a (positive) integer, float or datetime.
+func lexNumberOrDate(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == '-':
+		if lx.pos-lx.start != 5 {
+			return lx.errorf("All ISO8601 dates must be in full Zulu form.")
+		}
+		return lexDateAfterYear
+	case isDigit(r):
+		return lexNumberOrDate
+	case r == '.':
+		return lexFloatStart
+	}
+
+	lx.backup()
+	lx.emit(itemInteger)
+	return lx.pop()
+}
+
+// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format.
+// It assumes that "YYYY-" has already been consumed.
+func lexDateAfterYear(lx *lexer) stateFn {
+	formats := []rune{
+		// digits are '0'.
+		// everything else is direct equality.
+		'0', '0', '-', '0', '0',
+		'T',
+		'0', '0', ':', '0', '0', ':', '0', '0',
+		'Z',
+	}
+	for _, f := range formats {
+		r := lx.next()
+		if f == '0' {
+			if !isDigit(r) {
+				return lx.errorf("Expected digit in ISO8601 datetime, "+
+					"but found %q instead.", r)
+			}
+		} else if f != r {
+			return lx.errorf("Expected %q in ISO8601 datetime, "+
+				"but found %q instead.", f, r)
+		}
+	}
+	lx.emit(itemDatetime)
+	return lx.pop()
+}
+
+// lexNumberStart consumes either an integer or a float. It assumes that
+// a negative sign has already been read, but that *no* digits have been
+// consumed. lexNumberStart will move to the appropriate integer or float
+// states.
+func lexNumberStart(lx *lexer) stateFn {
+	// we MUST see a digit. Even floats have to start with a digit.
+	r := lx.next()
+	if !isDigit(r) {
+		if r == '.' {
+			return lx.errorf("Floats must start with a digit, not '.'.")
+		} else {
+			return lx.errorf("Expected a digit but got %q.", r)
+		}
+	}
+	return lexNumber
+}
+
+// lexNumber consumes an integer or a float after seeing the first digit.
+func lexNumber(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isDigit(r):
+		return lexNumber
+	case r == '.':
+		return lexFloatStart
+	}
+
+	lx.backup()
+	lx.emit(itemInteger)
+	return lx.pop()
+}
+
+// lexFloatStart starts the consumption of digits of a float after a '.'.
+// Namely, at least one digit is required.
+func lexFloatStart(lx *lexer) stateFn {
+	r := lx.next()
+	if !isDigit(r) {
+		return lx.errorf("Floats must have a digit after the '.', but got "+
+			"%q instead.", r)
+	}
+	return lexFloat
+}
+
+// lexFloat consumes the digits of a float after a '.'.
+// Assumes that one digit has been consumed after a '.' already.
+func lexFloat(lx *lexer) stateFn {
+	r := lx.next()
+	if isDigit(r) {
+		return lexFloat
+	}
+
+	lx.backup()
+	lx.emit(itemFloat)
+	return lx.pop()
+}
+
+// lexConst consumes the s[1:] in s. It assumes that s[0] has already been
+// consumed.
+func lexConst(lx *lexer, s string) stateFn {
+	for i := range s[1:] {
+		if r := lx.next(); r != rune(s[i+1]) {
+			return lx.errorf("Expected %q, but found %q instead.", s[:i+1],
+				s[:i]+string(r))
+		}
+	}
+	return nil
+}
+
+// lexTrue consumes the "rue" in "true". It assumes that 't' has already
+// been consumed.
+func lexTrue(lx *lexer) stateFn {
+	if fn := lexConst(lx, "true"); fn != nil {
+		return fn
+	}
+	lx.emit(itemBool)
+	return lx.pop()
+}
+
+// lexFalse consumes the "alse" in "false". It assumes that 'f' has already
+// been consumed.
+func lexFalse(lx *lexer) stateFn {
+	if fn := lexConst(lx, "false"); fn != nil {
+		return fn
+	}
+	lx.emit(itemBool)
+	return lx.pop()
+}
+
+// lexCommentStart begins the lexing of a comment. It will emit
+// itemCommentStart and consume no characters, passing control to lexComment.
+func lexCommentStart(lx *lexer) stateFn {
+	lx.ignore()
+	lx.emit(itemCommentStart)
+	return lexComment
+}
+
+// lexComment lexes an entire comment. It assumes that '#' has been consumed.
+// It will consume *up to* the first new line character, and pass control
+// back to the last state on the stack.
+func lexComment(lx *lexer) stateFn {
+	r := lx.peek()
+	if isNL(r) || r == eof {
+		lx.emit(itemText)
+		return lx.pop()
+	}
+	lx.next()
+	return lexComment
+}
+
+// lexSkip ignores all slurped input and moves on to the next state.
+func lexSkip(lx *lexer, nextState stateFn) stateFn {
+	return func(lx *lexer) stateFn {
+		lx.ignore()
+		return nextState
+	}
+}
+
+// isWhitespace returns true if `r` is a whitespace character according
+// to the spec.
+func isWhitespace(r rune) bool {
+	return r == '\t' || r == ' '
+}
+
+func isNL(r rune) bool {
+	return r == '\n' || r == '\r'
+}
+
+func isDigit(r rune) bool {
+	return r >= '0' && r <= '9'
+}
+
+func isHexadecimal(r rune) bool {
+	return (r >= '0' && r <= '9') ||
+		(r >= 'a' && r <= 'f') ||
+		(r >= 'A' && r <= 'F')
+}
+
+func isBareKeyChar(r rune) bool {
+	return (r >= 'A' && r <= 'Z') ||
+		(r >= 'a' && r <= 'z') ||
+		(r >= '0' && r <= '9') ||
+		r == '_' ||
+		r == '-'
+}
+
+func (itype itemType) String() string {
+	switch itype {
+	case itemError:
+		return "Error"
+	case itemNIL:
+		return "NIL"
+	case itemEOF:
+		return "EOF"
+	case itemText:
+		return "Text"
+	case itemString:
+		return "String"
+	case itemRawString:
+		return "String"
+	case itemMultilineString:
+		return "String"
+	case itemRawMultilineString:
+		return "String"
+	case itemBool:
+		return "Bool"
+	case itemInteger:
+		return "Integer"
+	case itemFloat:
+		return "Float"
+	case itemDatetime:
+		return "DateTime"
+	case itemTableStart:
+		return "TableStart"
+	case itemTableEnd:
+		return "TableEnd"
+	case itemKeyStart:
+		return "KeyStart"
+	case itemArray:
+		return "Array"
+	case itemArrayEnd:
+		return "ArrayEnd"
+	case itemCommentStart:
+		return "CommentStart"
+	}
+	panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype)))
+}
+
+func (item item) String() string {
+	return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val)
+}
@@ -0,0 +1,493 @@
+package toml
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+type parser struct {
+	mapping map[string]interface{}
+	types   map[string]tomlType
+	lx      *lexer
+
+	// A list of keys in the order that they appear in the TOML data.
+	ordered []Key
+
+	// the full key for the current hash in scope
+	context Key
+
+	// the base key name for everything except hashes
+	currentKey string
+
+	// rough approximation of line number
+	approxLine int
+
+	// A map of 'key.group.names' to whether they were created implicitly.
+	implicits map[string]bool
+}
+
+type parseError string
+
+func (pe parseError) Error() string {
+	return string(pe)
+}
+
+func parse(data string) (p *parser, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			var ok bool
+			if err, ok = r.(parseError); ok {
+				return
+			}
+			panic(r)
+		}
+	}()
+
+	p = &parser{
+		mapping:   make(map[string]interface{}),
+		types:     make(map[string]tomlType),
+		lx:        lex(data),
+		ordered:   make([]Key, 0),
+		implicits: make(map[string]bool),
+	}
+	for {
+		item := p.next()
+		if item.typ == itemEOF {
+			break
+		}
+		p.topLevel(item)
+	}
+
+	return p, nil
+}
+
+func (p *parser) panicf(format string, v ...interface{}) {
+	msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s",
+		p.approxLine, p.current(), fmt.Sprintf(format, v...))
+	panic(parseError(msg))
+}
+
+func (p *parser) next() item {
+	it := p.lx.nextItem()
+	if it.typ == itemError {
+		p.panicf("%s", it.val)
+	}
+	return it
+}
+
+func (p *parser) bug(format string, v ...interface{}) {
+	log.Panicf("BUG: %s\n\n", fmt.Sprintf(format, v...))
+}
+
+func (p *parser) expect(typ itemType) item {
+	it := p.next()
+	p.assertEqual(typ, it.typ)
+	return it
+}
+
+func (p *parser) assertEqual(expected, got itemType) {
+	if expected != got {
+		p.bug("Expected '%s' but got '%s'.", expected, got)
+	}
+}
+
+func (p *parser) topLevel(item item) {
+	switch item.typ {
+	case itemCommentStart:
+		p.approxLine = item.line
+		p.expect(itemText)
+	case itemTableStart:
+		kg := p.next()
+		p.approxLine = kg.line
+
+		var key Key
+		for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() {
+			key = append(key, p.keyString(kg))
+		}
+		p.assertEqual(itemTableEnd, kg.typ)
+
+		p.establishContext(key, false)
+		p.setType("", tomlHash)
+		p.ordered = append(p.ordered, key)
+	case itemArrayTableStart:
+		kg := p.next()
+		p.approxLine = kg.line
+
+		var key Key
+		for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() {
+			key = append(key, p.keyString(kg))
+		}
+		p.assertEqual(itemArrayTableEnd, kg.typ)
+
+		p.establishContext(key, true)
+		p.setType("", tomlArrayHash)
+		p.ordered = append(p.ordered, key)
+	case itemKeyStart:
+		kname := p.next()
+		p.approxLine = kname.line
+		p.currentKey = p.keyString(kname)
+
+		val, typ := p.value(p.next())
+		p.setValue(p.currentKey, val)
+		p.setType(p.currentKey, typ)
+		p.ordered = append(p.ordered, p.context.add(p.currentKey))
+		p.currentKey = ""
+	default:
+		p.bug("Unexpected type at top level: %s", item.typ)
+	}
+}
+
+// Gets a string for a key (or part of a key in a table name).
+func (p *parser) keyString(it item) string {
+	switch it.typ {
+	case itemText:
+		return it.val
+	case itemString, itemMultilineString,
+		itemRawString, itemRawMultilineString:
+		s, _ := p.value(it)
+		return s.(string)
+	default:
+		p.bug("Unexpected key type: %s", it.typ)
+		panic("unreachable")
+	}
+}
+
+// value translates an expected value from the lexer into a Go value wrapped
+// as an empty interface.
+func (p *parser) value(it item) (interface{}, tomlType) {
+	switch it.typ {
+	case itemString:
+		return p.replaceEscapes(it.val), p.typeOfPrimitive(it)
+	case itemMultilineString:
+		trimmed := stripFirstNewline(stripEscapedWhitespace(it.val))
+		return p.replaceEscapes(trimmed), p.typeOfPrimitive(it)
+	case itemRawString:
+		return it.val, p.typeOfPrimitive(it)
+	case itemRawMultilineString:
+		return stripFirstNewline(it.val), p.typeOfPrimitive(it)
+	case itemBool:
+		switch it.val {
+		case "true":
+			return true, p.typeOfPrimitive(it)
+		case "false":
+			return false, p.typeOfPrimitive(it)
+		}
+		p.bug("Expected boolean value, but got '%s'.", it.val)
+	case itemInteger:
+		num, err := strconv.ParseInt(it.val, 10, 64)
+		if err != nil {
+			// See comment below for floats describing why we make a
+			// distinction between a bug and a user error.
+			if e, ok := err.(*strconv.NumError); ok &&
+				e.Err == strconv.ErrRange {
+
+				p.panicf("Integer '%s' is out of the range of 64-bit "+
+					"signed integers.", it.val)
+			} else {
+				p.bug("Expected integer value, but got '%s'.", it.val)
+			}
+		}
+		return num, p.typeOfPrimitive(it)
+	case itemFloat:
+		num, err := strconv.ParseFloat(it.val, 64)
+		if err != nil {
+			// Distinguish float values. Normally, it'd be a bug if the lexer
+			// provides an invalid float, but it's possible that the float is
+			// out of range of valid values (which the lexer cannot determine).
+			// So mark the former as a bug but the latter as a legitimate user
+			// error.
+			//
+			// This is also true for integers.
+			if e, ok := err.(*strconv.NumError); ok &&
+				e.Err == strconv.ErrRange {
+
+				p.panicf("Float '%s' is out of the range of 64-bit "+
+					"IEEE-754 floating-point numbers.", it.val)
+			} else {
+				p.bug("Expected float value, but got '%s'.", it.val)
+			}
+		}
+		return num, p.typeOfPrimitive(it)
+	case itemDatetime:
+		t, err := time.Parse("2006-01-02T15:04:05Z", it.val)
+		if err != nil {
+			p.panicf("Invalid RFC3339 Zulu DateTime: '%s'.", it.val)
+		}
+		return t, p.typeOfPrimitive(it)
+	case itemArray:
+		array := make([]interface{}, 0)
+		types := make([]tomlType, 0)
+
+		for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
+			if it.typ == itemCommentStart {
+				p.expect(itemText)
+				continue
+			}
+
+			val, typ := p.value(it)
+			array = append(array, val)
+			types = append(types, typ)
+		}
+		return array, p.typeOfArray(types)
+	}
+	p.bug("Unexpected value type: %s", it.typ)
+	panic("unreachable")
+}
+
+// establishContext sets the current context of the parser,
+// where the context is either a hash or an array of hashes. Which one is
+// set depends on the value of the `array` parameter.
+//
+// Establishing the context also makes sure that the key isn't a duplicate, and
+// will create implicit hashes automatically.
+func (p *parser) establishContext(key Key, array bool) {
+	var ok bool
+
+	// Always start at the top level and drill down for our context.
+	hashContext := p.mapping
+	keyContext := make(Key, 0)
+
+	// We only need implicit hashes for key[0:-1]
+	for _, k := range key[0 : len(key)-1] {
+		_, ok = hashContext[k]
+		keyContext = append(keyContext, k)
+
+		// No key? Make an implicit hash and move on.
+		if !ok {
+			p.addImplicit(keyContext)
+			hashContext[k] = make(map[string]interface{})
+		}
+
+		// If the hash context is actually an array of tables, then set
+		// the hash context to the last element in that array.
+		//
+		// Otherwise, it better be a table, since this MUST be a key group (by
+		// virtue of it not being the last element in a key).
+		switch t := hashContext[k].(type) {
+		case []map[string]interface{}:
+			hashContext = t[len(t)-1]
+		case map[string]interface{}:
+			hashContext = t
+		default:
+			p.panicf("Key '%s' was already created as a hash.", keyContext)
+		}
+	}
+
+	p.context = keyContext
+	if array {
+		// If this is the first element for this array, then allocate a new
+		// list of tables for it.
+		k := key[len(key)-1]
+		if _, ok := hashContext[k]; !ok {
+			hashContext[k] = make([]map[string]interface{}, 0, 5)
+		}
+
+		// Add a new table. But make sure the key hasn't already been used
+		// for something else.
+		if hash, ok := hashContext[k].([]map[string]interface{}); ok {
+			hashContext[k] = append(hash, make(map[string]interface{}))
+		} else {
+			p.panicf("Key '%s' was already created and cannot be used as "+
+				"an array.", keyContext)
+		}
+	} else {
+		p.setValue(key[len(key)-1], make(map[string]interface{}))
+	}
+	p.context = append(p.context, key[len(key)-1])
+}
+
+// setValue sets the given key to the given value in the current context.
+// It will make sure that the key hasn't already been defined, account for
+// implicit key groups.
+func (p *parser) setValue(key string, value interface{}) {
+	var tmpHash interface{}
+	var ok bool
+
+	hash := p.mapping
+	keyContext := make(Key, 0)
+	for _, k := range p.context {
+		keyContext = append(keyContext, k)
+		if tmpHash, ok = hash[k]; !ok {
+			p.bug("Context for key '%s' has not been established.", keyContext)
+		}
+		switch t := tmpHash.(type) {
+		case []map[string]interface{}:
+			// The context is a table of hashes. Pick the most recent table
+			// defined as the current hash.
+			hash = t[len(t)-1]
+		case map[string]interface{}:
+			hash = t
+		default:
+			p.bug("Expected hash to have type 'map[string]interface{}', but "+
+				"it has '%T' instead.", tmpHash)
+		}
+	}
+	keyContext = append(keyContext, key)
+
+	if _, ok := hash[key]; ok {
+		// Typically, if the given key has already been set, then we have
+		// to raise an error since duplicate keys are disallowed. However,
+		// it's possible that a key was previously defined implicitly. In this
+		// case, it is allowed to be redefined concretely. (See the
+		// `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.)
+		//
+		// But we have to make sure to stop marking it as an implicit. (So that
+		// another redefinition provokes an error.)
+		//
+		// Note that since it has already been defined (as a hash), we don't
+		// want to overwrite it. So our business is done.
+		if p.isImplicit(keyContext) {
+			p.removeImplicit(keyContext)
+			return
+		}
+
+		// Otherwise, we have a concrete key trying to override a previous
+		// key, which is *always* wrong.
+		p.panicf("Key '%s' has already been defined.", keyContext)
+	}
+	hash[key] = value
+}
+
+// setType sets the type of a particular value at a given key.
+// It should be called immediately AFTER setValue.
+//
+// Note that if `key` is empty, then the type given will be applied to the
+// current context (which is either a table or an array of tables).
+func (p *parser) setType(key string, typ tomlType) {
+	keyContext := make(Key, 0, len(p.context)+1)
+	for _, k := range p.context {
+		keyContext = append(keyContext, k)
+	}
+	if len(key) > 0 { // allow type setting for hashes
+		keyContext = append(keyContext, key)
+	}
+	p.types[keyContext.String()] = typ
+}
+
+// addImplicit sets the given Key as having been created implicitly.
+func (p *parser) addImplicit(key Key) {
+	p.implicits[key.String()] = true
+}
+
+// removeImplicit stops tagging the given key as having been implicitly
+// created.
+func (p *parser) removeImplicit(key Key) {
+	p.implicits[key.String()] = false
+}
+
+// isImplicit returns true if the key group pointed to by the key was created
+// implicitly.
+func (p *parser) isImplicit(key Key) bool {
+	return p.implicits[key.String()]
+}
+
+// current returns the full key name of the current context.
+func (p *parser) current() string {
+	if len(p.currentKey) == 0 {
+		return p.context.String()
+	}
+	if len(p.context) == 0 {
+		return p.currentKey
+	}
+	return fmt.Sprintf("%s.%s", p.context, p.currentKey)
+}
+
+func stripFirstNewline(s string) string {
+	if len(s) == 0 || s[0] != '\n' {
+		return s
+	}
+	return s[1:]
+}
+
+func stripEscapedWhitespace(s string) string {
+	esc := strings.Split(s, "\\\n")
+	if len(esc) > 1 {
+		for i := 1; i < len(esc); i++ {
+			esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace)
+		}
+	}
+	return strings.Join(esc, "")
+}
+
+func (p *parser) replaceEscapes(str string) string {
+	var replaced []rune
+	s := []byte(str)
+	r := 0
+	for r < len(s) {
+		if s[r] != '\\' {
+			c, size := utf8.DecodeRune(s[r:])
+			r += size
+			replaced = append(replaced, c)
+			continue
+		}
+		r += 1
+		if r >= len(s) {
+			p.bug("Escape sequence at end of string.")
+			return ""
+		}
+		switch s[r] {
+		default:
+			p.bug("Expected valid escape code after \\, but got %q.", s[r])
+			return ""
+		case 'b':
+			replaced = append(replaced, rune(0x0008))
+			r += 1
+		case 't':
+			replaced = append(replaced, rune(0x0009))
+			r += 1
+		case 'n':
+			replaced = append(replaced, rune(0x000A))
+			r += 1
+		case 'f':
+			replaced = append(replaced, rune(0x000C))
+			r += 1
+		case 'r':
+			replaced = append(replaced, rune(0x000D))
+			r += 1
+		case '"':
+			replaced = append(replaced, rune(0x0022))
+			r += 1
+		case '\\':
+			replaced = append(replaced, rune(0x005C))
+			r += 1
+		case 'u':
+			// At this point, we know we have a Unicode escape of the form
+			// `uXXXX` at [r, r+5). (Because the lexer guarantees this
+			// for us.)
+			escaped := p.asciiEscapeToUnicode(s[r+1 : r+5])
+			replaced = append(replaced, escaped)
+			r += 5
+		case 'U':
+			// At this point, we know we have a Unicode escape of the form
+			// `uXXXX` at [r, r+9). (Because the lexer guarantees this
+			// for us.)
+			escaped := p.asciiEscapeToUnicode(s[r+1 : r+9])
+			replaced = append(replaced, escaped)
+			r += 9
+		}
+	}
+	return string(replaced)
+}
+
+func (p *parser) asciiEscapeToUnicode(bs []byte) rune {
+	s := string(bs)
+	hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32)
+	if err != nil {
+		p.bug("Could not parse '%s' as a hexadecimal number, but the "+
+			"lexer claims it's OK: %s", s, err)
+	}
+	if !utf8.ValidRune(rune(hex)) {
+		p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s)
+	}
+	return rune(hex)
+}
+
+func isStringType(ty itemType) bool {
+	return ty == itemString || ty == itemMultilineString ||
+		ty == itemRawString || ty == itemRawMultilineString
+}
@@ -0,0 +1 @@
+au BufWritePost *.go silent!make tags > /dev/null 2>&1
@@ -0,0 +1,91 @@
+package toml
+
+// tomlType represents any Go type that corresponds to a TOML type.
+// While the first draft of the TOML spec has a simplistic type system that
+// probably doesn't need this level of sophistication, we seem to be militating
+// toward adding real composite types.
+type tomlType interface {
+	typeString() string
+}
+
+// typeEqual accepts any two types and returns true if they are equal.
+func typeEqual(t1, t2 tomlType) bool {
+	if t1 == nil || t2 == nil {
+		return false
+	}
+	return t1.typeString() == t2.typeString()
+}
+
+func typeIsHash(t tomlType) bool {
+	return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash)
+}
+
+type tomlBaseType string
+
+func (btype tomlBaseType) typeString() string {
+	return string(btype)
+}
+
+func (btype tomlBaseType) String() string {
+	return btype.typeString()
+}
+
+var (
+	tomlInteger   tomlBaseType = "Integer"
+	tomlFloat     tomlBaseType = "Float"
+	tomlDatetime  tomlBaseType = "Datetime"
+	tomlString    tomlBaseType = "String"
+	tomlBool      tomlBaseType = "Bool"
+	tomlArray     tomlBaseType = "Array"
+	tomlHash      tomlBaseType = "Hash"
+	tomlArrayHash tomlBaseType = "ArrayHash"
+)
+
+// typeOfPrimitive returns a tomlType of any primitive value in TOML.
+// Primitive values are: Integer, Float, Datetime, String and Bool.
+//
+// Passing a lexer item other than the following will cause a BUG message
+// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime.
+func (p *parser) typeOfPrimitive(lexItem item) tomlType {
+	switch lexItem.typ {
+	case itemInteger:
+		return tomlInteger
+	case itemFloat:
+		return tomlFloat
+	case itemDatetime:
+		return tomlDatetime
+	case itemString:
+		return tomlString
+	case itemMultilineString:
+		return tomlString
+	case itemRawString:
+		return tomlString
+	case itemRawMultilineString:
+		return tomlString
+	case itemBool:
+		return tomlBool
+	}
+	p.bug("Cannot infer primitive type of lex item '%s'.", lexItem)
+	panic("unreachable")
+}
+
+// typeOfArray returns a tomlType for an array given a list of types of its
+// values.
+//
+// In the current spec, if an array is homogeneous, then its type is always
+// "Array". If the array is not homogeneous, an error is generated.
+func (p *parser) typeOfArray(types []tomlType) tomlType {
+	// Empty arrays are cool.
+	if len(types) == 0 {
+		return tomlArray
+	}
+
+	theType := types[0]
+	for _, t := range types[1:] {
+		if !typeEqual(theType, t) {
+			p.panicf("Array contains values of type '%s' and '%s', but "+
+				"arrays must be homogeneous.", theType, t)
+		}
+	}
+	return tomlArray
+}
@@ -0,0 +1,241 @@
+package toml
+
+// Struct field handling is adapted from code in encoding/json:
+//
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the Go distribution.
+
+import (
+	"reflect"
+	"sort"
+	"sync"
+)
+
+// A field represents a single field found in a struct.
+type field struct {
+	name  string       // the name of the field (`toml` tag included)
+	tag   bool         // whether field has a `toml` tag
+	index []int        // represents the depth of an anonymous field
+	typ   reflect.Type // the type of the field
+}
+
+// byName sorts field by name, breaking ties with depth,
+// then breaking ties with "name came from toml tag", then
+// breaking ties with index sequence.
+type byName []field
+
+func (x byName) Len() int { return len(x) }
+
+func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+
+func (x byName) Less(i, j int) bool {
+	if x[i].name != x[j].name {
+		return x[i].name < x[j].name
+	}
+	if len(x[i].index) != len(x[j].index) {
+		return len(x[i].index) < len(x[j].index)
+	}
+	if x[i].tag != x[j].tag {
+		return x[i].tag
+	}
+	return byIndex(x).Less(i, j)
+}
+
+// byIndex sorts field by index sequence.
+type byIndex []field
+
+func (x byIndex) Len() int { return len(x) }
+
+func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+
+func (x byIndex) Less(i, j int) bool {
+	for k, xik := range x[i].index {
+		if k >= len(x[j].index) {
+			return false
+		}
+		if xik != x[j].index[k] {
+			return xik < x[j].index[k]
+		}
+	}
+	return len(x[i].index) < len(x[j].index)
+}
+
+// typeFields returns a list of fields that TOML should recognize for the given
+// type. The algorithm is breadth-first search over the set of structs to
+// include - the top struct and then any reachable anonymous structs.
+func typeFields(t reflect.Type) []field {
+	// Anonymous fields to explore at the current level and the next.
+	current := []field{}
+	next := []field{{typ: t}}
+
+	// Count of queued names for current level and the next.
+	count := map[reflect.Type]int{}
+	nextCount := map[reflect.Type]int{}
+
+	// Types already visited at an earlier level.
+	visited := map[reflect.Type]bool{}
+
+	// Fields found.
+	var fields []field
+
+	for len(next) > 0 {
+		current, next = next, current[:0]
+		count, nextCount = nextCount, map[reflect.Type]int{}
+
+		for _, f := range current {
+			if visited[f.typ] {
+				continue
+			}
+			visited[f.typ] = true
+
+			// Scan f.typ for fields to include.
+			for i := 0; i < f.typ.NumField(); i++ {
+				sf := f.typ.Field(i)
+				if sf.PkgPath != "" && !sf.Anonymous { // unexported
+					continue
+				}
+				name, _ := getOptions(sf.Tag.Get("toml"))
+				if name == "-" {
+					continue
+				}
+				index := make([]int, len(f.index)+1)
+				copy(index, f.index)
+				index[len(f.index)] = i
+
+				ft := sf.Type
+				if ft.Name() == "" && ft.Kind() == reflect.Ptr {
+					// Follow pointer.
+					ft = ft.Elem()
+				}
+
+				// Record found field and index sequence.
+				if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
+					tagged := name != ""
+					if name == "" {
+						name = sf.Name
+					}
+					fields = append(fields, field{name, tagged, index, ft})
+					if count[f.typ] > 1 {
+						// If there were multiple instances, add a second,
+						// so that the annihilation code will see a duplicate.
+						// It only cares about the distinction between 1 or 2,
+						// so don't bother generating any more copies.
+						fields = append(fields, fields[len(fields)-1])
+					}
+					continue
+				}
+
+				// Record new anonymous struct to explore in next round.
+				nextCount[ft]++
+				if nextCount[ft] == 1 {
+					f := field{name: ft.Name(), index: index, typ: ft}
+					next = append(next, f)
+				}
+			}
+		}
+	}
+
+	sort.Sort(byName(fields))
+
+	// Delete all fields that are hidden by the Go rules for embedded fields,
+	// except that fields with TOML tags are promoted.
+
+	// The fields are sorted in primary order of name, secondary order
+	// of field index length. Loop over names; for each name, delete
+	// hidden fields by choosing the one dominant field that survives.
+	out := fields[:0]
+	for advance, i := 0, 0; i < len(fields); i += advance {
+		// One iteration per name.
+		// Find the sequence of fields with the name of this first field.
+		fi := fields[i]
+		name := fi.name
+		for advance = 1; i+advance < len(fields); advance++ {
+			fj := fields[i+advance]
+			if fj.name != name {
+				break
+			}
+		}
+		if advance == 1 { // Only one field with this name
+			out = append(out, fi)
+			continue
+		}
+		dominant, ok := dominantField(fields[i : i+advance])
+		if ok {
+			out = append(out, dominant)
+		}
+	}
+
+	fields = out
+	sort.Sort(byIndex(fields))
+
+	return fields
+}
+
+// dominantField looks through the fields, all of which are known to
+// have the same name, to find the single field that dominates the
+// others using Go's embedding rules, modified by the presence of
+// TOML tags. If there are multiple top-level fields, the boolean
+// will be false: This condition is an error in Go and we skip all
+// the fields.
+func dominantField(fields []field) (field, bool) {
+	// The fields are sorted in increasing index-length order. The winner
+	// must therefore be one with the shortest index length. Drop all
+	// longer entries, which is easy: just truncate the slice.
+	length := len(fields[0].index)
+	tagged := -1 // Index of first tagged field.
+	for i, f := range fields {
+		if len(f.index) > length {
+			fields = fields[:i]
+			break
+		}
+		if f.tag {
+			if tagged >= 0 {
+				// Multiple tagged fields at the same level: conflict.
+				// Return no field.
+				return field{}, false
+			}
+			tagged = i
+		}
+	}
+	if tagged >= 0 {
+		return fields[tagged], true
+	}
+	// All remaining fields have the same length. If there's more than one,
+	// we have a conflict (two fields named "X" at the same level) and we
+	// return no field.
+	if len(fields) > 1 {
+		return field{}, false
+	}
+	return fields[0], true
+}
+
+var fieldCache struct {
+	sync.RWMutex
+	m map[reflect.Type][]field
+}
+
+// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
+func cachedTypeFields(t reflect.Type) []field {
+	fieldCache.RLock()
+	f := fieldCache.m[t]
+	fieldCache.RUnlock()
+	if f != nil {
+		return f
+	}
+
+	// Compute fields without lock.
+	// Might duplicate effort but won't hold other computations back.
+	f = typeFields(t)
+	if f == nil {
+		f = []field{}
+	}
+
+	fieldCache.Lock()
+	if fieldCache.m == nil {
+		fieldCache.m = map[reflect.Type][]field{}
+	}
+	fieldCache.m[t] = f
+	fieldCache.Unlock()
+	return f
+}
@@ -0,0 +1,10 @@
+.*.swp
+.*.swo
+.swp
+*.test
+bootstrap/cmd/bootstrap-pigeon/bootstrap-pigeon
+bootstrap/cmd/bootstrap-build/bootstrap-build
+bootstrap/cmd/pegscan/pegscan
+bootstrap/cmd/pegparse/pegparse
+bin/
+pigeon
@@ -0,0 +1,8 @@
+language: go
+
+script: go test -v ./...
+
+go:
+    - 1.1
+    - 1.4
+    - tip
@@ -0,0 +1,33 @@
+# Contributing to pigeon
+
+There are various ways to help support this open source project:
+
+* if you use pigeon and find it useful, talk about it - that's probably the most basic way to help any open-source project: getting the word out that it exists and that it can be useful
+* if you use pigeon and find bugs, please [file an issue][0]
+* if something is poorly documented, or doesn't work as documented, this is also a bug, please [file an issue][0]
+* if you can fix the issue (whether it is documentation- or code-related), then [submit a pull-request][1] - but read on to see what should be done to get it merged
+* if you would like to see some new feature/behaviour being implemented, please first [open an issue][0] to discuss it because features are less likely to get merged compared to bug fixes
+
+## Submitting a pull request
+
+Assuming you already have a copy of the repository (either via `go get`, a github fork, a clone, etc.), you will also need `make` to regenerate all tools and files generated when a dependency changes. I use GNU make version 4.1, other versions of make may work too but haven't been tested.
+
+Run `make` in the root directory of the repository. That will create the bootstrap builder, the bootstrap parser, and the final parser, along with some generated Go files. Once `make` is run successfully, run `go test ./...` in the root directory to make sure all tests pass.
+
+Once this is done and tests pass, you can start implementing the bug fix (or the new feature provided **it has already been discussed and agreed in a github issue** first).
+
+For a bug fix, the best way to proceed is to first write a test that proves the bug, then write the code that fixes the bug and makes the test pass. All other tests should still pass too (unless it relied on the buggy behaviour, in which case existing tests must be fixed).
+
+For a new feature, it must be thoroughly tested. New code without new test(s) is unlikely to get merged.
+
+Respect the coding style of the repository, which means essentially to respect the [coding guidelines of the Go community][2]. Use `gofmt` to format your code, and `goimports` to add and format the list of imported packages (or do it manually, but in a `goimports`-style).
+
+Once all code is done and tests pass, regenerate the whole tree with `make`, run `make lint` to make sure the code is correct, and run tests again. You are now ready to submit the pull request.
+
+## Licensing
+
+All pull requests that get merged will be made available under the BSD 3-Clause license (see the LICENSE file for details), as the rest of the pigeon repository. Do not submit pull requests if you do not want your contributions to be made available under those terms.
+
+[0]: https://github.com/PuerkitoBio/pigeon/issues/new
+[1]: https://github.com/PuerkitoBio/pigeon/pulls
+[2]: https://github.com/golang/go/wiki/CodeReviewComments
@@ -0,0 +1,12 @@
+Copyright (c) 2015, Martin Angers & Contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,92 @@
+SHELL = /bin/sh
+
+# directories and source code lists
+ROOT = .
+ROOT_SRC = $(ROOT)/*.go
+BINDIR = ./bin
+EXAMPLES_DIR = $(ROOT)/examples
+TEST_DIR = $(ROOT)/test
+
+# builder and ast packages
+BUILDER_DIR = $(ROOT)/builder
+BUILDER_SRC = $(BUILDER_DIR)/*.go
+AST_DIR = $(ROOT)/ast
+AST_SRC = $(AST_DIR)/*.go
+
+# bootstrap tools variables
+BOOTSTRAP_DIR = $(ROOT)/bootstrap
+BOOTSTRAP_SRC = $(BOOTSTRAP_DIR)/*.go
+BOOTSTRAPBUILD_DIR = $(BOOTSTRAP_DIR)/cmd/bootstrap-build
+BOOTSTRAPBUILD_SRC = $(BOOTSTRAPBUILD_DIR)/*.go
+BOOTSTRAPPIGEON_DIR = $(BOOTSTRAP_DIR)/cmd/bootstrap-pigeon
+BOOTSTRAPPIGEON_SRC = $(BOOTSTRAPPIGEON_DIR)/*.go
+
+# grammar variables
+GRAMMAR_DIR = $(ROOT)/grammar
+BOOTSTRAP_GRAMMAR = $(GRAMMAR_DIR)/bootstrap.peg
+PIGEON_GRAMMAR = $(GRAMMAR_DIR)/pigeon.peg
+
+TEST_GENERATED_SRC = $(patsubst %.peg,%.go,$(shell echo ./{examples,test}/**/*.peg))
+
+all: $(BINDIR)/bootstrap-build $(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go \
+	$(BINDIR)/bootstrap-pigeon $(ROOT)/pigeon.go $(BINDIR)/pigeon \
+	$(TEST_GENERATED_SRC)
+
+$(BINDIR)/bootstrap-build: $(BOOTSTRAPBUILD_SRC) $(BOOTSTRAP_SRC) $(BUILDER_SRC) \
+	$(AST_SRC)
+	go build -o $@ $(BOOTSTRAPBUILD_DIR)
+
+$(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go: $(BINDIR)/bootstrap-build \
+	$(BOOTSTRAP_GRAMMAR)
+	$(BINDIR)/bootstrap-build $(BOOTSTRAP_GRAMMAR) | goimports > $@
+
+$(BINDIR)/bootstrap-pigeon: $(BOOTSTRAPPIGEON_SRC) \
+	$(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go
+	go build -o $@ $(BOOTSTRAPPIGEON_DIR)
+
+$(ROOT)/pigeon.go: $(BINDIR)/bootstrap-pigeon $(PIGEON_GRAMMAR)
+	$(BINDIR)/bootstrap-pigeon $(PIGEON_GRAMMAR) | goimports > $@
+
+$(BINDIR)/pigeon: $(ROOT_SRC) $(ROOT)/pigeon.go
+	go build -o $@ $(ROOT)
+
+$(BOOTSTRAP_GRAMMAR):
+$(PIGEON_GRAMMAR):
+
+# surely there's a better way to define the examples and test targets
+
+$(EXAMPLES_DIR)/json/json.go: $(EXAMPLES_DIR)/json/json.peg $(BINDIR)/pigeon
+	$(BINDIR)/pigeon $< | goimports > $@
+
+$(EXAMPLES_DIR)/calculator/calculator.go: $(EXAMPLES_DIR)/calculator/calculator.peg $(BINDIR)/pigeon
+	$(BINDIR)/pigeon $< | goimports > $@
+
+$(TEST_DIR)/andnot/andnot.go: $(TEST_DIR)/andnot/andnot.peg $(BINDIR)/pigeon
+	$(BINDIR)/pigeon $< | goimports > $@
+
+$(TEST_DIR)/predicates/predicates.go: $(TEST_DIR)/predicates/predicates.peg $(BINDIR)/pigeon
+	$(BINDIR)/pigeon $< | goimports > $@
+
+$(TEST_DIR)/issue_1/issue_1.go: $(TEST_DIR)/issue_1/issue_1.peg $(BINDIR)/pigeon
+	$(BINDIR)/pigeon $< | goimports > $@
+
+$(TEST_DIR)/linear/linear.go: $(TEST_DIR)/linear/linear.peg $(BINDIR)/pigeon
+	$(BINDIR)/pigeon $< | goimports > $@
+
+lint:
+	golint ./...
+	go vet ./...
+
+cmp:
+	@boot=$$(mktemp) && $(BINDIR)/bootstrap-pigeon $(PIGEON_GRAMMAR) | goimports > $$boot && \
+	official=$$(mktemp) && $(BINDIR)/pigeon $(PIGEON_GRAMMAR) | goimports > $$official && \
+	cmp $$boot $$official && \
+	unlink $$boot && \
+	unlink $$official
+
+clean:
+	rm $(BOOTSTRAPPIGEON_DIR)/bootstrap_pigeon.go $(ROOT)/pigeon.go $(TEST_GENERATED_SRC)
+	rm -rf $(BINDIR)
+
+.PHONY: all clean lint cmp
+
@@ -0,0 +1,144 @@
+# pigeon - a PEG parser generator for Go
+
+[![GoDoc](https://godoc.org/github.com/PuerkitoBio/pigeon?status.png)](https://godoc.org/github.com/PuerkitoBio/pigeon)
+[![build status](https://secure.travis-ci.org/PuerkitoBio/pigeon.png?branch=master)](http://travis-ci.org/PuerkitoBio/pigeon)
+[![Software License](https://img.shields.io/badge/license-BSD-blue.svg)](LICENSE)
+
+The pigeon command generates parsers based on a [parsing expression grammar (PEG)][0]. Its grammar and syntax is inspired by the [PEG.js project][1], while the implementation is loosely based on the [parsing expression grammar for C# 3.0][2] article. It parses Unicode text encoded in UTF-8.
+
+See the [godoc page][3] for detailed usage.
+
+## Installation
+
+Provided you have Go correctly installed with the $GOPATH and $GOBIN environment variables set, run:
+
+```
+$ go get -u github.com/PuerkitoBio/pigeon
+```
+
+This will install or update the package, and the `pigeon` command will be installed in your $GOBIN directory. Neither this package nor the parsers generated by this command require any third-party dependency, unless such a dependency is used in the code blocks of the grammar.
+
+## Basic usage
+
+```
+$ pigeon [options] [PEG_GRAMMAR_FILE]
+```
+
+By default, the input grammar is read from `stdin` and the generated code is printed to `stdout`. You may save it in a file using the `-o` flag, but pigeon makes no attempt to format the generated code, nor does it try to generate the required imports, because such a tool already exists. The recommended way to generate a properly formatted and working parser is to pipe the output of pigeon through the `goimports` tool:
+
+```
+$ pigeon my_revolutionary_programming_language.peg | goimports > main.go
+```
+
+This way, the generated code has all the necessary imports and is properly formatted. You can install `goimports` using:
+
+```
+$ go get golang.org/x/tools/cmd/goimports
+```
+
+See the [godoc page][3] for detailed usage.
+
+## Example
+
+Given the following grammar:
+
+```
+{
+// part of the initializer code block omitted for brevity
+
+var ops = map[string]func(int, int) int {
+    "+": func(l, r int) int {
+        return l + r
+    },
+    "-": func(l, r int) int {
+        return l - r
+    },
+    "*": func(l, r int) int {
+        return l * r
+    },
+    "/": func(l, r int) int {
+        return l / r
+    },
+}
+
+func toIfaceSlice(v interface{}) []interface{} {
+    if v == nil {
+        return nil
+    }
+    return v.([]interface{})
+}
+
+func eval(first, rest interface{}) int {
+    l := first.(int)
+    restSl := toIfaceSlice(rest)
+    for _, v := range restSl {
+        restExpr := toIfaceSlice(v)
+        r := restExpr[3].(int)
+        op := restExpr[1].(string)
+        l = ops[op](l, r)
+    }
+    return l
+}
+}
+
+
+Input <- expr:Expr EOF {
+    return expr, nil
+}
+
+Expr <- _ first:Term rest:( _ AddOp _ Term )* _ {
+    return eval(first, rest), nil
+}
+
+Term <- first:Factor rest:( _ MulOp _ Factor )* {
+    return eval(first, rest), nil
+}
+
+Factor <- '(' expr:Expr ')' {
+    return expr, nil
+} / integer:Integer {
+    return integer, nil
+}
+
+AddOp <- ( '+' / '-' ) {
+    return string(c.text), nil
+}
+
+MulOp <- ( '*' / '/' ) {
+    return string(c.text), nil
+}
+
+Integer <- '-'? [0-9]+ {
+    return strconv.Atoi(string(c.text))
+}
+
+_ "whitespace" <- [ \n\t\r]*
+
+EOF <- !.
+```
+
+The generated parser can parse simple arithmetic operations, e.g.:
+
+```
+18 + 3 - 27 * (-18 / -3)
+
+=> -141
+```
+
+More examples can be found in the `examples/` subdirectory.
+
+See the [godoc page][3] for detailed usage.
+
+## Contributing
+
+See the CONTRIBUTING.md file.
+
+## License
+
+The [BSD 3-Clause license][4]. See the LICENSE file.
+
+[0]: http://en.wikipedia.org/wiki/Parsing_expression_grammar
+[1]: http://pegjs.org/
+[2]: http://www.codeproject.com/Articles/29713/Parsing-Expression-Grammar-Support-for-C-Part
+[3]: https://godoc.org/github.com/PuerkitoBio/pigeon
+[4]: http://opensource.org/licenses/BSD-3-Clause
@@ -0,0 +1,3 @@
+- refactor implementation as a VM to avoid stack overflow in pathological cases (and maybe better performance): in branch wip-vm
+? options like current receiver name read directly from the grammar file
+? type annotations for generated code functions
@@ -0,0 +1,587 @@
+// Package ast defines the abstract syntax tree for the PEG grammar.
+//
+// The parser generator's PEG grammar generates a tree using this package
+// that is then converted by the builder to the simplified AST used in
+// the generated parser.
+package ast
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// Pos represents a position in a source file.
+type Pos struct {
+	Filename string
+	Line     int
+	Col      int
+	Off      int
+}
+
+// String returns the textual representation of a position.
+func (p Pos) String() string {
+	if p.Filename != "" {
+		return fmt.Sprintf("%s:%d:%d (%d)", p.Filename, p.Line, p.Col, p.Off)
+	}
+	return fmt.Sprintf("%d:%d (%d)", p.Line, p.Col, p.Off)
+}
+
+// Grammar is the top-level node of the AST for the PEG grammar.
+type Grammar struct {
+	p     Pos
+	Init  *CodeBlock
+	Rules []*Rule
+}
+
+// NewGrammar creates a new grammar at the specified position.
+func NewGrammar(p Pos) *Grammar {
+	return &Grammar{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (g *Grammar) Pos() Pos { return g.p }
+
+// String returns the textual representation of a node.
+func (g *Grammar) String() string {
+	var buf bytes.Buffer
+
+	buf.WriteString(fmt.Sprintf("%s: %T{Init: %v, Rules: [\n",
+		g.p, g, g.Init))
+	for _, r := range g.Rules {
+		buf.WriteString(fmt.Sprintf("%s,\n", r))
+	}
+	buf.WriteString("]}")
+	return buf.String()
+}
+
+// Rule represents a rule in the PEG grammar. It has a name, an optional
+// display name to be used in error messages, and an expression.
+type Rule struct {
+	p           Pos
+	Name        *Identifier
+	DisplayName *StringLit
+	Expr        Expression
+}
+
+// NewRule creates a rule with at the specified position and with the
+// specified name as identifier.
+func NewRule(p Pos, name *Identifier) *Rule {
+	return &Rule{p: p, Name: name}
+}
+
+// Pos returns the starting position of the node.
+func (r *Rule) Pos() Pos { return r.p }
+
+// String returns the textual representation of a node.
+func (r *Rule) String() string {
+	return fmt.Sprintf("%s: %T{Name: %v, DisplayName: %v, Expr: %v}",
+		r.p, r, r.Name, r.DisplayName, r.Expr)
+}
+
+// Expression is the interface implemented by all expression types.
+type Expression interface {
+	Pos() Pos
+}
+
+// ChoiceExpr is an ordered sequence of expressions. The parser tries to
+// match any of the alternatives in sequence and stops at the first one
+// that matches.
+type ChoiceExpr struct {
+	p            Pos
+	Alternatives []Expression
+}
+
+// NewChoiceExpr creates a choice expression at the specified position.
+func NewChoiceExpr(p Pos) *ChoiceExpr {
+	return &ChoiceExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (c *ChoiceExpr) Pos() Pos { return c.p }
+
+// String returns the textual representation of a node.
+func (c *ChoiceExpr) String() string {
+	var buf bytes.Buffer
+
+	buf.WriteString(fmt.Sprintf("%s: %T{Alternatives: [\n", c.p, c))
+	for _, e := range c.Alternatives {
+		buf.WriteString(fmt.Sprintf("%s,\n", e))
+	}
+	buf.WriteString("]}")
+	return buf.String()
+}
+
+// ActionExpr is an expression that has an associated block of code to
+// execute when the expression matches.
+type ActionExpr struct {
+	p      Pos
+	Expr   Expression
+	Code   *CodeBlock
+	FuncIx int
+}
+
+// NewActionExpr creates a new action expression at the specified position.
+func NewActionExpr(p Pos) *ActionExpr {
+	return &ActionExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (a *ActionExpr) Pos() Pos { return a.p }
+
+// String returns the textual representation of a node.
+func (a *ActionExpr) String() string {
+	return fmt.Sprintf("%s: %T{Expr: %v, Code: %v}", a.p, a, a.Expr, a.Code)
+}
+
+// SeqExpr is an ordered sequence of expressions, all of which must match
+// if the SeqExpr is to be a match itself.
+type SeqExpr struct {
+	p     Pos
+	Exprs []Expression
+}
+
+// NewSeqExpr creates a new sequence expression at the specified position.
+func NewSeqExpr(p Pos) *SeqExpr {
+	return &SeqExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (s *SeqExpr) Pos() Pos { return s.p }
+
+// String returns the textual representation of a node.
+func (s *SeqExpr) String() string {
+	var buf bytes.Buffer
+
+	buf.WriteString(fmt.Sprintf("%s: %T{Exprs: [\n", s.p, s))
+	for _, e := range s.Exprs {
+		buf.WriteString(fmt.Sprintf("%s,\n", e))
+	}
+	buf.WriteString("]}")
+	return buf.String()
+}
+
+// LabeledExpr is an expression that has an associated label. Code blocks
+// can access the value of the expression using that label, that becomes
+// a local variable in the code.
+type LabeledExpr struct {
+	p     Pos
+	Label *Identifier
+	Expr  Expression
+}
+
+// NewLabeledExpr creates a new labeled expression at the specified position.
+func NewLabeledExpr(p Pos) *LabeledExpr {
+	return &LabeledExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (l *LabeledExpr) Pos() Pos { return l.p }
+
+// String returns the textual representation of a node.
+func (l *LabeledExpr) String() string {
+	return fmt.Sprintf("%s: %T{Label: %v, Expr: %v}", l.p, l, l.Label, l.Expr)
+}
+
+// AndExpr is a zero-length matcher that is considered a match if the
+// expression it contains is a match.
+type AndExpr struct {
+	p    Pos
+	Expr Expression
+}
+
+// NewAndExpr creates a new and (&) expression at the specified position.
+func NewAndExpr(p Pos) *AndExpr {
+	return &AndExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (a *AndExpr) Pos() Pos { return a.p }
+
+// String returns the textual representation of a node.
+func (a *AndExpr) String() string {
+	return fmt.Sprintf("%s: %T{Expr: %v}", a.p, a, a.Expr)
+}
+
+// NotExpr is a zero-length matcher that is considered a match if the
+// expression it contains is not a match.
+type NotExpr struct {
+	p    Pos
+	Expr Expression
+}
+
+// NewNotExpr creates a new not (!) expression at the specified position.
+func NewNotExpr(p Pos) *NotExpr {
+	return &NotExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (n *NotExpr) Pos() Pos { return n.p }
+
+// String returns the textual representation of a node.
+func (n *NotExpr) String() string {
+	return fmt.Sprintf("%s: %T{Expr: %v}", n.p, n, n.Expr)
+}
+
+// ZeroOrOneExpr is an expression that can be matched zero or one time.
+type ZeroOrOneExpr struct {
+	p    Pos
+	Expr Expression
+}
+
+// NewZeroOrOneExpr creates a new zero or one expression at the specified
+// position.
+func NewZeroOrOneExpr(p Pos) *ZeroOrOneExpr {
+	return &ZeroOrOneExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (z *ZeroOrOneExpr) Pos() Pos { return z.p }
+
+// String returns the textual representation of a node.
+func (z *ZeroOrOneExpr) String() string {
+	return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr)
+}
+
+// ZeroOrMoreExpr is an expression that can be matched zero or more times.
+type ZeroOrMoreExpr struct {
+	p    Pos
+	Expr Expression
+}
+
+// NewZeroOrMoreExpr creates a new zero or more expression at the specified
+// position.
+func NewZeroOrMoreExpr(p Pos) *ZeroOrMoreExpr {
+	return &ZeroOrMoreExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (z *ZeroOrMoreExpr) Pos() Pos { return z.p }
+
+// String returns the textual representation of a node.
+func (z *ZeroOrMoreExpr) String() string {
+	return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr)
+}
+
+// OneOrMoreExpr is an expression that can be matched one or more times.
+type OneOrMoreExpr struct {
+	p    Pos
+	Expr Expression
+}
+
+// NewOneOrMoreExpr creates a new one or more expression at the specified
+// position.
+func NewOneOrMoreExpr(p Pos) *OneOrMoreExpr {
+	return &OneOrMoreExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (o *OneOrMoreExpr) Pos() Pos { return o.p }
+
+// String returns the textual representation of a node.
+func (o *OneOrMoreExpr) String() string {
+	return fmt.Sprintf("%s: %T{Expr: %v}", o.p, o, o.Expr)
+}
+
+// RuleRefExpr is an expression that references a rule by name.
+type RuleRefExpr struct {
+	p    Pos
+	Name *Identifier
+}
+
+// NewRuleRefExpr creates a new rule reference expression at the specified
+// position.
+func NewRuleRefExpr(p Pos) *RuleRefExpr {
+	return &RuleRefExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (r *RuleRefExpr) Pos() Pos { return r.p }
+
+// String returns the textual representation of a node.
+func (r *RuleRefExpr) String() string {
+	return fmt.Sprintf("%s: %T{Name: %v}", r.p, r, r.Name)
+}
+
+// AndCodeExpr is a zero-length matcher that is considered a match if the
+// code block returns true.
+type AndCodeExpr struct {
+	p      Pos
+	Code   *CodeBlock
+	FuncIx int
+}
+
+// NewAndCodeExpr creates a new and (&) code expression at the specified
+// position.
+func NewAndCodeExpr(p Pos) *AndCodeExpr {
+	return &AndCodeExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (a *AndCodeExpr) Pos() Pos { return a.p }
+
+// String returns the textual representation of a node.
+func (a *AndCodeExpr) String() string {
+	return fmt.Sprintf("%s: %T{Code: %v}", a.p, a, a.Code)
+}
+
+// NotCodeExpr is a zero-length matcher that is considered a match if the
+// code block returns false.
+type NotCodeExpr struct {
+	p      Pos
+	Code   *CodeBlock
+	FuncIx int
+}
+
+// NewNotCodeExpr creates a new not (!) code expression at the specified
+// position.
+func NewNotCodeExpr(p Pos) *NotCodeExpr {
+	return &NotCodeExpr{p: p}
+}
+
+// Pos returns the starting position of the node.
+func (n *NotCodeExpr) Pos() Pos { return n.p }
+
+// String returns the textual representation of a node.
+func (n *NotCodeExpr) String() string {
+	return fmt.Sprintf("%s: %T{Code: %v}", n.p, n, n.Code)
+}
+
+// LitMatcher is a string literal matcher. The value to match may be a
+// double-quoted string, a single-quoted single character, or a back-tick
+// quoted raw string.
+type LitMatcher struct {
+	posValue   // can be str, rstr or char
+	IgnoreCase bool
+}
+
+// NewLitMatcher creates a new literal matcher at the specified position and
+// with the specified value.
+func NewLitMatcher(p Pos, v string) *LitMatcher {
+	return &LitMatcher{posValue: posValue{p: p, Val: v}}
+}
+
+// Pos returns the starting position of the node.
+func (l *LitMatcher) Pos() Pos { return l.p }
+
+// String returns the textual representation of a node.
+func (l *LitMatcher) String() string {
+	return fmt.Sprintf("%s: %T{Val: %q, IgnoreCase: %t}", l.p, l, l.Val, l.IgnoreCase)
+}
+
+// CharClassMatcher is a character class matcher. The value to match must
+// be one of the specified characters, in a range of characters, or in the
+// Unicode classes of characters.
+type CharClassMatcher struct {
+	posValue
+	IgnoreCase     bool
+	Inverted       bool
+	Chars          []rune
+	Ranges         []rune // pairs of low/high range
+	UnicodeClasses []string
+}
+
+// NewCharClassMatcher creates a new character class matcher at the specified
+// position and with the specified raw value. It parses the raw value into
+// the list of characters, ranges and Unicode classes.
+func NewCharClassMatcher(p Pos, raw string) *CharClassMatcher {
+	c := &CharClassMatcher{posValue: posValue{p: p, Val: raw}}
+	c.parse()
+	return c
+}
+
+func (c *CharClassMatcher) parse() {
+	raw := c.Val
+	c.IgnoreCase = strings.HasSuffix(raw, "i")
+	if c.IgnoreCase {
+		raw = raw[:len(raw)-1]
+	}
+
+	// "unquote" the character classes
+	raw = raw[1 : len(raw)-1]
+	if len(raw) == 0 {
+		return
+	}
+
+	c.Inverted = raw[0] == '^'
+	if c.Inverted {
+		raw = raw[1:]
+		if len(raw) == 0 {
+			return
+		}
+	}
+
+	// content of char class is necessarily valid, so escapes are correct
+	r := strings.NewReader(raw)
+	var chars []rune
+	var buf bytes.Buffer
+outer:
+	for {
+		rn, _, err := r.ReadRune()
+		if err != nil {
+			break outer
+		}
+
+		consumeN := 0
+		switch rn {
+		case '\\':
+			rn, _, _ := r.ReadRune()
+			switch rn {
+			case ']':
+				chars = append(chars, rn)
+				continue
+
+			case 'p':
+				rn, _, _ := r.ReadRune()
+				if rn == '{' {
+					buf.Reset()
+					for {
+						rn, _, _ := r.ReadRune()
+						if rn == '}' {
+							break
+						}
+						buf.WriteRune(rn)
+					}
+					c.UnicodeClasses = append(c.UnicodeClasses, buf.String())
+				} else {
+					c.UnicodeClasses = append(c.UnicodeClasses, string(rn))
+				}
+				continue
+
+			case 'x':
+				consumeN = 2
+			case 'u':
+				consumeN = 4
+			case 'U':
+				consumeN = 8
+			case '0', '1', '2', '3', '4', '5', '6', '7':
+				consumeN = 2
+			}
+
+			buf.Reset()
+			buf.WriteRune(rn)
+			for i := 0; i < consumeN; i++ {
+				rn, _, _ := r.ReadRune()
+				buf.WriteRune(rn)
+			}
+			rn, _, _, _ = strconv.UnquoteChar("\\"+buf.String(), 0)
+			chars = append(chars, rn)
+
+		default:
+			chars = append(chars, rn)
+		}
+	}
+
+	// extract ranges and chars
+	inRange, wasRange := false, false
+	for i, r := range chars {
+		if inRange {
+			c.Ranges = append(c.Ranges, r)
+			inRange = false
+			wasRange = true
+			continue
+		}
+
+		if r == '-' && !wasRange && len(c.Chars) > 0 && i < len(chars)-1 {
+			inRange = true
+			wasRange = false
+			// start of range is the last Char added
+			c.Ranges = append(c.Ranges, c.Chars[len(c.Chars)-1])
+			c.Chars = c.Chars[:len(c.Chars)-1]
+			continue
+		}
+		wasRange = false
+		c.Chars = append(c.Chars, r)
+	}
+}
+
+// Pos returns the starting position of the node.
+func (c *CharClassMatcher) Pos() Pos { return c.p }
+
+// String returns the textual representation of a node.
+func (c *CharClassMatcher) String() string {
+	return fmt.Sprintf("%s: %T{Val: %q, IgnoreCase: %t, Inverted: %t}",
+		c.p, c, c.Val, c.IgnoreCase, c.Inverted)
+}
+
+// AnyMatcher is a matcher that matches any character except end-of-file.
+type AnyMatcher struct {
+	posValue
+}
+
+// NewAnyMatcher creates a new any matcher at the specified position. The
+// value is provided for completeness' sake, but it is always the dot.
+func NewAnyMatcher(p Pos, v string) *AnyMatcher {
+	return &AnyMatcher{posValue{p, v}}
+}
+
+// Pos returns the starting position of the node.
+func (a *AnyMatcher) Pos() Pos { return a.p }
+
+// String returns the textual representation of a node.
+func (a *AnyMatcher) String() string {
+	return fmt.Sprintf("%s: %T{Val: %q}", a.p, a, a.Val)
+}
+
+// CodeBlock represents a code block.
+type CodeBlock struct {
+	posValue
+}
+
+// NewCodeBlock creates a new code block at the specified position and with
+// the specified value. The value includes the outer braces.
+func NewCodeBlock(p Pos, code string) *CodeBlock {
+	return &CodeBlock{posValue{p, code}}
+}
+
+// Pos returns the starting position of the node.
+func (c *CodeBlock) Pos() Pos { return c.p }
+
+// String returns the textual representation of a node.
+func (c *CodeBlock) String() string {
+	return fmt.Sprintf("%s: %T{Val: %q}", c.p, c, c.Val)
+}
+
+// Identifier represents an identifier.
+type Identifier struct {
+	posValue
+}
+
+// NewIdentifier creates a new identifier at the specified position and
+// with the specified name.
+func NewIdentifier(p Pos, name string) *Identifier {
+	return &Identifier{posValue{p: p, Val: name}}
+}
+
+// Pos returns the starting position of the node.
+func (i *Identifier) Pos() Pos { return i.p }
+
+// String returns the textual representation of a node.
+func (i *Identifier) String() string {
+	return fmt.Sprintf("%s: %T{Val: %q}", i.p, i, i.Val)
+}
+
+// StringLit represents a string literal.
+type StringLit struct {
+	posValue
+}
+
+// NewStringLit creates a new string literal at the specified position and
+// with the specified value.
+func NewStringLit(p Pos, val string) *StringLit {
+	return &StringLit{posValue{p: p, Val: val}}
+}
+
+// Pos returns the starting position of the node.
+func (s *StringLit) Pos() Pos { return s.p }
+
+// String returns the textual representation of a node.
+func (s *StringLit) String() string {
+	return fmt.Sprintf("%s: %T{Val: %q}", s.p, s, s.Val)
+}
+
+type posValue struct {
+	p   Pos
+	Val string
+}
@@ -0,0 +1,107 @@
+package ast
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+var charClasses = []string{
+	"[]",
+	"[]i",
+	"[^]",
+	"[^]i",
+	"[a]",
+	"[ab]i",
+	"[^abc]i",
+	`[\a]`,
+	`[\b\nt]`,
+	`[\b\nt\pL]`,
+	`[\p{Greek}\tz\\\pN]`,
+	`[-]`,
+	`[--]`,
+	`[---]`,
+	`[a-z]`,
+	`[a-zB0-9]`,
+	`[A-Z]i`,
+	`[a-]`,
+	`[----]`,
+	`[\x00-\x05]`,
+}
+
+var expChars = []string{
+	"",
+	"",
+	"",
+	"",
+	"a",
+	"ab",
+	"abc",
+	"\a",
+	"\b\nt",
+	"\b\nt",
+	"\tz\\",
+	"-",
+	"--",
+	"",
+	"",
+	"B",
+	"",
+	"a-",
+	"-",
+	"",
+}
+
+var expUnicodeClasses = [][]string{
+	9:  {"L"},
+	10: {"Greek", "N"},
+	19: nil,
+}
+
+var expRanges = []string{
+	13: "--",
+	14: "az",
+	15: "az09",
+	16: "AZ",
+	18: "--",
+	19: "\x00\x05",
+}
+
+func TestCharClassParse(t *testing.T) {
+	for i, c := range charClasses {
+		m := NewCharClassMatcher(Pos{}, c)
+
+		ic := strings.HasSuffix(c, "i")
+		if m.IgnoreCase != ic {
+			t.Errorf("%q: want ignore case: %t, got %t", c, ic, m.IgnoreCase)
+		}
+		iv := c[1] == '^'
+		if m.Inverted != iv {
+			t.Errorf("%q: want inverted: %t, got %t", c, iv, m.Inverted)
+		}
+
+		if n := utf8.RuneCountInString(expChars[i]); len(m.Chars) != n {
+			t.Errorf("%q: want %d chars, got %d", c, n, len(m.Chars))
+		} else if string(m.Chars) != expChars[i] {
+			t.Errorf("%q: want %q, got %q", c, expChars[i], string(m.Chars))
+		}
+
+		if n := utf8.RuneCountInString(expRanges[i]); len(m.Ranges) != n {
+			t.Errorf("%q: want %d chars, got %d", c, n, len(m.Ranges))
+		} else if string(m.Ranges) != expRanges[i] {
+			t.Errorf("%q: want %q, got %q", c, expRanges[i], string(m.Ranges))
+		}
+
+		if n := len(expUnicodeClasses[i]); len(m.UnicodeClasses) != n {
+			t.Errorf("%q: want %d Unicode classes, got %d", c, n, len(m.UnicodeClasses))
+		} else if n > 0 {
+			want := expUnicodeClasses[i]
+			got := m.UnicodeClasses
+			for j, wantClass := range want {
+				if wantClass != got[j] {
+					t.Errorf("%q: range table %d: want %v, got %v", c, j, wantClass, got[j])
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,36 @@
+package main
+
+import "testing"
+
+// With Unicode classes in the grammar:
+// BenchmarkParseUnicodeClass          2000            548233 ns/op           96615 B/op        978 allocs/op
+//
+// With Unicode classes in a go map:
+// BenchmarkParseUnicodeClass          5000            272224 ns/op           37990 B/op        482 allocs/op
+func BenchmarkParseUnicodeClass(b *testing.B) {
+	input := []byte("a = [\\p{Latin}]")
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", input); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// With keywords in the grammar:
+// BenchmarkParseKeyword       5000            315189 ns/op           50175 B/op        530 allocs/op
+//
+// With keywords in a go map:
+// BenchmarkParseKeyword      10000            201175 ns/op           27017 B/op        331 allocs/op
+func BenchmarkParseKeyword(b *testing.B) {
+	input := []byte("a = uint32:'a'")
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", input); err == nil {
+			// error IS expected, fatal if none
+			b.Fatal(err)
+		}
+	}
+}
@@ -0,0 +1,52 @@
+// Command bootstrap-build bootstraps the PEG parser generator by
+// parsing the bootstrap grammar and creating a basic parser generator
+// sufficiently complete to parse the pigeon PEG grammar.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/PuerkitoBio/pigeon/bootstrap"
+	"github.com/PuerkitoBio/pigeon/builder"
+)
+
+func main() {
+	outFlag := flag.String("o", "", "output file, defaults to stdout")
+	flag.Parse()
+
+	if flag.NArg() != 1 {
+		fmt.Fprintln(os.Stderr, "USAGE: bootstrap-build [-o OUTPUT] FILE")
+		os.Exit(1)
+	}
+
+	outw := os.Stdout
+	if *outFlag != "" {
+		outf, err := os.Create(*outFlag)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		defer outf.Close()
+		outw = outf
+	}
+
+	f, err := os.Open(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(2)
+	}
+	defer f.Close()
+
+	p := bootstrap.NewParser()
+	g, err := p.Parse(os.Args[1], f)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if err := builder.BuildParser(outw, g); err != nil {
+		log.Fatal(err)
+	}
+}
@@ -0,0 +1,34 @@
+package main
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func BenchmarkParsePigeonNoMemo(b *testing.B) {
+	d, err := ioutil.ReadFile("../../../grammar/pigeon.peg")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", d, Memoize(false)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkParsePigeonMemo(b *testing.B) {
+	d, err := ioutil.ReadFile("../../../grammar/pigeon.peg")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", d, Memoize(true)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
@@ -0,0 +1,76 @@
+// Command bootstrap-pigeon generates a PEG parser from a PEG grammar
+// to bootstrap the pigeon command-line tool, as it is built using
+// a simplified bootstrapping grammar that understands just enough of the
+// pigeon grammar to parse it and build the tool.
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+	"github.com/PuerkitoBio/pigeon/builder"
+)
+
+func main() {
+	dbgFlag := flag.Bool("debug", false, "set debug mode")
+	noBuildFlag := flag.Bool("x", false, "do not build, only parse")
+	outputFlag := flag.String("o", "", "output file, defaults to stdout")
+	flag.Parse()
+
+	if flag.NArg() > 1 {
+		fmt.Fprintf(os.Stderr, "USAGE: %s [options] [FILE]\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	nm := "stdin"
+	inf := os.Stdin
+	if flag.NArg() == 1 {
+		f, err := os.Open(flag.Arg(0))
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		defer f.Close()
+		inf = f
+		nm = flag.Arg(0)
+	}
+	in := bufio.NewReader(inf)
+
+	g, err := ParseReader(nm, in, Debug(*dbgFlag))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "parse error: ", err)
+		os.Exit(3)
+	}
+
+	if !*noBuildFlag {
+		outw := os.Stdout
+		if *outputFlag != "" {
+			f, err := os.Create(*outputFlag)
+			if err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(4)
+			}
+			defer f.Close()
+			outw = f
+		}
+
+		if err := builder.BuildParser(outw, g.(*ast.Grammar)); err != nil {
+			fmt.Fprintln(os.Stderr, "build error: ", err)
+			os.Exit(5)
+		}
+	}
+}
+
+func (c *current) astPos() ast.Pos {
+	return ast.Pos{Line: c.pos.line, Col: c.pos.col, Off: c.pos.offset}
+}
+
+func toIfaceSlice(v interface{}) []interface{} {
+	if v == nil {
+		return nil
+	}
+	return v.([]interface{})
+}
@@ -0,0 +1,41 @@
+// Command pegparse is a helper command-line tool to test the bootstrap
+// parser.
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/PuerkitoBio/pigeon/bootstrap"
+)
+
+func main() {
+	if len(os.Args) > 2 {
+		fmt.Fprintln(os.Stderr, "USAGE: pegparse FILE")
+		os.Exit(1)
+	}
+
+	var in io.Reader
+
+	nm := "stdin"
+	if len(os.Args) == 2 {
+		f, err := os.Open(os.Args[1])
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		defer f.Close()
+		in = f
+		nm = os.Args[1]
+	} else {
+		in = bufio.NewReader(os.Stdin)
+	}
+
+	p := bootstrap.NewParser()
+	if _, err := p.Parse(nm, in); err != nil {
+		log.Fatal(err)
+	}
+}
@@ -0,0 +1,45 @@
+// Command pegscan is a helper command-line tool to test the bootstrap
+// scanner.
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/PuerkitoBio/pigeon/bootstrap"
+)
+
+func main() {
+	if len(os.Args) > 2 {
+		fmt.Fprintln(os.Stderr, "USAGE: pegscan FILE")
+		os.Exit(1)
+	}
+
+	var in io.Reader
+
+	nm := "stdin"
+	if len(os.Args) == 2 {
+		f, err := os.Open(os.Args[1])
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		defer f.Close()
+		in = f
+		nm = os.Args[1]
+	} else {
+		in = bufio.NewReader(os.Stdin)
+	}
+
+	var s bootstrap.Scanner
+	s.Init(nm, in, nil)
+	for {
+		tok, ok := s.Scan()
+		fmt.Println(tok)
+		if !ok {
+			break
+		}
+	}
+}
@@ -0,0 +1,7 @@
+// Package bootstrap implements the scanner and parser to bootstrap the
+// PEG parser generator.
+//
+// It parses the PEG grammar into an ast that is then used to generate
+// a parser generator based on this PEG grammar. The generated parser
+// can then parse the grammar again, without the bootstrap package.
+package bootstrap
@@ -0,0 +1,438 @@
+package bootstrap
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+type errList []error
+
+func (e *errList) reset() {
+	*e = (*e)[:0]
+}
+
+func (e *errList) add(p ast.Pos, err error) {
+	*e = append(*e, fmt.Errorf("%s: %v", p, err))
+}
+
+func (e *errList) err() error {
+	if len(*e) == 0 {
+		return nil
+	}
+	return e
+}
+
+func (e *errList) Error() string {
+	switch len(*e) {
+	case 0:
+		return ""
+	case 1:
+		return (*e)[0].Error()
+	default:
+		var buf bytes.Buffer
+
+		for i, err := range *e {
+			if i > 0 {
+				buf.WriteRune('\n')
+			}
+			buf.WriteString(err.Error())
+		}
+		return buf.String()
+	}
+}
+
+// Parser holds the state to parse the PEG grammar into
+// an abstract syntax tree (AST).
+type Parser struct {
+	s   Scanner
+	tok Token
+
+	errs *errList
+	dbg  bool
+	pk   Token
+}
+
+func (p *Parser) in(s string) string {
+	if p.dbg {
+		fmt.Println("IN  "+s, p.tok.id, p.tok.lit)
+	}
+	return s
+}
+
+func (p *Parser) out(s string) {
+	if p.dbg {
+		fmt.Println("OUT "+s, p.tok.id, p.tok.lit)
+	}
+}
+
+// NewParser creates a new Parser.
+func NewParser() *Parser {
+	return &Parser{errs: new(errList)}
+}
+
+// Parse parses the data from the reader r and generates the AST
+// or returns an error if it fails. The filename is used as information
+// in the error messages.
+func (p *Parser) Parse(filename string, r io.Reader) (*ast.Grammar, error) {
+	p.errs.reset()
+	p.s.Init(filename, r, p.errs.add)
+
+	g := p.grammar()
+	return g, p.errs.err()
+}
+
+func (p *Parser) read() {
+	if p.pk.pos.Line != 0 {
+		p.tok = p.pk
+		p.pk = Token{}
+		return
+	}
+	tok, _ := p.s.Scan()
+	p.tok = tok
+}
+
+func (p *Parser) peek() Token {
+	if p.pk.pos.Line == 0 {
+		p.pk, _ = p.s.Scan()
+	}
+	return p.pk
+}
+
+func (p *Parser) skip(ids ...tid) {
+outer:
+	for {
+		for _, id := range ids {
+			if p.tok.id == id {
+				p.read()
+				continue outer
+			}
+		}
+		return
+	}
+}
+
+func (p *Parser) grammar() *ast.Grammar {
+	defer p.out(p.in("grammar"))
+
+	// advance to the first token
+	p.read()
+	g := ast.NewGrammar(p.tok.pos)
+
+	p.skip(eol, semicolon)
+	if p.tok.id == code {
+		g.Init = ast.NewCodeBlock(p.tok.pos, p.tok.lit)
+		p.read()
+		p.skip(eol, semicolon)
+	}
+
+	for {
+		if p.tok.id == eof {
+			return g
+		}
+		r := p.rule()
+		if r != nil {
+			g.Rules = append(g.Rules, r)
+		}
+		p.read()
+		p.skip(eol, semicolon)
+	}
+}
+
+func (p *Parser) expect(ids ...tid) bool {
+	if len(ids) == 0 {
+		return true
+	}
+
+	for _, id := range ids {
+		if p.tok.id == id {
+			return true
+		}
+	}
+	if len(ids) == 1 {
+		p.errs.add(p.tok.pos, fmt.Errorf("expected %s, got %s", ids[0], p.tok.id))
+	} else {
+		p.errs.add(p.tok.pos, fmt.Errorf("expected any of %v, got %s", ids, p.tok.id))
+	}
+	return false
+}
+
+func (p *Parser) rule() *ast.Rule {
+	defer p.out(p.in("rule"))
+
+	if !p.expect(ident) {
+		return nil
+	}
+	r := ast.NewRule(p.tok.pos, ast.NewIdentifier(p.tok.pos, p.tok.lit))
+	p.read()
+
+	if p.tok.id == str || p.tok.id == rstr || p.tok.id == char {
+		if strings.HasSuffix(p.tok.lit, "i") {
+			p.errs.add(p.tok.pos, errors.New("invalid suffix 'i'"))
+			return nil
+		}
+		s, err := strconv.Unquote(p.tok.lit)
+		if err != nil {
+			p.errs.add(p.tok.pos, err)
+			return nil
+		}
+		r.DisplayName = ast.NewStringLit(p.tok.pos, s)
+		p.read()
+	}
+
+	if !p.expect(ruledef) {
+		return nil
+	}
+	p.read()
+	p.skip(eol)
+
+	expr := p.expression()
+	if expr == nil {
+		p.errs.add(p.tok.pos, errors.New("missing expression"))
+		return nil
+	}
+	r.Expr = expr
+
+	if !p.expect(eol, eof, semicolon) {
+		p.errs.add(p.tok.pos, errors.New("rule not terminated"))
+		return nil
+	}
+	return r
+}
+
+func (p *Parser) expression() ast.Expression {
+	defer p.out(p.in("expression"))
+
+	choice := ast.NewChoiceExpr(p.tok.pos)
+	for {
+		expr := p.actionExpr()
+		if expr != nil {
+			choice.Alternatives = append(choice.Alternatives, expr)
+		}
+		if p.tok.id != slash {
+			switch len(choice.Alternatives) {
+			case 0:
+				p.errs.add(p.tok.pos, errors.New("no expression in choice"))
+				return nil
+			case 1:
+				return choice.Alternatives[0]
+			default:
+				return choice
+			}
+		}
+		// move after the slash
+		p.read()
+	}
+}
+
+func (p *Parser) actionExpr() ast.Expression {
+	defer p.out(p.in("actionExpr"))
+
+	act := ast.NewActionExpr(p.tok.pos)
+	expr := p.seqExpr()
+	if expr == nil {
+		return nil
+	}
+	act.Expr = expr
+
+	if p.tok.id == code {
+		act.Code = ast.NewCodeBlock(p.tok.pos, p.tok.lit)
+		p.read()
+	}
+
+	if act.Code == nil {
+		return expr
+	}
+	return act
+}
+
+func (p *Parser) seqExpr() ast.Expression {
+	defer p.out(p.in("seqExpr"))
+
+	seq := ast.NewSeqExpr(p.tok.pos)
+	for {
+		expr := p.labeledExpr()
+		if expr == nil {
+			switch len(seq.Exprs) {
+			case 0:
+				p.errs.add(p.tok.pos, errors.New("no expression in sequence"))
+				return nil
+			case 1:
+				return seq.Exprs[0]
+			default:
+				return seq
+			}
+		}
+		seq.Exprs = append(seq.Exprs, expr)
+	}
+}
+
+func (p *Parser) labeledExpr() ast.Expression {
+	defer p.out(p.in("labeledExpr"))
+
+	lab := ast.NewLabeledExpr(p.tok.pos)
+	if p.tok.id == ident {
+		peek := p.peek()
+		if peek.id == colon {
+			label := ast.NewIdentifier(p.tok.pos, p.tok.lit)
+			lab.Label = label
+			p.read()
+			if !p.expect(colon) {
+				return nil
+			}
+			p.read()
+		}
+	}
+
+	expr := p.prefixedExpr()
+	if expr == nil {
+		if lab.Label != nil {
+			p.errs.add(p.tok.pos, errors.New("label without expression"))
+		}
+		return nil
+	}
+
+	if lab.Label != nil {
+		lab.Expr = expr
+		return lab
+	}
+	return expr
+}
+
+func (p *Parser) prefixedExpr() ast.Expression {
+	defer p.out(p.in("prefixedExpr"))
+
+	var pref ast.Expression
+	switch p.tok.id {
+	case ampersand:
+		pref = ast.NewAndExpr(p.tok.pos)
+		p.read()
+	case exclamation:
+		pref = ast.NewNotExpr(p.tok.pos)
+		p.read()
+	}
+
+	expr := p.suffixedExpr()
+	if expr == nil {
+		if pref != nil {
+			p.errs.add(p.tok.pos, errors.New("prefix operator without expression"))
+		}
+		return nil
+	}
+	switch p := pref.(type) {
+	case *ast.AndExpr:
+		p.Expr = expr
+		return p
+	case *ast.NotExpr:
+		p.Expr = expr
+		return p
+	default:
+		return expr
+	}
+}
+
+func (p *Parser) suffixedExpr() ast.Expression {
+	defer p.out(p.in("suffixedExpr"))
+
+	expr := p.primaryExpr()
+	if expr == nil {
+		if p.tok.id == question || p.tok.id == star || p.tok.id == plus {
+			p.errs.add(p.tok.pos, errors.New("suffix operator without expression"))
+		}
+		return nil
+	}
+
+	switch p.tok.id {
+	case question:
+		q := ast.NewZeroOrOneExpr(expr.Pos())
+		q.Expr = expr
+		p.read()
+		return q
+	case star:
+		s := ast.NewZeroOrMoreExpr(expr.Pos())
+		s.Expr = expr
+		p.read()
+		return s
+	case plus:
+		l := ast.NewOneOrMoreExpr(expr.Pos())
+		l.Expr = expr
+		p.read()
+		return l
+	default:
+		return expr
+	}
+}
+
+func (p *Parser) primaryExpr() ast.Expression {
+	defer p.out(p.in("primaryExpr"))
+
+	switch p.tok.id {
+	case str, rstr, char:
+		// literal matcher
+		ignore := strings.HasSuffix(p.tok.lit, "i")
+		if ignore {
+			p.tok.lit = p.tok.lit[:len(p.tok.lit)-1]
+		}
+		s, err := strconv.Unquote(p.tok.lit)
+		if err != nil {
+			p.errs.add(p.tok.pos, err)
+		}
+		lit := ast.NewLitMatcher(p.tok.pos, s)
+		lit.IgnoreCase = ignore
+		p.read()
+		return lit
+
+	case class:
+		// character class matcher
+		cl := ast.NewCharClassMatcher(p.tok.pos, p.tok.lit)
+		p.read()
+		return cl
+
+	case dot:
+		// any matcher
+		any := ast.NewAnyMatcher(p.tok.pos, p.tok.lit)
+		p.read()
+		return any
+
+	case ident:
+		// rule reference expression
+		return p.ruleRefExpr()
+
+	case lparen:
+		// expression in parenthesis
+		p.read()
+		expr := p.expression()
+		if expr == nil {
+			p.errs.add(p.tok.pos, errors.New("missing expression inside parenthesis"))
+			return nil
+		}
+		if !p.expect(rparen) {
+			return nil
+		}
+		p.read()
+		return expr
+
+	default:
+		// if p.tok.id != eof && p.tok.id != eol && p.tok.id != semicolon {
+		// 	p.errs.add(p.tok.pos, fmt.Errorf("invalid token %s (%q) for primary expression", p.tok.id, p.tok.lit))
+		// }
+		return nil
+	}
+}
+
+func (p *Parser) ruleRefExpr() ast.Expression {
+	defer p.out(p.in("ruleRefExpr"))
+
+	if !p.expect(ident) {
+		return nil
+	}
+	expr := ast.NewRuleRefExpr(p.tok.pos)
+	expr.Name = ast.NewIdentifier(p.tok.pos, p.tok.lit)
+	p.read()
+	return expr
+}
@@ -0,0 +1,97 @@
+package bootstrap
+
+import (
+	"strings"
+	"testing"
+)
+
+var parseValidCases = []string{
+	"",
+	"\n",
+	"\n{code}",
+	"\nR <- 'c'",
+	"\n\nR <- 'c'\n\n",
+	`
+A = ident:B / C+ / D?;`,
+	`{ code }
+
+R "name" <- "abc"i
+R2 = 'd'i
+R3 = ( R2+ ![;] )`,
+}
+
+var parseExpRes = []string{
+	`1:0 (0): *ast.Grammar{Init: <nil>, Rules: [
+]}`,
+	`2:0 (0): *ast.Grammar{Init: <nil>, Rules: [
+]}`,
+	`2:0 (0): *ast.Grammar{Init: 2:1 (1): *ast.CodeBlock{Val: "{code}"}, Rules: [
+]}`,
+	`2:0 (0): *ast.Grammar{Init: <nil>, Rules: [
+2:1 (1): *ast.Rule{Name: 2:1 (1): *ast.Identifier{Val: "R"}, DisplayName: <nil>, Expr: 2:6 (6): *ast.LitMatcher{Val: "c", IgnoreCase: false}},
+]}`,
+	`2:0 (0): *ast.Grammar{Init: <nil>, Rules: [
+3:1 (2): *ast.Rule{Name: 3:1 (2): *ast.Identifier{Val: "R"}, DisplayName: <nil>, Expr: 3:6 (7): *ast.LitMatcher{Val: "c", IgnoreCase: false}},
+]}`,
+	`2:0 (0): *ast.Grammar{Init: <nil>, Rules: [
+2:1 (1): *ast.Rule{Name: 2:1 (1): *ast.Identifier{Val: "A"}, DisplayName: <nil>, Expr: 2:5 (5): *ast.ChoiceExpr{Alternatives: [
+2:5 (5): *ast.LabeledExpr{Label: 2:5 (5): *ast.Identifier{Val: "ident"}, Expr: 2:11 (11): *ast.RuleRefExpr{Name: 2:11 (11): *ast.Identifier{Val: "B"}}},
+2:15 (15): *ast.OneOrMoreExpr{Expr: 2:15 (15): *ast.RuleRefExpr{Name: 2:15 (15): *ast.Identifier{Val: "C"}}},
+2:20 (20): *ast.ZeroOrOneExpr{Expr: 2:20 (20): *ast.RuleRefExpr{Name: 2:20 (20): *ast.Identifier{Val: "D"}}},
+]}},
+]}`,
+	`1:1 (0): *ast.Grammar{Init: 1:1 (0): *ast.CodeBlock{Val: "{ code }"}, Rules: [
+3:1 (10): *ast.Rule{Name: 3:1 (10): *ast.Identifier{Val: "R"}, DisplayName: 3:3 (12): *ast.StringLit{Val: "name"}, Expr: 3:13 (22): *ast.LitMatcher{Val: "abc", IgnoreCase: true}},
+4:1 (29): *ast.Rule{Name: 4:1 (29): *ast.Identifier{Val: "R2"}, DisplayName: <nil>, Expr: 4:6 (34): *ast.LitMatcher{Val: "d", IgnoreCase: true}},
+5:1 (39): *ast.Rule{Name: 5:1 (39): *ast.Identifier{Val: "R3"}, DisplayName: <nil>, Expr: 5:8 (46): *ast.SeqExpr{Exprs: [
+5:8 (46): *ast.OneOrMoreExpr{Expr: 5:8 (46): *ast.RuleRefExpr{Name: 5:8 (46): *ast.Identifier{Val: "R2"}}},
+5:12 (50): *ast.NotExpr{Expr: 5:13 (51): *ast.CharClassMatcher{Val: "[;]", IgnoreCase: false, Inverted: false}},
+]}},
+]}`,
+}
+
+func TestParseValid(t *testing.T) {
+	p := NewParser()
+	for i, c := range parseValidCases {
+		g, err := p.Parse("", strings.NewReader(c))
+		if err != nil {
+			t.Errorf("%d: got error %v", i, err)
+			continue
+		}
+
+		want := parseExpRes[i]
+		got := g.String()
+		if want != got {
+			t.Errorf("%d: want \n%s\n, got \n%s\n", i, want, got)
+		}
+	}
+}
+
+var parseInvalidCases = []string{
+	"a",
+	`R = )`,
+}
+
+var parseExpErrs = [][]string{
+	{"1:1 (0): expected ruledef, got eof"},
+	{"1:5 (4): no expression in sequence", "1:5 (4): no expression in choice", "1:5 (4): missing expression"},
+}
+
+func TestParseInvalid(t *testing.T) {
+	p := NewParser()
+	for i, c := range parseInvalidCases {
+		_, err := p.Parse("", strings.NewReader(c))
+		el := *(err.(*errList))
+		if len(el) != len(parseExpErrs[i]) {
+			t.Errorf("%d: want %d errors, got %d", i, len(parseExpErrs[i]), len(el))
+			continue
+		}
+		for j, err := range el {
+			want := parseExpErrs[i][j]
+			got := err.Error()
+			if want != got {
+				t.Errorf("%d: error %d: want %q, got %q", i, j, want, got)
+			}
+		}
+	}
+}
@@ -0,0 +1,25 @@
+// PEG grammar in EBNF form, to help implement the bootstrapping
+// parser. Terminals are tokens as defined in token.go and
+// returned by the Scanner implemented in scan.go.
+
+Grammar = [ code ] [ RuleList ] .
+RuleList = Rule { Rule } .
+Rule = ident [ str | rstr | char ] ruledef Expression ( eol | eof | semicolon ) .
+
+Expression = ChoiceExpr .
+ChoiceExpr = ActionExpr { "/" ActionExpr } .
+ActionExpr = SeqExpr [ code ] .
+SeqExpr = LabeledExpr { LabeledExpr } .
+LabeledExpr = [ ident colon ] PrefixedExpr .
+PrefixedExpr = [ PrefixedOp ] SuffixedExpr .
+PrefixedOp = ampersand | exclamation .
+SuffixedExpr = PrimaryExpr [ SuffixedOp ] .
+SuffixedOp = question | star | plus .
+PrimaryExpr = LiteralMatcher | CharClassMatcher | AnyMatcher | RuleRefExpr | 
+    lparen Expression rparen .
+
+RuleRefExpr = ident .
+
+LiteralMatcher = [ str | rstr | char ] .
+CharClassMatcher = class .
+AnyMatcher = dot .
@@ -0,0 +1,559 @@
+package bootstrap
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"unicode"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+// Scanner tokenizes an input source for the PEG grammar.
+type Scanner struct {
+	r    io.RuneReader
+	errh func(ast.Pos, error)
+
+	eof  bool
+	cpos ast.Pos
+	cur  rune
+	cw   int
+
+	tok bytes.Buffer
+}
+
+// Init initializes the scanner to read and tokenize text from r.
+func (s *Scanner) Init(filename string, r io.Reader, errh func(ast.Pos, error)) {
+	s.r = runeReader(r)
+	s.errh = errh
+
+	s.eof = false
+	s.cpos = ast.Pos{
+		Filename: filename,
+		Line:     1,
+	}
+
+	s.cur, s.cw = -1, 0
+	s.tok.Reset()
+}
+
+// Scan returns the next token, along with a boolean indicating if EOF was
+// reached (false means no more tokens).
+func (s *Scanner) Scan() (Token, bool) {
+	var tok Token
+
+	if !s.eof && s.cur == -1 {
+		// move to first rune
+		s.read()
+	}
+
+	s.skipWhitespace()
+	tok.pos = s.cpos
+
+	// the first switch cases all position the scanner on the next rune
+	// by their calls to scan*
+	switch {
+	case s.eof:
+		tok.id = eof
+	case isLetter(s.cur):
+		tok.id = ident
+		tok.lit = s.scanIdentifier()
+		if _, ok := blacklistedIdents[tok.lit]; ok {
+			s.errorpf(tok.pos, "illegal identifier %q", tok.lit)
+		}
+	case isRuleDefStart(s.cur):
+		tok.id = ruledef
+		tok.lit = s.scanRuleDef()
+	case s.cur == '\'':
+		tok.id = char
+		tok.lit = s.scanChar()
+	case s.cur == '"':
+		tok.id = str
+		tok.lit = s.scanString()
+	case s.cur == '`':
+		tok.id = rstr
+		tok.lit = s.scanRawString()
+	case s.cur == '[':
+		tok.id = class
+		tok.lit = s.scanClass()
+	case s.cur == '{':
+		tok.id = code
+		tok.lit = s.scanCode()
+
+	default:
+		r := s.cur
+		s.read()
+		switch r {
+		case '/':
+			if s.cur == '*' || s.cur == '/' {
+				tok.id, tok.lit = s.scanComment()
+				break
+			}
+			fallthrough
+		case ':', ';', '(', ')', '.', '&', '!', '?', '+', '*', '\n':
+			tok.id = tid(r)
+			tok.lit = string(r)
+		default:
+			s.errorf("invalid character %#U", r)
+			tok.id = invalid
+			tok.lit = string(r)
+		}
+	}
+
+	return tok, tok.id != eof
+}
+
+func (s *Scanner) scanIdentifier() string {
+	s.tok.Reset()
+	for isLetter(s.cur) || isDigit(s.cur) {
+		s.tok.WriteRune(s.cur)
+		s.read()
+	}
+	return s.tok.String()
+}
+
+func (s *Scanner) scanComment() (tid, string) {
+	s.tok.Reset()
+	s.tok.WriteRune('/') // initial '/' already consumed
+
+	var multiline bool
+	switch s.cur {
+	case '*':
+		multiline = true
+	case '\n', -1:
+		s.errorf("comment not terminated")
+		return lcomment, s.tok.String()
+	}
+
+	var closing bool
+	for {
+		s.tok.WriteRune(s.cur)
+		s.read()
+		switch s.cur {
+		case '\n':
+			if !multiline {
+				return lcomment, s.tok.String()
+			}
+		case -1:
+			if multiline {
+				s.errorf("comment not terminated")
+				return mlcomment, s.tok.String()
+			}
+			return lcomment, s.tok.String()
+		case '*':
+			if multiline {
+				closing = true
+			}
+		case '/':
+			if closing {
+				s.tok.WriteRune(s.cur)
+				s.read()
+				return mlcomment, s.tok.String()
+			}
+		}
+	}
+}
+
+func (s *Scanner) scanCode() string {
+	s.tok.Reset()
+	s.tok.WriteRune(s.cur)
+	depth := 1
+	for {
+		s.read()
+		s.tok.WriteRune(s.cur)
+		switch s.cur {
+		case -1:
+			s.errorf("code block not terminated")
+			return s.tok.String()
+		case '{':
+			depth++
+		case '}':
+			depth--
+			if depth == 0 {
+				s.read()
+				return s.tok.String()
+			}
+		}
+	}
+}
+
+func (s *Scanner) scanEscape(quote rune) bool {
+	// scanEscape is always called as part of a greater token, so do not
+	// reset s.tok, and write s.cur before calling s.read.
+	s.tok.WriteRune(s.cur)
+
+	var n int
+	var base, max uint32
+	var unicodeClass bool
+
+	s.read()
+	switch s.cur {
+	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
+		s.tok.WriteRune(s.cur)
+		return true
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		n, base, max = 3, 8, 255
+	case 'x':
+		s.tok.WriteRune(s.cur)
+		s.read()
+		n, base, max = 2, 16, 255
+	case 'u':
+		s.tok.WriteRune(s.cur)
+		s.read()
+		n, base, max = 4, 16, unicode.MaxRune
+	case 'U':
+		s.tok.WriteRune(s.cur)
+		s.read()
+		n, base, max = 8, 16, unicode.MaxRune
+	case 'p':
+		// unicode character class, only valid if quote is ']'
+		if quote == ']' {
+			s.tok.WriteRune(s.cur)
+			unicodeClass = true
+			s.read()
+			break
+		}
+		fallthrough
+	default:
+		s.tok.WriteRune(s.cur)
+		msg := "unknown escape sequence"
+		if s.cur == -1 || s.cur == '\n' {
+			msg = "escape sequence not terminated"
+			s.errorf(msg)
+		} else {
+			s.errorf(msg)
+			s.read()
+		}
+		return false
+	}
+
+	if unicodeClass {
+		switch s.cur {
+		case '\n', -1:
+			s.errorf("escape sequence not terminated")
+			return false
+		case '{':
+			// unicode class name, read until '}'
+			cnt := 0
+			for {
+				s.tok.WriteRune(s.cur)
+				s.read()
+				cnt++
+				switch s.cur {
+				case '\n', -1:
+					s.errorf("escape sequence not terminated")
+					return false
+				case '}':
+					if cnt < 2 {
+						s.errorf("empty Unicode character class escape sequence")
+					}
+					s.tok.WriteRune(s.cur)
+					return true
+				}
+			}
+		default:
+			// single letter class
+			s.tok.WriteRune(s.cur)
+			return true
+		}
+	}
+
+	var x uint32
+	for n > 0 {
+		s.tok.WriteRune(s.cur)
+		d := uint32(digitVal(s.cur))
+		if d >= base {
+			msg := fmt.Sprintf("illegal character %#U in escape sequence", s.cur)
+			if s.cur == -1 || s.cur == '\n' {
+				msg = "escape sequence not terminated"
+				s.errorf(msg)
+				return false
+			}
+			s.errorf(msg)
+			s.read()
+			return false
+		}
+		x = x*base + d
+		n--
+
+		if n > 0 {
+			s.read()
+		}
+	}
+
+	if x > max || 0xd800 <= x && x <= 0xe000 {
+		s.errorf("escape sequence is invalid Unicode code point")
+		s.read()
+		return false
+	}
+	return true
+}
+
+func (s *Scanner) scanClass() string {
+	s.tok.Reset()
+	s.tok.WriteRune(s.cur) // opening '['
+
+	var noread bool
+	for {
+		if !noread {
+			s.read()
+		}
+		noread = false
+		switch s.cur {
+		case '\\':
+			noread = !s.scanEscape(']')
+		case '\n', -1:
+			// \n not consumed
+			s.errorf("character class not terminated")
+			return s.tok.String()
+		case ']':
+			s.tok.WriteRune(s.cur)
+			s.read()
+			// can have an optional "i" ignore case suffix
+			if s.cur == 'i' {
+				s.tok.WriteRune(s.cur)
+				s.read()
+			}
+			return s.tok.String()
+		default:
+			s.tok.WriteRune(s.cur)
+		}
+	}
+}
+
+func (s *Scanner) scanRawString() string {
+	s.tok.Reset()
+	s.tok.WriteRune(s.cur) // opening '`'
+
+	var hasCR bool
+loop:
+	for {
+		s.read()
+		switch s.cur {
+		case -1:
+			s.errorf("raw string literal not terminated")
+			break loop
+		case '`':
+			s.tok.WriteRune(s.cur)
+			s.read()
+			// can have an optional "i" ignore case suffix
+			if s.cur == 'i' {
+				s.tok.WriteRune(s.cur)
+				s.read()
+			}
+			break loop
+		case '\r':
+			hasCR = true
+			fallthrough
+		default:
+			s.tok.WriteRune(s.cur)
+		}
+	}
+
+	b := s.tok.Bytes()
+	if hasCR {
+		b = stripCR(b)
+	}
+	return string(b)
+}
+
+func stripCR(b []byte) []byte {
+	c := make([]byte, len(b))
+	i := 0
+	for _, ch := range b {
+		if ch != '\r' {
+			c[i] = ch
+			i++
+		}
+	}
+	return c[:i]
+}
+
+func (s *Scanner) scanString() string {
+	s.tok.Reset()
+	s.tok.WriteRune(s.cur) // opening '"'
+
+	var noread bool
+	for {
+		if !noread {
+			s.read()
+		}
+		noread = false
+		switch s.cur {
+		case '\\':
+			noread = !s.scanEscape('"')
+		case '\n', -1:
+			// \n not consumed
+			s.errorf("string literal not terminated")
+			return s.tok.String()
+		case '"':
+			s.tok.WriteRune(s.cur)
+			s.read()
+			// can have an optional "i" ignore case suffix
+			if s.cur == 'i' {
+				s.tok.WriteRune(s.cur)
+				s.read()
+			}
+			return s.tok.String()
+		default:
+			s.tok.WriteRune(s.cur)
+		}
+	}
+}
+
+func (s *Scanner) scanChar() string {
+	s.tok.Reset()
+	s.tok.WriteRune(s.cur) // opening "'"
+
+	// must be followed by one char (which may be an escape) and a single
+	// quote, but read until we find that closing quote.
+	cnt := 0
+	var noread bool
+	for {
+		if !noread {
+			s.read()
+		}
+		noread = false
+		switch s.cur {
+		case '\\':
+			cnt++
+			noread = !s.scanEscape('\'')
+		case '\n', -1:
+			// \n not consumed
+			s.errorf("rune literal not terminated")
+			return s.tok.String()
+		case '\'':
+			s.tok.WriteRune(s.cur)
+			s.read()
+			if cnt != 1 {
+				s.errorf("rune literal is not a single rune")
+			}
+			// can have an optional "i" ignore case suffix
+			if s.cur == 'i' {
+				s.tok.WriteRune(s.cur)
+				s.read()
+			}
+			return s.tok.String()
+		default:
+			cnt++
+			s.tok.WriteRune(s.cur)
+		}
+	}
+}
+
+func (s *Scanner) scanRuleDef() string {
+	s.tok.Reset()
+	s.tok.WriteRune(s.cur)
+	r := s.cur
+	s.read()
+	if r == '<' {
+		if s.cur != -1 {
+			s.tok.WriteRune(s.cur)
+		}
+		if s.cur != '-' {
+			s.errorf("rule definition not terminated")
+		}
+		s.read()
+	}
+
+	return s.tok.String()
+}
+
+// read advances the Scanner to the next rune.
+func (s *Scanner) read() {
+	if s.eof {
+		return
+	}
+
+	r, w, err := s.r.ReadRune()
+	if err != nil {
+		s.fatalError(err)
+		return
+	}
+
+	s.cur = r
+	s.cpos.Off += s.cw
+	s.cw = w
+
+	// newline is '\n' as in Go
+	if r == '\n' {
+		s.cpos.Line++
+		s.cpos.Col = 0
+	} else {
+		s.cpos.Col++
+	}
+}
+
+// whitespace is the same as Go, except that it doesn't skip newlines,
+// those are returned as tokens.
+func (s *Scanner) skipWhitespace() {
+	for s.cur == ' ' || s.cur == '\t' || s.cur == '\r' {
+		s.read()
+	}
+}
+
+func isRuleDefStart(r rune) bool {
+	return r == '=' || r == '<' || r == '\u2190' /* leftwards arrow */ ||
+		r == '\u27f5' /* long leftwards arrow */
+}
+
+// isLetter has the same definition as Go.
+func isLetter(r rune) bool {
+	return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_' ||
+		r >= 0x80 && unicode.IsLetter(r)
+}
+
+// isDigit has the same definition as Go.
+func isDigit(r rune) bool {
+	return '0' <= r && r <= '9' || r >= 0x80 && unicode.IsDigit(r)
+}
+
+func digitVal(r rune) int {
+	switch {
+	case '0' <= r && r <= '9':
+		return int(r - '0')
+	case 'a' <= r && r <= 'f':
+		return int(r - 'a' + 10)
+	case 'A' <= r && r <= 'F':
+		return int(r - 'A' + 10)
+	}
+	return 16
+}
+
+// notify the handler of an error.
+func (s *Scanner) error(p ast.Pos, err error) {
+	if s.errh != nil {
+		s.errh(p, err)
+		return
+	}
+	fmt.Fprintf(os.Stderr, "%s: %v\n", p, err)
+}
+
+// helper to generate and notify of an error.
+func (s *Scanner) errorf(f string, args ...interface{}) {
+	s.errorpf(s.cpos, f, args...)
+}
+
+// helper to generate and notify of an error at a specific position.
+func (s *Scanner) errorpf(p ast.Pos, f string, args ...interface{}) {
+	s.error(p, fmt.Errorf(f, args...))
+}
+
+// notify a non-recoverable error that terminates the scanning.
+func (s *Scanner) fatalError(err error) {
+	s.cur = -1
+	s.eof = true
+	if err != io.EOF {
+		s.error(s.cpos, err)
+	}
+}
+
+// convert the reader to a rune reader if required.
+func runeReader(r io.Reader) io.RuneReader {
+	if rr, ok := r.(io.RuneReader); ok {
+		return rr
+	}
+	return bufio.NewReader(r)
+}
@@ -0,0 +1,358 @@
+package bootstrap
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+var scanValidCases = []string{
+	"",
+	"a",
+	"ab",
+	"abc",
+	"_",
+	"_0",
+	"abc_012",
+	`=`,
+	`<-`,
+	"\u2190",
+	"\u27f5",
+	"' '",
+	"'*'",
+	"'a'",
+	"'a'i",
+	"'a'b",
+	`'\n'`,
+	`'\t'`,
+	`'\''`,
+	`'\\'`,
+	`'\xab'`,
+	`'\x1F'`,
+	`'\u1234'`,
+	`'\U000B1234'`,
+	`""`,
+	`"a"`,
+	`"a"i`,
+	`"a"b`,
+	`"a\b"`,
+	`"a\b \n 1"`,
+	`"\xAbc\u1234d\U000011FF"`,
+	"``",
+	"`a`",
+	"`a`i",
+	"`a`b",
+	"`a \\n `", // `a \n `
+	"`a \n `",  // `a <newline> `
+	"`a \r\n `",
+	"[]",
+	"[[]",
+	"[[\\]]",
+	"[a]",
+	"[a]i",
+	"[a]b",
+	"[ab]",
+	"[a-b0-9]",
+	"[\\a]",
+	"[\\a\\pL_]",
+	"[\\a\\p{Greek}]",
+	"{}",
+	"{a}",
+	"{a}i",
+	"{\nif something {\n\tdoSomething()\n}\n}",
+	"// a",
+	"// a\nb",
+	"/a",
+	"/\n",
+	"/**/",
+	"/*a*/",
+	"/*a\nb*/",
+	":",
+	";",
+	"(",
+	")",
+	".",
+	"&",
+	"!",
+	"?",
+	"+",
+	"*",
+	"\n",
+	"pockage = a",
+	`Rule <-
+	E / ( 'a'? "bcd"i )+ / [efg-j]* { println() } // comment
+	/ &'\xff' /* and
+some
+comment
+*/`,
+}
+
+var scanExpTokens = [][]string{
+	{"1:0 (0): eof \"\""},
+	{"1:1 (0): ident \"a\"", "1:1 (0): eof \"\""},
+	{"1:1 (0): ident \"ab\"", "1:2 (1): eof \"\""},
+	{"1:1 (0): ident \"abc\"", "1:3 (2): eof \"\""},
+	{"1:1 (0): ident \"_\"", "1:1 (0): eof \"\""},
+	{"1:1 (0): ident \"_0\"", "1:2 (1): eof \"\""},
+	{"1:1 (0): ident \"abc_012\"", "1:7 (6): eof \"\""},
+	{"1:1 (0): ruledef \"=\"", "1:1 (0): eof \"\""},
+	{"1:1 (0): ruledef \"<-\"", "1:2 (1): eof \"\""},
+	{"1:1 (0): ruledef \"\u2190\"", "1:1 (0): eof \"\""},
+	{"1:1 (0): ruledef \"\u27f5\"", "1:1 (0): eof \"\""},
+	{"1:1 (0): char \"' '\"", "1:3 (2): eof \"\""},
+	{"1:1 (0): char \"'*'\"", "1:3 (2): eof \"\""},
+	{"1:1 (0): char \"'a'\"", "1:3 (2): eof \"\""},
+	{"1:1 (0): char \"'a'i\"", "1:4 (3): eof \"\""},
+	{"1:1 (0): char \"'a'\"", "1:4 (3): ident \"b\"", "1:4 (3): eof \"\""},
+	{`1:1 (0): char "'\\n'"`, `1:4 (3): eof ""`},
+	{`1:1 (0): char "'\\t'"`, `1:4 (3): eof ""`},
+	{`1:1 (0): char "'\\''"`, `1:4 (3): eof ""`},
+	{`1:1 (0): char "'\\\\'"`, `1:4 (3): eof ""`},
+	{`1:1 (0): char "'\\xab'"`, `1:6 (5): eof ""`},
+	{`1:1 (0): char "'\\x1F'"`, `1:6 (5): eof ""`},
+	{`1:1 (0): char "'\\u1234'"`, `1:8 (7): eof ""`},
+	{`1:1 (0): char "'\\U000B1234'"`, `1:12 (11): eof ""`},
+	{`1:1 (0): str "\"\""`, `1:2 (1): eof ""`},
+	{`1:1 (0): str "\"a\""`, `1:3 (2): eof ""`},
+	{`1:1 (0): str "\"a\"i"`, `1:4 (3): eof ""`},
+	{`1:1 (0): str "\"a\""`, `1:4 (3): ident "b"`, `1:4 (3): eof ""`},
+	{`1:1 (0): str "\"a\\b\""`, `1:5 (4): eof ""`},
+	{`1:1 (0): str "\"a\\b \\n 1\""`, `1:10 (9): eof ""`},
+	{`1:1 (0): str "\"\\xAbc\\u1234d\\U000011FF\""`, `1:24 (23): eof ""`},
+	{"1:1 (0): rstr \"``\"", `1:2 (1): eof ""`},
+	{"1:1 (0): rstr \"`a`\"", `1:3 (2): eof ""`},
+	{"1:1 (0): rstr \"`a`i\"", `1:4 (3): eof ""`},
+	{"1:1 (0): rstr \"`a`\"", "1:4 (3): ident \"b\"", `1:4 (3): eof ""`},
+	{"1:1 (0): rstr \"`a \\\\n `\"", `1:7 (6): eof ""`},
+	{"1:1 (0): rstr \"`a \\n `\"", `2:2 (5): eof ""`},
+	{"1:1 (0): rstr \"`a \\n `\"", `2:2 (6): eof ""`},
+	{"1:1 (0): class \"[]\"", `1:2 (1): eof ""`},
+	{"1:1 (0): class \"[[]\"", `1:3 (2): eof ""`},
+	{"1:1 (0): class \"[[\\\\]]\"", `1:5 (4): eof ""`},
+	{"1:1 (0): class \"[a]\"", `1:3 (2): eof ""`},
+	{"1:1 (0): class \"[a]i\"", `1:4 (3): eof ""`},
+	{"1:1 (0): class \"[a]\"", `1:4 (3): ident "b"`, `1:4 (3): eof ""`},
+	{"1:1 (0): class \"[ab]\"", `1:4 (3): eof ""`},
+	{"1:1 (0): class \"[a-b0-9]\"", `1:8 (7): eof ""`},
+	{"1:1 (0): class \"[\\\\a]\"", `1:4 (3): eof ""`},
+	{"1:1 (0): class \"[\\\\a\\\\pL_]\"", `1:8 (7): eof ""`},
+	{"1:1 (0): class \"[\\\\a\\\\p{Greek}]\"", `1:13 (12): eof ""`},
+	{"1:1 (0): code \"{}\"", `1:2 (1): eof ""`},
+	{"1:1 (0): code \"{a}\"", `1:3 (2): eof ""`},
+	{"1:1 (0): code \"{a}\"", "1:4 (3): ident \"i\"", `1:4 (3): eof ""`},
+	{"1:1 (0): code \"{\\nif something {\\n\\tdoSomething()\\n}\\n}\"", `5:1 (34): eof ""`},
+	{"1:1 (0): lcomment \"// a\"", `1:4 (3): eof ""`},
+	{"1:1 (0): lcomment \"// a\"", `2:0 (4): eol "\n"`, `2:1 (5): ident "b"`, `2:1 (5): eof ""`},
+	{"1:1 (0): slash \"/\"", `1:2 (1): ident "a"`, `1:2 (1): eof ""`},
+	{"1:1 (0): slash \"/\"", `2:0 (1): eol "\n"`, `2:0 (1): eof ""`},
+	{"1:1 (0): mlcomment \"/**/\"", `1:4 (3): eof ""`},
+	{"1:1 (0): mlcomment \"/*a*/\"", `1:5 (4): eof ""`},
+	{"1:1 (0): mlcomment \"/*a\\nb*/\"", `2:3 (6): eof ""`},
+	{"1:1 (0): colon \":\"", `1:1 (0): eof ""`},
+	{"1:1 (0): semicolon \";\"", `1:1 (0): eof ""`},
+	{"1:1 (0): lparen \"(\"", `1:1 (0): eof ""`},
+	{"1:1 (0): rparen \")\"", `1:1 (0): eof ""`},
+	{"1:1 (0): dot \".\"", `1:1 (0): eof ""`},
+	{"1:1 (0): ampersand \"&\"", `1:1 (0): eof ""`},
+	{"1:1 (0): exclamation \"!\"", `1:1 (0): eof ""`},
+	{"1:1 (0): question \"?\"", `1:1 (0): eof ""`},
+	{"1:1 (0): plus \"+\"", `1:1 (0): eof ""`},
+	{"1:1 (0): star \"*\"", `1:1 (0): eof ""`},
+	{"2:0 (0): eol \"\\n\"", `2:0 (0): eof ""`},
+	{"1:1 (0): ident \"pockage\"", `1:9 (8): ruledef "="`, `1:11 (10): ident "a"`, `1:11 (10): eof ""`},
+	{
+		`1:1 (0): ident "Rule"`,
+		`1:6 (5): ruledef "<-"`,
+		`2:0 (7): eol "\n"`,
+		`2:2 (9): ident "E"`,
+		`2:4 (11): slash "/"`,
+		`2:6 (13): lparen "("`,
+		`2:8 (15): char "'a'"`,
+		`2:11 (18): question "?"`,
+		`2:13 (20): str "\"bcd\"i"`,
+		`2:20 (27): rparen ")"`,
+		`2:21 (28): plus "+"`,
+		`2:23 (30): slash "/"`,
+		`2:25 (32): class "[efg-j]"`,
+		`2:32 (39): star "*"`,
+		`2:34 (41): code "{ println() }"`,
+		`2:48 (55): lcomment "// comment"`,
+		`3:0 (65): eol "\n"`,
+		`3:2 (67): slash "/"`,
+		`3:4 (69): ampersand "&"`,
+		`3:5 (70): char "'\\xff'"`,
+		`3:12 (77): mlcomment "/* and\nsome\ncomment\n*/"`,
+		`6:2 (98): eof ""`,
+	},
+}
+
+type errsink struct {
+	errs []error
+	pos  []ast.Pos
+}
+
+func (e *errsink) add(p ast.Pos, err error) {
+	e.errs = append(e.errs, err)
+	e.pos = append(e.pos, p)
+}
+
+func (e *errsink) reset() {
+	e.errs = e.errs[:0]
+	e.pos = e.pos[:0]
+}
+
+func (e *errsink) StringAt(i int) string {
+	if i < 0 || i >= len(e.errs) {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", e.pos[i], e.errs[i])
+}
+
+func TestScanValid(t *testing.T) {
+	old := tokenStringLen
+	tokenStringLen = 100
+	defer func() { tokenStringLen = old }()
+
+	var s Scanner
+	var errh errsink
+	for i, c := range scanValidCases {
+		errh.reset()
+		s.Init("", strings.NewReader(c), errh.add)
+
+		j := 0
+		for {
+			tok, ok := s.Scan()
+			if j < len(scanExpTokens[i]) {
+				got := tok.String()
+				want := scanExpTokens[i][j]
+				if got != want {
+					t.Errorf("%d: token %d: want %q, got %q", i, j, want, got)
+				}
+			} else {
+				t.Errorf("%d: want %d tokens, got #%d", i, len(scanExpTokens[i]), j+1)
+			}
+			if !ok {
+				if j < len(scanExpTokens[i])-1 {
+					t.Errorf("%d: wand %d tokens, got only %d", i, len(scanExpTokens[i]), j+1)
+				}
+				break
+			}
+			j++
+		}
+		if len(errh.errs) != 0 {
+			t.Errorf("%d: want no error, got %d", i, len(errh.errs))
+			t.Log(errh.errs)
+		}
+	}
+}
+
+var scanInvalidCases = []string{
+	"|",
+	"<",
+	"'",
+	"''",
+	"'ab'",
+	`'\xff\U00001234'`,
+	`'\pA'`,
+	`'\z'`,
+	"'\\\n",
+	`'\xg'`,
+	`'\129'`,
+	`'\12`,
+	`'\xa`,
+	`'\u123z'`,
+	`'\u12`,
+	`'\UFFFFffff'`,
+	`'\uD800'`,
+	`'\ue000'`,
+	`'\ud901'`,
+	`'\"'`,
+	"\"\n",
+	"\"",
+	"\"\\'\"",
+	"`",
+	"[",
+	"[\\\"",
+	`[\[]`,
+	`[\p]`,
+	`[\p{]`,
+	`[\p{`,
+	`[\p{}]`,
+	`{code{}`,
+	`/*a*`,
+	`/*a`,
+	`func`,
+}
+
+var scanExpErrs = [][]string{
+	{"1:1 (0): invalid character U+007C '|'"},
+	{"1:1 (0): rule definition not terminated"},
+	{"1:1 (0): rune literal not terminated"},
+	{"1:2 (1): rune literal is not a single rune"},
+	{"1:4 (3): rune literal is not a single rune"},
+	{"1:16 (15): rune literal is not a single rune"},
+	{"1:3 (2): unknown escape sequence",
+		"1:5 (4): rune literal is not a single rune"},
+	{"1:3 (2): unknown escape sequence"},
+	{"2:0 (2): escape sequence not terminated",
+		"2:0 (2): rune literal not terminated"},
+	{"1:4 (3): illegal character U+0067 'g' in escape sequence"},
+	{"1:5 (4): illegal character U+0039 '9' in escape sequence"},
+	{"1:4 (3): escape sequence not terminated",
+		"1:4 (3): rune literal not terminated"},
+	{"1:4 (3): escape sequence not terminated",
+		"1:4 (3): rune literal not terminated"},
+	{"1:7 (6): illegal character U+007A 'z' in escape sequence"},
+	{"1:5 (4): escape sequence not terminated",
+		"1:5 (4): rune literal not terminated"},
+	{"1:11 (10): escape sequence is invalid Unicode code point"},
+	{"1:7 (6): escape sequence is invalid Unicode code point"},
+	{"1:7 (6): escape sequence is invalid Unicode code point"},
+	{"1:7 (6): escape sequence is invalid Unicode code point"},
+	{"1:3 (2): unknown escape sequence"},
+	{"2:0 (1): string literal not terminated"},
+	{"1:1 (0): string literal not terminated"},
+	{"1:3 (2): unknown escape sequence"},
+	{"1:1 (0): raw string literal not terminated"},
+	{"1:1 (0): character class not terminated"},
+	{"1:3 (2): unknown escape sequence",
+		"1:3 (2): character class not terminated"},
+	{"1:3 (2): unknown escape sequence"},
+	{"1:4 (3): character class not terminated"},
+	{"1:5 (4): escape sequence not terminated",
+		"1:5 (4): character class not terminated"},
+	{"1:4 (3): escape sequence not terminated",
+		"1:4 (3): character class not terminated"},
+	{"1:5 (4): empty Unicode character class escape sequence"},
+	{"1:7 (6): code block not terminated"},
+	{"1:4 (3): comment not terminated"},
+	{"1:3 (2): comment not terminated"},
+	{"1:1 (0): illegal identifier \"func\""},
+}
+
+func TestScanInvalid(t *testing.T) {
+	var s Scanner
+	var errh errsink
+	for i, c := range scanInvalidCases {
+		errh.reset()
+		s.Init("", strings.NewReader(c), errh.add)
+		for {
+			if _, ok := s.Scan(); !ok {
+				break
+			}
+		}
+		if len(errh.errs) != len(scanExpErrs[i]) {
+			t.Errorf("%d: want %d errors, got %d", i, len(scanExpErrs[i]), len(errh.errs))
+			continue
+		}
+		for j := range errh.errs {
+			want := scanExpErrs[i][j]
+			got := errh.StringAt(j)
+			if want != got {
+				t.Errorf("%d: error %d: want %q, got %q", i, j, want, got)
+			}
+		}
+	}
+}
@@ -0,0 +1,161 @@
+package bootstrap
+
+import (
+	"fmt"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+type tid int
+
+const (
+	invalid tid = iota - 1
+	eof         // end-of-file token, id 0
+
+	ident   tid = iota + 127 // identifiers follow the same rules as Go
+	ruledef                  // rule definition token
+
+	// literals
+	char      // character literal, as in Go ('a'i?)
+	str       // double-quoted string literal, as in Go ("string"i?)
+	rstr      // back-tick quoted raw string literal, as in Go (`string`i?)
+	class     // square-brackets character classes ([a\n\t]i?)
+	lcomment  // line comment as in Go (// comment or /* comment */ with no newline)
+	mlcomment // multi-line comment as in Go (/* comment */)
+	code      // code blocks between '{' and '}'
+
+	// operators and delimiters have the value of their char
+	// smallest value in that category is 10, for '\n'
+	eol         tid = '\n' // end-of-line token, required in the parser
+	colon       tid = ':'  // separate variable name from expression ':'
+	semicolon   tid = ';'  // optional ';' to terminate rules
+	lparen      tid = '('  // parenthesis to group expressions '('
+	rparen      tid = ')'  // ')'
+	dot         tid = '.'  // any matcher '.'
+	ampersand   tid = '&'  // and-predicate '&'
+	exclamation tid = '!'  // not-predicate '!'
+	question    tid = '?'  // zero-or-one '?'
+	plus        tid = '+'  // one-or-more '+'
+	star        tid = '*'  // zero-or-more '*'
+	slash       tid = '/'  // ordered choice '/'
+)
+
+var lookup = map[tid]string{
+	invalid:     "invalid",
+	eof:         "eof",
+	ident:       "ident",
+	ruledef:     "ruledef",
+	char:        "char",
+	str:         "str",
+	rstr:        "rstr",
+	class:       "class",
+	lcomment:    "lcomment",
+	mlcomment:   "mlcomment",
+	code:        "code",
+	eol:         "eol",
+	colon:       "colon",
+	semicolon:   "semicolon",
+	lparen:      "lparen",
+	rparen:      "rparen",
+	dot:         "dot",
+	ampersand:   "ampersand",
+	exclamation: "exclamation",
+	question:    "question",
+	plus:        "plus",
+	star:        "star",
+	slash:       "slash",
+}
+
+func (t tid) String() string {
+	if s, ok := lookup[t]; ok {
+		return s
+	}
+	return fmt.Sprintf("tid(%d)", t)
+}
+
+var blacklistedIdents = map[string]struct{}{
+	// Go keywords http://golang.org/ref/spec#Keywords
+	"break":       struct{}{},
+	"case":        struct{}{},
+	"chan":        struct{}{},
+	"const":       struct{}{},
+	"continue":    struct{}{},
+	"default":     struct{}{},
+	"defer":       struct{}{},
+	"else":        struct{}{},
+	"fallthrough": struct{}{},
+	"for":         struct{}{},
+	"func":        struct{}{},
+	"go":          struct{}{},
+	"goto":        struct{}{},
+	"if":          struct{}{},
+	"import":      struct{}{},
+	"interface":   struct{}{},
+	"map":         struct{}{},
+	"package":     struct{}{},
+	"range":       struct{}{},
+	"return":      struct{}{},
+	"select":      struct{}{},
+	"struct":      struct{}{},
+	"switch":      struct{}{},
+	"type":        struct{}{},
+	"var":         struct{}{},
+
+	// predeclared identifiers http://golang.org/ref/spec#Predeclared_identifiers
+	"bool":       struct{}{},
+	"byte":       struct{}{},
+	"complex64":  struct{}{},
+	"complex128": struct{}{},
+	"error":      struct{}{},
+	"float32":    struct{}{},
+	"float64":    struct{}{},
+	"int":        struct{}{},
+	"int8":       struct{}{},
+	"int16":      struct{}{},
+	"int32":      struct{}{},
+	"int64":      struct{}{},
+	"rune":       struct{}{},
+	"string":     struct{}{},
+	"uint":       struct{}{},
+	"uint8":      struct{}{},
+	"uint16":     struct{}{},
+	"uint32":     struct{}{},
+	"uint64":     struct{}{},
+	"uintptr":    struct{}{},
+	"true":       struct{}{},
+	"false":      struct{}{},
+	"iota":       struct{}{},
+	"nil":        struct{}{},
+	"append":     struct{}{},
+	"cap":        struct{}{},
+	"close":      struct{}{},
+	"complex":    struct{}{},
+	"copy":       struct{}{},
+	"delete":     struct{}{},
+	"imag":       struct{}{},
+	"len":        struct{}{},
+	"make":       struct{}{},
+	"new":        struct{}{},
+	"panic":      struct{}{},
+	"print":      struct{}{},
+	"println":    struct{}{},
+	"real":       struct{}{},
+	"recover":    struct{}{},
+}
+
+// Token is a syntactic token generated by the scanner.
+type Token struct {
+	id  tid
+	lit string
+	pos ast.Pos
+}
+
+var tokenStringLen = 50
+
+func (t Token) String() string {
+	v := t.lit
+	if len(v) > tokenStringLen {
+		v = v[:tokenStringLen/2] + "[...]" + v[len(v)-(tokenStringLen/2):len(v)]
+	}
+	return fmt.Sprintf("%s: %s %q", t.pos, t.id, v)
+}
@@ -0,0 +1,573 @@
+// Package builder generates the parser code for a given grammar. It makes
+// no attempt to verify the correctness of the grammar.
+package builder
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+// generated function templates
+var (
+	onFuncTemplate = `func (%s *current) %s(%s) (interface{}, error) {
+%s
+}
+`
+	onPredFuncTemplate = `func (%s *current) %s(%s) (bool, error) {
+%s
+}
+`
+	callFuncTemplate = `func (p *parser) call%s() (interface{}, error) {
+	stack := p.vstack[len(p.vstack)-1]
+	_ = stack
+	return p.cur.%[1]s(%s)
+}
+`
+	callPredFuncTemplate = `func (p *parser) call%s() (bool, error) {
+	stack := p.vstack[len(p.vstack)-1]
+	_ = stack
+	return p.cur.%[1]s(%s)
+}
+`
+)
+
+// Option is a function that can set an option on the builder. It returns
+// the previous setting as an Option.
+type Option func(*builder) Option
+
+// ReceiverName returns an option that specifies the receiver name to
+// use for the current struct (which is the struct on which all code blocks
+// except the initializer are generated).
+func ReceiverName(nm string) Option {
+	return func(b *builder) Option {
+		prev := b.recvName
+		b.recvName = nm
+		return ReceiverName(prev)
+	}
+}
+
+// BuildParser builds the PEG parser using the provider grammar. The code is
+// written to the specified w.
+func BuildParser(w io.Writer, g *ast.Grammar, opts ...Option) error {
+	b := &builder{w: w, recvName: "c"}
+	b.setOptions(opts)
+	return b.buildParser(g)
+}
+
+type builder struct {
+	w   io.Writer
+	err error
+
+	// options
+	recvName string
+
+	ruleName  string
+	exprIndex int
+	argsStack [][]string
+}
+
+func (b *builder) setOptions(opts []Option) {
+	for _, opt := range opts {
+		opt(b)
+	}
+}
+
+func (b *builder) buildParser(g *ast.Grammar) error {
+	b.writeInit(g.Init)
+	b.writeGrammar(g)
+
+	for _, rule := range g.Rules {
+		b.writeRuleCode(rule)
+	}
+	b.writeStaticCode()
+
+	return b.err
+}
+
+func (b *builder) writeInit(init *ast.CodeBlock) {
+	if init == nil {
+		return
+	}
+
+	// remove opening and closing braces
+	val := init.Val[1 : len(init.Val)-1]
+	b.writelnf("%s", val)
+}
+
+func (b *builder) writeGrammar(g *ast.Grammar) {
+	// transform the ast grammar to the self-contained, no dependency version
+	// of the parser-generator grammar.
+	b.writelnf("var g = &grammar {")
+	b.writelnf("\trules: []*rule{")
+	for _, r := range g.Rules {
+		b.writeRule(r)
+	}
+	b.writelnf("\t},")
+	b.writelnf("}")
+}
+
+func (b *builder) writeRule(r *ast.Rule) {
+	if r == nil || r.Name == nil {
+		return
+	}
+
+	b.exprIndex = 0
+	b.ruleName = r.Name.Val
+
+	b.writelnf("{")
+	b.writelnf("\tname: %q,", r.Name.Val)
+	if r.DisplayName != nil && r.DisplayName.Val != "" {
+		b.writelnf("\tdisplayName: %q,", r.DisplayName.Val)
+	}
+	pos := r.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writef("\texpr: ")
+	b.writeExpr(r.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeExpr(expr ast.Expression) {
+	b.exprIndex++
+	switch expr := expr.(type) {
+	case *ast.ActionExpr:
+		b.writeActionExpr(expr)
+	case *ast.AndCodeExpr:
+		b.writeAndCodeExpr(expr)
+	case *ast.AndExpr:
+		b.writeAndExpr(expr)
+	case *ast.AnyMatcher:
+		b.writeAnyMatcher(expr)
+	case *ast.CharClassMatcher:
+		b.writeCharClassMatcher(expr)
+	case *ast.ChoiceExpr:
+		b.writeChoiceExpr(expr)
+	case *ast.LabeledExpr:
+		b.writeLabeledExpr(expr)
+	case *ast.LitMatcher:
+		b.writeLitMatcher(expr)
+	case *ast.NotCodeExpr:
+		b.writeNotCodeExpr(expr)
+	case *ast.NotExpr:
+		b.writeNotExpr(expr)
+	case *ast.OneOrMoreExpr:
+		b.writeOneOrMoreExpr(expr)
+	case *ast.RuleRefExpr:
+		b.writeRuleRefExpr(expr)
+	case *ast.SeqExpr:
+		b.writeSeqExpr(expr)
+	case *ast.ZeroOrMoreExpr:
+		b.writeZeroOrMoreExpr(expr)
+	case *ast.ZeroOrOneExpr:
+		b.writeZeroOrOneExpr(expr)
+	default:
+		b.err = fmt.Errorf("builder: unknown expression type %T", expr)
+	}
+}
+
+func (b *builder) writeActionExpr(act *ast.ActionExpr) {
+	if act == nil {
+		b.writelnf("nil,")
+		return
+	}
+	act.FuncIx = b.exprIndex
+	b.writelnf("&actionExpr{")
+	pos := act.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writelnf("\trun: (*parser).call%s,", b.funcName(act.FuncIx))
+	b.writef("\texpr: ")
+	b.writeExpr(act.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeAndCodeExpr(and *ast.AndCodeExpr) {
+	if and == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&andCodeExpr{")
+	pos := and.Pos()
+	and.FuncIx = b.exprIndex
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writelnf("\trun: (*parser).call%s,", b.funcName(and.FuncIx))
+	b.writelnf("},")
+}
+
+func (b *builder) writeAndExpr(and *ast.AndExpr) {
+	if and == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&andExpr{")
+	pos := and.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writef("\texpr: ")
+	b.writeExpr(and.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeAnyMatcher(any *ast.AnyMatcher) {
+	if any == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&anyMatcher{")
+	pos := any.Pos()
+	b.writelnf("\tline: %d, col: %d, offset: %d,", pos.Line, pos.Col, pos.Off)
+	b.writelnf("},")
+}
+
+func (b *builder) writeCharClassMatcher(ch *ast.CharClassMatcher) {
+	if ch == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&charClassMatcher{")
+	pos := ch.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writelnf("\tval: %q,", ch.Val)
+	if len(ch.Chars) > 0 {
+		b.writef("\tchars: []rune{")
+		for _, rn := range ch.Chars {
+			if ch.IgnoreCase {
+				b.writef("%q,", unicode.ToLower(rn))
+			} else {
+				b.writef("%q,", rn)
+			}
+		}
+		b.writelnf("},")
+	}
+	if len(ch.Ranges) > 0 {
+		b.writef("\tranges: []rune{")
+		for _, rn := range ch.Ranges {
+			if ch.IgnoreCase {
+				b.writef("%q,", unicode.ToLower(rn))
+			} else {
+				b.writef("%q,", rn)
+			}
+		}
+		b.writelnf("},")
+	}
+	if len(ch.UnicodeClasses) > 0 {
+		b.writef("\tclasses: []*unicode.RangeTable{")
+		for _, cl := range ch.UnicodeClasses {
+			b.writef("rangeTable(%q),", cl)
+		}
+		b.writelnf("},")
+	}
+	b.writelnf("\tignoreCase: %t,", ch.IgnoreCase)
+	b.writelnf("\tinverted: %t,", ch.Inverted)
+	b.writelnf("},")
+}
+
+func (b *builder) writeChoiceExpr(ch *ast.ChoiceExpr) {
+	if ch == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&choiceExpr{")
+	pos := ch.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	if len(ch.Alternatives) > 0 {
+		b.writelnf("\talternatives: []interface{}{")
+		for _, alt := range ch.Alternatives {
+			b.writeExpr(alt)
+		}
+		b.writelnf("\t},")
+	}
+	b.writelnf("},")
+}
+
+func (b *builder) writeLabeledExpr(lab *ast.LabeledExpr) {
+	if lab == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&labeledExpr{")
+	pos := lab.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	if lab.Label != nil && lab.Label.Val != "" {
+		b.writelnf("\tlabel: %q,", lab.Label.Val)
+	}
+	b.writef("\texpr: ")
+	b.writeExpr(lab.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeLitMatcher(lit *ast.LitMatcher) {
+	if lit == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&litMatcher{")
+	pos := lit.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	if lit.IgnoreCase {
+		b.writelnf("\tval: %q,", strings.ToLower(lit.Val))
+	} else {
+		b.writelnf("\tval: %q,", lit.Val)
+	}
+	b.writelnf("\tignoreCase: %t,", lit.IgnoreCase)
+	b.writelnf("},")
+}
+
+func (b *builder) writeNotCodeExpr(not *ast.NotCodeExpr) {
+	if not == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&notCodeExpr{")
+	pos := not.Pos()
+	not.FuncIx = b.exprIndex
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writelnf("\trun: (*parser).call%s,", b.funcName(not.FuncIx))
+	b.writelnf("},")
+}
+
+func (b *builder) writeNotExpr(not *ast.NotExpr) {
+	if not == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&notExpr{")
+	pos := not.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writef("\texpr: ")
+	b.writeExpr(not.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeOneOrMoreExpr(one *ast.OneOrMoreExpr) {
+	if one == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&oneOrMoreExpr{")
+	pos := one.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writef("\texpr: ")
+	b.writeExpr(one.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeRuleRefExpr(ref *ast.RuleRefExpr) {
+	if ref == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&ruleRefExpr{")
+	pos := ref.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	if ref.Name != nil && ref.Name.Val != "" {
+		b.writelnf("\tname: %q,", ref.Name.Val)
+	}
+	b.writelnf("},")
+}
+
+func (b *builder) writeSeqExpr(seq *ast.SeqExpr) {
+	if seq == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&seqExpr{")
+	pos := seq.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	if len(seq.Exprs) > 0 {
+		b.writelnf("\texprs: []interface{}{")
+		for _, e := range seq.Exprs {
+			b.writeExpr(e)
+		}
+		b.writelnf("\t},")
+	}
+	b.writelnf("},")
+}
+
+func (b *builder) writeZeroOrMoreExpr(zero *ast.ZeroOrMoreExpr) {
+	if zero == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&zeroOrMoreExpr{")
+	pos := zero.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writef("\texpr: ")
+	b.writeExpr(zero.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeZeroOrOneExpr(zero *ast.ZeroOrOneExpr) {
+	if zero == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.writelnf("&zeroOrOneExpr{")
+	pos := zero.Pos()
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writef("\texpr: ")
+	b.writeExpr(zero.Expr)
+	b.writelnf("},")
+}
+
+func (b *builder) writeRuleCode(rule *ast.Rule) {
+	if rule == nil || rule.Name == nil {
+		return
+	}
+
+	// keep trace of the current rule, as the code blocks are created
+	// in functions named "on<RuleName><#ExprIndex>".
+	b.ruleName = rule.Name.Val
+	b.pushArgsSet()
+	b.writeExprCode(rule.Expr)
+	b.popArgsSet()
+}
+
+func (b *builder) pushArgsSet() {
+	b.argsStack = append(b.argsStack, nil)
+}
+
+func (b *builder) popArgsSet() {
+	b.argsStack = b.argsStack[:len(b.argsStack)-1]
+}
+
+func (b *builder) addArg(arg *ast.Identifier) {
+	if arg == nil {
+		return
+	}
+	ix := len(b.argsStack) - 1
+	b.argsStack[ix] = append(b.argsStack[ix], arg.Val)
+}
+
+func (b *builder) writeExprCode(expr ast.Expression) {
+	switch expr := expr.(type) {
+	case *ast.ActionExpr:
+		b.writeExprCode(expr.Expr)
+		b.writeActionExprCode(expr)
+
+	case *ast.AndCodeExpr:
+		b.writeAndCodeExprCode(expr)
+
+	case *ast.LabeledExpr:
+		b.addArg(expr.Label)
+		b.pushArgsSet()
+		b.writeExprCode(expr.Expr)
+		b.popArgsSet()
+
+	case *ast.NotCodeExpr:
+		b.writeNotCodeExprCode(expr)
+
+	case *ast.AndExpr:
+		b.pushArgsSet()
+		b.writeExprCode(expr.Expr)
+		b.popArgsSet()
+	case *ast.ChoiceExpr:
+		for _, alt := range expr.Alternatives {
+			b.pushArgsSet()
+			b.writeExprCode(alt)
+			b.popArgsSet()
+		}
+	case *ast.NotExpr:
+		b.pushArgsSet()
+		b.writeExprCode(expr.Expr)
+		b.popArgsSet()
+	case *ast.OneOrMoreExpr:
+		b.pushArgsSet()
+		b.writeExprCode(expr.Expr)
+		b.popArgsSet()
+	case *ast.SeqExpr:
+		for _, sub := range expr.Exprs {
+			b.writeExprCode(sub)
+		}
+	case *ast.ZeroOrMoreExpr:
+		b.pushArgsSet()
+		b.writeExprCode(expr.Expr)
+		b.popArgsSet()
+	case *ast.ZeroOrOneExpr:
+		b.pushArgsSet()
+		b.writeExprCode(expr.Expr)
+		b.popArgsSet()
+	}
+}
+
+func (b *builder) writeActionExprCode(act *ast.ActionExpr) {
+	if act == nil {
+		return
+	}
+	b.writeFunc(act.FuncIx, act.Code, callFuncTemplate, onFuncTemplate)
+}
+
+func (b *builder) writeAndCodeExprCode(and *ast.AndCodeExpr) {
+	if and == nil {
+		return
+	}
+	b.writeFunc(and.FuncIx, and.Code, callPredFuncTemplate, onPredFuncTemplate)
+}
+
+func (b *builder) writeNotCodeExprCode(not *ast.NotCodeExpr) {
+	if not == nil {
+		return
+	}
+	b.writeFunc(not.FuncIx, not.Code, callPredFuncTemplate, onPredFuncTemplate)
+}
+
+func (b *builder) writeFunc(funcIx int, code *ast.CodeBlock, callTpl, funcTpl string) {
+	if code == nil {
+		return
+	}
+	val := strings.TrimSpace(code.Val)[1 : len(code.Val)-1]
+	if len(val) > 0 && val[0] == '\n' {
+		val = val[1:]
+	}
+	if len(val) > 0 && val[len(val)-1] == '\n' {
+		val = val[:len(val)-1]
+	}
+	var args bytes.Buffer
+	ix := len(b.argsStack) - 1
+	if ix >= 0 {
+		for i, arg := range b.argsStack[ix] {
+			if i > 0 {
+				args.WriteString(", ")
+			}
+			args.WriteString(arg)
+		}
+	}
+	if args.Len() > 0 {
+		args.WriteString(" interface{}")
+	}
+
+	fnNm := b.funcName(funcIx)
+	b.writelnf(funcTpl, b.recvName, fnNm, args.String(), val)
+
+	args.Reset()
+	if ix >= 0 {
+		for i, arg := range b.argsStack[ix] {
+			if i > 0 {
+				args.WriteString(", ")
+			}
+			args.WriteString(fmt.Sprintf(`stack[%q]`, arg))
+		}
+	}
+	b.writelnf(callTpl, fnNm, args.String())
+}
+
+func (b *builder) writeStaticCode() {
+	b.writelnf(staticCode)
+}
+
+func (b *builder) funcName(ix int) string {
+	return "on" + b.ruleName + strconv.Itoa(ix)
+}
+
+func (b *builder) writef(f string, args ...interface{}) {
+	if b.err == nil {
+		_, b.err = fmt.Fprintf(b.w, f, args...)
+	}
+}
+
+func (b *builder) writelnf(f string, args ...interface{}) {
+	b.writef(f+"\n", args...)
+}
@@ -0,0 +1,40 @@
+package builder
+
+import (
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/pigeon/bootstrap"
+)
+
+var grammar = ` 
+{
+var test = "some string"
+
+func init() {
+	fmt.Println("this is inside the init")
+}
+}
+
+start = additive eof
+additive = left:multiplicative "+" space right:additive {
+	fmt.Println(left, right)
+} / mul:multiplicative { fmt.Println(mul) }
+multiplicative = left:primary op:"*" space right:multiplicative { fmt.Println(left, right, op) } / primary
+primary = integer / "(" space additive:additive ")" space { fmt.Println(additive) }
+integer "integer" = digits:[0123456789]+ space { fmt.Println(digits) }
+space = ' '*
+eof = !. { fmt.Println("eof") }
+`
+
+func TestBuildParser(t *testing.T) {
+	p := bootstrap.NewParser()
+	g, err := p.Parse("", strings.NewReader(grammar))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := BuildParser(ioutil.Discard, g); err != nil {
+		t.Fatal(err)
+	}
+}
@@ -0,0 +1,867 @@
+package builder
+
+var staticCode = `
+var (
+	// errNoRule is returned when the grammar to parse has no rule.
+	errNoRule          = errors.New("grammar has no rule")
+
+	// errInvalidEncoding is returned when the source is not properly
+	// utf8-encoded.
+	errInvalidEncoding = errors.New("invalid encoding")
+
+	// errNoMatch is returned if no match could be found.
+	errNoMatch         = errors.New("no match found")
+)
+
+// Option is a function that can set an option on the parser. It returns
+// the previous setting as an Option.
+type Option func(*parser) Option
+
+// Debug creates an Option to set the debug flag to b. When set to true,
+// debugging information is printed to stdout while parsing.
+//
+// The default is false.
+func Debug(b bool) Option {
+	return func(p *parser) Option {
+		old := p.debug
+		p.debug = b
+		return Debug(old)
+	}
+}
+
+// Memoize creates an Option to set the memoize flag to b. When set to true,
+// the parser will cache all results so each expression is evaluated only
+// once. This guarantees linear parsing time even for pathological cases,
+// at the expense of more memory and slower times for typical cases.
+//
+// The default is false.
+func Memoize(b bool) Option {
+	return func(p *parser) Option {
+		old := p.memoize
+		p.memoize = b
+		return Memoize(old)
+	}
+}
+
+// Recover creates an Option to set the recover flag to b. When set to
+// true, this causes the parser to recover from panics and convert it
+// to an error. Setting it to false can be useful while debugging to
+// access the full stack trace.
+//
+// The default is true.
+func Recover(b bool) Option {
+	return func(p *parser) Option {
+		old := p.recover
+		p.recover = b
+		return Recover(old)
+	}
+}
+
+// ParseFile parses the file identified by filename.
+func ParseFile(filename string, opts ...Option) (interface{}, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ParseReader(filename, f, opts...)
+}
+
+// ParseReader parses the data from r using filename as information in the
+// error messages.
+func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return Parse(filename, b, opts...)
+}
+
+// Parse parses the data from b using filename as information in the
+// error messages.
+func Parse(filename string, b []byte, opts ...Option) (interface{}, error) {
+	return newParser(filename, b, opts...).parse(g)
+}
+
+// position records a position in the text.
+type position struct {
+	line, col, offset int
+}
+
+func (p position) String() string {
+	return fmt.Sprintf("%%d:%%d [%%d]", p.line, p.col, p.offset)
+}
+
+// savepoint stores all state required to go back to this point in the
+// parser.
+type savepoint struct {
+	position
+	rn rune
+	w  int
+}
+
+type current struct {
+	pos  position // start position of the match
+	text []byte   // raw text of the match
+}
+
+// the AST types...
+
+type grammar struct {
+	pos   position
+	rules []*rule
+}
+
+type rule struct {
+	pos         position
+	name        string
+	displayName string
+	expr        interface{}
+}
+
+type choiceExpr struct {
+	pos          position
+	alternatives []interface{}
+}
+
+type actionExpr struct {
+	pos    position
+	expr   interface{}
+	run    func(*parser) (interface{}, error)
+}
+
+type seqExpr struct {
+	pos   position
+	exprs []interface{}
+}
+
+type labeledExpr struct {
+	pos   position
+	label string
+	expr  interface{}
+}
+
+type expr struct {
+	pos  position
+	expr interface{}
+}
+
+type andExpr expr
+type notExpr expr
+type zeroOrOneExpr expr
+type zeroOrMoreExpr expr
+type oneOrMoreExpr expr
+
+type ruleRefExpr struct {
+	pos  position
+	name string
+}
+
+type andCodeExpr struct {
+	pos position
+	run func(*parser) (bool, error)
+}
+
+type notCodeExpr struct {
+	pos position
+	run func(*parser) (bool, error)
+}
+
+type litMatcher struct {
+	pos        position
+	val        string
+	ignoreCase bool
+}
+
+type charClassMatcher struct {
+	pos        position
+	val        string
+	chars      []rune
+	ranges     []rune
+	classes    []*unicode.RangeTable
+	ignoreCase bool
+	inverted   bool
+}
+
+type anyMatcher position
+
+// errList cumulates the errors found by the parser.
+type errList []error
+
+func (e *errList) add(err error) {
+	*e = append(*e, err)
+}
+
+func (e errList) err() error {
+	if len(e) == 0 {
+		return nil
+	}
+	e.dedupe()
+	return e
+}
+
+func (e *errList) dedupe() {
+	var cleaned []error
+	set := make(map[string]bool)
+	for _, err := range *e {
+		if msg := err.Error(); !set[msg] {
+			set[msg] = true
+			cleaned = append(cleaned, err)
+		}
+	}
+	*e = cleaned
+}
+
+func (e errList) Error() string {
+	switch len(e) {
+	case 0:
+		return ""
+	case 1:
+		return e[0].Error()
+	default:
+		var buf bytes.Buffer
+
+		for i, err := range e {
+			if i > 0 {
+				buf.WriteRune('\n')
+			}
+			buf.WriteString(err.Error())
+		}
+		return buf.String()
+	}
+}
+
+// parserError wraps an error with a prefix indicating the rule in which
+// the error occurred. The original error is stored in the Inner field.
+type parserError struct {
+	Inner  error
+	pos    position
+	prefix string
+}
+
+// Error returns the error message.
+func (p *parserError) Error() string {
+	return p.prefix + ": " + p.Inner.Error()
+}
+
+// newParser creates a parser with the specified input source and options.
+func newParser(filename string, b []byte, opts ...Option) *parser {
+	p := &parser{
+		filename: filename,
+		errs: new(errList),
+		data: b,
+		pt: savepoint{position: position{line: 1}},
+		recover: true,
+	}
+	p.setOptions(opts)
+	return p
+}
+
+// setOptions applies the options to the parser.
+func (p *parser) setOptions(opts []Option) {
+	for _, opt := range opts {
+		opt(p)
+	}
+}
+
+type resultTuple struct {
+	v interface{}
+	b bool
+	end savepoint
+}
+
+type parser struct {
+	filename string
+	pt       savepoint
+	cur      current
+
+	data []byte
+	errs *errList
+
+	recover bool
+	debug bool
+	depth  int
+
+	memoize bool
+	// memoization table for the packrat algorithm:
+	// map[offset in source] map[expression or rule] {value, match}
+	memo map[int]map[interface{}]resultTuple
+
+	// rules table, maps the rule identifier to the rule node
+	rules  map[string]*rule
+	// variables stack, map of label to value
+	vstack []map[string]interface{}
+	// rule stack, allows identification of the current rule in errors
+	rstack []*rule
+
+	// stats
+	exprCnt int
+}
+
+// push a variable set on the vstack.
+func (p *parser) pushV() {
+	if cap(p.vstack) == len(p.vstack) {
+		// create new empty slot in the stack
+		p.vstack = append(p.vstack, nil)
+	} else {
+		// slice to 1 more
+		p.vstack = p.vstack[:len(p.vstack)+1]
+	}
+
+	// get the last args set
+	m := p.vstack[len(p.vstack)-1]
+	if m != nil && len(m) == 0 {
+		// empty map, all good
+		return
+	}
+
+	m = make(map[string]interface{})
+	p.vstack[len(p.vstack)-1] = m
+}
+
+// pop a variable set from the vstack.
+func (p *parser) popV() {
+	// if the map is not empty, clear it
+	m := p.vstack[len(p.vstack)-1]
+	if len(m) > 0 {
+		// GC that map
+		p.vstack[len(p.vstack)-1] = nil
+	}
+	p.vstack = p.vstack[:len(p.vstack)-1]
+}
+
+func (p *parser) print(prefix, s string) string {
+	if !p.debug {
+		return s
+	}
+
+	fmt.Printf("%%s %%d:%%d:%%d: %%s [%%#U]\n",
+		prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn)
+	return s
+}
+
+func (p *parser) in(s string) string {
+	p.depth++
+	return p.print(strings.Repeat(" ", p.depth) + ">", s)
+}
+
+func (p *parser) out(s string) string {
+	p.depth--
+	return p.print(strings.Repeat(" ", p.depth) + "<", s)
+}
+
+func (p *parser) addErr(err error) {
+	p.addErrAt(err, p.pt.position)
+}
+
+func (p *parser) addErrAt(err error, pos position) {
+	var buf bytes.Buffer
+	if p.filename != "" {
+		buf.WriteString(p.filename)
+	}
+	if buf.Len() > 0 {
+		buf.WriteString(":")
+	}
+	buf.WriteString(fmt.Sprintf("%%d:%%d (%%d)", pos.line, pos.col, pos.offset))
+	if len(p.rstack) > 0 {
+		if buf.Len() > 0 {
+			buf.WriteString(": ")
+		}
+		rule := p.rstack[len(p.rstack)-1]
+		if rule.displayName != "" {
+			buf.WriteString("rule " + rule.displayName)
+		} else {
+			buf.WriteString("rule " + rule.name)
+		}
+	}
+	pe := &parserError{Inner: err, prefix: buf.String()}
+	p.errs.add(pe)
+}
+
+// read advances the parser to the next rune.
+func (p *parser) read() {
+	p.pt.offset += p.pt.w
+	rn, n := utf8.DecodeRune(p.data[p.pt.offset:])
+	p.pt.rn = rn
+	p.pt.w = n
+	p.pt.col++
+	if rn == '\n' {
+		p.pt.line++
+		p.pt.col = 0
+	}
+
+	if rn == utf8.RuneError {
+		if n > 0 {
+			p.addErr(errInvalidEncoding)
+		}
+	}
+}
+
+// restore parser position to the savepoint pt.
+func (p *parser) restore(pt savepoint) {
+	if p.debug {
+		defer p.out(p.in("restore"))
+	}
+	if pt.offset == p.pt.offset {
+		return
+	}
+	p.pt = pt
+}
+
+// get the slice of bytes from the savepoint start to the current position.
+func (p *parser) sliceFrom(start savepoint) []byte {
+	return p.data[start.position.offset:p.pt.position.offset]
+}
+
+func (p *parser) getMemoized(node interface{}) (resultTuple, bool) {
+	if len(p.memo) == 0 {
+		return resultTuple{}, false
+	}
+	m := p.memo[p.pt.offset]
+	if len(m) == 0 {
+		return resultTuple{}, false
+	}
+	res, ok := m[node]
+	return res, ok
+}
+
+func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) {
+	if p.memo == nil {
+		p.memo = make(map[int]map[interface{}]resultTuple)
+	}
+	m := p.memo[pt.offset]
+	if m == nil {
+		m = make(map[interface{}]resultTuple)
+		p.memo[pt.offset] = m
+	}
+	m[node] = tuple
+}
+
+func (p *parser) buildRulesTable(g *grammar) {
+	p.rules = make(map[string]*rule, len(g.rules))
+	for _, r := range g.rules {
+		p.rules[r.name] = r
+	}
+}
+
+func (p *parser) parse(g *grammar) (val interface{}, err error) {
+	if len(g.rules) == 0 {
+		p.addErr(errNoRule)
+		return nil, p.errs.err()
+	}
+
+	// TODO : not super critical but this could be generated
+	p.buildRulesTable(g)
+
+	if p.recover {
+		// panic can be used in action code to stop parsing immediately
+		// and return the panic as an error.
+		defer func() {
+			if e := recover(); e != nil {
+				if p.debug {
+					defer p.out(p.in("panic handler"))
+				}
+				val = nil
+				switch e := e.(type) {
+				case error:
+					p.addErr(e)
+				default:
+					p.addErr(fmt.Errorf("%%v", e))
+				}
+				err = p.errs.err()
+			}
+		}()
+	}
+
+	// start rule is rule [0]
+	p.read() // advance to first rune
+	val, ok := p.parseRule(g.rules[0])
+	if !ok {
+		if len(*p.errs) == 0 {
+			// make sure this doesn't go out silently
+			p.addErr(errNoMatch)
+		}
+		return nil, p.errs.err()
+	}
+	return val, p.errs.err()
+}
+
+func (p *parser) parseRule(rule *rule) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseRule " + rule.name))
+	}
+
+	if p.memoize {
+		res, ok := p.getMemoized(rule)
+		if ok {
+			p.restore(res.end)
+			return res.v, res.b
+		}
+	}
+
+	start := p.pt
+	p.rstack = append(p.rstack, rule)
+	p.pushV()
+	val, ok := p.parseExpr(rule.expr)
+	p.popV()
+	p.rstack = p.rstack[:len(p.rstack)-1]
+	if ok && p.debug {
+		p.print(strings.Repeat(" ", p.depth) + "MATCH", string(p.sliceFrom(start)))
+	}
+
+	if p.memoize {
+		p.setMemoized(start, rule, resultTuple{val, ok, p.pt})
+	}
+	return val, ok
+}
+
+func (p *parser) parseExpr(expr interface{}) (interface{}, bool) {
+	var pt savepoint
+	var ok bool
+
+	if p.memoize {
+		res, ok := p.getMemoized(expr)
+		if ok {
+			p.restore(res.end)
+			return res.v, res.b
+		}
+		pt = p.pt
+	}
+
+	p.exprCnt++
+	var val interface{}
+	switch expr := expr.(type) {
+	case *actionExpr:
+		val, ok = p.parseActionExpr(expr)
+	case *andCodeExpr:
+		val, ok = p.parseAndCodeExpr(expr)
+	case *andExpr:
+		val, ok = p.parseAndExpr(expr)
+	case *anyMatcher:
+		val, ok = p.parseAnyMatcher(expr)
+	case *charClassMatcher:
+		val, ok = p.parseCharClassMatcher(expr)
+	case *choiceExpr:
+		val, ok = p.parseChoiceExpr(expr)
+	case *labeledExpr:
+		val, ok = p.parseLabeledExpr(expr)
+	case *litMatcher:
+		val, ok = p.parseLitMatcher(expr)
+	case *notCodeExpr:
+		val, ok = p.parseNotCodeExpr(expr)
+	case *notExpr:
+		val, ok = p.parseNotExpr(expr)
+	case *oneOrMoreExpr:
+		val, ok = p.parseOneOrMoreExpr(expr)
+	case *ruleRefExpr:
+		val, ok = p.parseRuleRefExpr(expr)
+	case *seqExpr:
+		val, ok = p.parseSeqExpr(expr)
+	case *zeroOrMoreExpr:
+		val, ok = p.parseZeroOrMoreExpr(expr)
+	case *zeroOrOneExpr:
+		val, ok = p.parseZeroOrOneExpr(expr)
+	default:
+		panic(fmt.Sprintf("unknown expression type %%T", expr))
+	}
+	if p.memoize {
+		p.setMemoized(pt, expr, resultTuple{val, ok, p.pt})
+	}
+	return val, ok
+}
+
+func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseActionExpr"))
+	}
+
+	start := p.pt
+	val, ok := p.parseExpr(act.expr)
+	if ok {
+		p.cur.pos = start.position
+		p.cur.text = p.sliceFrom(start)
+		actVal, err := act.run(p)
+		if err != nil {
+			p.addErrAt(err, start.position)
+		}
+		val = actVal
+	}
+	if ok && p.debug {
+		p.print(strings.Repeat(" ", p.depth) + "MATCH", string(p.sliceFrom(start)))
+	}
+	return val, ok
+}
+
+func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseAndCodeExpr"))
+	}
+
+	ok, err := and.run(p)
+	if err != nil {
+		p.addErr(err)
+	}
+	return nil, ok
+}
+
+func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseAndExpr"))
+	}
+
+	pt := p.pt
+	p.pushV()
+	_, ok := p.parseExpr(and.expr)
+	p.popV()
+	p.restore(pt)
+	return nil, ok
+}
+
+func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseAnyMatcher"))
+	}
+
+	if p.pt.rn != utf8.RuneError {
+		start := p.pt
+		p.read()
+		return p.sliceFrom(start), true
+	}
+	return nil, false
+}
+
+func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseCharClassMatcher"))
+	}
+
+	cur := p.pt.rn
+	// can't match EOF
+	if cur == utf8.RuneError {
+		return nil, false
+	}
+	start := p.pt
+	if chr.ignoreCase {
+		cur = unicode.ToLower(cur)
+	}
+
+	// try to match in the list of available chars
+	for _, rn := range chr.chars {
+		if rn == cur {
+			if chr.inverted {
+				return nil, false
+			}
+			p.read()
+			return p.sliceFrom(start), true
+		}
+	}
+
+	// try to match in the list of ranges
+	for i := 0; i < len(chr.ranges); i += 2 {
+		if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] {
+			if chr.inverted {
+				return nil, false
+			}
+			p.read()
+			return p.sliceFrom(start), true
+		}
+	}
+
+	// try to match in the list of Unicode classes
+	for _, cl := range chr.classes {
+		if unicode.Is(cl, cur) {
+			if chr.inverted {
+				return nil, false
+			}
+			p.read()
+			return p.sliceFrom(start), true
+		}
+	}
+
+	if chr.inverted {
+		p.read()
+		return p.sliceFrom(start), true
+	}
+	return nil, false
+}
+
+func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseChoiceExpr"))
+	}
+
+	for _, alt := range ch.alternatives {
+		p.pushV()
+		val, ok := p.parseExpr(alt)
+		p.popV()
+		if ok {
+			return val, ok
+		}
+	}
+	return nil, false
+}
+
+func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseLabeledExpr"))
+	}
+
+	p.pushV()
+	val, ok := p.parseExpr(lab.expr)
+	p.popV()
+	if ok && lab.label != "" {
+		m := p.vstack[len(p.vstack)-1]
+		m[lab.label] = val
+	}
+	return val, ok
+}
+
+func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseLitMatcher"))
+	}
+
+	start := p.pt
+	for _, want := range lit.val {
+		cur := p.pt.rn
+		if lit.ignoreCase {
+			cur = unicode.ToLower(cur)
+		}
+		if cur != want {
+			p.restore(start)
+			return nil, false
+		}
+		p.read()
+	}
+	return p.sliceFrom(start), true
+}
+
+func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseNotCodeExpr"))
+	}
+
+	ok, err := not.run(p)
+	if err != nil {
+		p.addErr(err)
+	}
+	return nil, !ok
+}
+
+func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseNotExpr"))
+	}
+
+	pt := p.pt
+	p.pushV()
+	_, ok := p.parseExpr(not.expr)
+	p.popV()
+	p.restore(pt)
+	return nil, !ok
+}
+
+func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseOneOrMoreExpr"))
+	}
+
+	var vals []interface{}
+
+	for {
+		p.pushV()
+		val, ok := p.parseExpr(expr.expr)
+		p.popV()
+		if !ok {
+			if len(vals) == 0 {
+				// did not match once, no match
+				return nil, false
+			}
+			return vals, true
+		}
+		vals = append(vals, val)
+	}
+}
+
+func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseRuleRefExpr " + ref.name))
+	}
+
+	if ref.name == "" {
+		panic(fmt.Sprintf("%%s: invalid rule: missing name", ref.pos))
+	}
+
+	rule := p.rules[ref.name]
+	if rule == nil {
+		p.addErr(fmt.Errorf("undefined rule: %%s", ref.name))
+		return nil, false
+	}
+	return p.parseRule(rule)
+}
+
+func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseSeqExpr"))
+	}
+
+	var vals []interface{}
+
+	pt := p.pt
+	for _, expr := range seq.exprs {
+		val, ok := p.parseExpr(expr)
+		if !ok {
+			p.restore(pt)
+			return nil, false
+		}
+		vals = append(vals, val)
+	}
+	return vals, true
+}
+
+func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseZeroOrMoreExpr"))
+	}
+
+	var vals []interface{}
+
+	for {
+		p.pushV()
+		val, ok := p.parseExpr(expr.expr)
+		p.popV()
+		if !ok {
+			return vals, true
+		}
+		vals = append(vals, val)
+	}
+}
+
+func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) {
+	if p.debug {
+		defer p.out(p.in("parseZeroOrOneExpr"))
+	}
+
+	p.pushV()
+	val, _ := p.parseExpr(expr.expr)
+	p.popV()
+	// whether it matched or not, consider it a match
+	return val, true
+}
+
+func rangeTable(class string) *unicode.RangeTable {
+	if rt, ok := unicode.Categories[class]; ok {
+		return rt
+	}
+	if rt, ok := unicode.Properties[class]; ok {
+		return rt
+	}
+	if rt, ok := unicode.Scripts[class]; ok {
+		return rt
+	}
+
+	// cannot happen
+	panic(fmt.Sprintf("invalid Unicode class: %%s", class))
+}
+`
@@ -0,0 +1,304 @@
+package main
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+func compareGrammars(t *testing.T, src string, exp, got *ast.Grammar) bool {
+	if (exp.Init != nil) != (got.Init != nil) {
+		t.Errorf("%q: want Init? %t, got %t", src, exp.Init != nil, got.Init != nil)
+		return false
+	}
+	if exp.Init != nil {
+		if exp.Init.Val != got.Init.Val {
+			t.Errorf("%q: want Init %q, got %q", src, exp.Init.Val, got.Init.Val)
+			return false
+		}
+	}
+
+	rn, rm := len(exp.Rules), len(got.Rules)
+	if rn != rm {
+		t.Errorf("%q: want %d rules, got %d", src, rn, rm)
+		return false
+	}
+
+	for i, r := range got.Rules {
+		if !compareRule(t, src+": "+exp.Rules[i].Name.Val, exp.Rules[i], r) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareRule(t *testing.T, prefix string, exp, got *ast.Rule) bool {
+	if exp.Name.Val != got.Name.Val {
+		t.Errorf("%q: want rule name %q, got %q", prefix, exp.Name.Val, got.Name.Val)
+		return false
+	}
+	if (exp.DisplayName != nil) != (got.DisplayName != nil) {
+		t.Errorf("%q: want DisplayName? %t, got %t", prefix, exp.DisplayName != nil, got.DisplayName != nil)
+		return false
+	}
+	if exp.DisplayName != nil {
+		if exp.DisplayName.Val != got.DisplayName.Val {
+			t.Errorf("%q: want DisplayName %q, got %q", prefix, exp.DisplayName.Val, got.DisplayName.Val)
+			return false
+		}
+	}
+	return compareExpr(t, prefix, 0, exp.Expr, got.Expr)
+}
+
+func compareExpr(t *testing.T, prefix string, ix int, exp, got ast.Expression) bool {
+	ixPrefix := prefix + " (" + strconv.Itoa(ix) + ")"
+
+	switch exp := exp.(type) {
+	case *ast.ActionExpr:
+		got, ok := got.(*ast.ActionExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if (exp.Code != nil) != (got.Code != nil) {
+			t.Errorf("%q: want Code?: %t, got %t", ixPrefix, exp.Code != nil, got.Code != nil)
+			return false
+		}
+		if exp.Code != nil {
+			if exp.Code.Val != got.Code.Val {
+				t.Errorf("%q: want code %q, got %q", ixPrefix, exp.Code.Val, got.Code.Val)
+				return false
+			}
+		}
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	case *ast.AndCodeExpr:
+		got, ok := got.(*ast.AndCodeExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if (exp.Code != nil) != (got.Code != nil) {
+			t.Errorf("%q: want Code?: %t, got %t", ixPrefix, exp.Code != nil, got.Code != nil)
+			return false
+		}
+		if exp.Code != nil {
+			if exp.Code.Val != got.Code.Val {
+				t.Errorf("%q: want code %q, got %q", ixPrefix, exp.Code.Val, got.Code.Val)
+				return false
+			}
+		}
+
+	case *ast.AndExpr:
+		got, ok := got.(*ast.AndExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	case *ast.AnyMatcher:
+		got, ok := got.(*ast.AnyMatcher)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		// for completion's sake...
+		if exp.Val != got.Val {
+			t.Errorf("%q: want value %q, got %q", ixPrefix, exp.Val, got.Val)
+		}
+
+	case *ast.CharClassMatcher:
+		got, ok := got.(*ast.CharClassMatcher)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if exp.IgnoreCase != got.IgnoreCase {
+			t.Errorf("%q: want IgnoreCase %t, got %t", ixPrefix, exp.IgnoreCase, got.IgnoreCase)
+			return false
+		}
+		if exp.Inverted != got.Inverted {
+			t.Errorf("%q: want Inverted %t, got %t", ixPrefix, exp.Inverted, got.Inverted)
+			return false
+		}
+
+		ne, ng := len(exp.Chars), len(got.Chars)
+		if ne != ng {
+			t.Errorf("%q: want %d Chars, got %d (%v)", ixPrefix, ne, ng, got.Chars)
+			return false
+		}
+		for i, r := range exp.Chars {
+			if r != got.Chars[i] {
+				t.Errorf("%q: want Chars[%d] %#U, got %#U", ixPrefix, i, r, got.Chars[i])
+				return false
+			}
+		}
+
+		ne, ng = len(exp.Ranges), len(got.Ranges)
+		if ne != ng {
+			t.Errorf("%q: want %d Ranges, got %d", ixPrefix, ne, ng)
+			return false
+		}
+		for i, r := range exp.Ranges {
+			if r != got.Ranges[i] {
+				t.Errorf("%q: want Ranges[%d] %#U, got %#U", ixPrefix, i, r, got.Ranges[i])
+				return false
+			}
+		}
+
+		ne, ng = len(exp.UnicodeClasses), len(got.UnicodeClasses)
+		if ne != ng {
+			t.Errorf("%q: want %d UnicodeClasses, got %d", ixPrefix, ne, ng)
+			return false
+		}
+		for i, s := range exp.UnicodeClasses {
+			if s != got.UnicodeClasses[i] {
+				t.Errorf("%q: want UnicodeClasses[%d] %q, got %q", ixPrefix, i, s, got.UnicodeClasses[i])
+				return false
+			}
+		}
+
+	case *ast.ChoiceExpr:
+		got, ok := got.(*ast.ChoiceExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		ne, ng := len(exp.Alternatives), len(got.Alternatives)
+		if ne != ng {
+			t.Errorf("%q: want %d Alternatives, got %d", ixPrefix, ne, ng)
+			return false
+		}
+
+		for i, alt := range exp.Alternatives {
+			if !compareExpr(t, prefix, ix+1, alt, got.Alternatives[i]) {
+				return false
+			}
+		}
+
+	case *ast.LabeledExpr:
+		got, ok := got.(*ast.LabeledExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if (exp.Label != nil) != (got.Label != nil) {
+			t.Errorf("%q: want Label?: %t, got %t", ixPrefix, exp.Label != nil, got.Label != nil)
+			return false
+		}
+		if exp.Label != nil {
+			if exp.Label.Val != got.Label.Val {
+				t.Errorf("%q: want label %q, got %q", ixPrefix, exp.Label.Val, got.Label.Val)
+				return false
+			}
+		}
+
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	case *ast.LitMatcher:
+		got, ok := got.(*ast.LitMatcher)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if exp.IgnoreCase != got.IgnoreCase {
+			t.Errorf("%q: want IgnoreCase %t, got %t", ixPrefix, exp.IgnoreCase, got.IgnoreCase)
+			return false
+		}
+		if exp.Val != got.Val {
+			t.Errorf("%q: want value %q, got %q", ixPrefix, exp.Val, got.Val)
+			return false
+		}
+
+	case *ast.NotCodeExpr:
+		got, ok := got.(*ast.NotCodeExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if (exp.Code != nil) != (got.Code != nil) {
+			t.Errorf("%q: want Code?: %t, got %t", ixPrefix, exp.Code != nil, got.Code != nil)
+			return false
+		}
+		if exp.Code != nil {
+			if exp.Code.Val != got.Code.Val {
+				t.Errorf("%q: want code %q, got %q", ixPrefix, exp.Code.Val, got.Code.Val)
+				return false
+			}
+		}
+
+	case *ast.NotExpr:
+		got, ok := got.(*ast.NotExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	case *ast.OneOrMoreExpr:
+		got, ok := got.(*ast.OneOrMoreExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	case *ast.RuleRefExpr:
+		got, ok := got.(*ast.RuleRefExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		if (exp.Name != nil) != (got.Name != nil) {
+			t.Errorf("%q: want Name?: %t, got %t", ixPrefix, exp.Name != nil, got.Name != nil)
+			return false
+		}
+		if exp.Name != nil {
+			if exp.Name.Val != got.Name.Val {
+				t.Errorf("%q: want name %q, got %q", ixPrefix, exp.Name.Val, got.Name.Val)
+				return false
+			}
+		}
+
+	case *ast.SeqExpr:
+		got, ok := got.(*ast.SeqExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		ne, ng := len(exp.Exprs), len(got.Exprs)
+		if ne != ng {
+			t.Errorf("%q: want %d Exprs, got %d", ixPrefix, ne, ng)
+			return false
+		}
+
+		for i, expr := range exp.Exprs {
+			if !compareExpr(t, prefix, ix+1, expr, got.Exprs[i]) {
+				return false
+			}
+		}
+
+	case *ast.ZeroOrMoreExpr:
+		got, ok := got.(*ast.ZeroOrMoreExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	case *ast.ZeroOrOneExpr:
+		got, ok := got.(*ast.ZeroOrOneExpr)
+		if !ok {
+			t.Errorf("%q: want expression type %T, got %T", ixPrefix, exp, got)
+			return false
+		}
+		return compareExpr(t, prefix, ix+1, exp.Expr, got.Expr)
+
+	default:
+		t.Fatalf("unexpected expression type %T", exp)
+	}
+	return true
+}
@@ -0,0 +1,438 @@
+/*
+Command pigeon generates parsers in Go from a PEG grammar.
+
+From Wikipedia [0]:
+
+	A parsing expression grammar is a type of analytic formal grammar, i.e.
+	it describes a formal language in terms of a set of rules for recognizing
+	strings in the language.
+
+Its features and syntax are inspired by the PEG.js project [1], while
+the implementation is loosely based on [2]. Formal presentation of the
+PEG theory by Bryan Ford is also an important reference [3]. An introductory
+blog post can be found at [4].
+
+	[0]: http://en.wikipedia.org/wiki/Parsing_expression_grammar
+	[1]: http://pegjs.org/
+	[2]: http://www.codeproject.com/Articles/29713/Parsing-Expression-Grammar-Support-for-C-Part
+	[3]: http://pdos.csail.mit.edu/~baford/packrat/popl04/peg-popl04.pdf
+	[4]: http://0value.com/A-PEG-parser-generator-for-Go
+
+Command-line usage
+
+The pigeon tool must be called with PEG input as defined
+by the accepted PEG syntax below. The grammar may be provided by a
+file or read from stdin. The generated parser is written to stdout
+by default.
+
+	pigeon [options] [GRAMMAR_FILE]
+
+The following options can be specified:
+
+	-cache : cache parser results to avoid exponential parsing time in
+	pathological cases. Can make the parsing slower for typical
+	cases and uses more memory (default: false).
+
+	-debug : boolean, print debugging info to stdout (default: false).
+
+	-no-recover : boolean, if set, do not recover from a panic. Useful
+	to access the panic stack when debugging, otherwise the panic
+	is converted to an error (default: false).
+
+	-o=FILE : string, output file where the generated parser will be
+	written (default: stdout).
+
+	-x : boolean, if set, do not build the parser, just parse the input grammar
+	(default: false).
+
+	-receiver-name=NAME : string, name of the receiver variable for the generated
+	code blocks. Non-initializer code blocks in the grammar end up as methods on the
+	*current type, and this option sets the name of the receiver (default: c).
+
+The tool makes no attempt to format the code, nor to detect the
+required imports. It is recommended to use goimports to properly generate
+the output code:
+	pigeon GRAMMAR_FILE | goimports > output_file.go
+
+The goimports tool can be installed with:
+	go get golang.org/x/tools/cmd/goimports
+
+If the code blocks in the grammar (see below, section "Code block") are golint-
+and go vet-compliant, then the resulting generated code will also be golint-
+and go vet-compliant.
+
+The generated code doesn't use any third-party dependency unless code blocks
+in the grammar require such a dependency.
+
+PEG syntax
+
+The accepted syntax for the grammar is formally defined in the
+grammar/pigeon.peg file, using the PEG syntax. What follows is an informal
+description of this syntax.
+
+Identifiers, whitespace, comments and literals follow the same
+notation as the Go language, as defined in the language specification
+(http://golang.org/ref/spec#Source_code_representation):
+
+	// single line comment*/
+//	/* multi-line comment */
+/*	'x' (single quotes for single char literal)
+	"double quotes for string literal"
+	`backtick quotes for raw string literal`
+	RuleName (a valid identifier)
+
+The grammar must be Unicode text encoded in UTF-8. New lines are identified
+by the \n character (U+000A). Space (U+0020), horizontal tabs (U+0009) and
+carriage returns (U+000D) are considered whitespace and are ignored except
+to separate tokens.
+
+Rules
+
+A PEG grammar consists of a set of rules. A rule is an identifier followed
+by a rule definition operator and an expression. An optional display name -
+a string literal used in error messages instead of the rule identifier - can
+be specified after the rule identifier. E.g.:
+	RuleA "friendly name" = 'a'+ // RuleA is one or more lowercase 'a's
+
+The rule definition operator can be any one of those:
+	=, <-, ← (U+2190), ⟵ (U+27F5)
+
+Expressions
+
+A rule is defined by an expression. The following sections describe the
+various expression types. Expressions can be grouped by using parentheses,
+and a rule can be referenced by its identifier in place of an expression.
+
+Choice expression
+
+The choice expression is a list of expressions that will be tested in the
+order they are defined. The first one that matches will be used. Expressions
+are separated by the forward slash character "/". E.g.:
+	ChoiceExpr = A / B / C // A, B and C should be rules declared in the grammar
+
+Because the first match is used, it is important to think about the order
+of expressions. For example, in this rule, "<=" would never be used because
+the "<" expression comes first:
+	BadChoiceExpr = "<" / "<="
+
+Sequence expression
+
+The sequence expression is a list of expressions that must all match in
+that same order for the sequence expression to be considered a match.
+Expressions are separated by whitespace. E.g.:
+	SeqExpr = "A" "b" "c" // matches "Abc", but not "Acb"
+
+Labeled expression
+
+A labeled expression consists of an identifier followed by a colon ":"
+and an expression. A labeled expression introduces a variable named with
+the label that can be referenced in the code blocks in the same scope.
+The variable will have the value of the expression that follows the colon.
+E.g.:
+	LabeledExpr = value:[a-z]+ {
+		fmt.Println(value)
+		return value, nil
+	}
+
+The variable is typed as an empty interface, and the underlying type depends
+on the following:
+
+For terminals (character and string literals, character classes and
+the any matcher), the value is []byte. E.g.:
+    Rule = label:'a' { // label is []byte }
+
+For predicates (& and !), the value is always nil. E.g.:
+	Rule = label:&'a' { // label is nil }
+
+For a sequence, the value is a slice of empty interfaces, one for each
+expression value in the sequence. The underlying types of each value
+in the slice follow the same rules described here, recursively. E.g.:
+	Rule = label:('a' 'b') { // label is []interface{} }
+
+For a repetition (+ and *), the value is a slice of empty interfaces, one for
+each repetition. The underlying types of each value in the slice follow
+the same rules described here, recursively. E.g.:
+	Rule = label:[a-z]+ { // label is []interface{} }
+
+For a choice expression, the value is that of the matching choice. E.g.:
+	Rule = label:('a' / 'b') { // label is []byte }
+
+For the optional expression (?), the value is nil or the value of the
+expression. E.g.:
+	Rule = label:'a'? { // label is nil or []byte }
+
+Of course, the type of the value can be anything once an action code block
+is used. E.g.:
+	RuleA = label:'3' {
+		return 3, nil
+	}
+	RuleB = label:RuleA { // label is int }
+
+And and not expressions
+
+An expression prefixed with the ampersand "&" is the "and" predicate
+expression: it is considered a match if the following expression is a match,
+but it does not consume any input.
+
+An expression prefixed with the exclamation point "!" is the "not" predicate
+expression: it is considered a match if the following expression is not
+a match, but it does not consume any input. E.g.:
+	AndExpr = "A" &"B" // matches "A" if followed by a "B" (does not consume "B")
+	NotExpr = "A" !"B" // matches "A" if not followed by a "B" (does not consume "B")
+
+The expression following the & and ! operators can be a code block. In that
+case, the code block must return a bool and an error. The operator's semantic
+is the same, & is a match if the code block returns true, ! is a match if the
+code block returns false. The code block has access to any labeled value
+defined in its scope. E.g.:
+	CodeAndExpr = value:[a-z] &{
+		// can access the value local variable...
+		return true, nil
+	}
+
+Repeating expressions
+
+An expression followed by "*", "?" or "+" is a match if the expression
+occurs zero or more times ("*"), zero or one time "?" or one or more times
+("+") respectively. The match is greedy, it will match as many times as
+possible. E.g.
+	ZeroOrMoreAs = "A"*
+
+Literal matcher
+
+A literal matcher tries to match the input against a single character or a
+string literal. The literal may be a single-quoted single character, a
+double-quoted string or a backtick-quoted raw string. The same rules as in Go
+apply regarding the allowed characters and escapes.
+
+The literal may be followed by a lowercase "i" (outside the ending quote)
+to indicate that the match is case-insensitive. E.g.:
+	LiteralMatch = "Awesome\n"i // matches "awesome" followed by a newline
+
+Character class matcher
+
+A character class matcher tries to match the input against a class of characters
+inside square brackets "[...]". Inside the brackets, characters represent
+themselves and the same escapes as in string literals are available, except
+that the single- and double-quote escape is not valid, instead the closing
+square bracket "]" must be escaped to be used.
+
+Character ranges can be specified using the "[a-z]" notation. Unicode
+classes can be specified using the "[\pL]" notation, where L is a
+single-letter Unicode class of characters, or using the "[\p{Class}]"
+notation where Class is a valid Unicode class (e.g. "Latin").
+
+As for string literals, a lowercase "i" may follow the matcher (outside
+the ending square bracket) to indicate that the match is case-insensitive.
+A "^" as first character inside the square brackets indicates that the match
+is inverted (it is a match if the input does not match the character class
+matcher). E.g.:
+	NotAZ = [^a-z]i
+
+Any matcher
+
+The any matcher is represented by the dot ".". It matches any character
+except the end of file, thus the "!." expression is used to indicate "match
+the end of file". E.g.:
+	AnyChar = . // match a single character
+	EOF = !.
+
+Code block
+
+Code blocks can be added to generate custom Go code. There are three kinds
+of code blocks: the initializer, the action and the predicate. All code blocks
+appear inside curly braces "{...}".
+
+The initializer must appear first in the grammar, before any rule. It is
+copied as-is (minus the wrapping curly braces) at the top of the generated
+parser. It may contain function declarations, types, variables, etc. just
+like any Go file. Every symbol declared here will be available to all other
+code blocks.  Although the initializer is optional in a valid grammar, it is
+usually required to generate a valid Go source code file (for the package
+clause). E.g.:
+	{
+		package main
+
+		func someHelper() {
+			// ...
+		}
+	}
+
+Action code blocks are code blocks declared after an expression in a rule.
+Those code blocks are turned into a method on the "*current" type in the
+generated source code. The method receives any labeled expression's value
+as argument (as interface{}) and must return two values, the first being
+the value of the expression (an interface{}), and the second an error.
+If a non-nil error is returned, it is added to the list of errors that the
+parser will return. E.g.:
+	RuleA = "A"+ {
+		// return the matched string, "c" is the default name for
+		// the *current receiver variable.
+		return string(c.text), nil
+	}
+
+Predicate code blocks are code blocks declared immediately after the and "&"
+or the not "!" operators. Like action code blocks, predicate code blocks
+are turned into a method on the "*current" type in the generated source code.
+The method receives any labeled expression's value as argument (as interface{})
+and must return two values, the first being a bool and the second an error.
+If a non-nil error is returned, it is added to the list of errors that the
+parser will return. E.g.:
+	RuleAB = [ab]i+ &{
+		return true, nil
+	}
+
+The current type is a struct that provides two useful fields that can be
+accessed in action and predicate code blocks: "pos" and "text".
+
+The "pos" field indicates the current position of the parser in the source
+input. It is itself a struct with three fields: "line", "col" and "offset".
+Line is a 1-based line number, col is a 1-based column number that counts
+runes from the start of the line, and offset is a 0-based byte offset.
+
+The "text" field is the slice of bytes of the current match. It is empty
+in a predicate code block.
+
+Using the generated parser
+
+The parser generated by pigeon exports a few symbols so that it can be used
+as a package with public functions to parse input text. The exported API is:
+	- Parse(string, []byte, ...Option) (interface{}, error)
+	- ParseFile(string, ...Option) (interface{}, error)
+	- ParseReader(string, io.Reader, ...Option) (interface{}, error)
+	- Debug(bool) Option
+	- Memoize(bool) Option
+	- Recover(bool) Option
+
+See the godoc page of the generated parser for the test/predicates grammar
+for an example documentation page of the exported API:
+http://godoc.org/github.com/PuerkitoBio/pigeon/test/predicates.
+
+Like the grammar used to generate the parser, the input text must be
+UTF-8-encoded Unicode.
+
+The start rule of the parser is the first rule in the PEG grammar used
+to generate the parser. A call to any of the Parse* functions returns
+the value generated by executing the grammar on the provided input text,
+and an optional error.
+
+Typically, the grammar should generate some kind of abstract syntax tree (AST),
+but for simple grammars it may evaluate the result immediately, such as in
+the examples/calculator example. There are no constraints imposed on the
+author of the grammar, it can return whatever is needed.
+
+Error reporting
+
+When the parser returns a non-nil error, the error is always of type errList,
+which is defined as a slice of errors ([]error). Each error in the list is
+of type *parserError. This is a struct that has an "Inner" field that can be
+used to access the original error.
+
+So if a code block returns some well-known error like:
+	{
+		return nil, io.EOF
+	}
+
+The original error can be accessed this way:
+	_, err := ParseFile("some_file")
+	if err != nil {
+		list := err.(errList)
+		for _, err := range list {
+			pe := err.(*parserError)
+			if pe.Inner == io.EOF {
+				// ...
+			}
+		}
+	}
+
+By defaut the parser will continue after an error is returned and will
+cumulate all errors found during parsing. If the grammar reaches a point
+where it shouldn't continue, a panic statement can be used to terminate
+parsing. The panic will be caught at the top-level of the Parse* call
+and will be converted into a *parserError like any error, and an errList
+will still be returned to the caller.
+
+The divide by zero error in the examples/calculator grammar leverages this
+feature (no special code is needed to handle division by zero, if it
+happens, the runtime panics and it is recovered and returned as a parsing
+error).
+
+Providing good error reporting in a parser is not a trivial task. Part
+of it is provided by the pigeon tool, by offering features such as
+filename, position and rule name in the error message, but an
+important part of good error reporting needs to be done by the grammar
+author.
+
+For example, many programming languages use double-quotes for string literals.
+Usually, if the opening quote is found, the closing quote is expected, and if
+none is found, there won't be any other rule that will match, there's no need
+to backtrack and try other choices, an error should be added to the list
+and the match should be consumed.
+
+In order to do this, the grammar can look something like this:
+
+	StringLiteral = '"' ValidStringChar* '"' {
+		// this is the valid case, build string literal node
+		// node = ...
+		return node, nil
+	} / '"'  ValidStringChar* !'"' {
+		// invalid case, build a replacement string literal node or build a BadNode
+		// node = ...
+		return node, errors.New("string literal not terminated")
+	}
+
+This is just one example, but it illustrates the idea that error reporting
+needs to be thought out when designing the grammar.
+
+API stability
+
+Generated parsers have user-provided code mixed with pigeon code
+in the same package, so there is no package
+boundary in the resulting code to prevent access to unexported symbols.
+What is meant to be implementation
+details in pigeon is also available to user code - which doesn't mean
+it should be used.
+
+For this reason, it is important to precisely define what is intended to be
+the supported API of pigeon, the parts that will be stable
+in future versions.
+
+The "stability" of the API attempts to make a similar guarantee as the
+Go 1 compatibility [5]. The following lists what part of the
+current pigeon code falls under that guarantee (features may be added in
+the future):
+
+    - The pigeon command-line flags and arguments: those will not be removed
+    and will maintain the same semantics.
+
+    - The explicitly exported API generated by pigeon. See [6] for the
+    documentation of this API on a generated parser.
+
+    - The PEG syntax, as documented above.
+
+    - The code blocks (except the initializer) will always be generated as
+    methods on the *current type, and this type is guaranteed to have
+    the fields pos (type position) and text (type []byte). There are no
+    guarantees on other fields and methods of this type.
+
+    - The position type will always have the fields line, col and offset,
+    all defined as int. There are no guarantees on other fields and methods
+    of this type.
+
+    - The type of the error value returned by the Parse* functions, when
+    not nil, will always be errList defined as a []error. There are no
+    guarantees on methods of this type, other than the fact it implements the
+    error interface.
+
+    - Individual errors in the errList will always be of type *parserError,
+    and this type is guaranteed to have an Inner field that contains the
+    original error value. There are no guarantees on other fields and methods
+    of this type.
+
+References:
+
+    [5]: https://golang.org/doc/go1compat
+    [6]: http://godoc.org/github.com/PuerkitoBio/pigeon/test/predicates
+
+*/
+package main
@@ -0,0 +1,101 @@
+{
+// Command calculator is a small PEG-generated parser that computes
+// simple math using integers.
+//
+// Example usage: $ calculator "3 + (2 - 5 * 12)"
+//
+// Inspired by pegjs arithmetic example:
+// https://github.com/pegjs/pegjs/blob/master/examples/arithmetics.pegjs
+//
+package main
+
+var ops = map[string]func(int, int) int {
+    "+": func(l, r int) int {
+        return l + r
+    },
+    "-": func(l, r int) int {
+        return l - r
+    },
+    "*": func(l, r int) int {
+        return l * r
+    },
+    "/": func(l, r int) int {
+        return l / r
+    },
+}
+
+// for testing purpose
+var cntCodeBlocks int
+
+func main() {
+    if len(os.Args) != 2 {
+        log.Fatal("Usage: calculator 'EXPR'")
+    }
+    got, err := ParseReader("", strings.NewReader(os.Args[1]))
+    if err != nil {
+        log.Fatal(err)
+    }
+    fmt.Println("=", got)
+}
+
+func toIfaceSlice(v interface{}) []interface{} {
+    if v == nil {
+        return nil
+    }
+    return v.([]interface{})
+}
+
+func eval(first, rest interface{}) int {
+    l := first.(int)
+    restSl := toIfaceSlice(rest)
+    for _, v := range restSl {
+        restExpr := toIfaceSlice(v)
+        r := restExpr[3].(int)
+        op := restExpr[1].(string)
+        l = ops[op](l, r)
+    }
+    return l
+}
+}
+
+Input <- expr:Expr EOF {
+    cntCodeBlocks++
+    return expr, nil
+}
+
+Expr <- _ first:Term rest:( _ AddOp _ Term )* _ {
+    cntCodeBlocks++
+    return eval(first, rest), nil
+}
+
+Term <- first:Factor rest:( _ MulOp _ Factor )* {
+    cntCodeBlocks++
+    return eval(first, rest), nil
+}
+
+Factor <- '(' expr:Expr ')' {
+    cntCodeBlocks++
+    return expr, nil
+} / integer:Integer {
+    cntCodeBlocks++
+    return integer, nil
+}
+
+AddOp <- ( '+' / '-' ) {
+    cntCodeBlocks++
+    return string(c.text), nil
+}
+
+MulOp <- ( '*' / '/' ) {
+    cntCodeBlocks++
+    return string(c.text), nil
+}
+
+Integer <- '-'? [0-9]+ {
+    cntCodeBlocks++
+    return strconv.Atoi(string(c.text))
+}
+
+_ "whitespace" <- [ \n\t\r]*
+
+EOF <- !.
@@ -0,0 +1,176 @@
+package main
+
+import "testing"
+
+var longishExpr = `
+18 + 3 - 27012 * ( (1234 - 43) / 7 ) + -4 * 8129
+`
+
+var validCases = map[string]int{
+	"0":   0,
+	"1":   1,
+	"-1":  -1,
+	"10":  10,
+	"-10": -10,
+
+	"(0)":   0,
+	"(1)":   1,
+	"(-1)":  -1,
+	"(10)":  10,
+	"(-10)": -10,
+
+	"1+1":   2,
+	"1-1":   0,
+	"1*1":   1,
+	"1/1":   1,
+	"1 + 1": 2,
+	"1 - 1": 0,
+	"1 * 1": 1,
+	"1 / 1": 1,
+
+	"1+0":   1,
+	"1-0":   1,
+	"1*0":   0,
+	"1 + 0": 1,
+	"1 - 0": 1,
+	"1 * 0": 0,
+
+	"1\n+\t2\r\n +\n3\n": 6,
+	"(2) * 3":            6,
+
+	" 1 + 2 - 3 * 4 / 5 ":       1,
+	" 1 + (2 - 3) * 4 / 5 ":     1,
+	" (1 + 2 - 3) * 4 / 5 ":     0,
+	" 1 + 2 - (3 * 4) / 5 ":     1,
+	" 18 + 3 - 27 * (-18 / -3)": -141,
+	longishExpr:                 -4624535,
+}
+
+func TestValidCases(t *testing.T) {
+	for tc, exp := range validCases {
+		got, err := Parse("", []byte(tc))
+		if err != nil {
+			t.Errorf("%q: want no error, got %v", tc, err)
+			continue
+		}
+		goti, ok := got.(int)
+		if !ok {
+			t.Errorf("%q: want type %T, got %T", tc, exp, got)
+			continue
+		}
+		if exp != goti {
+			t.Errorf("%q: want %d, got %d", tc, exp, goti)
+		}
+	}
+}
+
+var invalidCases = map[string]string{
+	"":        "1:1 (0): no match found",
+	"(":       "1:1 (0): no match found",
+	")":       "1:1 (0): no match found",
+	"()":      "1:1 (0): no match found",
+	"+":       "1:1 (0): no match found",
+	"-":       "1:1 (0): no match found",
+	"*":       "1:1 (0): no match found",
+	"/":       "1:1 (0): no match found",
+	"+1":      "1:1 (0): no match found",
+	"*1":      "1:1 (0): no match found",
+	"/1":      "1:1 (0): no match found",
+	"1/0":     "1:4 (3): rule Term: runtime error: integer divide by zero",
+	"1+":      "1:1 (0): no match found",
+	"1-":      "1:1 (0): no match found",
+	"1*":      "1:1 (0): no match found",
+	"1/":      "1:1 (0): no match found",
+	"1 (+ 2)": "1:1 (0): no match found",
+	"1 (2)":   "1:1 (0): no match found",
+	"\xfe":    "1:1 (0): invalid encoding",
+}
+
+func TestInvalidCases(t *testing.T) {
+	for tc, exp := range invalidCases {
+		got, err := Parse("", []byte(tc))
+		if err == nil {
+			t.Errorf("%q: want error, got none (%v)", tc, got)
+			continue
+		}
+		el, ok := err.(errList)
+		if !ok {
+			t.Errorf("%q: want error type %T, got %T", tc, &errList{}, err)
+			continue
+		}
+		for _, e := range el {
+			if _, ok := e.(*parserError); !ok {
+				t.Errorf("%q: want all individual errors to be %T, got %T (%[3]v)", tc, &parserError{}, e)
+			}
+		}
+		if exp != err.Error() {
+			t.Errorf("%q: want \n%s\n, got \n%s\n", tc, exp, err)
+		}
+	}
+}
+
+func TestPanicNoRecover(t *testing.T) {
+	defer func() {
+		if e := recover(); e != nil {
+			// all good
+			return
+		}
+		t.Fatal("want panic, got none")
+	}()
+
+	// should panic
+	Parse("", []byte("1 / 0"), Recover(false))
+}
+
+func TestMemoization(t *testing.T) {
+	in := " 2 + 35 * ( 18 - -4 / ( 5 + 1) ) * 456 + -1"
+	want := 287281
+
+	p := newParser("", []byte(in), Memoize(false))
+	got, err := p.parse(g)
+	if err != nil {
+		t.Fatal(err)
+	}
+	goti := got.(int)
+	if goti != want {
+		t.Errorf("want %d, got %d", want, goti)
+	}
+	if p.exprCnt != 415 {
+		t.Errorf("with Memoize=false, want %d expressions evaluated, got %d", 415, p.exprCnt)
+	}
+
+	p = newParser("", []byte(in), Memoize(true))
+	got, err = p.parse(g)
+	if err != nil {
+		t.Fatal(err)
+	}
+	goti = got.(int)
+	if goti != want {
+		t.Errorf("want %d, got %d", want, goti)
+	}
+	if p.exprCnt != 389 {
+		t.Errorf("with Memoize=true, want %d expressions evaluated, got %d", 389, p.exprCnt)
+	}
+}
+
+func BenchmarkPigeonCalculatorNoMemo(b *testing.B) {
+	d := []byte(longishExpr)
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", d, Memoize(false)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkPigeonCalculatorMemo(b *testing.B) {
+	d := []byte(longishExpr)
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", d, Memoize(true)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
@@ -0,0 +1,118 @@
+{
+// Command json parses JSON as defined by [1].
+//
+// BUGS: the escaped forward solidus (`\/`) is not currently handled.
+//
+// [1]: http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+package main
+
+func main() {
+    in := os.Stdin
+    nm := "stdin"
+    if len(os.Args) > 1 {
+        f, err := os.Open(os.Args[1])
+        if err != nil {
+            log.Fatal(err)
+        }
+        defer f.Close()
+        in = f
+        nm = os.Args[1]
+    }
+
+    got, err := ParseReader(nm, in)
+    if err != nil {
+        log.Fatal(err)
+    }
+    fmt.Println(got)
+}
+
+func toIfaceSlice(v interface{}) []interface{} {
+    if v == nil {
+        return nil
+    }
+    return v.([]interface{})
+}
+}
+
+JSON ← _ vals:Value+ EOF {
+    valsSl := toIfaceSlice(vals)
+    switch len(valsSl) {
+    case 0:
+        return nil, nil
+    case 1:
+        return valsSl[0], nil
+    default:
+        return valsSl, nil
+    }
+}
+
+Value ← val:( Object / Array / Number / String / Bool / Null ) _ {
+    return val, nil
+}
+
+Object ← '{' _ vals:( String _ ':' _ Value ( ',' _ String _ ':' _ Value )* )? '}' {
+    res := make(map[string]interface{})
+    valsSl := toIfaceSlice(vals)
+    if len(valsSl) == 0 {
+        return res, nil
+    }
+    res[valsSl[0].(string)] = valsSl[4]
+    restSl := toIfaceSlice(valsSl[5])
+    for _, v := range restSl {
+        vSl := toIfaceSlice(v)
+        res[vSl[2].(string)] = vSl[6]
+    }
+    return res, nil
+}
+
+Array ← '[' _ vals:( Value ( ',' _ Value )* )? ']' {
+    valsSl := toIfaceSlice(vals)
+    if len(valsSl) == 0 {
+        return []interface{}{}, nil
+    }
+    res := []interface{}{valsSl[0]}
+    restSl := toIfaceSlice(valsSl[1])
+    for _, v := range restSl {
+        vSl := toIfaceSlice(v)
+        res = append(res, vSl[2])
+    }
+    return res, nil
+}
+
+Number ← '-'? Integer ( '.' DecimalDigit+ )? Exponent? {
+    // JSON numbers have the same syntax as Go's, and are parseable using
+    // strconv.
+    return strconv.ParseFloat(string(c.text), 64)
+}
+
+Integer ← '0' / NonZeroDecimalDigit DecimalDigit*
+
+Exponent ← 'e'i [+-]? DecimalDigit+
+
+String ← '"' ( !EscapedChar . / '\\' EscapeSequence )* '"' {
+    // TODO : the forward slash (solidus) is not a valid escape in Go, it will
+    // fail if there's one in the string
+    return strconv.Unquote(string(c.text))
+}
+
+EscapedChar ← [\x00-\x1f"\\]
+
+EscapeSequence ← SingleCharEscape / UnicodeEscape
+
+SingleCharEscape ← ["\\/bfnrt]
+
+UnicodeEscape ← 'u' HexDigit HexDigit HexDigit HexDigit
+
+DecimalDigit ← [0-9]
+
+NonZeroDecimalDigit ← [1-9]
+
+HexDigit ← [0-9a-f]i
+
+Bool ← "true" { return true, nil } / "false" { return false, nil }
+
+Null ← "null" { return nil, nil }
+
+_ "whitespace" ← [ \t\r\n]*
+
+EOF ← !.
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+func TestCmpStdlib(t *testing.T) {
+	files := testJSONFiles(t)
+	for _, file := range files {
+		pgot, err := ParseFile(file)
+		if err != nil {
+			t.Errorf("%s: pigeon.ParseFile: %v", file, err)
+			continue
+		}
+
+		b, err := ioutil.ReadFile(file)
+		if err != nil {
+			t.Errorf("%s: ioutil.ReadAll: %v", file, err)
+			continue
+		}
+		var jgot interface{}
+		if err := json.Unmarshal(b, &jgot); err != nil {
+			t.Errorf("%s: json.Unmarshal: %v", file, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(pgot, jgot) {
+			t.Errorf("%s: not equal", file)
+			continue
+		}
+	}
+}
+
+func testJSONFiles(t *testing.T) []string {
+	const rootDir = "testdata"
+
+	fis, err := ioutil.ReadDir(rootDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := make([]string, 0, len(fis))
+	for _, fi := range fis {
+		if filepath.Ext(fi.Name()) == ".json" {
+			files = append(files, filepath.Join(rootDir, fi.Name()))
+		}
+	}
+	return files
+}
+
+func BenchmarkPigeonJSONNoMemo(b *testing.B) {
+	d, err := ioutil.ReadFile("testdata/github-octokit-repos.json")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", d, Memoize(false)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkPigeonJSONMemo(b *testing.B) {
+	d, err := ioutil.ReadFile("testdata/github-octokit-repos.json")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("", d, Memoize(true)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStdlibJSON(b *testing.B) {
+	d, err := ioutil.ReadFile("testdata/github-octokit-repos.json")
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		var iface interface{}
+		if err := json.Unmarshal(d, &iface); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
@@ -0,0 +1,3 @@
+[
+
+]
@@ -0,0 +1,370 @@
+[
+  {
+    "id": 417862,
+    "name": "octokit.rb",
+    "full_name": "octokit/octokit.rb",
+    "owner": {
+      "login": "octokit",
+      "id": 3430433,
+      "avatar_url": "https://avatars.githubusercontent.com/u/3430433?v=3",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octokit",
+      "html_url": "https://github.com/octokit",
+      "followers_url": "https://api.github.com/users/octokit/followers",
+      "following_url": "https://api.github.com/users/octokit/following{/other_user}",
+      "gists_url": "https://api.github.com/users/octokit/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/octokit/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/octokit/subscriptions",
+      "organizations_url": "https://api.github.com/users/octokit/orgs",
+      "repos_url": "https://api.github.com/users/octokit/repos",
+      "events_url": "https://api.github.com/users/octokit/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/octokit/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "private": false,
+    "html_url": "https://github.com/octokit/octokit.rb",
+    "description": "Ruby toolkit for the GitHub API",
+    "fork": false,
+    "url": "https://api.github.com/repos/octokit/octokit.rb",
+    "forks_url": "https://api.github.com/repos/octokit/octokit.rb/forks",
+    "keys_url": "https://api.github.com/repos/octokit/octokit.rb/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/octokit/octokit.rb/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/octokit/octokit.rb/teams",
+    "hooks_url": "https://api.github.com/repos/octokit/octokit.rb/hooks",
+    "issue_events_url": "https://api.github.com/repos/octokit/octokit.rb/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/octokit/octokit.rb/events",
+    "assignees_url": "https://api.github.com/repos/octokit/octokit.rb/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/octokit/octokit.rb/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/octokit/octokit.rb/tags",
+    "blobs_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/octokit/octokit.rb/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/octokit/octokit.rb/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/octokit/octokit.rb/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/octokit/octokit.rb/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/octokit/octokit.rb/languages",
+    "stargazers_url": "https://api.github.com/repos/octokit/octokit.rb/stargazers",
+    "contributors_url": "https://api.github.com/repos/octokit/octokit.rb/contributors",
+    "subscribers_url": "https://api.github.com/repos/octokit/octokit.rb/subscribers",
+    "subscription_url": "https://api.github.com/repos/octokit/octokit.rb/subscription",
+    "commits_url": "https://api.github.com/repos/octokit/octokit.rb/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/octokit/octokit.rb/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/octokit/octokit.rb/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/octokit/octokit.rb/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/octokit/octokit.rb/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/octokit/octokit.rb/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/octokit/octokit.rb/merges",
+    "archive_url": "https://api.github.com/repos/octokit/octokit.rb/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/octokit/octokit.rb/downloads",
+    "issues_url": "https://api.github.com/repos/octokit/octokit.rb/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/octokit/octokit.rb/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/octokit/octokit.rb/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/octokit/octokit.rb/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/octokit/octokit.rb/labels{/name}",
+    "releases_url": "https://api.github.com/repos/octokit/octokit.rb/releases{/id}",
+    "created_at": "2009-12-10T21:41:49Z",
+    "updated_at": "2015-04-02T15:26:33Z",
+    "pushed_at": "2015-03-25T01:12:36Z",
+    "git_url": "git://github.com/octokit/octokit.rb.git",
+    "ssh_url": "git@github.com:octokit/octokit.rb.git",
+    "clone_url": "https://github.com/octokit/octokit.rb.git",
+    "svn_url": "https://github.com/octokit/octokit.rb",
+    "homepage": "http://octokit.github.io/octokit.rb/",
+    "size": 16088,
+    "stargazers_count": 1845,
+    "watchers_count": 1845,
+    "language": "Ruby",
+    "has_issues": true,
+    "has_downloads": true,
+    "has_wiki": false,
+    "has_pages": true,
+    "forks_count": 401,
+    "mirror_url": null,
+    "open_issues_count": 5,
+    "forks": 401,
+    "open_issues": 5,
+    "watchers": 1845,
+    "default_branch": "master",
+    "permissions": {
+      "admin": false,
+      "push": false,
+      "pull": true
+    }
+  },
+  {
+    "id": 7528679,
+    "name": "octokit.net",
+    "full_name": "octokit/octokit.net",
+    "owner": {
+      "login": "octokit",
+      "id": 3430433,
+      "avatar_url": "https://avatars.githubusercontent.com/u/3430433?v=3",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octokit",
+      "html_url": "https://github.com/octokit",
+      "followers_url": "https://api.github.com/users/octokit/followers",
+      "following_url": "https://api.github.com/users/octokit/following{/other_user}",
+      "gists_url": "https://api.github.com/users/octokit/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/octokit/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/octokit/subscriptions",
+      "organizations_url": "https://api.github.com/users/octokit/orgs",
+      "repos_url": "https://api.github.com/users/octokit/repos",
+      "events_url": "https://api.github.com/users/octokit/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/octokit/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "private": false,
+    "html_url": "https://github.com/octokit/octokit.net",
+    "description": "A GitHub API client library for .NET ",
+    "fork": false,
+    "url": "https://api.github.com/repos/octokit/octokit.net",
+    "forks_url": "https://api.github.com/repos/octokit/octokit.net/forks",
+    "keys_url": "https://api.github.com/repos/octokit/octokit.net/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/octokit/octokit.net/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/octokit/octokit.net/teams",
+    "hooks_url": "https://api.github.com/repos/octokit/octokit.net/hooks",
+    "issue_events_url": "https://api.github.com/repos/octokit/octokit.net/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/octokit/octokit.net/events",
+    "assignees_url": "https://api.github.com/repos/octokit/octokit.net/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/octokit/octokit.net/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/octokit/octokit.net/tags",
+    "blobs_url": "https://api.github.com/repos/octokit/octokit.net/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/octokit/octokit.net/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/octokit/octokit.net/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/octokit/octokit.net/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/octokit/octokit.net/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/octokit/octokit.net/languages",
+    "stargazers_url": "https://api.github.com/repos/octokit/octokit.net/stargazers",
+    "contributors_url": "https://api.github.com/repos/octokit/octokit.net/contributors",
+    "subscribers_url": "https://api.github.com/repos/octokit/octokit.net/subscribers",
+    "subscription_url": "https://api.github.com/repos/octokit/octokit.net/subscription",
+    "commits_url": "https://api.github.com/repos/octokit/octokit.net/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/octokit/octokit.net/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/octokit/octokit.net/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/octokit/octokit.net/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/octokit/octokit.net/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/octokit/octokit.net/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/octokit/octokit.net/merges",
+    "archive_url": "https://api.github.com/repos/octokit/octokit.net/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/octokit/octokit.net/downloads",
+    "issues_url": "https://api.github.com/repos/octokit/octokit.net/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/octokit/octokit.net/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/octokit/octokit.net/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/octokit/octokit.net/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/octokit/octokit.net/labels{/name}",
+    "releases_url": "https://api.github.com/repos/octokit/octokit.net/releases{/id}",
+    "created_at": "2013-01-09T20:48:45Z",
+    "updated_at": "2015-04-02T18:10:11Z",
+    "pushed_at": "2015-04-03T11:47:52Z",
+    "git_url": "git://github.com/octokit/octokit.net.git",
+    "ssh_url": "git@github.com:octokit/octokit.net.git",
+    "clone_url": "https://github.com/octokit/octokit.net.git",
+    "svn_url": "https://github.com/octokit/octokit.net",
+    "homepage": null,
+    "size": 70529,
+    "stargazers_count": 637,
+    "watchers_count": 637,
+    "language": "C#",
+    "has_issues": true,
+    "has_downloads": true,
+    "has_wiki": false,
+    "has_pages": false,
+    "forks_count": 270,
+    "mirror_url": null,
+    "open_issues_count": 63,
+    "forks": 270,
+    "open_issues": 63,
+    "watchers": 637,
+    "default_branch": "master",
+    "permissions": {
+      "admin": false,
+      "push": false,
+      "pull": true
+    }
+  },
+  {
+    "id": 7530454,
+    "name": "octokit.objc",
+    "full_name": "octokit/octokit.objc",
+    "owner": {
+      "login": "octokit",
+      "id": 3430433,
+      "avatar_url": "https://avatars.githubusercontent.com/u/3430433?v=3",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octokit",
+      "html_url": "https://github.com/octokit",
+      "followers_url": "https://api.github.com/users/octokit/followers",
+      "following_url": "https://api.github.com/users/octokit/following{/other_user}",
+      "gists_url": "https://api.github.com/users/octokit/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/octokit/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/octokit/subscriptions",
+      "organizations_url": "https://api.github.com/users/octokit/orgs",
+      "repos_url": "https://api.github.com/users/octokit/repos",
+      "events_url": "https://api.github.com/users/octokit/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/octokit/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "private": false,
+    "html_url": "https://github.com/octokit/octokit.objc",
+    "description": "GitHub API client for Objective-C",
+    "fork": false,
+    "url": "https://api.github.com/repos/octokit/octokit.objc",
+    "forks_url": "https://api.github.com/repos/octokit/octokit.objc/forks",
+    "keys_url": "https://api.github.com/repos/octokit/octokit.objc/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/octokit/octokit.objc/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/octokit/octokit.objc/teams",
+    "hooks_url": "https://api.github.com/repos/octokit/octokit.objc/hooks",
+    "issue_events_url": "https://api.github.com/repos/octokit/octokit.objc/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/octokit/octokit.objc/events",
+    "assignees_url": "https://api.github.com/repos/octokit/octokit.objc/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/octokit/octokit.objc/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/octokit/octokit.objc/tags",
+    "blobs_url": "https://api.github.com/repos/octokit/octokit.objc/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/octokit/octokit.objc/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/octokit/octokit.objc/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/octokit/octokit.objc/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/octokit/octokit.objc/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/octokit/octokit.objc/languages",
+    "stargazers_url": "https://api.github.com/repos/octokit/octokit.objc/stargazers",
+    "contributors_url": "https://api.github.com/repos/octokit/octokit.objc/contributors",
+    "subscribers_url": "https://api.github.com/repos/octokit/octokit.objc/subscribers",
+    "subscription_url": "https://api.github.com/repos/octokit/octokit.objc/subscription",
+    "commits_url": "https://api.github.com/repos/octokit/octokit.objc/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/octokit/octokit.objc/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/octokit/octokit.objc/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/octokit/octokit.objc/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/octokit/octokit.objc/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/octokit/octokit.objc/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/octokit/octokit.objc/merges",
+    "archive_url": "https://api.github.com/repos/octokit/octokit.objc/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/octokit/octokit.objc/downloads",
+    "issues_url": "https://api.github.com/repos/octokit/octokit.objc/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/octokit/octokit.objc/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/octokit/octokit.objc/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/octokit/octokit.objc/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/octokit/octokit.objc/labels{/name}",
+    "releases_url": "https://api.github.com/repos/octokit/octokit.objc/releases{/id}",
+    "created_at": "2013-01-09T22:42:53Z",
+    "updated_at": "2015-04-03T06:16:41Z",
+    "pushed_at": "2015-03-21T17:10:20Z",
+    "git_url": "git://github.com/octokit/octokit.objc.git",
+    "ssh_url": "git@github.com:octokit/octokit.objc.git",
+    "clone_url": "https://github.com/octokit/octokit.objc.git",
+    "svn_url": "https://github.com/octokit/octokit.objc",
+    "homepage": "",
+    "size": 3779,
+    "stargazers_count": 1131,
+    "watchers_count": 1131,
+    "language": "Objective-C",
+    "has_issues": true,
+    "has_downloads": true,
+    "has_wiki": true,
+    "has_pages": false,
+    "forks_count": 170,
+    "mirror_url": null,
+    "open_issues_count": 26,
+    "forks": 170,
+    "open_issues": 26,
+    "watchers": 1131,
+    "default_branch": "master",
+    "permissions": {
+      "admin": false,
+      "push": false,
+      "pull": true
+    }
+  },
+  {
+    "id": 10575811,
+    "name": "go-octokit",
+    "full_name": "octokit/go-octokit",
+    "owner": {
+      "login": "octokit",
+      "id": 3430433,
+      "avatar_url": "https://avatars.githubusercontent.com/u/3430433?v=3",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octokit",
+      "html_url": "https://github.com/octokit",
+      "followers_url": "https://api.github.com/users/octokit/followers",
+      "following_url": "https://api.github.com/users/octokit/following{/other_user}",
+      "gists_url": "https://api.github.com/users/octokit/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/octokit/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/octokit/subscriptions",
+      "organizations_url": "https://api.github.com/users/octokit/orgs",
+      "repos_url": "https://api.github.com/users/octokit/repos",
+      "events_url": "https://api.github.com/users/octokit/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/octokit/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "private": false,
+    "html_url": "https://github.com/octokit/go-octokit",
+    "description": "Simple Go wrapper for the GitHub API",
+    "fork": false,
+    "url": "https://api.github.com/repos/octokit/go-octokit",
+    "forks_url": "https://api.github.com/repos/octokit/go-octokit/forks",
+    "keys_url": "https://api.github.com/repos/octokit/go-octokit/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/octokit/go-octokit/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/octokit/go-octokit/teams",
+    "hooks_url": "https://api.github.com/repos/octokit/go-octokit/hooks",
+    "issue_events_url": "https://api.github.com/repos/octokit/go-octokit/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/octokit/go-octokit/events",
+    "assignees_url": "https://api.github.com/repos/octokit/go-octokit/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/octokit/go-octokit/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/octokit/go-octokit/tags",
+    "blobs_url": "https://api.github.com/repos/octokit/go-octokit/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/octokit/go-octokit/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/octokit/go-octokit/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/octokit/go-octokit/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/octokit/go-octokit/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/octokit/go-octokit/languages",
+    "stargazers_url": "https://api.github.com/repos/octokit/go-octokit/stargazers",
+    "contributors_url": "https://api.github.com/repos/octokit/go-octokit/contributors",
+    "subscribers_url": "https://api.github.com/repos/octokit/go-octokit/subscribers",
+    "subscription_url": "https://api.github.com/repos/octokit/go-octokit/subscription",
+    "commits_url": "https://api.github.com/repos/octokit/go-octokit/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/octokit/go-octokit/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/octokit/go-octokit/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/octokit/go-octokit/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/octokit/go-octokit/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/octokit/go-octokit/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/octokit/go-octokit/merges",
+    "archive_url": "https://api.github.com/repos/octokit/go-octokit/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/octokit/go-octokit/downloads",
+    "issues_url": "https://api.github.com/repos/octokit/go-octokit/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/octokit/go-octokit/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/octokit/go-octokit/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/octokit/go-octokit/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/octokit/go-octokit/labels{/name}",
+    "releases_url": "https://api.github.com/repos/octokit/go-octokit/releases{/id}",
+    "created_at": "2013-06-08T23:50:29Z",
+    "updated_at": "2015-04-02T18:47:34Z",
+    "pushed_at": "2015-04-02T18:48:16Z",
+    "git_url": "git://github.com/octokit/go-octokit.git",
+    "ssh_url": "git@github.com:octokit/go-octokit.git",
+    "clone_url": "https://github.com/octokit/go-octokit.git",
+    "svn_url": "https://github.com/octokit/go-octokit",
+    "homepage": "https://github.com/octokit/go-octokit",
+    "size": 3693,
+    "stargazers_count": 106,
+    "watchers_count": 106,
+    "language": "Go",
+    "has_issues": true,
+    "has_downloads": true,
+    "has_wiki": false,
+    "has_pages": false,
+    "forks_count": 29,
+    "mirror_url": null,
+    "open_issues_count": 16,
+    "forks": 29,
+    "open_issues": 16,
+    "watchers": 106,
+    "default_branch": "master",
+    "permissions": {
+      "admin": false,
+      "push": false,
+      "pull": true
+    }
+  }
+]
@@ -0,0 +1,238 @@
+{
+package main
+}
+
+Grammar ← __ initializer:( Initializer __ )? rules:( Rule __ )+ {
+    pos := c.astPos()
+
+    // create the grammar, assign its initializer
+    g := ast.NewGrammar(pos)
+    initSlice := toIfaceSlice(initializer)
+    if len(initSlice) > 0 {
+        g.Init = initSlice[0].(*ast.CodeBlock)
+    }
+
+    rulesSlice := toIfaceSlice(rules)
+    g.Rules = make([]*ast.Rule, len(rulesSlice))
+    for i, duo := range rulesSlice {
+        g.Rules[i] = duo.([]interface{})[0].(*ast.Rule)
+    }
+
+    return g, nil
+}
+
+Initializer ← code:CodeBlock EOS {
+    return code, nil
+}
+
+Rule ← name:IdentifierName __ display:( StringLiteral __ )? RuleDefOp __ expr:Expression EOS {
+    pos := c.astPos()
+
+    rule := ast.NewRule(pos, name.(*ast.Identifier))
+    displaySlice := toIfaceSlice(display)
+    if len(displaySlice) > 0 {
+        rule.DisplayName = displaySlice[0].(*ast.StringLit)
+    }
+    rule.Expr = expr.(ast.Expression)
+
+    return rule, nil
+}
+
+Expression ← ChoiceExpr
+
+ChoiceExpr ← first:ActionExpr rest:( __ "/" __ ActionExpr )* {
+    restSlice := toIfaceSlice(rest)
+    if len(restSlice) == 0 {
+        return first, nil
+    }
+
+    pos := c.astPos()
+    choice := ast.NewChoiceExpr(pos)
+    choice.Alternatives = []ast.Expression{first.(ast.Expression)}
+    for _, sl := range restSlice {
+        choice.Alternatives = append(choice.Alternatives, sl.([]interface{})[3].(ast.Expression))
+    }
+    return choice, nil
+}
+
+ActionExpr ← expr:SeqExpr code:( __ CodeBlock )? {
+    if code == nil {
+        return expr, nil
+    }
+
+    pos := c.astPos()
+    act := ast.NewActionExpr(pos)
+    act.Expr = expr.(ast.Expression)
+    codeSlice := toIfaceSlice(code)
+    act.Code = codeSlice[1].(*ast.CodeBlock)
+
+    return act, nil
+}
+
+SeqExpr ← first:LabeledExpr rest:( __ LabeledExpr )* {
+    restSlice := toIfaceSlice(rest)
+    if len(restSlice) == 0 {
+        return first, nil
+    }
+    seq := ast.NewSeqExpr(c.astPos())
+    seq.Exprs = []ast.Expression{first.(ast.Expression)}
+    for _, sl := range restSlice {
+        seq.Exprs = append(seq.Exprs, sl.([]interface{})[1].(ast.Expression))
+    }
+    return seq, nil
+}
+
+LabeledExpr ← label:Identifier __ ':' __ expr:PrefixedExpr {
+    pos := c.astPos()
+    lab := ast.NewLabeledExpr(pos)
+    lab.Label = label.(*ast.Identifier)
+    lab.Expr = expr.(ast.Expression)
+    return lab, nil
+} / PrefixedExpr
+
+PrefixedExpr ← op:PrefixedOp __ expr:SuffixedExpr {
+    pos := c.astPos()
+    opStr := op.(string)
+    if opStr == "&" {
+        and := ast.NewAndExpr(pos)
+        and.Expr = expr.(ast.Expression)
+        return and, nil
+    }
+    not := ast.NewNotExpr(pos)
+    not.Expr = expr.(ast.Expression)
+    return not, nil
+} / SuffixedExpr
+
+PrefixedOp ← ( '&' / '!' ) {
+    return string(c.text), nil
+}
+
+SuffixedExpr ← expr:PrimaryExpr __ op:SuffixedOp {
+    pos := c.astPos()
+    opStr := op.(string)
+    switch opStr {
+    case "?":
+        zero := ast.NewZeroOrOneExpr(pos)
+        zero.Expr = expr.(ast.Expression)
+        return zero, nil
+    case "*":
+        zero := ast.NewZeroOrMoreExpr(pos)
+        zero.Expr = expr.(ast.Expression)
+        return zero, nil
+    case "+":
+        one := ast.NewOneOrMoreExpr(pos)
+        one.Expr = expr.(ast.Expression)
+        return one, nil
+    default:
+        return nil, errors.New("unknown operator: " + opStr)
+    }
+} / PrimaryExpr 
+
+SuffixedOp ← ( '?' / '*' / '+' ) {
+    return string(c.text), nil
+}
+
+PrimaryExpr ← LitMatcher / CharClassMatcher / AnyMatcher / RuleRefExpr / SemanticPredExpr / "(" __ expr:Expression __ ")" {
+    return expr, nil
+}
+RuleRefExpr ← name:IdentifierName !( __ ( StringLiteral __ )? RuleDefOp ) {
+    ref := ast.NewRuleRefExpr(c.astPos())
+    ref.Name = name.(*ast.Identifier)
+    return ref, nil
+}
+SemanticPredExpr ← op:SemanticPredOp __ code:CodeBlock {
+    opStr := op.(string)
+    if opStr == "&" {
+        and := ast.NewAndCodeExpr(c.astPos())
+        and.Code = code.(*ast.CodeBlock)
+        return and, nil
+    }
+    not := ast.NewNotCodeExpr(c.astPos())
+    not.Code = code.(*ast.CodeBlock)
+    return not, nil
+}
+SemanticPredOp ← ( '&' / '!' ) {
+    return string(c.text), nil
+}
+
+RuleDefOp ← '=' / "<-" / '\u2190' / '\u27f5'
+
+SourceChar ← .
+Comment ← MultiLineComment / SingleLineComment
+MultiLineComment ← "/*" ( !"*/" SourceChar )* "*/"
+MultiLineCommentNoLineTerminator ← "/*" ( !( "*/" / EOL ) SourceChar )* "*/"
+SingleLineComment ← "//" ( !EOL SourceChar )*
+
+Identifier ← IdentifierName
+IdentifierName ← IdentifierStart IdentifierPart* {
+    return ast.NewIdentifier(c.astPos(), string(c.text)), nil
+}
+IdentifierStart ← [a-z_]i
+IdentifierPart ← IdentifierStart / [0-9]
+
+LitMatcher ← lit:StringLiteral ignore:"i"? {
+    rawStr := lit.(*ast.StringLit).Val
+	s, err := strconv.Unquote(rawStr)
+    if err != nil {
+        return nil, err
+    }
+    m := ast.NewLitMatcher(c.astPos(), s)
+    m.IgnoreCase = ignore != nil
+    return m, nil
+}
+StringLiteral ← ( '"' DoubleStringChar* '"' / "'" SingleStringChar "'" / '`' RawStringChar '`' ) {
+    return ast.NewStringLit(c.astPos(), string(c.text)), nil
+}
+DoubleStringChar ← !( '"' / "\\" / EOL ) SourceChar / "\\" DoubleStringEscape
+SingleStringChar ← !( "'" / "\\" / EOL ) SourceChar / "\\" SingleStringEscape
+RawStringChar ← !'`' SourceChar
+
+DoubleStringEscape ← "'" / CommonEscapeSequence
+SingleStringEscape ← '"' / CommonEscapeSequence
+
+CommonEscapeSequence ← SingleCharEscape / OctalEscape / HexEscape / LongUnicodeEscape / ShortUnicodeEscape
+SingleCharEscape ← 'a' / 'b' / 'n' / 'f' / 'r' / 't' / 'v' / '\\'
+OctalEscape ← OctalDigit OctalDigit OctalDigit
+HexEscape ← 'x' HexDigit HexDigit
+LongUnicodeEscape ← 'U' HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit
+ShortUnicodeEscape ← 'u' HexDigit HexDigit HexDigit HexDigit
+
+OctalDigit ← [0-7]
+DecimalDigit ← [0-9]
+HexDigit ← [0-9a-f]i
+
+CharClassMatcher ← '[' ( ClassCharRange / ClassChar / "\\" UnicodeClassEscape )* ']' 'i'? {
+    pos := c.astPos()
+    cc := ast.NewCharClassMatcher(pos, string(c.text))
+    return cc, nil
+}
+ClassCharRange ← ClassChar '-' ClassChar
+ClassChar ← !( "]" / "\\" / EOL ) SourceChar / "\\" CharClassEscape
+CharClassEscape ← ']' / CommonEscapeSequence
+
+UnicodeClassEscape ← 'p' ( SingleCharUnicodeClass / '{' UnicodeClass '}' )
+SingleCharUnicodeClass ← [LMNCPZS]
+UnicodeClass ← [a-z_]i+
+
+AnyMatcher ← "." {
+    any := ast.NewAnyMatcher(c.astPos(), ".")
+    return any, nil
+}
+
+CodeBlock ← "{" Code "}" {
+    pos := c.astPos()
+    cb := ast.NewCodeBlock(pos, string(c.text))
+    return cb, nil
+}
+
+Code ← ( ( ![{}] SourceChar )+ / "{" Code "}" )*
+
+__ ← ( Whitespace / EOL / Comment )*
+_ ← ( Whitespace / MultiLineCommentNoLineTerminator )*
+
+Whitespace ← [ \t\r]
+EOL ← '\n'
+EOS ← __ ';' / _ SingleLineComment? EOL / __ EOF
+
+EOF ← !.
+
@@ -0,0 +1,294 @@
+{
+package main
+}
+
+Grammar ← __ initializer:( Initializer __ )? rules:( Rule __ )+ EOF {
+    pos := c.astPos()
+
+    // create the grammar, assign its initializer
+    g := ast.NewGrammar(pos)
+    initSlice := toIfaceSlice(initializer)
+    if len(initSlice) > 0 {
+        g.Init = initSlice[0].(*ast.CodeBlock)
+    }
+
+    rulesSlice := toIfaceSlice(rules)
+    g.Rules = make([]*ast.Rule, len(rulesSlice))
+    for i, duo := range rulesSlice {
+        g.Rules[i] = duo.([]interface{})[0].(*ast.Rule)
+    }
+
+    return g, nil
+}
+
+Initializer ← code:CodeBlock EOS {
+    return code, nil
+}
+
+Rule ← name:IdentifierName __ display:( StringLiteral __ )? RuleDefOp __ expr:Expression EOS {
+    pos := c.astPos()
+
+    rule := ast.NewRule(pos, name.(*ast.Identifier))
+    displaySlice := toIfaceSlice(display)
+    if len(displaySlice) > 0 {
+        rule.DisplayName = displaySlice[0].(*ast.StringLit)
+    }
+    rule.Expr = expr.(ast.Expression)
+
+    return rule, nil
+}
+
+Expression ← ChoiceExpr
+
+ChoiceExpr ← first:ActionExpr rest:( __ "/" __ ActionExpr )* {
+    restSlice := toIfaceSlice(rest)
+    if len(restSlice) == 0 {
+        return first, nil
+    }
+
+    pos := c.astPos()
+    choice := ast.NewChoiceExpr(pos)
+    choice.Alternatives = []ast.Expression{first.(ast.Expression)}
+    for _, sl := range restSlice {
+        choice.Alternatives = append(choice.Alternatives, sl.([]interface{})[3].(ast.Expression))
+    }
+    return choice, nil
+}
+
+ActionExpr ← expr:SeqExpr code:( __ CodeBlock )? {
+    if code == nil {
+        return expr, nil
+    }
+
+    pos := c.astPos()
+    act := ast.NewActionExpr(pos)
+    act.Expr = expr.(ast.Expression)
+    codeSlice := toIfaceSlice(code)
+    act.Code = codeSlice[1].(*ast.CodeBlock)
+
+    return act, nil
+}
+
+SeqExpr ← first:LabeledExpr rest:( __ LabeledExpr )* {
+    restSlice := toIfaceSlice(rest)
+    if len(restSlice) == 0 {
+        return first, nil
+    }
+    seq := ast.NewSeqExpr(c.astPos())
+    seq.Exprs = []ast.Expression{first.(ast.Expression)}
+    for _, sl := range restSlice {
+        seq.Exprs = append(seq.Exprs, sl.([]interface{})[1].(ast.Expression))
+    }
+    return seq, nil
+}
+
+LabeledExpr ← label:Identifier __ ':' __ expr:PrefixedExpr {
+    pos := c.astPos()
+    lab := ast.NewLabeledExpr(pos)
+    lab.Label = label.(*ast.Identifier)
+    lab.Expr = expr.(ast.Expression)
+    return lab, nil
+} / PrefixedExpr
+
+PrefixedExpr ← op:PrefixedOp __ expr:SuffixedExpr {
+    pos := c.astPos()
+    opStr := op.(string)
+    if opStr == "&" {
+        and := ast.NewAndExpr(pos)
+        and.Expr = expr.(ast.Expression)
+        return and, nil
+    }
+    not := ast.NewNotExpr(pos)
+    not.Expr = expr.(ast.Expression)
+    return not, nil
+} / SuffixedExpr
+
+PrefixedOp ← ( '&' / '!' ) {
+    return string(c.text), nil
+}
+
+SuffixedExpr ← expr:PrimaryExpr __ op:SuffixedOp {
+    pos := c.astPos()
+    opStr := op.(string)
+    switch opStr {
+    case "?":
+        zero := ast.NewZeroOrOneExpr(pos)
+        zero.Expr = expr.(ast.Expression)
+        return zero, nil
+    case "*":
+        zero := ast.NewZeroOrMoreExpr(pos)
+        zero.Expr = expr.(ast.Expression)
+        return zero, nil
+    case "+":
+        one := ast.NewOneOrMoreExpr(pos)
+        one.Expr = expr.(ast.Expression)
+        return one, nil
+    default:
+        return nil, errors.New("unknown operator: " + opStr)
+    }
+} / PrimaryExpr 
+
+SuffixedOp ← ( '?' / '*' / '+' ) {
+    return string(c.text), nil
+}
+
+PrimaryExpr ← LitMatcher / CharClassMatcher / AnyMatcher / RuleRefExpr / SemanticPredExpr / "(" __ expr:Expression __ ")" {
+    return expr, nil
+}
+RuleRefExpr ← name:IdentifierName !( __ ( StringLiteral __ )? RuleDefOp ) {
+    ref := ast.NewRuleRefExpr(c.astPos())
+    ref.Name = name.(*ast.Identifier)
+    return ref, nil
+}
+SemanticPredExpr ← op:SemanticPredOp __ code:CodeBlock {
+    opStr := op.(string)
+    if opStr == "&" {
+        and := ast.NewAndCodeExpr(c.astPos())
+        and.Code = code.(*ast.CodeBlock)
+        return and, nil
+    }
+    not := ast.NewNotCodeExpr(c.astPos())
+    not.Code = code.(*ast.CodeBlock)
+    return not, nil
+}
+SemanticPredOp ← ( '&' / '!' ) {
+    return string(c.text), nil
+}
+
+RuleDefOp ← '=' / "<-" / '\u2190' / '\u27f5'
+
+SourceChar ← .
+Comment ← MultiLineComment / SingleLineComment
+MultiLineComment ← "/*" ( !"*/" SourceChar )* "*/"
+MultiLineCommentNoLineTerminator ← "/*" ( !( "*/" / EOL ) SourceChar )* "*/"
+SingleLineComment ← "//" ( !EOL SourceChar )*
+
+Identifier ← ident:IdentifierName {
+    astIdent := ast.NewIdentifier(c.astPos(), string(c.text))
+    if reservedWords[astIdent.Val] {
+        return astIdent, errors.New("identifier is a reserved word")
+    }
+    return astIdent, nil
+}
+
+IdentifierName ← IdentifierStart IdentifierPart* {
+    return ast.NewIdentifier(c.astPos(), string(c.text)), nil
+}
+IdentifierStart ← [\pL_]
+IdentifierPart ← IdentifierStart / [\p{Nd}]
+
+LitMatcher ← lit:StringLiteral ignore:"i"? {
+    rawStr := lit.(*ast.StringLit).Val
+	s, err := strconv.Unquote(rawStr)
+    if err != nil {
+        // an invalid string literal raises an error in the escape rules,
+        // so simply replace the literal with an empty string here to
+        // avoid a cascade of errors.
+        s = ""
+    }
+    m := ast.NewLitMatcher(c.astPos(), s)
+    m.IgnoreCase = ignore != nil
+    return m, nil
+}
+StringLiteral ← ( '"' DoubleStringChar* '"' / "'" SingleStringChar "'" / '`' RawStringChar* '`' ) {
+    return ast.NewStringLit(c.astPos(), string(c.text)), nil
+} / ( ( '"' DoubleStringChar* ( EOL / EOF ) ) / ( "'" SingleStringChar? ( EOL / EOF ) ) / '`' RawStringChar* EOF ) {
+    return ast.NewStringLit(c.astPos(), "``"), errors.New("string literal not terminated")
+}
+
+DoubleStringChar ← !( '"' / "\\" / EOL ) SourceChar / "\\" DoubleStringEscape
+SingleStringChar ← !( "'" / "\\" / EOL ) SourceChar / "\\" SingleStringEscape
+RawStringChar ← !'`' SourceChar
+
+DoubleStringEscape ← ( '"' / CommonEscapeSequence )
+    / ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid escape character")
+}
+SingleStringEscape ← ( "'" / CommonEscapeSequence )
+    / ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid escape character")
+}
+
+CommonEscapeSequence ← SingleCharEscape / OctalEscape / HexEscape / LongUnicodeEscape / ShortUnicodeEscape
+SingleCharEscape ← 'a' / 'b' / 'n' / 'f' / 'r' / 't' / 'v' / '\\'
+OctalEscape ← OctalDigit OctalDigit OctalDigit
+    / OctalDigit ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid octal escape")
+}
+HexEscape ← 'x' HexDigit HexDigit
+    / 'x' ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid hexadecimal escape")
+}
+LongUnicodeEscape ← 
+    'U' HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit {
+        return validateUnicodeEscape(string(c.text), "invalid Unicode escape")
+    }
+    / 'U' ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid Unicode escape")
+}
+ShortUnicodeEscape ← 
+    'u' HexDigit HexDigit HexDigit HexDigit {
+        return validateUnicodeEscape(string(c.text), "invalid Unicode escape")
+    }
+    / 'u' ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid Unicode escape")
+}
+
+OctalDigit ← [0-7]
+DecimalDigit ← [0-9]
+HexDigit ← [0-9a-f]i
+
+CharClassMatcher ← '[' ( ClassCharRange / ClassChar / "\\" UnicodeClassEscape )* ']' 'i'? {
+    pos := c.astPos()
+    cc := ast.NewCharClassMatcher(pos, string(c.text))
+    return cc, nil
+} / '[' ( !( EOL ) SourceChar )* ( EOL / EOF ) {
+    return ast.NewCharClassMatcher(c.astPos(), "[]"), errors.New("character class not terminated")
+}
+
+ClassCharRange ← ClassChar '-' ClassChar
+ClassChar ← !( "]" / "\\" / EOL ) SourceChar / "\\" CharClassEscape
+CharClassEscape ← ( ']' / CommonEscapeSequence )
+    / !'p' ( SourceChar / EOL / EOF ) {
+    return nil, errors.New("invalid escape character")
+}
+
+UnicodeClassEscape ← 'p' ( 
+      SingleCharUnicodeClass
+    / !'{' ( SourceChar / EOL / EOF ) { return nil, errors.New("invalid Unicode class escape") }
+    / '{' ident:IdentifierName '}' {
+        if !unicodeClasses[ident.(*ast.Identifier).Val] {
+            return nil, errors.New("invalid Unicode class escape")
+        }
+        return nil, nil
+    }
+    / '{' IdentifierName ( ']' / EOL / EOF ) {
+        return nil, errors.New("Unicode class not terminated")
+    }
+    )
+SingleCharUnicodeClass ← [LMNCPZS]
+
+AnyMatcher ← "." {
+    any := ast.NewAnyMatcher(c.astPos(), ".")
+    return any, nil
+}
+
+CodeBlock ← '{' Code '}' {
+    pos := c.astPos()
+    cb := ast.NewCodeBlock(pos, string(c.text))
+    return cb, nil
+} / '{' Code EOF {
+    return nil, errors.New("code block not terminated")
+}
+
+Code ← ( ( ![{}] SourceChar )+ / '{' Code '}' )*
+
+__ ← ( Whitespace / EOL / Comment )*
+_ ← ( Whitespace / MultiLineCommentNoLineTerminator )*
+
+Whitespace ← [ \t\r]
+EOL ← '\n'
+EOS ← __ ';' / _ SingleLineComment? EOL / __ EOF
+
+EOF ← !.
+
@@ -0,0 +1,195 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+	"github.com/PuerkitoBio/pigeon/builder"
+)
+
+var exit = os.Exit
+
+func main() {
+	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// define command-line flags
+	var (
+		cacheFlag     = fs.Bool("cache", false, "cache parsing results")
+		dbgFlag       = fs.Bool("debug", false, "set debug mode")
+		shortHelpFlag = fs.Bool("h", false, "show help page")
+		longHelpFlag  = fs.Bool("help", false, "show help page")
+		noRecoverFlag = fs.Bool("no-recover", false, "do not recover from panic")
+		outputFlag    = fs.String("o", "", "output file, defaults to stdout")
+		recvrNmFlag   = fs.String("receiver-name", "c", "receiver name for the generated methods")
+		noBuildFlag   = fs.Bool("x", false, "do not build, only parse")
+	)
+
+	fs.Usage = usage
+	fs.Parse(os.Args[1:])
+
+	if *shortHelpFlag || *longHelpFlag {
+		fs.Usage()
+		exit(0)
+	}
+
+	if fs.NArg() > 1 {
+		argError(1, "expected one argument, got %q", strings.Join(fs.Args(), " "))
+	}
+
+	// get input source
+	infile := ""
+	if fs.NArg() == 1 {
+		infile = fs.Arg(0)
+	}
+	nm, rc := input(infile)
+	defer rc.Close()
+
+	// parse input
+	g, err := ParseReader(nm, rc, Debug(*dbgFlag), Memoize(*cacheFlag), Recover(!*noRecoverFlag))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "parse error(s):\n", err)
+		exit(3)
+	}
+
+	if !*noBuildFlag {
+		// generate parser
+		out := output(*outputFlag)
+		defer out.Close()
+
+		curNmOpt := builder.ReceiverName(*recvrNmFlag)
+		if err := builder.BuildParser(out, g.(*ast.Grammar), curNmOpt); err != nil {
+			fmt.Fprintln(os.Stderr, "build error: ", err)
+			exit(5)
+		}
+	}
+}
+
+var usagePage = `usage: %s [options] [GRAMMAR_FILE]
+
+Pigeon generates a parser based on a PEG grammar. It doesn't try
+to format the generated code nor to detect required imports -
+it is recommended to pipe the output of pigeon through a tool
+such as goimports to do this, e.g.:
+
+	pigeon GRAMMAR_FILE | goimports > output.go
+
+Use the following command to install goimports:
+
+	go get golang.org/x/tools/cmd/goimports
+
+By default, pigeon reads the grammar from stdin and writes the
+generated parser to stdout. If GRAMMAR_FILE is specified, the
+grammar is read from this file instead. If the -o flag is set,
+the generated code is written to this file instead.
+
+	-cache
+		cache parser results to avoid exponential parsing time in
+		pathological cases. Can make the parsing slower for typical
+		cases and uses more memory.
+	-debug
+		output debugging information while parsing the grammar.
+	-h -help
+		display this help message.
+	-no-recover
+		do not recover from a panic. Useful to access the panic stack
+		when debugging, otherwise the panic is converted to an error.
+	-o OUTPUT_FILE
+		write the generated parser to OUTPUT_FILE. Defaults to stdout.
+	-receiver-name NAME
+		use NAME as for the receiver name of the generated methods
+		for the grammar's code blocks. Defaults to "c".
+	-x
+		do not generate the parser, only parse the grammar.
+
+See https://godoc.org/github.com/PuerkitoBio/pigeon for more
+information.
+`
+
+// usage prints the help page of the command-line tool.
+func usage() {
+	fmt.Printf(usagePage, os.Args[0])
+}
+
+// argError prints an error message to stderr, prints the command usage
+// and exits with the specified exit code.
+func argError(exitCode int, msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg, args...)
+	fmt.Fprintln(os.Stderr)
+	usage()
+	exit(exitCode)
+}
+
+// input gets the name and reader to get input text from.
+func input(filename string) (nm string, rc io.ReadCloser) {
+	nm = "stdin"
+	inf := os.Stdin
+	if filename != "" {
+		f, err := os.Open(filename)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			exit(2)
+		}
+		inf = f
+		nm = filename
+	}
+	r := bufio.NewReader(inf)
+	return nm, makeReadCloser(r, inf)
+}
+
+// output gets the writer to write the generated parser to.
+func output(filename string) io.WriteCloser {
+	out := os.Stdout
+	if filename != "" {
+		f, err := os.Create(filename)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			exit(4)
+		}
+		out = f
+	}
+	return out
+}
+
+// create a ReadCloser that reads from r and closes c.
+func makeReadCloser(r io.Reader, c io.Closer) io.ReadCloser {
+	rc := struct {
+		io.Reader
+		io.Closer
+	}{r, c}
+	return io.ReadCloser(rc)
+}
+
+// astPos is a helper method for the PEG grammar parser. It returns the
+// position of the current match as an ast.Pos.
+func (c *current) astPos() ast.Pos {
+	return ast.Pos{Line: c.pos.line, Col: c.pos.col, Off: c.pos.offset}
+}
+
+// toIfaceSlice is a helper function for the PEG grammar parser. It converts
+// v to a slice of empty interfaces.
+func toIfaceSlice(v interface{}) []interface{} {
+	if v == nil {
+		return nil
+	}
+	return v.([]interface{})
+}
+
+// validateUnicodeEscape checks that the provided escape sequence is a
+// valid Unicode escape sequence.
+func validateUnicodeEscape(escape, errMsg string) (interface{}, error) {
+	r, _, _, err := strconv.UnquoteChar("\\"+escape, '"')
+	if err != nil {
+		return nil, errors.New(errMsg)
+	}
+	if 0xD800 <= r && r <= 0xDFFF {
+		return nil, errors.New(errMsg)
+	}
+	return nil, nil
+}
@@ -0,0 +1,54 @@
+package main
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestMain(t *testing.T) {
+	stdout, stderr := os.Stdout, os.Stderr
+	os.Stdout, _ = os.Open(os.DevNull)
+	os.Stderr, _ = os.Open(os.DevNull)
+	defer func() {
+		exit = os.Exit
+		os.Stdout = stdout
+		os.Stderr = stderr
+	}()
+	exit = func(code int) {
+		panic(code)
+	}
+
+	cases := []struct {
+		args string
+		code int
+	}{
+		{args: "", code: 3},            // stdin: no match found
+		{args: "-h", code: 0},          // help
+		{args: "FILE1 FILE2", code: 1}, // want only 1 non-flag arg
+		{args: "-x", code: 3},          // stdin: no match found
+	}
+
+	for _, tc := range cases {
+		os.Args = append([]string{"pigeon"}, strings.Fields(tc.args)...)
+
+		got := runMainRecover()
+		if got != tc.code {
+			t.Errorf("%q: want code %d, got %d", tc.args, tc.code, got)
+		}
+	}
+}
+
+func runMainRecover() (code int) {
+	defer func() {
+		if e := recover(); e != nil {
+			if i, ok := e.(int); ok {
+				code = i
+				return
+			}
+			panic(e)
+		}
+	}()
+	main()
+	return 0
+}
@@ -0,0 +1,53 @@
+// Command unicode-classes generates a set-like map of all valid
+// Unicode classes.
+package main
+
+import (
+	"fmt"
+	"sort"
+	"unicode"
+)
+
+func main() {
+	set := make(map[string]bool)
+	for k := range unicode.Categories {
+		set[k] = true
+	}
+	for k := range unicode.Properties {
+		set[k] = true
+	}
+	for k := range unicode.Scripts {
+		set[k] = true
+	}
+	classes := make([]string, 0, len(set))
+	for k := range set {
+		classes = append(classes, k)
+	}
+
+	sort.Strings(classes)
+	fmt.Println(`// This file is generated by the misc/cmd/unicode-classes tool.
+// Do not edit.
+`)
+	fmt.Println("package main")
+	fmt.Println("\nvar unicodeClasses = map[string]bool{")
+	for _, s := range classes {
+		fmt.Printf("\t%q: true,\n", s)
+	}
+	fmt.Println("}")
+}
+
+// lenSorter was used to generate Unicode classes directly in the PEG
+// grammar (where longer classes had to come first).
+type lenSorter []string
+
+func (l lenSorter) Len() int      { return len(l) }
+func (l lenSorter) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
+func (l lenSorter) Less(i, j int) bool {
+	li, lj := len(l[i]), len(l[j])
+	if lj < li {
+		return true
+	} else if li < lj {
+		return false
+	}
+	return l[j] < l[i]
+}
@@ -0,0 +1,37 @@
+#!/bin/sh
+# Copyright 2012 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# git gofmt pre-commit hook
+#
+# To use, store as .git/hooks/pre-commit inside your repository and make sure
+# it has execute permissions.
+#
+# This script does not handle file names that contain spaces.
+
+# golint is purely informational, it doesn't fail with exit code != 0 if it finds something,
+# because it may find a lot of false positives. Just print out its result for information.
+echo "lint result (informational only):"
+golint ./...
+
+# go vet returns 1 if an error was found. Exit the hook with this exit code.
+go vet ./...
+vetres=$?
+
+# Check for gofmt problems and report if any.
+gofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.go$')
+[ -z "$gofiles" ] && echo "EXIT $vetres" && exit $vetres
+
+unformatted=$(gofmt -l $gofiles)
+[ -z "$unformatted" ] && echo "EXIT $vetres" && exit $vetres
+
+# Some files are not gofmt'd. Print message and fail.
+
+echo >&2 "Go files must be formatted with gofmt. Please run:"
+for fn in $unformatted; do
+    echo >&2 "  gofmt -w $PWD/$fn"
+done
+
+echo "EXIT 1"
+exit 1
@@ -0,0 +1,439 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/PuerkitoBio/pigeon/ast"
+)
+
+var invalidParseCases = map[string]string{
+	"":           "file:1:1 (0): no match found",
+	"a":          "file:1:1 (0): no match found",
+	"abc":        "file:1:1 (0): no match found",
+	" ":          "file:1:1 (0): no match found",
+	`a = +`:      "file:1:1 (0): no match found",
+	`a = *`:      "file:1:1 (0): no match found",
+	`a = ?`:      "file:1:1 (0): no match found",
+	"a ←":        "file:1:1 (0): no match found",
+	"a ← b\nb ←": "file:1:1 (0): no match found",
+	"a ← nil:b":  "file:1:5 (6): rule Identifier: identifier is a reserved word",
+	"\xfe":       "file:1:1 (0): invalid encoding",
+	"{}{}":       "file:1:1 (0): no match found",
+
+	// non-terminated, empty, EOF "quoted" tokens
+	"{":         "file:1:1 (0): rule CodeBlock: code block not terminated",
+	"\n{":       "file:2:1 (1): rule CodeBlock: code block not terminated",
+	`a = "`:     "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = `":     "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = '":     "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	`a = [`:     "file:1:5 (4): rule CharClassMatcher: character class not terminated",
+	`a = [\p{]`: `file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+
+	// non-terminated, empty, EOL "quoted" tokens
+	"{\n":          "file:1:1 (0): rule CodeBlock: code block not terminated",
+	"\n{\n":        "file:2:1 (1): rule CodeBlock: code block not terminated",
+	"a = \"\n":     "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = `\n":      "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = '\n":      "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = [\n":      "file:1:5 (4): rule CharClassMatcher: character class not terminated",
+	"a = [\\p{\n]": `file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+
+	// non-terminated quoted tokens with escaped closing char
+	`a = "\"`: "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	`a = '\'`: "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	`a = [\]`: "file:1:5 (4): rule CharClassMatcher: character class not terminated",
+
+	// non-terminated, non-empty, EOF "quoted" tokens
+	"{a":     "file:1:1 (0): rule CodeBlock: code block not terminated",
+	"\n{{}":  "file:2:1 (1): rule CodeBlock: code block not terminated",
+	`a = "b`: "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = `b": "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	"a = 'b": "file:1:5 (4): rule StringLiteral: string literal not terminated",
+	`a = [b`: "file:1:5 (4): rule CharClassMatcher: character class not terminated",
+	`a = [\p{W]`: `file:1:8 (7): rule UnicodeClassEscape: Unicode class not terminated
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+
+	// invalid escapes
+	`a ← [\pA]`:    "file:1:8 (9): rule UnicodeClassEscape: invalid Unicode class escape",
+	`a ← [\p{WW}]`: "file:1:8 (9): rule UnicodeClassEscape: invalid Unicode class escape",
+	`a = '\"'`:     "file:1:7 (6): rule SingleStringEscape: invalid escape character",
+	`a = "\'"`:     "file:1:7 (6): rule DoubleStringEscape: invalid escape character",
+	`a = [\']`:     "file:1:7 (6): rule CharClassEscape: invalid escape character",
+	`a = '\xz'`:    "file:1:7 (6): rule HexEscape: invalid hexadecimal escape",
+	`a = '\0z'`:    "file:1:7 (6): rule OctalEscape: invalid octal escape",
+	`a = '\uz'`:    "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = '\Uz'`:    "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+
+	// escapes followed by newline
+	"a = '\\\n": `file:2:0 (6): rule SingleStringEscape: invalid escape character
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\x\n": `file:1:7 (6): rule HexEscape: invalid hexadecimal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\0\n": `file:1:7 (6): rule OctalEscape: invalid octal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\u\n": `file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\U\n": `file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\\n": `file:2:0 (6): rule DoubleStringEscape: invalid escape character
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\x\n": `file:1:7 (6): rule HexEscape: invalid hexadecimal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\0\n": `file:1:7 (6): rule OctalEscape: invalid octal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\u\n": `file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\U\n": `file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = [\\\n": `file:2:0 (6): rule CharClassEscape: invalid escape character
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\x\n": `file:1:7 (6): rule HexEscape: invalid hexadecimal escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\0\n": `file:1:7 (6): rule OctalEscape: invalid octal escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\u\n": `file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\U\n": `file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\p\n": `file:2:0 (7): rule UnicodeClassEscape: invalid Unicode class escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\p{\n": `file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+
+	// escapes followed by EOF
+	"a = '\\": `file:1:7 (6): rule SingleStringEscape: invalid escape character
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\x": `file:1:7 (6): rule HexEscape: invalid hexadecimal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\0": `file:1:7 (6): rule OctalEscape: invalid octal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\u": `file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = '\\U": `file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\": `file:1:7 (6): rule DoubleStringEscape: invalid escape character
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\x": `file:1:7 (6): rule HexEscape: invalid hexadecimal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\0": `file:1:7 (6): rule OctalEscape: invalid octal escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\u": `file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = \"\\U": `file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule StringLiteral: string literal not terminated`,
+	"a = [\\": `file:1:7 (6): rule CharClassEscape: invalid escape character
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\x": `file:1:7 (6): rule HexEscape: invalid hexadecimal escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\0": `file:1:7 (6): rule OctalEscape: invalid octal escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\u": `file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\U": `file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\p": `file:1:8 (7): rule UnicodeClassEscape: invalid Unicode class escape
+file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+	"a = [\\p{": `file:1:5 (4): rule CharClassMatcher: character class not terminated`,
+
+	// multi-char escapes, fail after 2 chars
+	`a = '\x0z'`: "file:1:7 (6): rule HexEscape: invalid hexadecimal escape",
+	`a = '\00z'`: "file:1:7 (6): rule OctalEscape: invalid octal escape",
+	`a = '\u0z'`: "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = '\U0z'`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	// multi-char escapes, fail after 3 chars
+	`a = '\u00z'`: "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = '\U00z'`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	// multi-char escapes, fail after 4 chars
+	`a = '\u000z'`: "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = '\U000z'`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	// multi-char escapes, fail after 5 chars
+	`a = '\U0000z'`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	// multi-char escapes, fail after 6 chars
+	`a = '\U00000z'`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	// multi-char escapes, fail after 7 chars
+	`a = '\U000000z'`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+
+	// combine escape errors
+	`a = "\a\b\c\t\n\r\xab\xz\ux"`: `file:1:11 (10): rule DoubleStringEscape: invalid escape character
+file:1:23 (22): rule HexEscape: invalid hexadecimal escape
+file:1:26 (25): rule ShortUnicodeEscape: invalid Unicode escape`,
+
+	// syntactically valid escapes, but invalid values
+	`a = "\udfff"`:     "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = "\ud800"`:     "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = "\ud801"`:     "file:1:7 (6): rule ShortUnicodeEscape: invalid Unicode escape",
+	`a = "\U00110000"`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	`a = "\U0000DFFF"`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	`a = "\U0000D800"`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+	`a = "\U0000D801"`: "file:1:7 (6): rule LongUnicodeEscape: invalid Unicode escape",
+}
+
+func TestInvalidParseCases(t *testing.T) {
+	memo := false
+again:
+	for tc, exp := range invalidParseCases {
+		_, err := Parse("file", []byte(tc), Memoize(memo))
+		if err == nil {
+			t.Errorf("%q: want error, got none", tc)
+			continue
+		}
+		if err.Error() != exp {
+			t.Errorf("%q: want \n%s\n, got \n%s\n", tc, exp, err)
+		}
+	}
+	if !memo {
+		memo = true
+		goto again
+	}
+}
+
+var validParseCases = map[string]*ast.Grammar{
+	"a = b": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+		},
+	},
+	"a ← b\nc=d \n e <- f \ng\u27f5h": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "c"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "d")},
+			},
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "e"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "f")},
+			},
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "g"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "h")},
+			},
+		},
+	},
+	`a "A"← b`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name:        ast.NewIdentifier(ast.Pos{}, "a"),
+				DisplayName: ast.NewStringLit(ast.Pos{}, `"A"`),
+				Expr:        &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+		},
+	},
+	"{ init \n}\na 'A'← b": &ast.Grammar{
+		Init: ast.NewCodeBlock(ast.Pos{}, "{ init \n}"),
+		Rules: []*ast.Rule{
+			{
+				Name:        ast.NewIdentifier(ast.Pos{}, "a"),
+				DisplayName: ast.NewStringLit(ast.Pos{}, `'A'`),
+				Expr:        &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+		},
+	},
+	"a\n<-\nb": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+		},
+	},
+	"a\n<-\nb\nc": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.SeqExpr{
+					Exprs: []ast.Expression{
+						&ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+						&ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "c")},
+					},
+				},
+			},
+		},
+	},
+	"a\n<-\nb\nc\n=\nd": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "c"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "d")},
+			},
+		},
+	},
+	"a\n<-\nb\nc\n'C'\n=\nd": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "b")},
+			},
+			{
+				Name:        ast.NewIdentifier(ast.Pos{}, "c"),
+				DisplayName: ast.NewStringLit(ast.Pos{}, `'C'`),
+				Expr:        &ast.RuleRefExpr{Name: ast.NewIdentifier(ast.Pos{}, "d")},
+			},
+		},
+	},
+	`a = [a-def]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'e', 'f'},
+					Ranges: []rune{'a', 'd'},
+				},
+			},
+		},
+	},
+	`a = [abc-f]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'a', 'b'},
+					Ranges: []rune{'c', 'f'},
+				},
+			},
+		},
+	},
+	`a = [abc-fg]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'a', 'b', 'g'},
+					Ranges: []rune{'c', 'f'},
+				},
+			},
+		},
+	},
+	`a = [abc-fgh-l]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'a', 'b', 'g'},
+					Ranges: []rune{'c', 'f', 'h', 'l'},
+				},
+			},
+		},
+	},
+	`a = [\x00-\xabc]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'c'},
+					Ranges: []rune{'\x00', '\xab'},
+				},
+			},
+		},
+	},
+	`a = [-a-b]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'-'},
+					Ranges: []rune{'a', 'b'},
+				},
+			},
+		},
+	},
+	`a = [a-b-d]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'-', 'd'},
+					Ranges: []rune{'a', 'b'},
+				},
+			},
+		},
+	},
+	`a = [\u0012\123]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars: []rune{'\u0012', '\123'},
+				},
+			},
+		},
+	},
+	`a = [-\u0012-\U00001234]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					Chars:  []rune{'-'},
+					Ranges: []rune{'\u0012', '\U00001234'},
+				},
+			},
+		},
+	},
+	`a = [\p{Latin}]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					UnicodeClasses: []string{"Latin"},
+				},
+			},
+		},
+	},
+	`a = [\p{Latin}\pZ]`: &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: &ast.CharClassMatcher{
+					UnicodeClasses: []string{"Latin", "Z"},
+				},
+			},
+		},
+	},
+	"a = `a\nb\nc`": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: ast.NewLitMatcher(ast.Pos{}, "a\nb\nc"),
+			},
+		},
+	},
+	"a = ``": &ast.Grammar{
+		Rules: []*ast.Rule{
+			{
+				Name: ast.NewIdentifier(ast.Pos{}, "a"),
+				Expr: ast.NewLitMatcher(ast.Pos{}, ""),
+			},
+		},
+	},
+}
+
+func TestValidParseCases(t *testing.T) {
+	memo := false
+again:
+	for tc, exp := range validParseCases {
+		got, err := Parse("", []byte(tc))
+		if err != nil {
+			t.Errorf("%q: got error %v", tc, err)
+			continue
+		}
+		gotg, ok := got.(*ast.Grammar)
+		if !ok {
+			t.Errorf("%q: want grammar type %T, got %T", tc, exp, got)
+			continue
+		}
+		compareGrammars(t, tc, exp, gotg)
+	}
+	if !memo {
+		memo = true
+		goto again
+	}
+}
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`au BufWritePost *.go silent!make tags > /dev/null 2>&1`