Add agda, bash, c, nix, python and rust

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 xenia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,9 @@
+# xenia/tree-sitters — a collection of tree-sitter grammars, built with nix for static linking
+
+Languages:
+* c: https://github.com/tree-sitter/tree-sitter-c (MIT)
+* agda: https://github.com/tree-sitter/tree-sitter-agda (MIT)
+* bash: https://github.com/tree-sitter/tree-sitter-bash (MIT)
+* nix: https://github.com/nix-community/tree-sitter-nix (MIT)
+* python: https://github.com/tree-sitter/tree-sitter-python (MIT)
+* rust: https://github.com/tree-sitter/tree-sitter-rust (MIT)

agda/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 LUA Ting-Gan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agda/examples/example.txt

@@ -0,0 +1,8 @@
+record RawMonoid c ℓ : Set (suc (c ⊔ ℓ)) where
+  infixl 7 _∙_
+  infix  4 _≈_
+  field
+    Carrier : Set c
+    _≈_     : Rel Carrier ℓ
+    _b_     : Op Carrier
+    a       : Carrier

agda/src/scanner.c

@@ -0,0 +1,291 @@
+#include "tree_sitter/parser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#define VEC_RESIZE(vec, _cap)                                                  \
+    void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0]));           \
+    assert(tmp != NULL);                                                       \
+    (vec).data = tmp;                                                          \
+    (vec).cap = (_cap);
+
+#define VEC_GROW(vec, _cap)                                                    \
+    if ((vec).cap < (_cap)) {                                                  \
+        VEC_RESIZE((vec), (_cap));                                             \
+    }
+
+#define VEC_PUSH(vec, el)                                                      \
+    if ((vec).cap == (vec).len) {                                              \
+        VEC_RESIZE((vec), MAX(16, (vec).len * 2));                             \
+    }                                                                          \
+    (vec).data[(vec).len++] = (el);
+
+#define VEC_POP(vec) (vec).len--;
+
+#define VEC_NEW                                                                \
+    { .len = 0, .cap = 0, .data = NULL }
+
+#define VEC_BACK(vec) ((vec).data[(vec).len - 1])
+
+#define VEC_FREE(vec)                                                          \
+    {                                                                          \
+        if ((vec).data != NULL)                                                \
+            free((vec).data);                                                  \
+    }
+
+#define VEC_CLEAR(vec) (vec).len = 0;
+
+#define QUEUE_RESIZE(queue, _cap)                                              \
+    do {                                                                       \
+        void *tmp = realloc((queue).data, (_cap) * sizeof((queue).data[0]));   \
+        assert(tmp != NULL);                                                   \
+        (queue).data = tmp;                                                    \
+        (queue).cap = (_cap);                                                  \
+    } while (0)
+
+#define QUEUE_GROW(queue, _cap)                                                \
+    do {                                                                       \
+        if ((queue).cap < (_cap)) {                                            \
+            QUEUE_RESIZE((queue), (_cap));                                     \
+        }                                                                      \
+    } while (0)
+
+#define QUEUE_PUSH(queue, el)                                                  \
+    do {                                                                       \
+        if ((queue).cap == 0) {                                                \
+            QUEUE_RESIZE((queue), 16);                                         \
+        } else if ((queue).cap == ((queue).tail - (queue).head)) {             \
+            QUEUE_RESIZE((queue), (queue).cap * 2);                            \
+        }                                                                      \
+        (queue).data[(queue).tail % (queue).cap] = (el);                       \
+        (queue).tail++;                                                        \
+    } while (0)
+
+#define QUEUE_POP(queue)                                                       \
+    do {                                                                       \
+        assert((queue).head < (queue).tail);                                   \
+        (queue).head++;                                                        \
+    } while (0)
+
+#define QUEUE_FRONT(queue) (queue).data[(queue).head % (queue).cap]
+
+#define QUEUE_EMPTY(queue) ((queue).head == (queue).tail)
+
+#define QUEUE_NEW                                                              \
+    { .head = 0, .tail = 0, .cap = 0, .data = NULL }
+
+#define QUEUE_FREE(queue)                                                      \
+    do {                                                                       \
+        if ((queue).data != NULL)                                              \
+            free((queue).data);                                                \
+    } while (0)
+
+#define QUEUE_CLEAR(queue)                                                     \
+    do {                                                                       \
+        (queue).head = 0;                                                      \
+        (queue).tail = 0;                                                      \
+    } while (0)
+
+enum TokenType {
+    NEWLINE,
+    INDENT,
+    DEDENT,
+};
+
+typedef struct {
+    uint32_t len;
+    uint32_t cap;
+    uint16_t *data;
+} indent_vec;
+
+static indent_vec indent_vec_new() {
+    indent_vec vec = VEC_NEW;
+    vec.data = calloc(1, sizeof(uint16_t));
+    vec.cap = 1;
+    return vec;
+}
+
+typedef struct {
+    uint32_t head;
+    uint32_t tail;
+    uint32_t cap;
+    uint16_t *data;
+} token_queue;
+
+static token_queue token_queue_new() {
+    token_queue queue = QUEUE_NEW;
+    queue.data = calloc(1, sizeof(uint16_t));
+    queue.cap = 1;
+    return queue;
+}
+
+typedef struct {
+    indent_vec indents;
+    uint32_t queued_dedent_count;
+    token_queue tokens;
+} Scanner;
+
+static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+
+static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+bool tree_sitter_agda_external_scanner_scan(void *payload, TSLexer *lexer,
+                                            const bool *valid_symbols) {
+    Scanner *scanner = (Scanner *)payload;
+
+    if (QUEUE_EMPTY(scanner->tokens)) {
+        if (valid_symbols[DEDENT] && scanner->queued_dedent_count > 0) {
+            scanner->queued_dedent_count--;
+            QUEUE_PUSH(scanner->tokens, DEDENT);
+            QUEUE_PUSH(scanner->tokens, NEWLINE);
+        } else {
+            bool skipped_newline = false;
+
+            while (lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
+                   lexer->lookahead == '\r' || lexer->lookahead == '\n') {
+                if (lexer->lookahead == '\n') {
+                    skipped_newline = true;
+                    skip(lexer);
+                } else {
+                    skip(lexer);
+                }
+            }
+
+            if (lexer->eof(lexer)) {
+                if (valid_symbols[DEDENT] && scanner->indents.len > 1) {
+                    VEC_POP(scanner->indents);
+                    QUEUE_PUSH(scanner->tokens, DEDENT);
+                    QUEUE_PUSH(scanner->tokens, NEWLINE);
+                } else if (valid_symbols[NEWLINE]) {
+                    QUEUE_PUSH(scanner->tokens, NEWLINE);
+                }
+            } else {
+                bool next_token_is_comment = false;
+
+                uint16_t indent_length = (uint16_t)lexer->get_column(lexer);
+
+                bool indent = indent_length > VEC_BACK(scanner->indents);
+                bool dedent = indent_length < VEC_BACK(scanner->indents);
+
+                if (!next_token_is_comment) {
+                    if (skipped_newline) {
+                        if (indent) {
+                            if (valid_symbols[INDENT]) {
+                                VEC_PUSH(scanner->indents, indent_length);
+                                QUEUE_PUSH(scanner->tokens, INDENT);
+                            }
+                        } else if (dedent) {
+                            if (valid_symbols[NEWLINE]) {
+                                QUEUE_PUSH(scanner->tokens, NEWLINE);
+                            }
+                        } else {
+                            if (valid_symbols[NEWLINE]) {
+                                QUEUE_PUSH(scanner->tokens, NEWLINE);
+                            }
+                        }
+                    } else {
+                        if (indent) {
+                            if (valid_symbols[INDENT]) {
+                                VEC_PUSH(scanner->indents, indent_length);
+                                QUEUE_PUSH(scanner->tokens, INDENT);
+                            }
+                        } else if (dedent) {
+                            VEC_POP(scanner->indents);
+                            while (indent_length < VEC_BACK(scanner->indents)) {
+                                VEC_POP(scanner->indents);
+                                scanner->queued_dedent_count++;
+                            }
+                            if (valid_symbols[DEDENT]) {
+                                QUEUE_PUSH(scanner->tokens, DEDENT);
+                                QUEUE_PUSH(scanner->tokens, NEWLINE);
+                            } else {
+                                scanner->queued_dedent_count++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    if (QUEUE_EMPTY(scanner->tokens)) {
+        return false;
+    }
+
+    lexer->result_symbol = QUEUE_FRONT(scanner->tokens);
+    QUEUE_POP(scanner->tokens);
+    return true;
+}
+
+unsigned tree_sitter_agda_external_scanner_serialize(void *payload,
+                                                     char *buffer) {
+    Scanner *scanner = (Scanner *)payload;
+
+    if (scanner->indents.len * sizeof(uint16_t) + 1 >
+        TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
+        return 0;
+    }
+
+    unsigned size = 0;
+
+    buffer[size++] = (char)scanner->queued_dedent_count;
+
+    memcpy(&buffer[size], scanner->indents.data,
+           scanner->indents.len * sizeof(uint16_t));
+    size += (unsigned)(scanner->indents.len * sizeof(uint16_t));
+
+    return size;
+}
+
+void tree_sitter_agda_external_scanner_deserialize(void *payload,
+                                                   const char *buffer,
+                                                   unsigned length) {
+    Scanner *scanner = (Scanner *)payload;
+
+    scanner->queued_dedent_count = 0;
+    VEC_CLEAR(scanner->indents);
+
+    if (length == 0) {
+        if (buffer == NULL) {
+            VEC_PUSH(scanner->indents, 0);
+        }
+        return;
+    }
+
+    scanner->queued_dedent_count = (uint8_t)buffer[0];
+
+    unsigned size = 1;
+
+    if (length > size) {
+        VEC_GROW(scanner->indents,
+                 (uint32_t)(length - size) / sizeof(uint16_t));
+        scanner->indents.len = (length - size) / sizeof(uint16_t);
+        memcpy(scanner->indents.data, &buffer[size],
+               scanner->indents.len * sizeof(uint16_t));
+        size += (unsigned)(scanner->indents.len * sizeof(uint16_t));
+    }
+
+    if (scanner->indents.len == 0) {
+        VEC_PUSH(scanner->indents, 0);
+        return;
+    }
+
+    assert(size == length);
+}
+
+void *tree_sitter_agda_external_scanner_create() {
+    Scanner *scanner = calloc(1, sizeof(Scanner));
+    scanner->indents = indent_vec_new();
+    scanner->tokens = token_queue_new();
+    tree_sitter_agda_external_scanner_deserialize(scanner, NULL, 0);
+    return scanner;
+}
+
+void tree_sitter_agda_external_scanner_destroy(void *payload) {
+    Scanner *scanner = (Scanner *)payload;
+    VEC_FREE(scanner->indents);
+    QUEUE_FREE(scanner->tokens);
+    free(scanner);
+}

bash/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bash/examples/atom.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+if [ "$(uname)" == 'Darwin' ]; then
+  OS='Mac'
+elif [ "$(expr substr $(uname -s) 1 5)" == 'Linux' ]; then
+  OS='Linux'
+else
+  echo "Your platform ($(uname -a)) is not supported."
+  exit 1
+fi
+
+if [ "$(basename $0)" == 'atom-beta' ]; then
+  BETA_VERSION=true
+else
+  BETA_VERSION=
+fi
+
+export ATOM_DISABLE_SHELLING_OUT_FOR_ENVIRONMENT=true
+
+while getopts ":wtfvh-:" opt; do
+  case "$opt" in
+    -)
+      case "${OPTARG}" in
+        wait)
+          WAIT=1
+          ;;
+        help|version)
+          REDIRECT_STDERR=1
+          EXPECT_OUTPUT=1
+          ;;
+        foreground|benchmark|benchmark-test|test)
+          EXPECT_OUTPUT=1
+          ;;
+      esac
+      ;;
+    w)
+      WAIT=1
+      ;;
+    h|v)
+      REDIRECT_STDERR=1
+      EXPECT_OUTPUT=1
+      ;;
+    f|t)
+      EXPECT_OUTPUT=1
+      ;;
+  esac
+done
+
+if [ $REDIRECT_STDERR ]; then
+  exec 2> /dev/null
+fi
+
+if [ $EXPECT_OUTPUT ]; then
+  export ELECTRON_ENABLE_LOGGING=1
+fi
+
+if [ $OS == 'Mac' ]; then
+  if [ -L "$0" ]; then
+    SCRIPT="$(readlink "$0")"
+  else
+    SCRIPT="$0"
+  fi
+  ATOM_APP="$(dirname "$(dirname "$(dirname "$(dirname "$SCRIPT")")")")"
+  if [ "$ATOM_APP" == . ]; then
+    unset ATOM_APP
+  else
+    ATOM_PATH="$(dirname "$ATOM_APP")"
+    ATOM_APP_NAME="$(basename "$ATOM_APP")"
+  fi
+
+  if [ -n "$BETA_VERSION" ]; then
+    ATOM_EXECUTABLE_NAME="Atom Beta"
+  else
+    ATOM_EXECUTABLE_NAME="Atom"
+  fi
+
+  if [ -z "${ATOM_PATH}" ]; then
+    # If ATOM_PATH isn't set, check /Applications and then ~/Applications for Atom.app
+    if [ -x "/Applications/$ATOM_APP_NAME" ]; then
+      ATOM_PATH="/Applications"
+    elif [ -x "$HOME/Applications/$ATOM_APP_NAME" ]; then
+      ATOM_PATH="$HOME/Applications"
+    else
+      # We haven't found an Atom.app, use spotlight to search for Atom
+      ATOM_PATH="$(mdfind "kMDItemCFBundleIdentifier == 'com.github.atom'" | grep -v ShipIt | head -1 | xargs -0 dirname)"
+
+      # Exit if Atom can't be found
+      if [ ! -x "$ATOM_PATH/$ATOM_APP_NAME" ]; then
+        echo "Cannot locate ${ATOM_APP_NAME}, it is usually located in /Applications. Set the ATOM_PATH environment variable to the directory containing ${ATOM_APP_NAME}."
+        exit 1
+      fi
+    fi
+  fi
+
+  if [ $EXPECT_OUTPUT ]; then
+    "$ATOM_PATH/$ATOM_APP_NAME/Contents/MacOS/$ATOM_EXECUTABLE_NAME" --executed-from="$(pwd)" --pid=$$ "$@"
+    exit $?
+  else
+    open -a "$ATOM_PATH/$ATOM_APP_NAME" -n --args --executed-from="$(pwd)" --pid=$$ --path-environment="$PATH" "$@"
+  fi
+elif [ $OS == 'Linux' ]; then
+  SCRIPT=$(readlink -f "$0")
+  USR_DIRECTORY=$(readlink -f $(dirname $SCRIPT)/..)
+
+  if [ -n "$BETA_VERSION" ]; then
+    ATOM_PATH="$USR_DIRECTORY/share/atom-beta/atom"
+  else
+    ATOM_PATH="$USR_DIRECTORY/share/atom/atom"
+  fi
+
+  ATOM_HOME="${ATOM_HOME:-$HOME/.atom}"
+  mkdir -p "$ATOM_HOME"
+
+  : ${TMPDIR:=/tmp}
+
+  [ -x "$ATOM_PATH" ] || ATOM_PATH="$TMPDIR/atom-build/Atom/atom"
+
+  if [ $EXPECT_OUTPUT ]; then
+    "$ATOM_PATH" --executed-from="$(pwd)" --pid=$$ "$@"
+    exit $?
+  else
+    (
+    nohup "$ATOM_PATH" --executed-from="$(pwd)" --pid=$$ "$@" > "$ATOM_HOME/nohup.out" 2>&1
+    if [ $? -ne 0 ]; then
+      cat "$ATOM_HOME/nohup.out"
+      exit $?
+    fi
+    ) &
+  fi
+fi
+
+# Exits this process when Atom is used as $EDITOR
+on_die() {
+  exit 0
+}
+trap 'on_die' SIGQUIT SIGTERM
+
+# If the wait flag is set, don't exit this process until Atom tells it to.
+if [ $WAIT ]; then
+  while true; do
+    sleep 1
+  done
+fi

bash/examples/clean-old.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# look for old 0.x cruft, and get rid of it.
+# Should already be sitting in the npm folder.
+
+# This doesn't have to be quite as cross-platform as install.sh.
+# There are some bash-isms, because maintaining *two*
+# fully-portable posix/bourne sh scripts is too much for
+# one project with a sane maintainer.
+
+# If readlink isn't available, then this is just too tricky.
+# However, greadlink is fine, so Solaris can join the party, too.
+readlink="readlink"
+which $readlink >/dev/null 2>/dev/null
+if [ $? -ne 0 ]; then
+  readlink="greadlink"
+  which $readlink >/dev/null 2>/dev/null
+  if [ $? -ne 0 ]; then
+    echo "Can't find the readlink or greadlink command. Aborting."
+    exit 1
+  fi
+fi
+
+if [ "x$npm_config_prefix" != "x" ]; then
+  PREFIXES=$npm_config_prefix
+else
+  node="$NODE"
+  if [ "x$node" = "x" ]; then
+    node=`which node`
+  fi
+  if [ "x$node" = "x" ]; then
+    echo "Can't find node to determine prefix. Aborting."
+    exit 1
+  fi
+
+
+  PREFIX=`dirname $node`
+  PREFIX=`dirname $PREFIX`
+  echo "cleanup prefix=$PREFIX"
+  PREFIXES=$PREFIX
+
+  altprefix=`"$node" -e process.installPrefix`
+  if [ "x$altprefix" != "x" ] && [ "x$altprefix" != "x$PREFIX" ]; then
+    echo "altprefix=$altprefix"
+    PREFIXES="$PREFIX $altprefix"
+  fi
+fi
+
+# now prefix is where npm would be rooted by default
+# go hunting.
+
+packages=
+for prefix in $PREFIXES; do
+  packages="$packages
+    "`ls "$prefix"/lib/node/.npm 2>/dev/null | grep -v .cache`
+done
+
+packages=`echo $packages`
+
+filelist=()
+fid=0
+
+for prefix in $PREFIXES; do
+  # remove any links into the .npm dir, or links to
+  # version-named shims/symlinks.
+  for folder in share/man bin lib/node; do
+    find $prefix/$folder -type l | while read file; do
+      target=`$readlink $file | grep '/\.npm/'`
+      if [ "x$target" != "x" ]; then
+        # found one!
+        filelist[$fid]="$file"
+        let 'fid++'
+        # also remove any symlinks to this file.
+        base=`basename "$file"`
+        base=`echo "$base" | awk -F@ '{print $1}'`
+        if [ "x$base" != "x" ]; then
+          find "`dirname $file`" -type l -name "$base"'*' \
+          | while read l; do
+              target=`$readlink "$l" | grep "$base"`
+              if [ "x$target" != "x" ]; then
+                filelist[$fid]="$1"
+                let 'fid++'
+              fi
+            done
+        fi
+      fi
+    done
+
+    # Scour for shim files.  These are relics of 0.2 npm installs.
+    # note: grep -r is not portable.
+    find $prefix/$folder -type f \
+      | xargs grep -sl '// generated by npm' \
+      | while read file; do
+          filelist[$fid]="$file"
+          let 'fid++'
+        done
+  done
+
+  # now remove the package modules, and the .npm folder itself.
+  if [ "x$packages" != "x" ]; then
+    for pkg in $packages; do
+      filelist[$fid]="$prefix/lib/node/$pkg"
+      let 'fid++'
+      for i in $prefix/lib/node/$pkg\@*; do
+        filelist[$fid]="$i"
+        let 'fid++'
+      done
+    done
+  fi
+
+  for folder in lib/node/.npm lib/npm share/npm; do
+    if [ -d $prefix/$folder ]; then
+      filelist[$fid]="$prefix/$folder"
+      let 'fid++'
+    fi
+  done
+done
+
+# now actually clean, but only if there's anything TO clean
+if [ "${#filelist[@]}" -gt 0 ]; then
+  echo ""
+  echo "This script will find and eliminate any shims, symbolic"
+  echo "links, and other cruft that was installed by npm 0.x."
+  echo ""
+
+  if [ "x$packages" != "x" ]; then
+    echo "The following packages appear to have been installed with"
+    echo "an old version of npm, and will be removed forcibly:"
+    for pkg in $packages; do
+      echo "    $pkg"
+    done
+    echo "Make a note of these. You may want to install them"
+    echo "with npm 1.0 when this process is completed."
+    echo ""
+  fi
+
+  OK=
+  if [ "x$1" = "x-y" ]; then
+    OK="yes"
+  fi
+
+  while [ "$OK" != "y" ] && [ "$OK" != "yes" ] && [ "$OK" != "no" ]; do
+    echo "Is this OK?"
+    echo "  enter 'yes' or 'no'"
+    echo "  or 'show' to see a list of files "
+    read OK
+    if [ "x$OK" = "xshow" ] || [ "x$OK" = "xs" ]; then
+      for i in "${filelist[@]}"; do
+        echo "$i"
+      done
+    fi
+  done
+  if [ "$OK" = "no" ]; then
+    echo "Aborting"
+    exit 1
+  fi
+  for i in "${filelist[@]}"; do
+    rm -rf "$i"
+  done
+fi
+
+echo ""
+echo 'All clean!'
+
+exit 0

bash/examples/doc-build.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+if [[ $DEBUG != "" ]]; then
+  set -x
+fi
+set -o errexit
+set -o pipefail
+
+if ! [ -x node_modules/.bin/marked-man ]; then
+  ps=0
+  if [ -f .building_marked-man ]; then
+    pid=$(cat .building_marked-man)
+    ps=$(ps -p $pid | grep $pid | wc -l) || true
+  fi
+
+  if [ -f .building_marked-man ] && [ $ps != 0 ]; then
+    while [ -f .building_marked-man ]; do
+      sleep 1
+    done
+  else
+    # a race to see which make process will be the one to install marked-man
+    echo $$ > .building_marked-man
+    sleep 1
+    if [ $(cat .building_marked-man) == $$ ]; then
+      make node_modules/.bin/marked-man
+      rm .building_marked-man
+    else
+      while [ -f .building_marked-man ]; do
+        sleep 1
+      done
+    fi
+  fi
+fi
+
+if ! [ -x node_modules/.bin/marked ]; then
+  ps=0
+  if [ -f .building_marked ]; then
+    pid=$(cat .building_marked)
+    ps=$(ps -p $pid | grep $pid | wc -l) || true
+  fi
+
+  if [ -f .building_marked ] && [ $ps != 0 ]; then
+    while [ -f .building_marked ]; do
+      sleep 1
+    done
+  else
+    # a race to see which make process will be the one to install marked
+    echo $$ > .building_marked
+    sleep 1
+    if [ $(cat .building_marked) == $$ ]; then
+      make node_modules/.bin/marked
+      rm .building_marked
+    else
+      while [ -f .building_marked ]; do
+        sleep 1
+      done
+    fi
+  fi
+fi
+
+src=$1
+dest=$2
+name=$(basename ${src%.*})
+date=$(date -u +'%Y-%m-%d %H:%M:%S')
+version=$(node cli.js -v)
+
+mkdir -p $(dirname $dest)
+
+html_replace_tokens () {
+	local url=$1
+	sed "s|@NAME@|$name|g" \
+	| sed "s|@DATE@|$date|g" \
+	| sed "s|@URL@|$url|g" \
+	| sed "s|@VERSION@|$version|g" \
+	| perl -p -e 's/<h1([^>]*)>([^\(]*\([0-9]\)) -- (.*?)<\/h1>/<h1>\2<\/h1> <p>\3<\/p>/g' \
+	| perl -p -e 's/npm-npm/npm/g' \
+	| perl -p -e 's/([^"-])(npm-)?README(?!\.html)(\(1\))?/\1<a href="..\/..\/doc\/README.html">README<\/a>/g' \
+	| perl -p -e 's/<title><a href="[^"]+README.html">README<\/a><\/title>/<title>README<\/title>/g' \
+	| perl -p -e 's/([^"-])([^\(> ]+)(\(1\))/\1<a href="..\/cli\/\2.html">\2\3<\/a>/g' \
+	| perl -p -e 's/([^"-])([^\(> ]+)(\(3\))/\1<a href="..\/api\/\2.html">\2\3<\/a>/g' \
+	| perl -p -e 's/([^"-])([^\(> ]+)(\(5\))/\1<a href="..\/files\/\2.html">\2\3<\/a>/g' \
+	| perl -p -e 's/([^"-])([^\(> ]+)(\(7\))/\1<a href="..\/misc\/\2.html">\2\3<\/a>/g' \
+	| perl -p -e 's/\([1357]\)<\/a><\/h1>/<\/a><\/h1>/g' \
+	| (if [ $(basename $(dirname $dest)) == "doc" ]; then
+			perl -p -e 's/ href="\.\.\// href="/g'
+		else
+			cat
+		fi)
+}
+
+man_replace_tokens () {
+	sed "s|@VERSION@|$version|g" \
+	| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(1\)/npm help \2/g' \
+	| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(([57])\)/npm help \3 \2/g' \
+	| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(3\)/npm apihelp \2/g' \
+	| perl -p -e 's/npm\(1\)/npm help npm/g' \
+	| perl -p -e 's/npm\(3\)/npm apihelp npm/g'
+}
+
+case $dest in
+  *.[1357])
+    ./node_modules/.bin/marked-man --roff $src \
+    | man_replace_tokens > $dest
+    exit $?
+    ;;
+  *.html)
+    url=${dest/html\//}
+    (cat html/dochead.html && \
+     cat $src | ./node_modules/.bin/marked &&
+     cat html/docfoot.html)\
+    | html_replace_tokens $url \
+    > $dest
+    exit $?
+    ;;
+  *)
+    echo "Invalid destination type: $dest" >&2
+    exit 1
+    ;;
+esac

bash/examples/install.sh

@@ -0,0 +1,270 @@
+#!/bin/sh
+
+# A word about this shell script:
+#
+# It must work everywhere, including on systems that lack
+# a /bin/bash, map 'sh' to ksh, ksh97, bash, ash, or zsh,
+# and potentially have either a posix shell or bourne
+# shell living at /bin/sh.
+#
+# See this helpful document on writing portable shell scripts:
+# http://www.gnu.org/s/hello/manual/autoconf/Portable-Shell.html
+#
+# The only shell it won't ever work on is cmd.exe.
+
+if [ "x$0" = "xsh" ]; then
+  # run as curl | sh
+  # on some systems, you can just do cat>npm-install.sh
+  # which is a bit cuter.  But on others, &1 is already closed,
+  # so catting to another script file won't do anything.
+  # Follow Location: headers, and fail on errors
+  curl -f -L -s https://www.npmjs.org/install.sh > npm-install-$$.sh
+  ret=$?
+  if [ $ret -eq 0 ]; then
+    (exit 0)
+  else
+    rm npm-install-$$.sh
+    echo "Failed to download script" >&2
+    exit $ret
+  fi
+  sh npm-install-$$.sh
+  ret=$?
+  rm npm-install-$$.sh
+  exit $ret
+fi
+
+# See what "npm_config_*" things there are in the env,
+# and make them permanent.
+# If this fails, it's not such a big deal.
+configures="`env | grep 'npm_config_' | sed -e 's|^npm_config_||g'`"
+
+npm_config_loglevel="error"
+if [ "x$npm_debug" = "x" ]; then
+  (exit 0)
+else
+  echo "Running in debug mode."
+  echo "Note that this requires bash or zsh."
+  set -o xtrace
+  set -o pipefail
+  npm_config_loglevel="verbose"
+fi
+export npm_config_loglevel
+
+# make sure that node exists
+node=`which node 2>&1`
+ret=$?
+if [ $ret -eq 0 ] && [ -x "$node" ]; then
+  (exit 0)
+else
+  echo "npm cannot be installed without node.js." >&2
+  echo "Install node first, and then try again." >&2
+  echo "" >&2
+  echo "Maybe node is installed, but not in the PATH?" >&2
+  echo "Note that running as sudo can change envs." >&2
+  echo ""
+  echo "PATH=$PATH" >&2
+  exit $ret
+fi
+
+# set the temp dir
+TMP="${TMPDIR}"
+if [ "x$TMP" = "x" ]; then
+  TMP="/tmp"
+fi
+TMP="${TMP}/npm.$$"
+rm -rf "$TMP" || true
+mkdir "$TMP"
+if [ $? -ne 0 ]; then
+  echo "failed to mkdir $TMP" >&2
+  exit 1
+fi
+
+BACK="$PWD"
+
+ret=0
+tar="${TAR}"
+if [ -z "$tar" ]; then
+  tar="${npm_config_tar}"
+fi
+if [ -z "$tar" ]; then
+  tar=`which tar 2>&1`
+  ret=$?
+fi
+
+if [ $ret -eq 0 ] && [ -x "$tar" ]; then
+  echo "tar=$tar"
+  echo "version:"
+  $tar --version
+  ret=$?
+fi
+
+if [ $ret -eq 0 ]; then
+  (exit 0)
+else
+  echo "No suitable tar program found."
+  exit 1
+fi
+
+
+
+# Try to find a suitable make
+# If the MAKE environment var is set, use that.
+# otherwise, try to find gmake, and then make.
+# If no make is found, then just execute the necessary commands.
+
+# XXX For some reason, make is building all the docs every time.  This
+# is an annoying source of bugs. Figure out why this happens.
+MAKE=NOMAKE
+
+if [ "x$MAKE" = "x" ]; then
+  make=`which gmake 2>&1`
+  if [ $? -eq 0 ] && [ -x "$make" ]; then
+    (exit 0)
+  else
+    make=`which make 2>&1`
+    if [ $? -eq 0 ] && [ -x "$make" ]; then
+      (exit 0)
+    else
+      make=NOMAKE
+    fi
+  fi
+else
+  make="$MAKE"
+fi
+
+if [ -x "$make" ]; then
+  (exit 0)
+else
+  # echo "Installing without make. This may fail." >&2
+  make=NOMAKE
+fi
+
+# If there's no bash, then don't even try to clean
+if [ -x "/bin/bash" ]; then
+  (exit 0)
+else
+  clean="no"
+fi
+
+node_version=`"$node" --version 2>&1`
+ret=$?
+if [ $ret -ne 0 ]; then
+  echo "You need node to run this program." >&2
+  echo "node --version reports: $node_version" >&2
+  echo "with exit code = $ret" >&2
+  echo "Please install node before continuing." >&2
+  exit $ret
+fi
+
+t="${npm_install}"
+if [ -z "$t" ]; then
+  # switch based on node version.
+  # note that we can only use strict sh-compatible patterns here.
+  case $node_version in
+    0.[01234567].* | v0.[01234567].*)
+      echo "You are using an outdated and unsupported version of" >&2
+      echo "node ($node_version).  Please update node and try again." >&2
+      exit 99
+      ;;
+    *)
+      echo "install npm@latest"
+      t="latest"
+      ;;
+  esac
+fi
+
+# need to echo "" after, because Posix sed doesn't treat EOF
+# as an implied end of line.
+url=`(curl -SsL https://registry.npmjs.org/npm/$t; echo "") \
+     | sed -e 's/^.*tarball":"//' \
+     | sed -e 's/".*$//'`
+
+ret=$?
+if [ "x$url" = "x" ]; then
+  ret=125
+  # try without the -e arg to sed.
+  url=`(curl -SsL https://registry.npmjs.org/npm/$t; echo "") \
+       | sed 's/^.*tarball":"//' \
+       | sed 's/".*$//'`
+  ret=$?
+  if [ "x$url" = "x" ]; then
+    ret=125
+  fi
+fi
+if [ $ret -ne 0 ]; then
+  echo "Failed to get tarball url for npm/$t" >&2
+  exit $ret
+fi
+
+
+echo "fetching: $url" >&2
+
+cd "$TMP" \
+  && curl -SsL "$url" \
+     | $tar -xzf - \
+  && cd "$TMP"/* \
+  && (ver=`"$node" bin/read-package-json.js package.json version`
+      isnpm10=0
+      if [ $ret -eq 0 ]; then
+        if [ -d node_modules ]; then
+          if "$node" node_modules/semver/bin/semver -v "$ver" -r "1"
+          then
+            isnpm10=1
+          fi
+        else
+          if "$node" bin/semver -v "$ver" -r ">=1.0"; then
+            isnpm10=1
+          fi
+        fi
+      fi
+
+      ret=0
+      if [ $isnpm10 -eq 1 ] && [ -f "scripts/clean-old.sh" ]; then
+        if [ "x$skipclean" = "x" ]; then
+          (exit 0)
+        else
+          clean=no
+        fi
+        if [ "x$clean" = "xno" ] \
+            || [ "x$clean" = "xn" ]; then
+          echo "Skipping 0.x cruft clean" >&2
+          ret=0
+        elif [ "x$clean" = "xy" ] || [ "x$clean" = "xyes" ]; then
+          NODE="$node" /bin/bash "scripts/clean-old.sh" "-y"
+          ret=$?
+        else
+          NODE="$node" /bin/bash "scripts/clean-old.sh" </dev/tty
+          ret=$?
+        fi
+      fi
+
+      if [ $ret -ne 0 ]; then
+        echo "Aborted 0.x cleanup.  Exiting." >&2
+        exit $ret
+      fi) \
+  && (if [ "x$configures" = "x" ]; then
+        (exit 0)
+      else
+        echo "./configure $configures"
+        echo "$configures" > npmrc
+      fi) \
+  && (if [ "$make" = "NOMAKE" ]; then
+        (exit 0)
+      elif "$make" uninstall install; then
+        (exit 0)
+      else
+        make="NOMAKE"
+      fi
+      if [ "$make" = "NOMAKE" ]; then
+        "$node" cli.js rm npm -gf
+        "$node" cli.js install -gf
+      fi) \
+  && cd "$BACK" \
+  && rm -rf "$TMP" \
+  && echo "It worked"
+
+ret=$?
+if [ $ret -ne 0 ]; then
+  echo "It failed" >&2
+fi
+exit $ret

bash/examples/release.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# script for creating a zip and tarball for inclusion in node
+
+unset CDPATH
+
+set -e
+
+rm -rf release *.tgz || true
+mkdir release
+node ./cli.js pack --loglevel error >/dev/null
+mv *.tgz release
+cd release
+tar xzf *.tgz
+
+mkdir node_modules
+mv package node_modules/npm
+
+# make the zip for windows users
+cp node_modules/npm/bin/*.cmd .
+zipname=npm-$(node ../cli.js -v).zip
+zip -q -9 -r -X "$zipname" *.cmd node_modules
+
+# make the tar for node's deps
+cd node_modules
+tarname=npm-$(node ../../cli.js -v).tgz
+tar czf "$tarname" npm
+
+cd ..
+mv "node_modules/$tarname" .
+
+rm -rf *.cmd
+rm -rf node_modules
+
+echo "release/$tarname"
+echo "release/$zipname"

bash/examples/relocate.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Change the cli shebang to point at the specified node
+# Useful for when the program is moved around after install.
+# Also used by the default 'make install' in node to point
+# npm at the newly installed node, rather than the first one
+# in the PATH, which would be the default otherwise.
+
+# bash /path/to/npm/scripts/relocate.sh $nodepath
+# If $nodepath is blank, then it'll use /usr/bin/env
+
+dir="$(dirname "$(dirname "$0")")"
+cli="$dir"/bin/npm-cli.js
+tmp="$cli".tmp
+
+node="$1"
+if [ "x$node" = "x" ]; then
+  node="/usr/bin/env node"
+fi
+node="#!$node"
+
+sed -e 1d "$cli" > "$tmp"
+echo "$node" > "$cli"
+cat "$tmp" >> "$cli"
+rm "$tmp"
+chmod ogu+x $cli

bash/examples/test.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+set -e
+
+function usage {
+  cat <<-EOF
+USAGE
+
+  $0  [-dgGhv] [-f focus-string] [-s seed]
+
+OPTIONS
+
+  -h  print this message
+
+  -b  run make under scan-build static analyzer
+
+  -d  run tests in a debugger (either lldb or gdb)
+
+  -g  run tests with valgrind's memcheck tool
+
+  -G  run tests with valgrind's memcheck tool, including a full leak check
+
+  -v  run tests with verbose output
+
+  -f  run only tests whose description contain the given string
+
+  -s  set the seed used to control random behavior
+
+  -z  pipe tests' stderr to \`dot(1)\` to render an SVG log
+
+
+EOF
+}
+
+profile=
+leak_check=no
+mode=normal
+verbose=
+args=()
+target=tests
+export BUILDTYPE=Test
+cmd="out/${BUILDTYPE}/${target}"
+run_scan_build=
+
+if [ "$(uname -s)" == "Darwin" ]; then
+  export LINK="clang++ -fsanitize=address"
+fi
+
+while getopts "bdf:s:gGhpvS" option; do
+  case ${option} in
+    h)
+      usage
+      exit
+      ;;
+    d)
+      mode=debug
+      ;;
+    g)
+      mode=valgrind
+      ;;
+    G)
+      mode=valgrind
+      leak_check=full
+      ;;
+    p)
+      profile=true
+      ;;
+    f)
+      args+=("--only=${OPTARG}")
+      ;;
+    v)
+      verbose=true
+      ;;
+    s)
+      export TREE_SITTER_SEED=${OPTARG}
+      ;;
+    S)
+      mode=SVG
+      ;;
+    b)
+      run_scan_build=true
+      ;;
+  esac
+done
+
+if [[ -n $verbose ]]; then
+  args+=("--reporter=spec")
+else
+  args+=("--reporter=singleline")
+fi
+
+if [[ -n "$run_scan_build" ]]; then
+  . script/util/scan-build.sh
+  scan_build make -j2 $target
+else
+  make -j2 $target
+fi
+args=${args:-""}
+
+if [[ -n $profile ]]; then
+  export CPUPROFILE=/tmp/${target}-$(date '+%s').prof
+fi
+
+case ${mode} in
+  valgrind)
+    valgrind                                     \
+      --suppressions=./script/util/valgrind.supp \
+      --dsymutil=yes                             \
+      --leak-check=${leak_check}                 \
+      $cmd "${args[@]}" 2>&1 |                   \
+      grep --color -E '\w+_tests?.cc:\d+|$'
+    ;;
+
+  debug)
+    if which -s lldb; then
+      lldb $cmd -- "${args[@]}"
+    elif which -s gdb; then
+      gdb $cmd -- "${args[@]}"
+    else
+      echo "No debugger found"
+      exit 1
+    fi
+    ;;
+
+  SVG)
+    echo "<!DOCTYPE html><style>svg { width: 100%; margin-bottom: 20px; }</style>" > index.html
+    $cmd "${args[@]}" 2> >(grep -v 'Assertion failed' | dot -Tsvg >> index.html)
+    echo "Wrote index.html"
+    ;;
+
+  normal)
+    time $cmd "${args[@]}"
+    ;;
+esac
+
+if [[ -n $profile ]]; then
+  pprof $cmd $CPUPROFILE
+fi

bash/examples/update-authors.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+git log --reverse --format='%aN <%aE>' | perl -wnE '
+BEGIN {
+  say "# Authors sorted by whether or not they\x27re me";
+}
+
+print $seen{$_} = $_ unless $seen{$_}
+' > AUTHORS

bash/test/corpus/commands.txt

@@ -0,0 +1,708 @@
+===============================
+Commands
+===============================
+
+whoami
+
+---
+
+(program
+  (command (command_name (word))))
+
+===============================
+Commands with arguments
+===============================
+
+cat file1.txt
+git diff --word-diff=color -- file1.txt file2.txt
+echo $sing\
+levar
+
+---
+
+(program
+  (command (command_name (word)) (word))
+  (command (command_name (word)) (word) (word) (word) (word) (word))
+  (command (command_name (word)) (simple_expansion (variable_name)) (word)))
+
+===============================
+Quoted command names
+===============================
+
+"$a/$b" c
+
+---
+
+(program
+  (command
+    (command_name (string (simple_expansion (variable_name)) (string_content) (simple_expansion (variable_name))))
+    (word)))
+
+===============================
+Commands with numeric arguments
+===============================
+
+exit 1
+
+---
+
+(program
+  (command (command_name (word)) (number)))
+
+===================================
+Commands with environment variables
+===================================
+
+VAR1=1 ./script/test
+VAR1=a VAR2="ok" git diff --word-diff=color
+
+---
+
+(program
+  (command
+    (variable_assignment (variable_name) (number))
+    (command_name (word)))
+  (command
+    (variable_assignment (variable_name) (word))
+    (variable_assignment (variable_name) (string (string_content)))
+    (command_name (word))
+    (word)
+    (word)))
+
+===================================
+Empty environment variables
+===================================
+
+VAR1=
+VAR2= echo
+
+---
+
+(program
+  (variable_assignment (variable_name))
+  (command (variable_assignment (variable_name)) (command_name (word))))
+
+===============================
+File redirects
+===============================
+
+whoami > /dev/null
+cat a b > /dev/null
+2>&1 whoami
+echo "foobar" >&2
+[ ! command -v go &>/dev/null ] && return
+
+if [ ]; then
+	>aa >bb
+fi
+
+exec {VIRTWL[0]} {VIRTWL[1]} <&- >&-
+exec {VIRTWL[0]}<&- {VIRTWL[1]}>&-
+
+grep 2>/dev/null -q "^/usr/bin/scponly$" /etc/shells
+
+---
+
+(program
+  (redirected_statement
+    (command (command_name (word)))
+    (file_redirect (word)))
+  (redirected_statement
+    (command (command_name (word)) (word) (word))
+    (file_redirect (word)))
+  (command
+    (file_redirect (file_descriptor) (number))
+    (command_name (word)))
+  (redirected_statement
+    (command (command_name (word)) (string (string_content)))
+    (file_redirect (number)))
+  (list
+    (test_command
+      (redirected_statement
+        (negated_command
+          (command (command_name (word)) (word) (word)))
+        (file_redirect (word))))
+    (command (command_name (word))))
+  (if_statement
+    (test_command)
+    (redirected_statement
+      (file_redirect (word))
+      (file_redirect (word))))
+      (redirected_statement
+    (command
+      (command_name (word))
+      (concatenation (word) (word) (word) (number) (word) (word))
+      (concatenation (word) (word) (word) (number) (word) (word)))
+    (file_redirect)
+    (file_redirect))
+  (redirected_statement
+    (command
+      (command_name (word))
+      (concatenation (word) (word) (word) (number) (word) (word)))
+    (file_redirect
+      (concatenation (word) (word) (word) (number) (word) (word)))
+    (file_redirect))
+  (redirected_statement
+    (command (command_name (word)))
+    (file_redirect (file_descriptor) (word) (word) (string (string_content)) (word))))
+
+===============================
+File redirects (noclobber override)
+===============================
+
+whoami >| /dev/null
+cat a b >| /dev/null
+
+---
+
+(program
+  (redirected_statement
+    (command (command_name (word)))
+    (file_redirect (word)))
+  (redirected_statement
+    (command (command_name (word)) (word) (word))
+    (file_redirect (word))))
+
+===============================
+Heredoc redirects
+===============================
+
+node <<JS
+console.log("hi")
+JS
+
+bash -c <<JS
+echo hi
+JS
+
+newins <<-EOF - org.freedesktop.Notifications.service
+	[D-BUS Service]
+	Name=org.freedesktop.Notifications
+	Exec=/usr/libexec/notification-daemon
+EOF
+
+---
+
+(program
+  (redirected_statement
+    (command (command_name (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body)
+      (heredoc_end)))
+  (redirected_statement
+    (command (command_name (word)) (word))
+    (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body)
+      (heredoc_end)))
+  (redirected_statement
+    (command (command_name (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (word)
+      (word)
+      (heredoc_body)
+      (heredoc_end))))
+
+===============================
+Heredocs with variables
+===============================
+
+node <<JS
+a $B ${C}
+JS
+
+exit
+
+---
+
+(program
+  (redirected_statement
+    (command
+      (command_name
+        (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body
+        (simple_expansion
+          (variable_name))
+        (heredoc_content)
+        (expansion
+          (variable_name))
+        (heredoc_content))
+      (heredoc_end)))
+  (command
+    (command_name
+      (word))))
+
+=================================
+Heredocs with file redirects
+=================================
+
+cat <<EOF > $tmpfile
+a $B ${C}
+EOF
+
+wc -l $tmpfile
+
+---
+
+(program
+  (redirected_statement
+    (command
+      (command_name
+        (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (file_redirect
+        (simple_expansion
+          (variable_name)))
+      (heredoc_body
+        (simple_expansion
+          (variable_name))
+        (heredoc_content)
+        (expansion
+          (variable_name))
+        (heredoc_content))
+      (heredoc_end)))
+  (command
+    (command_name
+      (word))
+    (word)
+    (simple_expansion
+      (variable_name))))
+
+=================================
+Heredocs with many file redirects
+=================================
+
+FOO=bar echo <<EOF 2> err.txt > hello.txt
+hello
+EOF
+
+---
+
+(program
+  (redirected_statement
+    body: (command
+      (variable_assignment
+        name: (variable_name)
+        value: (word))
+      name: (command_name
+        (word)))
+    redirect: (heredoc_redirect
+      (heredoc_start)
+      redirect: (file_redirect
+        descriptor: (file_descriptor)
+        destination: (word))
+      redirect: (file_redirect
+        destination: (word))
+      (heredoc_body)
+      (heredoc_end))))
+
+=================================
+Heredocs with pipes
+=================================
+
+one <<EOF | grep two
+three
+EOF
+
+---
+
+(program
+  (redirected_statement
+    (command
+      (command_name
+        (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (pipeline
+        (command
+          (command_name
+            (word))
+          (word)))
+      (heredoc_body)
+      (heredoc_end))))
+
+======================================
+Heredocs with escaped expansions
+======================================
+
+cat  << EOF
+DEV_NAME=\$(lsblk)
+EOF
+
+---
+
+(program (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end))))
+
+======================================
+Quoted Heredocs
+======================================
+
+cat << 'EOF'
+a=$b
+EOF
+
+cat << "EOF"
+a=$b
+EOF
+
+cat <<"END OF FILE"
+hello,
+world
+END OF FILE
+
+cat << \EOF
+EOF
+
+---
+
+(program
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end))))
+
+==========================================
+Heredocs with indented closing delimiters
+==========================================
+
+usage() {
+	cat <<-EOF
+		Usage: ${0##*/} FOO BAR
+	EOF
+}
+
+---
+
+(program
+  (function_definition
+    (word)
+    (compound_statement
+      (redirected_statement
+        (command (command_name (word)))
+        (heredoc_redirect
+          (heredoc_start)
+          (heredoc_body (expansion (special_variable_name) (regex)) (heredoc_content))
+          (heredoc_end))))))
+
+==========================================
+Heredocs with empty bodies
+==========================================
+
+node <<JS
+JS
+
+node << 'SJ'
+SJ
+
+usage() {
+	cat <<-EOF
+	EOF
+}
+
+node << 'EOF' > temp
+EOF
+
+---
+
+(program
+  (redirected_statement
+    body: (command
+      name: (command_name
+        (word)))
+    redirect: (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body)
+      (heredoc_end)))
+  (redirected_statement
+    body: (command
+      name: (command_name
+        (word)))
+    redirect: (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body)
+      (heredoc_end)))
+  (function_definition
+    name: (word)
+    body: (compound_statement
+      (redirected_statement
+        body: (command
+          name: (command_name
+            (word)))
+        redirect: (heredoc_redirect
+          (heredoc_start)
+          (heredoc_body)
+          (heredoc_end)))))
+  (redirected_statement
+    body: (command
+      name: (command_name
+        (word)))
+    redirect: (heredoc_redirect
+      (heredoc_start)
+      redirect: (file_redirect
+        destination: (word))
+      (heredoc_body)
+      (heredoc_end))))
+
+==========================================
+Heredocs with weird characters
+==========================================
+
+node <<_DELIMITER_WITH_UNDERSCORES_
+Hello.
+_DELIMITER_WITH_UNDERSCORES_
+
+node <<'```'
+Hello.
+```
+
+node <<!HEREDOC!
+Hello.
+!HEREDOC!
+
+node <<\'
+Hello.
+'
+
+node <<\\
+Hello.
+\
+
+---
+
+(program
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end))))
+
+==========================================
+Heredocs with a rhs statement
+==========================================
+
+cat <<-_EOF_ || die "cat EOF failed"
+	#!/bin/sh
+	echo hello
+_EOF_
+
+---
+
+(program
+  (redirected_statement
+    (command (command_name (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (command (command_name (word)) (string (string_content)))
+      (heredoc_body)
+      (heredoc_end))))
+
+==========================================
+Heredocs with a $ that is not an expansion
+==========================================
+
+cat <<EOF
+# check out this regex '^EOF$'
+EOF
+
+---
+
+(program
+  (redirected_statement
+    (command (command_name (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body)
+      (heredoc_end))))
+
+==========================================
+Nested Heredocs
+==========================================
+
+cat <<OUTER
+Outer Heredoc Start
+$(cat <<INNER
+Inner Heredoc Content
+$(cat <<INNERMOST
+Innermost Heredoc Content
+INNERMOST
+)
+INNER)
+Outer Heredoc End
+OUTER
+
+---
+
+(program
+  (redirected_statement
+    (command (command_name (word)))
+    (heredoc_redirect
+      (heredoc_start)
+      (heredoc_body
+        (command_substitution
+          (redirected_statement
+            (command (command_name (word)))
+            (heredoc_redirect
+              (heredoc_start)
+              (heredoc_body
+                (command_substitution
+                  (redirected_statement
+                    (command (command_name (word)))
+                    (heredoc_redirect
+                      (heredoc_start)
+                      (heredoc_body)
+                      (heredoc_end))))
+                (heredoc_content))
+              (heredoc_end))))
+        (heredoc_content))
+      (heredoc_end))))
+
+==========================================
+Herestrings
+==========================================
+
+node <<< foo
+
+while read -u 3 entry; do
+  echo $entry
+done 3<<<"$ENTRIES"
+
+$(tc-getCC) -Werror -Wl,-l:libobjc.so.${ver} -x objective-c \
+		- <<<$'int main() {}' -o /dev/null 2> /dev/null;
+
+<<<string cmd arg
+
+cmd arg <<<string
+
+cmd <<<string arg
+
+<<<string
+
+---
+
+(program
+  (command (command_name (word)) (herestring_redirect (word)))
+  (redirected_statement
+    (while_statement
+      (command (command_name (word)) (word) (number) (word))
+      (do_group
+        (command (command_name (word)) (simple_expansion (variable_name)))))
+    (herestring_redirect
+      (file_descriptor)
+      (string (simple_expansion (variable_name)))))
+  (redirected_statement
+    (command
+      (command_name (command_substitution (command (command_name (word)))))
+      (word)
+      (concatenation (word) (expansion (variable_name)))
+      (word)
+      (word)
+      (word)
+      (herestring_redirect (ansi_c_string))
+      (word)
+      (word))
+    (file_redirect (file_descriptor) (word)))
+  (command (herestring_redirect (word)) (command_name (word)) (word))
+  (command (command_name (word)) (word) (herestring_redirect (word)))
+  (command (command_name (word)) (herestring_redirect (word)) (word))
+  (redirected_statement (herestring_redirect (word))))
+
+==========================================
+Subscripts
+==========================================
+
+echo ${a[1 + 2]}
+
+echo ${b[1234 % 2]}
+
+${words[++counter]}
+
+${array[(($number+1))]}
+
+${array[((number+1))]}
+
+---
+
+(program
+  (command
+    (command_name (word))
+    (expansion
+      (subscript (variable_name) (binary_expression (number) (number)))))
+  (command
+    (command_name (word))
+    (expansion
+      (subscript (variable_name) (binary_expression (number) (number)))))
+  (command
+    (command_name
+      (expansion
+        (subscript (variable_name) (unary_expression (word))))))
+  (command
+    (command_name
+      (expansion
+        (subscript
+          (variable_name)
+          (arithmetic_expansion (binary_expression (simple_expansion (variable_name)) (number)))))))
+  (command
+    (command_name
+      (expansion
+        (subscript
+          (variable_name)
+          (arithmetic_expansion (binary_expression (variable_name) (number))))))))
+
+==========================================
+Bare $
+==========================================
+
+echo $
+echo "${module}"$
+echo $$
+
+---
+
+(program
+  (command (command_name (word)))
+  (command
+    (command_name (word))
+    (concatenation (string (expansion (variable_name)))))
+  (command
+    (command_name (word))
+    (simple_expansion (special_variable_name))))
+
+==========================================
+Arithmetic with command substitution
+==========================================
+
+$(( $( ver_cut 2 ) - 1 ))
+
+---
+
+(program
+  (command
+    (command_name
+      (arithmetic_expansion
+        (binary_expression
+          (command_substitution
+            (command
+              (command_name
+                (word))
+              (number)))
+          (number))))))
+
+
+==========================================
+Ralative path without dots
+==========================================
+
+bin/ls /usr/bin
+
+---
+
+(program
+  (command
+    (command_name
+      (word))
+    (word)))

bash/test/corpus/crlf.txt

@@ -0,0 +1,13 @@
+================================
+Variables with CRLF line endings
+================================
+
+A=one
+
+B=two
+
+---
+
+(program
+  (variable_assignment (variable_name) (word))
+  (variable_assignment (variable_name) (word)))

bash/test/corpus/programs.txt

@@ -0,0 +1,108 @@
+===============================
+Comments
+===============================
+
+#!/bin/bash
+# hi
+
+---
+
+(program
+  (comment)
+  (comment))
+
+===============================
+Escaped newlines
+===============================
+
+abc \
+  d \
+  e
+
+f=g   \
+  h=i \
+  j   \
+  --k
+
+---
+
+(program
+  (command
+    (command_name
+      (word))
+    (word)
+    (word))
+  (command
+    (variable_assignment
+      (variable_name)
+      (word))
+    (variable_assignment
+      (variable_name)
+      (word))
+    (command_name
+      (word))
+    (word)))
+
+=============================
+escaped newline immediately after a char
+=============================
+
+echo a \
+    b
+
+echo a\
+    b
+
+echo a\
+    b\
+    c
+
+
+-----------------------------
+
+(program
+  (command
+    (command_name
+      (word))
+    (word)
+    (word))
+  (command
+    (command_name
+      (word))
+    (word)
+    (word))
+  (command
+    (command_name
+      (word))
+    (word)
+    (word)
+    (word)))
+
+=============================
+Escaped whitespace
+=============================
+
+echo 1 \ 2 \ 3
+
+---
+
+(program
+  (command
+    (command_name
+      (word))
+    (number)
+    (number)
+    (number)))
+
+====================================
+Files without trailing terminators
+====================================
+
+echo hi
+---
+
+(program
+  (command
+    (command_name
+      (word))
+    (word)))

c/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

c/examples/malloc.c

@@ -0,0 +1,532 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include "libc.h"
+#include "atomic.h"
+#include "pthread_impl.h"
+
+#if defined(__GNUC__) && defined(__PIC__)
+#define inline inline __attribute__((always_inline))
+#endif
+
+void *__mmap(void *, size_t, int, int, int, off_t);
+int __munmap(void *, size_t);
+void *__mremap(void *, size_t, size_t, int, ...);
+int __madvise(void *, size_t, int);
+
+struct chunk {
+	size_t psize, csize;
+	struct chunk *next, *prev;
+};
+
+struct bin {
+	volatile int lock[2];
+	struct chunk *head;
+	struct chunk *tail;
+};
+
+static struct {
+	volatile uint64_t binmap;
+	struct bin bins[64];
+	volatile int free_lock[2];
+} mal;
+
+
+#define SIZE_ALIGN (4*sizeof(size_t))
+#define SIZE_MASK (-SIZE_ALIGN)
+#define OVERHEAD (2*sizeof(size_t))
+#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN)
+#define DONTCARE 16
+#define RECLAIM 163840
+
+#define CHUNK_SIZE(c) ((c)->csize & -2)
+#define CHUNK_PSIZE(c) ((c)->psize & -2)
+#define PREV_CHUNK(c) ((struct chunk *)((char *)(c) - CHUNK_PSIZE(c)))
+#define NEXT_CHUNK(c) ((struct chunk *)((char *)(c) + CHUNK_SIZE(c)))
+#define MEM_TO_CHUNK(p) (struct chunk *)((char *)(p) - OVERHEAD)
+#define CHUNK_TO_MEM(c) (void *)((char *)(c) + OVERHEAD)
+#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head))
+
+#define C_INUSE  ((size_t)1)
+
+#define IS_MMAPPED(c) !((c)->csize & (C_INUSE))
+
+
+/* Synchronization tools */
+
+static inline void lock(volatile int *lk)
+{
+	if (libc.threads_minus_1)
+		while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
+}
+
+static inline void unlock(volatile int *lk)
+{
+	if (lk[0]) {
+		a_store(lk, 0);
+		if (lk[1]) __wake(lk, 1, 1);
+	}
+}
+
+static inline void lock_bin(int i)
+{
+	lock(mal.bins[i].lock);
+	if (!mal.bins[i].head)
+		mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i);
+}
+
+static inline void unlock_bin(int i)
+{
+	unlock(mal.bins[i].lock);
+}
+
+static int first_set(uint64_t x)
+{
+#if 1
+	return a_ctz_64(x);
+#else
+	static const char debruijn64[64] = {
+		0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
+		62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
+		63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
+		51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
+	};
+	static const char debruijn32[32] = {
+		0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
+		31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
+	};
+	if (sizeof(long) < 8) {
+		uint32_t y = x;
+		if (!y) {
+			y = x>>32;
+			return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
+		}
+		return debruijn32[(y&-y)*0x076be629 >> 27];
+	}
+	return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
+#endif
+}
+
+static const unsigned char bin_tab[60] = {
+	            32,33,34,35,36,36,37,37,38,38,39,39,
+	40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43,
+	44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,
+	46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47,
+};
+
+static int bin_index(size_t x)
+{
+	x = x / SIZE_ALIGN - 1;
+	if (x <= 32) return x;
+	if (x < 512) return bin_tab[x/8-4];
+	if (x > 0x1c00) return 63;
+	return bin_tab[x/128-4] + 16;
+}
+
+static int bin_index_up(size_t x)
+{
+	x = x / SIZE_ALIGN - 1;
+	if (x <= 32) return x;
+	x--;
+	if (x < 512) return bin_tab[x/8-4] + 1;
+	return bin_tab[x/128-4] + 17;
+}
+
+#if 0
+void __dump_heap(int x)
+{
+	struct chunk *c;
+	int i;
+	for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c))
+		fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n",
+			c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)),
+			c->csize & 15,
+			NEXT_CHUNK(c)->psize & 15);
+	for (i=0; i<64; i++) {
+		if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) {
+			fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head);
+			if (!(mal.binmap & 1ULL<<i))
+				fprintf(stderr, "missing from binmap!\n");
+		} else if (mal.binmap & 1ULL<<i)
+			fprintf(stderr, "binmap wrongly contains %d!\n", i);
+	}
+}
+#endif
+
+void *__expand_heap(size_t *);
+
+static struct chunk *expand_heap(size_t n)
+{
+	static int heap_lock[2];
+	static void *end;
+	void *p;
+	struct chunk *w;
+
+	/* The argument n already accounts for the caller's chunk
+	 * overhead needs, but if the heap can't be extended in-place,
+	 * we need room for an extra zero-sized sentinel chunk. */
+	n += SIZE_ALIGN;
+
+	lock(heap_lock);
+
+	p = __expand_heap(&n);
+	if (!p) {
+		unlock(heap_lock);
+		return 0;
+	}
+
+	/* If not just expanding existing space, we need to make a
+	 * new sentinel chunk below the allocated space. */
+	if (p != end) {
+		/* Valid/safe because of the prologue increment. */
+		n -= SIZE_ALIGN;
+		p = (char *)p + SIZE_ALIGN;
+		w = MEM_TO_CHUNK(p);
+		w->psize = 0 | C_INUSE;
+	}
+
+	/* Record new heap end and fill in footer. */
+	end = (char *)p + n;
+	w = MEM_TO_CHUNK(end);
+	w->psize = n | C_INUSE;
+	w->csize = 0 | C_INUSE;
+
+	/* Fill in header, which may be new or may be replacing a
+	 * zero-size sentinel header at the old end-of-heap. */
+	w = MEM_TO_CHUNK(p);
+	w->csize = n | C_INUSE;
+
+	unlock(heap_lock);
+
+	return w;
+}
+
+static int adjust_size(size_t *n)
+{
+	/* Result of pointer difference must fit in ptrdiff_t. */
+	if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) {
+		if (*n) {
+			errno = ENOMEM;
+			return -1;
+		} else {
+			*n = SIZE_ALIGN;
+			return 0;
+		}
+	}
+	*n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK;
+	return 0;
+}
+
+static void unbin(struct chunk *c, int i)
+{
+	if (c->prev == c->next)
+		a_and_64(&mal.binmap, ~(1ULL<<i));
+	c->prev->next = c->next;
+	c->next->prev = c->prev;
+	c->csize |= C_INUSE;
+	NEXT_CHUNK(c)->psize |= C_INUSE;
+}
+
+static int alloc_fwd(struct chunk *c)
+{
+	int i;
+	size_t k;
+	while (!((k=c->csize) & C_INUSE)) {
+		i = bin_index(k);
+		lock_bin(i);
+		if (c->csize == k) {
+			unbin(c, i);
+			unlock_bin(i);
+			return 1;
+		}
+		unlock_bin(i);
+	}
+	return 0;
+}
+
+static int alloc_rev(struct chunk *c)
+{
+	int i;
+	size_t k;
+	while (!((k=c->psize) & C_INUSE)) {
+		i = bin_index(k);
+		lock_bin(i);
+		if (c->psize == k) {
+			unbin(PREV_CHUNK(c), i);
+			unlock_bin(i);
+			return 1;
+		}
+		unlock_bin(i);
+	}
+	return 0;
+}
+
+
+/* pretrim - trims a chunk _prior_ to removing it from its bin.
+ * Must be called with i as the ideal bin for size n, j the bin
+ * for the _free_ chunk self, and bin j locked. */
+static int pretrim(struct chunk *self, size_t n, int i, int j)
+{
+	size_t n1;
+	struct chunk *next, *split;
+
+	/* We cannot pretrim if it would require re-binning. */
+	if (j < 40) return 0;
+	if (j < i+3) {
+		if (j != 63) return 0;
+		n1 = CHUNK_SIZE(self);
+		if (n1-n <= MMAP_THRESHOLD) return 0;
+	} else {
+		n1 = CHUNK_SIZE(self);
+	}
+	if (bin_index(n1-n) != j) return 0;
+
+	next = NEXT_CHUNK(self);
+	split = (void *)((char *)self + n);
+
+	split->prev = self->prev;
+	split->next = self->next;
+	split->prev->next = split;
+	split->next->prev = split;
+	split->psize = n | C_INUSE;
+	split->csize = n1-n;
+	next->psize = n1-n;
+	self->csize = n | C_INUSE;
+	return 1;
+}
+
+static void trim(struct chunk *self, size_t n)
+{
+	size_t n1 = CHUNK_SIZE(self);
+	struct chunk *next, *split;
+
+	if (n >= n1 - DONTCARE) return;
+
+	next = NEXT_CHUNK(self);
+	split = (void *)((char *)self + n);
+
+	split->psize = n | C_INUSE;
+	split->csize = n1-n | C_INUSE;
+	next->psize = n1-n | C_INUSE;
+	self->csize = n | C_INUSE;
+
+	free(CHUNK_TO_MEM(split));
+}
+
+void *malloc(size_t n)
+{
+	struct chunk *c;
+	int i, j;
+
+	if (adjust_size(&n) < 0) return 0;
+
+	if (n > MMAP_THRESHOLD) {
+		size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE;
+		char *base = __mmap(0, len, PROT_READ|PROT_WRITE,
+			MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+		if (base == (void *)-1) return 0;
+		c = (void *)(base + SIZE_ALIGN - OVERHEAD);
+		c->csize = len - (SIZE_ALIGN - OVERHEAD);
+		c->psize = SIZE_ALIGN - OVERHEAD;
+		return CHUNK_TO_MEM(c);
+	}
+
+	i = bin_index_up(n);
+	for (;;) {
+		uint64_t mask = mal.binmap & -(1ULL<<i);
+		if (!mask) {
+			c = expand_heap(n);
+			if (!c) return 0;
+			if (alloc_rev(c)) {
+				struct chunk *x = c;
+				c = PREV_CHUNK(c);
+				NEXT_CHUNK(x)->psize = c->csize =
+					x->csize + CHUNK_SIZE(c);
+			}
+			break;
+		}
+		j = first_set(mask);
+		lock_bin(j);
+		c = mal.bins[j].head;
+		if (c != BIN_TO_CHUNK(j)) {
+			if (!pretrim(c, n, i, j)) unbin(c, j);
+			unlock_bin(j);
+			break;
+		}
+		unlock_bin(j);
+	}
+
+	/* Now patch up in case we over-allocated */
+	trim(c, n);
+
+	return CHUNK_TO_MEM(c);
+}
+
+void *__malloc0(size_t n)
+{
+	void *p = malloc(n);
+	if (p && !IS_MMAPPED(MEM_TO_CHUNK(p))) {
+		size_t *z;
+		n = (n + sizeof *z - 1)/sizeof *z;
+		for (z=p; n; n--, z++) if (*z) *z=0;
+	}
+	return p;
+}
+
+void *realloc(void *p, size_t n)
+{
+	struct chunk *self, *next;
+	size_t n0, n1;
+	void *new;
+
+	if (!p) return malloc(n);
+
+	if (adjust_size(&n) < 0) return 0;
+
+	self = MEM_TO_CHUNK(p);
+	n1 = n0 = CHUNK_SIZE(self);
+
+	if (IS_MMAPPED(self)) {
+		size_t extra = self->psize;
+		char *base = (char *)self - extra;
+		size_t oldlen = n0 + extra;
+		size_t newlen = n + extra;
+		/* Crash on realloc of freed chunk */
+		if (extra & 1) a_crash();
+		if (newlen < PAGE_SIZE && (new = malloc(n))) {
+			memcpy(new, p, n-OVERHEAD);
+			free(p);
+			return new;
+		}
+		newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE;
+		if (oldlen == newlen) return p;
+		base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE);
+		if (base == (void *)-1)
+			goto copy_realloc;
+		self = (void *)(base + extra);
+		self->csize = newlen - extra;
+		return CHUNK_TO_MEM(self);
+	}
+
+	next = NEXT_CHUNK(self);
+
+	/* Crash on corrupted footer (likely from buffer overflow) */
+	if (next->psize != self->csize) a_crash();
+
+	/* Merge adjacent chunks if we need more space. This is not
+	 * a waste of time even if we fail to get enough space, because our
+	 * subsequent call to free would otherwise have to do the merge. */
+	if (n > n1 && alloc_fwd(next)) {
+		n1 += CHUNK_SIZE(next);
+		next = NEXT_CHUNK(next);
+	}
+	/* FIXME: find what's wrong here and reenable it..? */
+	if (0 && n > n1 && alloc_rev(self)) {
+		self = PREV_CHUNK(self);
+		n1 += CHUNK_SIZE(self);
+	}
+	self->csize = n1 | C_INUSE;
+	next->psize = n1 | C_INUSE;
+
+	/* If we got enough space, split off the excess and return */
+	if (n <= n1) {
+		//memmove(CHUNK_TO_MEM(self), p, n0-OVERHEAD);
+		trim(self, n);
+		return CHUNK_TO_MEM(self);
+	}
+
+copy_realloc:
+	/* As a last resort, allocate a new chunk and copy to it. */
+	new = malloc(n-OVERHEAD);
+	if (!new) return 0;
+	memcpy(new, p, n0-OVERHEAD);
+	free(CHUNK_TO_MEM(self));
+	return new;
+}
+
+void free(void *p)
+{
+	struct chunk *self = MEM_TO_CHUNK(p);
+	struct chunk *next;
+	size_t final_size, new_size, size;
+	int reclaim=0;
+	int i;
+
+	if (!p) return;
+
+	if (IS_MMAPPED(self)) {
+		size_t extra = self->psize;
+		char *base = (char *)self - extra;
+		size_t len = CHUNK_SIZE(self) + extra;
+		/* Crash on double free */
+		if (extra & 1) a_crash();
+		__munmap(base, len);
+		return;
+	}
+
+	final_size = new_size = CHUNK_SIZE(self);
+	next = NEXT_CHUNK(self);
+
+	/* Crash on corrupted footer (likely from buffer overflow) */
+	if (next->psize != self->csize) a_crash();
+
+	for (;;) {
+		if (self->psize & next->csize & C_INUSE) {
+			self->csize = final_size | C_INUSE;
+			next->psize = final_size | C_INUSE;
+			i = bin_index(final_size);
+			lock_bin(i);
+			lock(mal.free_lock);
+			if (self->psize & next->csize & C_INUSE)
+				break;
+			unlock(mal.free_lock);
+			unlock_bin(i);
+		}
+
+		if (alloc_rev(self)) {
+			self = PREV_CHUNK(self);
+			size = CHUNK_SIZE(self);
+			final_size += size;
+			if (new_size+size > RECLAIM && (new_size+size^size) > size)
+				reclaim = 1;
+		}
+
+		if (alloc_fwd(next)) {
+			size = CHUNK_SIZE(next);
+			final_size += size;
+			if (new_size+size > RECLAIM && (new_size+size^size) > size)
+				reclaim = 1;
+			next = NEXT_CHUNK(next);
+		}
+	}
+
+	if (!(mal.binmap & 1ULL<<i))
+		a_or_64(&mal.binmap, 1ULL<<i);
+
+	self->csize = final_size;
+	next->psize = final_size;
+	unlock(mal.free_lock);
+
+	self->next = BIN_TO_CHUNK(i);
+	self->prev = mal.bins[i].tail;
+	self->next->prev = self;
+	self->prev->next = self;
+
+	/* Replace middle of large chunks with fresh zero pages */
+	if (reclaim) {
+		uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE;
+		uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE;
+#if 1
+		__madvise((void *)a, b-a, MADV_DONTNEED);
+#else
+		__mmap((void *)a, b-a, PROT_READ|PROT_WRITE,
+			MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
+#endif
+	}
+
+	unlock_bin(i);
+}

c/test/corpus/ambiguities.txt

@@ -0,0 +1,274 @@
+================================================================================
+pointer declarations vs expressions
+================================================================================
+
+TSLanguage *(*lang_parser)(void);
+
+char (*ptr_to_array)[];
+
+int main() {
+  // declare a function pointer
+  T1 * b(T2 a);
+
+  // evaluate expressions
+  c * d(5);
+  e(f * g);
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (declaration
+    (type_identifier)
+    (pointer_declarator
+      (function_declarator
+        (parenthesized_declarator
+          (pointer_declarator
+            (identifier)))
+        (parameter_list
+          (parameter_declaration
+            (primitive_type))))))
+  (declaration
+    (primitive_type)
+    (array_declarator
+      (parenthesized_declarator
+        (pointer_declarator
+          (identifier)))))
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (comment)
+      (declaration
+        (type_identifier)
+        (pointer_declarator
+          (function_declarator
+            (identifier)
+            (parameter_list
+              (parameter_declaration
+                (type_identifier)
+                (identifier))))))
+      (comment)
+      (expression_statement
+        (binary_expression
+          (identifier)
+          (call_expression
+            (identifier)
+            (argument_list
+              (number_literal)))))
+      (expression_statement
+        (call_expression
+          (identifier)
+          (argument_list
+            (binary_expression
+              (identifier)
+              (identifier))))))))
+
+================================================================================
+casts vs multiplications
+================================================================================
+
+/*
+ * ambiguities
+ */
+
+int main() {
+  // cast
+  a((B *)c);
+
+  // parenthesized product
+  d((e * f));
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (comment)
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (comment)
+      (expression_statement
+        (call_expression
+          (identifier)
+          (argument_list
+            (cast_expression
+              (type_descriptor
+                (type_identifier)
+                (abstract_pointer_declarator))
+              (identifier)))))
+      (comment)
+      (expression_statement
+        (call_expression
+          (identifier)
+          (argument_list
+            (parenthesized_expression
+              (binary_expression
+                (identifier)
+                (identifier)))))))))
+
+================================================================================
+function-like type macros vs function calls
+================================================================================
+
+// this is a macro
+GIT_INLINE(int *) x = 5;
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (comment)
+  (declaration
+    (macro_type_specifier
+      (identifier)
+      (type_descriptor
+        (primitive_type)
+        (abstract_pointer_declarator)))
+    (init_declarator
+      (identifier)
+      (number_literal))))
+
+================================================================================
+function calls vs parenthesized declarators vs macro types
+================================================================================
+
+int main() {
+  /*
+   * Could be either:
+   *   - function call
+   *   - declaration w/ parenthesized declarator
+   *   - declaration w/ macro type, no declarator
+   */
+  ABC(d);
+
+  /*
+   * Normal declaration
+   */
+  efg hij;
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (comment)
+      (expression_statement
+        (call_expression
+          (identifier)
+          (argument_list
+            (identifier))))
+      (comment)
+      (declaration
+        (type_identifier)
+        (identifier)))))
+
+================================================================================
+Call expressions vs empty declarations w/ macros as types
+================================================================================
+
+int main() {
+  int a = 1;
+  b(a);
+  A(A *);
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (declaration
+        (primitive_type)
+        (init_declarator
+          (identifier)
+          (number_literal)))
+      (expression_statement
+        (call_expression
+          (identifier)
+          (argument_list
+            (identifier))))
+      (macro_type_specifier
+        (identifier)
+        (type_descriptor
+          (type_identifier)
+          (abstract_pointer_declarator))))))
+
+================================================================================
+Comments after for loops with ambiguities
+================================================================================
+
+int main() {
+  for (a *b = c; d; e) {
+    aff;
+  }
+
+  // a-comment
+
+  g;
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (for_statement
+        (declaration
+          (type_identifier)
+          (init_declarator
+            (pointer_declarator
+              (identifier))
+            (identifier)))
+        (identifier)
+        (identifier)
+        (compound_statement
+          (expression_statement
+            (identifier))))
+      (comment)
+      (expression_statement
+        (identifier)))))
+
+================================================================================
+Top-level macro invocations
+================================================================================
+
+DEFINE_SOMETHING(THING_A, "this is a thing a");
+DEFINE_SOMETHING(THING_B, "this is a thing b", "thanks");
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (expression_statement
+    (call_expression
+      (identifier)
+      (argument_list
+        (identifier)
+        (string_literal
+          (string_content)))))
+  (expression_statement
+    (call_expression
+      (identifier)
+      (argument_list
+        (identifier)
+        (string_literal
+          (string_content))
+        (string_literal
+          (string_content))))))

c/test/corpus/crlf.txt

@@ -0,0 +1,13 @@
+============================================
+Line comments with escaped CRLF line endings
+============================================
+
+// hello \
+   this is still a comment
+this_is_not a_comment;
+
+---
+
+(translation_unit
+  (comment)
+  (declaration (type_identifier) (identifier)))

c/test/corpus/microsoft.txt

@@ -0,0 +1,253 @@
+================================
+declaration specs
+================================
+
+struct __declspec(dllexport) s2
+{
+};
+
+union __declspec(noinline) u2 {
+};
+
+---
+
+(translation_unit
+  (struct_specifier
+    (ms_declspec_modifier
+      (identifier))
+    name: (type_identifier)
+    body: (field_declaration_list))
+  (union_specifier
+    (ms_declspec_modifier
+      (identifier))
+    name: (type_identifier)
+    body: (field_declaration_list)))
+
+================================
+pointers
+================================
+
+struct s2
+{
+    int * __restrict x;
+    int * __sptr psp;
+    int * __uptr pup;
+    int * __unaligned pup;
+};
+
+void sum2(int n, int * __restrict a, int * __restrict b,
+          int * c, int * d) {
+   int i;
+   for (i = 0; i < n; i++) {
+      a[i] = b[i] + c[i];
+      c[i] = b[i] + d[i];
+    }
+}
+
+void MyFunction(char * __uptr myValue);
+
+---
+
+(translation_unit
+  (struct_specifier
+    name: (type_identifier)
+    body: (field_declaration_list
+      (field_declaration
+        type: (primitive_type)
+        declarator: (pointer_declarator
+          (ms_pointer_modifier
+            (ms_restrict_modifier))
+          declarator: (field_identifier)))
+      (field_declaration
+        type: (primitive_type)
+        declarator: (pointer_declarator
+          (ms_pointer_modifier
+            (ms_signed_ptr_modifier))
+          declarator: (field_identifier)))
+      (field_declaration
+        type: (primitive_type)
+        declarator: (pointer_declarator
+          (ms_pointer_modifier
+            (ms_unsigned_ptr_modifier))
+          declarator: (field_identifier)))
+      (field_declaration
+        type: (primitive_type)
+        declarator: (pointer_declarator
+          (ms_pointer_modifier
+            (ms_unaligned_ptr_modifier))
+          declarator: (field_identifier)))))
+  (function_definition
+    type: (primitive_type)
+    declarator: (function_declarator
+      declarator: (identifier)
+      parameters: (parameter_list
+        (parameter_declaration
+          type: (primitive_type)
+          declarator: (identifier))
+        (parameter_declaration
+          type: (primitive_type)
+          declarator: (pointer_declarator
+            (ms_pointer_modifier
+              (ms_restrict_modifier))
+            declarator: (identifier)))
+        (parameter_declaration
+          type: (primitive_type)
+          declarator: (pointer_declarator
+            (ms_pointer_modifier
+              (ms_restrict_modifier))
+            declarator: (identifier)))
+        (parameter_declaration
+          type: (primitive_type)
+          declarator: (pointer_declarator
+            declarator: (identifier)))
+        (parameter_declaration
+          type: (primitive_type)
+          declarator: (pointer_declarator
+            declarator: (identifier)))))
+    body: (compound_statement
+      (declaration
+        type: (primitive_type)
+        declarator: (identifier))
+      (for_statement
+        initializer: (assignment_expression
+          left: (identifier)
+          right: (number_literal))
+        condition: (binary_expression
+          left: (identifier)
+          right: (identifier))
+        update: (update_expression
+          argument: (identifier))
+        body: (compound_statement
+          (expression_statement
+            (assignment_expression
+              left: (subscript_expression
+                argument: (identifier)
+                index: (identifier))
+              right: (binary_expression
+                left: (subscript_expression
+                  argument: (identifier)
+                  index: (identifier))
+                right: (subscript_expression
+                  argument: (identifier)
+                  index: (identifier)))))
+          (expression_statement
+            (assignment_expression
+              left: (subscript_expression
+                argument: (identifier)
+                index: (identifier))
+              right: (binary_expression
+                left: (subscript_expression
+                  argument: (identifier)
+                  index: (identifier))
+                right: (subscript_expression
+                  argument: (identifier)
+                  index: (identifier)))))))))
+  (declaration
+    type: (primitive_type)
+    declarator: (function_declarator
+      declarator: (identifier)
+      parameters: (parameter_list
+        (parameter_declaration
+          type: (primitive_type)
+          declarator: (pointer_declarator
+            (ms_pointer_modifier
+              (ms_unsigned_ptr_modifier))
+            declarator: (identifier)))))))
+
+================================
+call modifiers
+================================
+
+__cdecl void mymethod(){
+    return;
+}
+
+__fastcall void mymethod(){
+    return;
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (ms_call_modifier)
+    type: (primitive_type)
+    declarator: (function_declarator
+      declarator: (identifier)
+      parameters: (parameter_list))
+    body: (compound_statement
+      (return_statement)))
+  (function_definition
+    (ms_call_modifier)
+    type: (primitive_type)
+    declarator: (function_declarator
+      declarator: (identifier)
+      parameters: (parameter_list))
+    body: (compound_statement
+      (return_statement))))
+
+
+================================
+SEH exception handling
+================================
+
+int main() {
+    int arg;
+    __try {
+        __try {
+            arg = 1;
+            __leave;
+        } __except (-1) {
+            arg = 2;
+        }
+        __leave;
+        arg = 3;
+    } __finally {
+        printf("arg: %d\n", arg);
+    }
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (declaration
+        (primitive_type)
+        (identifier))
+      (seh_try_statement
+        (compound_statement
+          (seh_try_statement
+            (compound_statement
+              (expression_statement
+                (assignment_expression
+                  (identifier)
+                  (number_literal)))
+              (seh_leave_statement))
+            (seh_except_clause
+              (parenthesized_expression
+                (number_literal))
+              (compound_statement
+                (expression_statement
+                  (assignment_expression
+                    (identifier)
+                    (number_literal))))))
+          (seh_leave_statement)
+          (expression_statement
+            (assignment_expression
+              (identifier)
+              (number_literal))))
+        (seh_finally_clause
+          (compound_statement
+            (expression_statement
+              (call_expression
+                (identifier)
+                (argument_list
+                  (string_literal
+                    (string_content)
+                    (escape_sequence))
+                  (identifier))))))))))

c/test/corpus/preprocessor.txt

@@ -0,0 +1,401 @@
+================================================================================
+Include directives
+================================================================================
+
+#include "some/path.h"
+#include <stdint.h>
+#include MACRO
+#include MACRO(arg1, arg2)
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_include
+    path: (string_literal
+      (string_content)))
+  (preproc_include
+    path: (system_lib_string))
+  (preproc_include
+    path: (identifier))
+  (preproc_include
+    path: (call_expression
+      function: (identifier)
+      arguments: (argument_list
+        (identifier)
+        (identifier)))))
+
+================================================================================
+Object-like macro definitions
+================================================================================
+
+#define ONE
+    #define TWO int a = b;
+#define THREE \
+  c == d ? \
+  e : \
+  f
+#define FOUR (mno * pq)
+#define FIVE(a,b) x \
+                  + y
+#define SIX(a,   \
+            b) x \
+               + y
+#define SEVEN 7/* seven has an
+                * annoying comment */
+#define EIGHT(x) do { \
+        x = x + 1;  \
+        x = x / 2;  \
+    } while (x > 0);
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_def
+    name: (identifier))
+  (preproc_def
+    name: (identifier)
+    value: (preproc_arg))
+  (preproc_def
+    name: (identifier)
+    value: (preproc_arg))
+  (preproc_def
+    name: (identifier)
+    value: (preproc_arg))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params
+      (identifier)
+      (identifier))
+    value: (preproc_arg))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params
+      (identifier)
+      (identifier))
+    value: (preproc_arg))
+  (preproc_def
+    name: (identifier)
+    value: (preproc_arg)
+    (comment))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params
+      (identifier))
+    value: (preproc_arg)))
+
+================================================================================
+Function-like macro definitions
+================================================================================
+
+#define ONE() a
+#define TWO(b) c
+#define THREE(d, e) f
+#define FOUR(...) g
+#define FIVE(h, i, ...) j
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params)
+    value: (preproc_arg))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params
+      (identifier))
+    value: (preproc_arg))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params
+      (identifier)
+      (identifier))
+    value: (preproc_arg))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params)
+    value: (preproc_arg))
+  (preproc_function_def
+    name: (identifier)
+    parameters: (preproc_params
+      (identifier)
+      (identifier))
+    value: (preproc_arg)))
+
+================================================================================
+Ifdefs
+================================================================================
+
+#ifndef DEFINE1
+int j;
+#endif
+
+#ifdef DEFINE2
+ssize_t b;
+#define c 32
+#elif defined DEFINE3
+#else
+int b;
+#define c 16
+#endif
+
+#ifdef DEFINE2
+#else
+#  ifdef DEFINE3
+#  else
+#  endif
+#endif
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_ifdef
+    name: (identifier)
+    (declaration
+      type: (primitive_type)
+      declarator: (identifier)))
+  (preproc_ifdef
+    name: (identifier)
+    (declaration
+      type: (primitive_type)
+      declarator: (identifier))
+    (preproc_def
+      name: (identifier)
+      value: (preproc_arg))
+    alternative: (preproc_elif
+      condition: (preproc_defined
+        (identifier))
+      alternative: (preproc_else
+        (declaration
+          type: (primitive_type)
+          declarator: (identifier))
+        (preproc_def
+          name: (identifier)
+          value: (preproc_arg)))))
+  (preproc_ifdef
+    name: (identifier)
+    alternative: (preproc_else
+      (preproc_ifdef
+        name: (identifier)
+        alternative: (preproc_else)))))
+
+================================================================================
+Elifdefs
+================================================================================
+
+#ifndef DEFINE1
+int j;
+#elifndef DEFINE2
+int k;
+#endif
+
+#ifdef DEFINE2
+ssize_t b;
+#elifdef DEFINE3
+ssize_t c;
+#else
+int b;
+#endif
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_ifdef
+    (identifier)
+    (declaration
+      (primitive_type)
+      (identifier))
+    (preproc_elifdef
+      (identifier)
+      (declaration
+        (primitive_type)
+        (identifier))))
+  (preproc_ifdef
+    (identifier)
+    (declaration
+      (primitive_type)
+      (identifier))
+    (preproc_elifdef
+      (identifier)
+      (declaration
+        (primitive_type)
+        (identifier))
+      (preproc_else
+        (declaration
+          (primitive_type)
+          (identifier))))))
+
+================================================================================
+General if blocks
+================================================================================
+
+#if defined(__GNUC__) && defined(__PIC__)
+#define inline inline __attribute__((always_inline))
+#elif defined(_WIN32)
+#define something
+#elif !defined(SOMETHING_ELSE)
+#define SOMETHING_ELSE
+#else
+#include <something>
+#endif
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_if
+    condition: (binary_expression
+      left: (preproc_defined
+        (identifier))
+      right: (preproc_defined
+        (identifier)))
+    (preproc_def
+      name: (identifier)
+      value: (preproc_arg))
+    alternative: (preproc_elif
+      condition: (preproc_defined
+        (identifier))
+      (preproc_def
+        name: (identifier))
+      alternative: (preproc_elif
+        condition: (unary_expression
+          argument: (preproc_defined
+            (identifier)))
+        (preproc_def
+          name: (identifier))
+        alternative: (preproc_else
+          (preproc_include
+            path: (system_lib_string)))))))
+
+================================================================================
+Preprocessor conditionals in functions
+================================================================================
+
+int main() {
+  #if d
+    puts("1");
+  #else
+    puts("2");
+  #endif
+
+  #if a
+    return 0;
+  #elif b
+    return 1;
+  #elif c
+    return 2;
+  #else
+    return 3;
+  #endif
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (preproc_if
+        (identifier)
+        (expression_statement
+          (call_expression
+            (identifier)
+            (argument_list
+              (string_literal
+                (string_content)))))
+        (preproc_else
+          (expression_statement
+            (call_expression
+              (identifier)
+              (argument_list
+                (string_literal
+                  (string_content)))))))
+      (preproc_if
+        (identifier)
+        (return_statement
+          (number_literal))
+        (preproc_elif
+          (identifier)
+          (return_statement
+            (number_literal))
+          (preproc_elif
+            (identifier)
+            (return_statement
+              (number_literal))
+            (preproc_else
+              (return_statement
+                (number_literal)))))))))
+
+================================================================================
+Preprocessor conditionals in struct/union bodies
+================================================================================
+
+struct S {
+#ifdef _WIN32
+  LONG f2;
+#else
+  uint32_t f2;
+#endif
+};
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (struct_specifier
+    (type_identifier)
+    (field_declaration_list
+      (preproc_ifdef
+        (identifier)
+        (field_declaration
+          (type_identifier)
+          (field_identifier))
+        (preproc_else
+          (field_declaration
+            (primitive_type)
+            (field_identifier)))))))
+
+================================================================================
+Unknown preprocessor directives
+================================================================================
+
+#pragma mark - UIViewController
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_call
+    directive: (preproc_directive)
+    argument: (preproc_arg)))
+
+================================================================================
+Preprocessor expressions
+================================================================================
+
+#if A(B || C) && \
+    !D(F)
+
+uint32_t a;
+
+#endif
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (preproc_if
+    (binary_expression
+      (call_expression
+        (identifier)
+        (argument_list
+          (binary_expression
+            (identifier)
+            (identifier))))
+      (unary_expression
+        (call_expression
+          (identifier)
+          (argument_list
+            (identifier)))))
+    (declaration
+      (primitive_type)
+      (identifier))))

c/test/corpus/statements.txt

@@ -0,0 +1,535 @@
+================================================================================
+If statements
+================================================================================
+
+int main() {
+  if (a)
+    1;
+
+  if (!a) {
+    2;
+  } else {
+    3;
+  }
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (if_statement
+        (parenthesized_expression
+          (identifier))
+        (expression_statement
+          (number_literal)))
+      (if_statement
+        (parenthesized_expression
+          (unary_expression
+            (identifier)))
+        (compound_statement
+          (expression_statement
+            (number_literal)))
+        (else_clause
+          (compound_statement
+            (expression_statement
+              (number_literal))))))))
+
+================================================================================
+For loops
+================================================================================
+
+int main() {
+  for (;;)
+    1;
+
+  for (int i = 0; i < 5; next(), i++) {
+    2;
+  }
+
+  for (start(); check(); step())
+    3;
+
+  for (i = 0, j = 0, k = 0, l = 0; i < 1, j < 1; i++, j++, k++, l++)
+    1;
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (for_statement
+        (expression_statement
+          (number_literal)))
+      (for_statement
+        (declaration
+          (primitive_type)
+          (init_declarator
+            (identifier)
+            (number_literal)))
+        (binary_expression
+          (identifier)
+          (number_literal))
+        (comma_expression
+          (call_expression
+            (identifier)
+            (argument_list))
+          (update_expression
+            (identifier)))
+        (compound_statement
+          (expression_statement
+            (number_literal))))
+      (for_statement
+        (call_expression
+          (identifier)
+          (argument_list))
+        (call_expression
+          (identifier)
+          (argument_list))
+        (call_expression
+          (identifier)
+          (argument_list))
+        (expression_statement
+          (number_literal)))
+      (for_statement
+        (comma_expression
+          (assignment_expression
+            (identifier)
+            (number_literal))
+          (comma_expression
+            (assignment_expression
+              (identifier)
+              (number_literal))
+            (comma_expression
+              (assignment_expression
+                (identifier)
+                (number_literal))
+              (assignment_expression
+                (identifier)
+                (number_literal)))))
+        (comma_expression
+          (binary_expression
+            (identifier)
+            (number_literal))
+          (binary_expression
+            (identifier)
+            (number_literal)))
+        (comma_expression
+          (update_expression
+            (identifier))
+          (comma_expression
+            (update_expression
+              (identifier))
+            (comma_expression
+              (update_expression
+                (identifier))
+              (update_expression
+                (identifier)))))
+        (expression_statement
+          (number_literal))))))
+
+================================================================================
+While loops
+================================================================================
+
+int main() {
+  while (x)
+    printf("hi");
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (while_statement
+        (parenthesized_expression
+          (identifier))
+        (expression_statement
+          (call_expression
+            (identifier)
+            (argument_list
+              (string_literal
+                (string_content)))))))))
+
+================================================================================
+Labeled statements
+================================================================================
+
+void foo(T *t) {
+recur:
+  t = t->next();
+  if (t) goto recur;
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list
+        (parameter_declaration
+          (type_identifier)
+          (pointer_declarator
+            (identifier)))))
+    (compound_statement
+      (labeled_statement
+        (statement_identifier)
+        (expression_statement
+          (assignment_expression
+            (identifier)
+            (call_expression
+              (field_expression
+                (identifier)
+                (field_identifier))
+              (argument_list)))))
+      (if_statement
+        (parenthesized_expression
+          (identifier))
+        (goto_statement
+          (statement_identifier))))))
+
+================================================================================
+Switch statements
+================================================================================
+
+void foo(int a) {
+  switch (a) {
+    puts("entered switch!");
+
+    case 3:
+    case 5:
+      if (b) {
+        c();
+      }
+      break;
+
+    default:
+      c();
+      break;
+  }
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list
+        (parameter_declaration
+          (primitive_type)
+          (identifier))))
+    (compound_statement
+      (switch_statement
+        (parenthesized_expression
+          (identifier))
+        (compound_statement
+          (expression_statement
+            (call_expression
+              (identifier)
+              (argument_list
+                (string_literal
+                  (string_content)))))
+          (case_statement
+            (number_literal))
+          (case_statement
+            (number_literal)
+            (if_statement
+              (parenthesized_expression
+                (identifier))
+              (compound_statement
+                (expression_statement
+                  (call_expression
+                    (identifier)
+                    (argument_list)))))
+            (break_statement))
+          (case_statement
+            (expression_statement
+              (call_expression
+                (identifier)
+                (argument_list)))
+            (break_statement)))))))
+
+================================================================================
+Case statements separate from switch statements
+================================================================================
+
+int main() {
+  switch (count % 8) {
+  case 0:
+    do {
+      *to = *from++;
+      case 2: *to = *from++;
+      case 1: *to = *from++;
+    } while (--n > 0);
+  }
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (switch_statement
+        (parenthesized_expression
+          (binary_expression
+            (identifier)
+            (number_literal)))
+        (compound_statement
+          (case_statement
+            (number_literal)
+            (do_statement
+              (compound_statement
+                (expression_statement
+                  (assignment_expression
+                    (pointer_expression
+                      (identifier))
+                    (pointer_expression
+                      (update_expression
+                        (identifier)))))
+                (case_statement
+                  (number_literal)
+                  (expression_statement
+                    (assignment_expression
+                      (pointer_expression
+                        (identifier))
+                      (pointer_expression
+                        (update_expression
+                          (identifier))))))
+                (case_statement
+                  (number_literal)
+                  (expression_statement
+                    (assignment_expression
+                      (pointer_expression
+                        (identifier))
+                      (pointer_expression
+                        (update_expression
+                          (identifier)))))))
+              (parenthesized_expression
+                (binary_expression
+                  (update_expression
+                    (identifier))
+                  (number_literal))))))))))
+
+================================================================================
+Return statements
+================================================================================
+
+void foo() {
+  return;
+  return a;
+  return a, b;
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (return_statement)
+      (return_statement
+        (identifier))
+      (return_statement
+        (comma_expression
+          (identifier)
+          (identifier))))))
+
+================================================================================
+Comments with asterisks
+================================================================================
+
+/*************************
+ * odd number of asterisks
+ *************************/
+int a;
+
+/**************************
+ * even number of asterisks
+ **************************/
+int b;
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (comment)
+  (declaration
+    (primitive_type)
+    (identifier))
+  (comment)
+  (declaration
+    (primitive_type)
+    (identifier)))
+
+================================================================================
+Comment with multiple backslashes
+================================================================================
+
+int a = 3; // Hello \\
+World
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (declaration
+    (primitive_type)
+    (init_declarator
+      (identifier)
+      (number_literal)))
+  (comment))
+
+================================================================================
+Attributes
+================================================================================
+
+void f() {
+  [[a]] switch (b) {
+   [[c]] case 1: {}
+   case 2:
+     [[fallthrough]];
+   default:
+  }
+  [[a]] while (true) {}
+  [[a]] if (true) {}
+  [[a]] for (;;) {}
+  [[a]] return;
+  [[a]] a;
+  [[a]];
+  [[a]] label: {}
+  [[a]] goto label;
+
+  // these are c++ specific, but their bind locations should be c-compatible
+  if (true) [[likely]] {} else [[unlikely]] {}
+  do [[likely]] {} while (true);
+}
+
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (function_definition
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list))
+    (compound_statement
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (switch_statement
+          (parenthesized_expression
+            (identifier))
+          (compound_statement
+            (attributed_statement
+              (attribute_declaration
+                (attribute
+                  (identifier)))
+              (case_statement
+                (number_literal)
+                (compound_statement)))
+            (case_statement
+              (number_literal)
+              (attributed_statement
+                (attribute_declaration
+                  (attribute
+                    (identifier)))
+                (expression_statement)))
+            (case_statement))))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (while_statement
+          (parenthesized_expression
+            (true))
+          (compound_statement)))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (if_statement
+          (parenthesized_expression
+            (true))
+          (compound_statement)))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (for_statement
+          (compound_statement)))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (return_statement))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (expression_statement
+          (identifier)))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (expression_statement))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (labeled_statement
+          (statement_identifier)
+          (compound_statement)))
+      (attributed_statement
+        (attribute_declaration
+          (attribute
+            (identifier)))
+        (goto_statement
+          (statement_identifier)))
+      (comment)
+      (if_statement
+        (parenthesized_expression
+          (true))
+        (attributed_statement
+          (attribute_declaration
+            (attribute
+              (identifier)))
+          (compound_statement))
+        (else_clause
+          (attributed_statement
+            (attribute_declaration
+              (attribute
+                (identifier)))
+            (compound_statement))))
+      (do_statement
+        (attributed_statement
+          (attribute_declaration
+            (attribute
+              (identifier)))
+          (compound_statement))
+        (parenthesized_expression
+          (true))))))

c/test/corpus/types.txt

@@ -0,0 +1,80 @@
+========================================
+Primitive types
+========================================
+
+int a;
+uint8_t a;
+uint16_t a;
+uint32_t a;
+uint64_t a;
+uintptr_t a;
+
+int8_t a;
+int16_t a;
+int32_t a;
+int64_t a;
+intptr_t a;
+
+char16_t a;
+char32_t a;
+
+size_t a;
+ssize_t a;
+
+---
+
+(translation_unit
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier))
+  (declaration (primitive_type) (identifier)))
+
+========================================
+Type modifiers
+========================================
+
+void f(unsigned);
+void f(unsigned int);
+void f(signed long int);
+void f(unsigned v1);
+void f(unsigned long v2);
+
+---
+
+(translation_unit
+  (declaration
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list (parameter_declaration (sized_type_specifier)))))
+  (declaration
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list (parameter_declaration (sized_type_specifier (primitive_type))))))
+  (declaration
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list (parameter_declaration (sized_type_specifier (primitive_type))))))
+  (declaration
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list (parameter_declaration (sized_type_specifier) (identifier)))))
+  (declaration
+    (primitive_type)
+    (function_declarator
+      (identifier)
+      (parameter_list (parameter_declaration (sized_type_specifier) (identifier))))))

c/test/highlight/keywords.c

@@ -0,0 +1,6 @@
+#include <stdlib.h>
+// ^ keyword
+//        ^ string
+
+#include "something.h"
+//        ^ string

c/test/highlight/names.c

@@ -0,0 +1,33 @@
+typedef struct {
+  // ^ keyword
+  //    ^ keyword
+  a_t b;
+  // <- type
+  //  ^ property
+
+  unsigned c_t (*d)[2];
+  // ^ type
+  //       ^ type
+  //             ^ property
+}, T, V;
+// ^ type
+//    ^ type
+
+int main(const char string[SIZE]) {
+// <- type
+//  ^ function
+//        ^ keyword
+//             ^ type
+//                   ^ variable
+//                         ^ constant
+
+  return foo.bar + foo.baz();
+  // ^ keyword
+  //     ^ variable
+  //         ^ property
+  //                   ^ function
+
+error:
+  // <- label
+  return 0;
+}

nix/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Charles Strahan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nix/grammar.js

@@ -0,0 +1,411 @@
+const PREC = {
+  impl: 1,
+  or: 2,
+  and: 3,
+  eq: 4,
+  neq: 4,
+  "<": 5,
+  ">": 5,
+  leq: 5,
+  geq: 5,
+  update: 6,
+  not: 7,
+  "+": 8,
+  "-": 8,
+  "*": 9,
+  "/": 9,
+  concat: 10,
+  "?": 11,
+  negate: 12,
+};
+
+module.exports = grammar({
+  name: "nix",
+
+  extras: ($) => [/\s/, $.comment],
+
+  supertypes: ($) => [$._expression],
+
+  inline: ($) => [],
+
+  externals: ($) => [
+    $.string_fragment,
+    $._indented_string_fragment,
+    $._path_start,
+    $.path_fragment,
+    $.dollar_escape,
+    $._indented_dollar_escape,
+  ],
+
+  word: ($) => $.keyword,
+
+  conflicts: ($) => [],
+
+  rules: {
+    source_code: ($) => optional(field("expression", $._expression)),
+    _expression: ($) => $._expr_function_expression,
+
+    // Keywords go before identifiers to let them take precedence when both are expected.
+    // Workaround before https://github.com/tree-sitter/tree-sitter/pull/246
+    keyword: ($) => /if|then|else|let|inherit|in|rec|with|assert/,
+    identifier: ($) => /[a-zA-Z_][a-zA-Z0-9_\'\-]*/,
+
+    variable_expression: ($) => field("name", $.identifier),
+    integer_expression: ($) => /[0-9]+/,
+    float_expression: ($) =>
+      /(([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?/,
+
+    path_expression: ($) =>
+      seq(
+        alias($._path_start, $.path_fragment),
+        repeat(
+          choice(
+            $.path_fragment,
+            alias($._immediate_interpolation, $.interpolation)
+          )
+        )
+      ),
+
+    _hpath_start: ($) => /\~\/[a-zA-Z0-9\._\-\+\/]+/,
+    hpath_expression: ($) =>
+      seq(
+        alias($._hpath_start, $.path_fragment),
+        repeat(
+          choice(
+            $.path_fragment,
+            alias($._immediate_interpolation, $.interpolation)
+          )
+        )
+      ),
+
+    spath_expression: ($) => /<[a-zA-Z0-9\._\-\+]+(\/[a-zA-Z0-9\._\-\+]+)*>/,
+    uri_expression: ($) =>
+      /[a-zA-Z][a-zA-Z0-9\+\-\.]*:[a-zA-Z0-9%\/\?:@\&=\+\$,\-_\.\!\~\*\']+/,
+
+    _expr_function_expression: ($) =>
+      choice(
+        $.function_expression,
+        $.assert_expression,
+        $.with_expression,
+        $.let_expression,
+        $._expr_if
+      ),
+
+    function_expression: ($) =>
+      choice(
+        seq(
+          field("universal", $.identifier),
+          ":",
+          field("body", $._expr_function_expression)
+        ),
+        seq(
+          field("formals", $.formals),
+          ":",
+          field("body", $._expr_function_expression)
+        ),
+        seq(
+          field("formals", $.formals),
+          "@",
+          field("universal", $.identifier),
+          ":",
+          field("body", $._expr_function_expression)
+        ),
+        seq(
+          field("universal", $.identifier),
+          "@",
+          field("formals", $.formals),
+          ":",
+          field("body", $._expr_function_expression)
+        )
+      ),
+
+    formals: ($) =>
+      choice(
+        seq("{", "}"),
+        seq("{", commaSep1(field("formal", $.formal)), "}"),
+        seq(
+          "{",
+          commaSep1(field("formal", $.formal)),
+          ",",
+          field("ellipses", $.ellipses),
+          "}"
+        ),
+        seq("{", field("ellipses", $.ellipses), "}")
+      ),
+    formal: ($) =>
+      seq(
+        field("name", $.identifier),
+        optional(seq("?", field("default", $._expression)))
+      ),
+    ellipses: ($) => "...",
+
+    assert_expression: ($) =>
+      seq(
+        "assert",
+        field("condition", $._expression),
+        ";",
+        field("body", $._expr_function_expression)
+      ),
+    with_expression: ($) =>
+      seq(
+        "with",
+        field("environment", $._expression),
+        ";",
+        field("body", $._expr_function_expression)
+      ),
+    let_expression: ($) =>
+      seq(
+        "let",
+        optional($.binding_set),
+        "in",
+        field("body", $._expr_function_expression)
+      ),
+
+    _expr_if: ($) => choice($.if_expression, $._expr_op),
+
+    if_expression: ($) =>
+      seq(
+        "if",
+        field("condition", $._expression),
+        "then",
+        field("consequence", $._expression),
+        "else",
+        field("alternative", $._expression)
+      ),
+
+    _expr_op: ($) =>
+      choice(
+        $.has_attr_expression,
+        $.unary_expression,
+        $.binary_expression,
+        $._expr_apply_expression
+      ),
+
+    // I choose to *not* have this among the binary operators because
+    // this is the sole exception that takes an attrpath (instead of expression)
+    // as its right operand.
+    // My gut feeling is that this is:
+    //   1) better in theory, and
+    //   2) will be easier to work with in practice.
+    has_attr_expression: ($) =>
+      prec(
+        PREC["?"],
+        seq(
+          field("expression", $._expr_op),
+          field("operator", "?"),
+          field("attrpath", $.attrpath)
+        )
+      ),
+
+    unary_expression: ($) =>
+      choice(
+        ...[
+          ["!", PREC.not],
+          ["-", PREC.negate],
+        ].map(([operator, precedence]) =>
+          prec(
+            precedence,
+            seq(field("operator", operator), field("argument", $._expr_op))
+          )
+        )
+      ),
+
+    binary_expression: ($) =>
+      choice(
+        // left assoc.
+        ...[
+          ["==", PREC.eq],
+          ["!=", PREC.neq],
+          ["<", PREC["<"]],
+          ["<=", PREC.leq],
+          [">", PREC[">"]],
+          [">=", PREC.geq],
+          ["&&", PREC.and],
+          ["||", PREC.or],
+          ["+", PREC["+"]],
+          ["-", PREC["-"]],
+          ["*", PREC["*"]],
+          ["/", PREC["/"]],
+        ].map(([operator, precedence]) =>
+          prec.left(
+            precedence,
+            seq(
+              field("left", $._expr_op),
+              field("operator", operator),
+              field("right", $._expr_op)
+            )
+          )
+        ),
+        // right assoc.
+        ...[
+          ["->", PREC.impl],
+          ["//", PREC.update],
+          ["++", PREC.concat],
+        ].map(([operator, precedence]) =>
+          prec.right(
+            precedence,
+            seq(
+              field("left", $._expr_op),
+              field("operator", operator),
+              field("right", $._expr_op)
+            )
+          )
+        )
+      ),
+
+    _expr_apply_expression: ($) =>
+      choice($.apply_expression, $._expr_select_expression),
+
+    apply_expression: ($) =>
+      seq(
+        field("function", $._expr_apply_expression),
+        field("argument", $._expr_select_expression)
+      ),
+
+    _expr_select_expression: ($) => choice($.select_expression, $._expr_simple),
+
+    select_expression: ($) =>
+      choice(
+        seq(
+          field("expression", $._expr_simple),
+          ".",
+          field("attrpath", $.attrpath)
+        ),
+        seq(
+          field("expression", $._expr_simple),
+          ".",
+          field("attrpath", $.attrpath),
+          "or",
+          field("default", $._expr_select_expression)
+        )
+      ),
+
+    _expr_simple: ($) =>
+      choice(
+        $.variable_expression,
+        $.integer_expression,
+        $.float_expression,
+        $.string_expression,
+        $.indented_string_expression,
+        $.path_expression,
+        $.hpath_expression,
+        $.spath_expression,
+        $.uri_expression,
+        $.parenthesized_expression,
+        $.attrset_expression,
+        $.let_attrset_expression,
+        $.rec_attrset_expression,
+        $.list_expression
+      ),
+
+    parenthesized_expression: ($) =>
+      seq("(", field("expression", $._expression), ")"),
+
+    attrset_expression: ($) => seq("{", optional($.binding_set), "}"),
+    let_attrset_expression: ($) =>
+      seq("let", "{", optional($.binding_set), "}"),
+    rec_attrset_expression: ($) =>
+      seq("rec", "{", optional($.binding_set), "}"),
+
+    string_expression: ($) =>
+      seq(
+        '"',
+        repeat(
+          choice(
+            $.string_fragment,
+            $.interpolation,
+            choice(
+              $.escape_sequence,
+              seq($.dollar_escape, alias("$", $.string_fragment))
+            )
+          )
+        ),
+        '"'
+      ),
+
+    escape_sequence: ($) => token.immediate(/\\([^$]|\s)/), // Can also escape newline.
+
+    indented_string_expression: ($) =>
+      seq(
+        "''",
+        repeat(
+          choice(
+            alias($._indented_string_fragment, $.string_fragment),
+            $.interpolation,
+            choice(
+              alias($._indented_escape_sequence, $.escape_sequence),
+              seq(
+                alias($._indented_dollar_escape, $.dollar_escape),
+                alias("$", $.string_fragment)
+              )
+            )
+          )
+        ),
+        "''"
+      ),
+    _indented_escape_sequence: ($) => token.immediate(/'''|''\\([^$]|\s)/), // Can also escape newline.
+
+    binding_set: ($) =>
+      repeat1(field("binding", choice($.binding, $.inherit, $.inherit_from))),
+    binding: ($) =>
+      seq(
+        field("attrpath", $.attrpath),
+        "=",
+        field("expression", $._expression),
+        ";"
+      ),
+    inherit: ($) => seq("inherit", field("attrs", $.inherited_attrs), ";"),
+    inherit_from: ($) =>
+      seq(
+        "inherit",
+        "(",
+        field("expression", $._expression),
+        ")",
+        field("attrs", $.inherited_attrs),
+        ";"
+      ),
+
+    attrpath: ($) =>
+      sep1(
+        field(
+          "attr",
+          choice($.identifier, $.string_expression, $.interpolation)
+        ),
+        "."
+      ),
+
+    inherited_attrs: ($) =>
+      repeat1(
+        field(
+          "attr",
+          choice($.identifier, $.string_expression, $.interpolation)
+        )
+      ),
+
+    _immediate_interpolation: ($) =>
+      seq(token.immediate("${"), field("expression", $._expression), "}"),
+    interpolation: ($) => seq("${", field("expression", $._expression), "}"),
+
+    list_expression: ($) =>
+      seq("[", repeat(field("element", $._expr_select_expression)), "]"),
+
+    comment: ($) =>
+      token(choice(seq("#", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/"))),
+  },
+});
+
+function sep(rule, separator) {
+  return optional(sep1(rule, separator));
+}
+
+function sep1(rule, separator) {
+  return seq(rule, repeat(seq(separator, rule)));
+}
+
+function commaSep1(rule) {
+  return sep1(rule, ",");
+}
+
+function commaSep(rule) {
+  return optional(commaSep1(rule));
+}

nix/src/scanner.c

@@ -0,0 +1,238 @@
+#include <tree_sitter/parser.h>
+
+enum TokenType {
+  STRING_FRAGMENT,
+  INDENTED_STRING_FRAGMENT,
+  PATH_START,
+  PATH_FRAGMENT,
+  DOLLAR_ESCAPE,
+  INDENTED_DOLLAR_ESCAPE,
+};
+
+static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+
+static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+static bool scan_dollar_escape(TSLexer *lexer) {
+  lexer->result_symbol = DOLLAR_ESCAPE;
+  advance(lexer);
+  lexer->mark_end(lexer);
+  if (lexer->lookahead == '$') {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+static bool scan_indented_dollar_escape(TSLexer *lexer) {
+  lexer->result_symbol = INDENTED_DOLLAR_ESCAPE;
+  advance(lexer);
+  lexer->mark_end(lexer);
+  if (lexer->lookahead == '$') {
+    return true;
+  } else {
+    if (lexer->lookahead == '\\') {
+      advance(lexer);
+      if (lexer->lookahead == '$') {
+        lexer->mark_end(lexer);
+        return true;
+      }
+    }
+    return false;
+  }
+}
+
+// Here we only parse literal fragment inside a string.
+// Delimiter, interpolation and escape sequence are handled by the parser and we
+// simply stop at them.
+//
+// The implementation is inspired by tree-sitter-javascript:
+// https://github.com/tree-sitter/tree-sitter-javascript/blob/fdeb68ac8d2bd5a78b943528bb68ceda3aade2eb/src/scanner.c#L19
+static bool scan_string_fragment(TSLexer *lexer) {
+  lexer->result_symbol = STRING_FRAGMENT;
+  for (bool has_content = false;; has_content = true) {
+    lexer->mark_end(lexer);
+    switch (lexer->lookahead) {
+    case '"':
+    case '\\':
+      return has_content;
+    case '$':
+      advance(lexer);
+      if (lexer->lookahead == '{') {
+        return has_content;
+      } else if (lexer->lookahead != '"' && lexer->lookahead != '\\') {
+        // Any char following '$' other than '"', '\\' and '{' (which was
+        // handled above) should be consumed as additional string content. This
+        // means `$${` doesn't start an interpolation, but `$$${` does.
+        advance(lexer);
+      }
+      break;
+    // Simply give up on EOF or '\0'.
+    case '\0':
+      return false;
+    default:
+      advance(lexer);
+    }
+  }
+}
+
+// See comments of scan_string_fragment.
+static bool scan_indented_string_fragment(TSLexer *lexer) {
+  lexer->result_symbol = INDENTED_STRING_FRAGMENT;
+  for (bool has_content = false;; has_content = true) {
+    lexer->mark_end(lexer);
+    switch (lexer->lookahead) {
+    case '$':
+      advance(lexer);
+      if (lexer->lookahead == '{') {
+        return has_content;
+      } else if (lexer->lookahead != '\'') {
+        // Any char following '$' other than '\'' and '{' (which was handled
+        // above) should be consumed as additional string content. This means
+        // `$${` doesn't start an interpolation, but `$$${` does.
+        advance(lexer);
+      }
+      break;
+    case '\'':
+      advance(lexer);
+      if (lexer->lookahead == '\'') {
+        // Two single quotes always stop current string fragment.
+        // It can be either an end delimiter '', or escape sequences ''', ''$,
+        // ''\<any>
+        return has_content;
+      }
+      break;
+    // Simply give up on EOF or '\0'.
+    case '\0':
+      return false;
+    default:
+      advance(lexer);
+    }
+  }
+}
+
+static bool is_path_char(int32_t c) {
+  return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+         (c >= 'A' && c <= 'Z') || c == '-' || c == '+' || c == '_' ||
+         c == '.' || c == '/';
+}
+
+static bool scan_path_start(TSLexer *lexer) {
+  lexer->result_symbol = PATH_START;
+
+  bool have_sep = false;
+  bool have_after_sep = false;
+  int32_t c = lexer->lookahead;
+
+  // unlike string_fragments which which are preceded by initial token (i.e.
+  // '"') and thus will have all leading external whitespace consumed, we have
+  // no such luxury with the path_start token.
+  //
+  // so we must skip over any leading whitespace here.
+  while (c == ' ' || c == '\n' || c == '\r' || c == '\t') {
+    skip(lexer);
+    c = lexer->lookahead;
+  }
+
+  while (true) {
+    lexer->mark_end(lexer);
+    c = lexer->lookahead;
+
+    if (c == '/') {
+      have_sep = true;
+    } else if (is_path_char(c)) {
+      if (have_sep) {
+        have_after_sep = true;
+      }
+    } else if (c == '$') {
+      // starting a interpolation,
+      // so we have a valid token as long as we've seen a separator.
+      // example: a/${x}
+      return have_sep;
+    } else {
+      // we have a valid token if we've consumed anything after a separator.
+      // example: a/b
+      return have_after_sep;
+    }
+
+    advance(lexer);
+  }
+}
+
+static bool scan_path_fragment(TSLexer *lexer) {
+  lexer->result_symbol = PATH_FRAGMENT;
+
+  for (bool has_content = false;; has_content = true) {
+    lexer->mark_end(lexer);
+    if (!is_path_char(lexer->lookahead)) {
+      return has_content;
+    }
+    advance(lexer);
+  }
+}
+
+void *tree_sitter_nix_external_scanner_create() { return NULL; }
+
+bool tree_sitter_nix_external_scanner_scan(void *payload, TSLexer *lexer,
+                                           const bool *valid_symbols) {
+  // This never happens in valid grammar. Only during error recovery, everything
+  // becomes valid. See: https://github.com/tree-sitter/tree-sitter/issues/1259
+  //
+  // We should not consume any content as string fragment during error recovery,
+  // or we'll break more valid grammar below. The test 'attrset typing field
+  // following string' covers this.
+  if (valid_symbols[STRING_FRAGMENT] &&
+      valid_symbols[INDENTED_STRING_FRAGMENT] && valid_symbols[PATH_START] &&
+      valid_symbols[PATH_FRAGMENT] && valid_symbols[DOLLAR_ESCAPE] &&
+      valid_symbols[INDENTED_DOLLAR_ESCAPE]) {
+    return false;
+  } else if (valid_symbols[STRING_FRAGMENT]) {
+    if (lexer->lookahead == '\\') {
+      return scan_dollar_escape(lexer);
+    }
+    return scan_string_fragment(lexer);
+  } else if (valid_symbols[INDENTED_STRING_FRAGMENT]) {
+    if (lexer->lookahead == '\'') {
+      lexer->mark_end(lexer);
+      advance(lexer);
+      if (lexer->lookahead == '\'') {
+        return scan_indented_dollar_escape(lexer);
+      }
+    }
+    return scan_indented_string_fragment(lexer);
+  } else if (valid_symbols[PATH_FRAGMENT] && is_path_char(lexer->lookahead)) {
+    // path_fragments should be scanned as immediate tokens, with no preceding
+    // extras. so we assert that the very first token is a path character, and
+    // otherwise we fall through to the case below. example:
+    //   a/b${c} d/e${f}
+    //          ^--- note that scanning for the path_fragment will start here.
+    //               this *should* be parsed as a function application.
+    //               so we want to fall through to the path_start case below,
+    //               which will skip the whitespace and correctly scan the
+    //               following path_start.
+    //
+    // also, we want this above path_start, because wherever there's ambiguity
+    // we want to parse another fragment instead of starting a new path.
+    // example:
+    //   a/b${c}d/e${f}
+    // if we swap the precedence, we'd effectively parse the above as the
+    // following function application:
+    //   (a/b${c}) (d/e${f})
+    return scan_path_fragment(lexer);
+  } else if (valid_symbols[PATH_START]) {
+    return scan_path_start(lexer);
+  }
+
+  return false;
+}
+
+unsigned tree_sitter_nix_external_scanner_serialize(void *payload,
+                                                    char *buffer) {
+  return 0;
+}
+
+void tree_sitter_nix_external_scanner_deserialize(void *payload,
+                                                  const char *buffer,
+                                                  unsigned length) {}
+
+void tree_sitter_nix_external_scanner_destroy(void *payload) {}

nix/test/highlight/basic.nix

@@ -0,0 +1,80 @@
+{
+  or = { or = 1; }.or or 42;
+  # <- property
+  #  ^ punctuation.delimiter
+  #      ^ property
+  #                 ^ property
+  #                    ^ keyword
+  the-question = if builtins.true then "to be" else "not to be";
+  # <- property
+  #  ^ property
+  #    ^ property
+  #               ^ keyword
+  #                  ^ variable.builtin
+  #                           ^ property
+  #                                ^ keyword
+  #                                      ^ string
+  #                                             ^ keyword
+  #                                                    ^ string
+  null = if null then true else false;
+  # <- property
+  #          ^ variable.builtin
+  #                    ^ variable.builtin
+  #                              ^ variable.builtin
+  pkgs' = { inherit (pkgs) stdenv lib; };
+  # <- property
+  #   ^ property
+  #          ^ keyword
+  #                   ^ variable
+  #                         ^ property
+  #                                ^ property
+  thing' =
+    # <- property
+    let inherit (pkgs) stdenv lib;
+    # <- keyword
+    #    ^ keyword
+    #             ^ variable
+    #                   ^ property
+    #                          ^ property
+    in derivation rec {
+    # <- keyword
+      # ^ function.builtin
+      #            ^ keyword
+      pname = "thing";
+      # <- property
+      #         ^ string
+      version = "v1.2.3";
+      name = "${pname}-${version}";
+      # <- property
+      #      ^ string
+      #       ^ punctuation.special
+      #          ^ variable
+      #              ^ punctuation.special
+      #               ^ string
+      #                   ^ variable
+      #                          ^ string
+      buildInputs = with pkgs; [ thing_a thing_b ];
+      # <- property
+      #              ^ keyword
+      #                   ^ variable
+      #                           ^ variable
+      #                                    ^ variable
+    };
+  assert_bool = bool: assert lib.isBool bool; bool;
+  # <- property
+  #               ^ variable.parameter
+  #                     ^ keyword
+  #                            ^ variable
+  #                                ^ function
+  #                                       ^ variable
+  #                                             ^ variable
+  import = import ./overlays.nix { inherit pkgs; };
+  # <- property
+  #         ^ function.builtin
+  #                 ^ string.special.path
+  #                                 ^ keyword
+  #                                         ^ property
+  uri = https://github.com;
+  #      ^ string.special.uri
+  #                ^ string.special.uri
+}

python/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python/examples/compound-statement-without-trailing-newline.py

@@ -0,0 +1,3 @@
+class Foo:
+  def bar():
+    print "hi"

python/examples/crlf-line-endings.py

@@ -0,0 +1,6 @@
+print a
+
+if b:    
+    if c:
+        d
+    e

python/examples/mixed-spaces-tabs.py

@@ -0,0 +1,4 @@
+def main():
+	print "hello"
+	# 1 tab = 8 spaces in Python 2
+        return

python/examples/multiple-newlines.py

@@ -0,0 +1,25 @@
+def hi():
+
+
+
+    print "hi"
+
+
+def bye():
+    print "bye"
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

python/examples/python2-grammar-crlf.py

@@ -0,0 +1,973 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.test_support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0377, 255)
+        self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxint
+        if maxint == 2147483647:
+            self.assertEquals(-2147483647-1, -020000000000)
+            # XXX -2147483648
+            self.assert_(037777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            for s in '2147483648', '040000000000', '0x100000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxint == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
+            self.assert_(01777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            for s in '9223372036854775808', '02000000000000000000000','0x10000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxint value %r' % maxint)
+
+    def testLongIntegers(self):
+        x = 0L
+        x = 0l
+        x = 0xffffffffffffffffL
+        x = 0xffffffffffffffffl
+        x = 077777777777777777L
+        x = 077777777777777777l
+        x = 123456789012345678901234567890L
+        x = 123456789012345678901234567890l
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### 'def' NAME parameters ':' suite
+        ### parameters: '(' [varargslist] ')'
+        ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
+        ###            | ('**'|'*' '*') NAME)
+        ###            | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+        ### fpdef: NAME | '(' fplist ')'
+        ### fplist: fpdef (',' fpdef)* [',']
+        ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
+        ### argument: [test '='] test   # Really [keyword '='] test
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        def f4(two, (compound, (argument, list))): pass
+        def f5((compound, first), two): pass
+        self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
+        self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
+        if sys.platform.startswith('java'):
+            self.assertEquals(f4.func_code.co_varnames,
+                   ('two', '(compound, (argument, list))', 'compound', 'argument',
+                                'list',))
+            self.assertEquals(f5.func_code.co_varnames,
+                   ('(compound, first)', 'two', 'compound', 'first'))
+        else:
+            self.assertEquals(f4.func_code.co_varnames,
+                  ('two', '.1', 'compound', 'argument',  'list'))
+            self.assertEquals(f5.func_code.co_varnames,
+                  ('.0', 'two', 'compound', 'first'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+        def v3(a, (b, c), *rest): return a, b, c, rest
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        f4(1, (2, (3, 4)))
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+        v3(1,(2,3))
+        v3(1,(2,3),4)
+        v3(1,(2,3),4,5,6,7,8,9,0)
+
+        # ceval unpacks the formal arguments into the first argcount names;
+        # thus, the names nested inside tuples must appear after these names.
+        if sys.platform.startswith('java'):
+            self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
+        else:
+            self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
+        self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+        def d31v((x)): pass
+        d31v(1)
+        def d32v((x,)): pass
+        d32v((1,))
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0L]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | print_stmt  | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testPrintStmt(self):
+        # 'print' (test ',')* [test]
+        import StringIO
+
+        # Can't test printing to real stdout without comparing output
+        # which is not available in unittest.
+        save_stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+        print 1, 2, 3
+        print 1, 2, 3,
+        print
+        print 0 or 1, 0 or 1,
+        print 0 or 1
+
+        # 'print' '>>' test ','
+        print >> sys.stdout, 1, 2, 3
+        print >> sys.stdout, 1, 2, 3,
+        print >> sys.stdout
+        print >> sys.stdout, 0 or 1, 0 or 1,
+        print >> sys.stdout, 0 or 1
+
+        # test printing to an instance
+        class Gulp:
+            def write(self, msg): pass
+
+        gulp = Gulp()
+        print >> gulp, 1, 2, 3
+        print >> gulp, 1, 2, 3,
+        print >> gulp
+        print >> gulp, 0 or 1, 0 or 1,
+        print >> gulp, 0 or 1
+
+        # test print >> None
+        def driver():
+            oldstdout = sys.stdout
+            sys.stdout = Gulp()
+            try:
+                tellme(Gulp())
+                tellme()
+            finally:
+                sys.stdout = oldstdout
+
+        # we should see this once
+        def tellme(file=sys.stdout):
+            print >> file, 'hello world'
+
+        driver()
+
+        # we should not see this at all
+        def tellme(file=None):
+            print >> file, 'goodbye universe'
+
+        driver()
+
+        self.assertEqual(sys.stdout.getvalue(), '''\
+1 2 3
+1 2 3
+1 1 1
+1 2 3
+1 2 3
+1 1 1
+hello world
+''')
+        sys.stdout = save_stdout
+
+        # syntax errors
+        check_syntax_error(self, 'print ,')
+        check_syntax_error(self, 'print >> x,')
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo <> 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError, 'just testing'
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testExec(self):
+        # 'exec' expr ['in' expr [',' expr]]
+        z = None
+        del z
+        exec 'z=1+1\n'
+        if z != 2: self.fail('exec \'z=1+1\'\\n')
+        del z
+        exec 'z=1+1'
+        if z != 2: self.fail('exec \'z=1+1\'')
+        z = None
+        del z
+        import types
+        if hasattr(types, "UnicodeType"):
+            exec r"""if 1:
+            exec u'z=1+1\n'
+            if z != 2: self.fail('exec u\'z=1+1\'\\n')
+            del z
+            exec u'z=1+1'
+            if z != 2: self.fail('exec u\'z=1+1\'')"""
+        g = {}
+        exec 'z = 1' in g
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
+        g = {}
+        l = {}
+
+        import warnings
+        warnings.filterwarnings("ignore", "global statement", module="<string>")
+        exec 'global a; a = 1; b = 2' in g, l
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if l.has_key('__builtins__'): del l['__builtins__']
+        if (g, l) != ({'a':1}, {'b':2}):
+            self.fail('exec ... in g (%s), l (%s)' %(g,l))
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError, e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr [('as' | ',') expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError, msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError), msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 <> 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort()
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
+        ### dictmaker: test ':' test (',' test ':' test)* [',']
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = `x`
+        x = `1 or 2 or 3`
+        self.assertEqual(`1,2`, '(1, 2)')
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x):
+            x.decorated = True
+            return x
+        @class_decorator
+        class G:
+            pass
+        self.assertEqual(G.decorated, True)
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [None < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(g.next(), [x for x in range(10)])
+        try:
+            g.next()
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            g.next()
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print x
+            return ret
+
+        self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

python/examples/python2-grammar.py

@@ -0,0 +1,975 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.test_support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0377, 255)
+        self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxint
+        if maxint == 2147483647:
+            self.assertEquals(-2147483647-1, -020000000000)
+            # XXX -2147483648
+            self.assert_(037777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            for s in '2147483648', '040000000000', '0x100000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxint == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
+            self.assert_(01777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            for s in '9223372036854775808', '02000000000000000000000', \
+                     '0x10000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxint value %r' % maxint)
+
+    def testLongIntegers(self):
+        x = 0L
+        x = 0l
+        x = 0xffffffffffffffffL
+        x = 0xffffffffffffffffl
+        x = 077777777777777777L
+        x = 077777777777777777l
+        x = 123456789012345678901234567890L
+        x = 123456789012345678901234567890l
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### 'def' NAME parameters ':' suite
+        ### parameters: '(' [varargslist] ')'
+        ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
+        ###            | ('**'|'*' '*') NAME)
+        ###            | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+        ### fpdef: NAME | '(' fplist ')'
+        ### fplist: fpdef (',' fpdef)* [',']
+        ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
+        ### argument: [test '='] test   # Really [keyword '='] test
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        def f4(two, (compound, (argument, list))): pass
+        def f5((compound, first), two): pass
+        self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
+        self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
+        if sys.platform.startswith('java'):
+            self.assertEquals(f4.func_code.co_varnames,
+                   ('two', '(compound, (argument, list))', 'compound', 'argument',
+                                'list',))
+            self.assertEquals(f5.func_code.co_varnames,
+                   ('(compound, first)', 'two', 'compound', 'first'))
+        else:
+            self.assertEquals(f4.func_code.co_varnames,
+                  ('two', '.1', 'compound', 'argument',  'list'))
+            self.assertEquals(f5.func_code.co_varnames,
+                  ('.0', 'two', 'compound', 'first'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+        def v3(a, (b, c), *rest): return a, b, c, rest
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        f4(1, (2, (3, 4)))
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+        v3(1,(2,3))
+        v3(1,(2,3),4)
+        v3(1,(2,3),4,5,6,7,8,9,0)
+
+        # ceval unpacks the formal arguments into the first argcount names;
+        # thus, the names nested inside tuples must appear after these names.
+        if sys.platform.startswith('java'):
+            self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
+        else:
+            self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
+        self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+        def d31v((x)): pass
+        d31v(1)
+        def d32v((x,)): pass
+        d32v((1,))
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0L]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | print_stmt  | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testPrintStmt(self):
+        # 'print' (test ',')* [test]
+        import StringIO
+
+        # Can't test printing to real stdout without comparing output
+        # which is not available in unittest.
+        save_stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+        print 1, 2, 3
+        print 1, 2, 3,
+        print
+        print 0 or 1, 0 or 1,
+        print 0 or 1
+
+        # 'print' '>>' test ','
+        print >> sys.stdout, 1, 2, 3
+        print >> sys.stdout, 1, 2, 3,
+        print >> sys.stdout
+        print >> sys.stdout, 0 or 1, 0 or 1,
+        print >> sys.stdout, 0 or 1
+
+        # test printing to an instance
+        class Gulp:
+            def write(self, msg): pass
+
+        gulp = Gulp()
+        print >> gulp, 1, 2, 3
+        print >> gulp, 1, 2, 3,
+        print >> gulp
+        print >> gulp, 0 or 1, 0 or 1,
+        print >> gulp, 0 or 1
+
+        # test print >> None
+        def driver():
+            oldstdout = sys.stdout
+            sys.stdout = Gulp()
+            try:
+                tellme(Gulp())
+                tellme()
+            finally:
+                sys.stdout = oldstdout
+
+        # we should see this once
+        def tellme(file=sys.stdout):
+            print >> file, 'hello world'
+
+        driver()
+
+        # we should not see this at all
+        def tellme(file=None):
+            print >> file, 'goodbye universe'
+
+        driver()
+
+        self.assertEqual(sys.stdout.getvalue(), '''\
+1 2 3
+1 2 3
+1 1 1
+1 2 3
+1 2 3
+1 1 1
+hello world
+''')
+        sys.stdout = save_stdout
+
+        # syntax errors
+        check_syntax_error(self, 'print ,')
+        check_syntax_error(self, 'print >> x,')
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo <> 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError, 'just testing'
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testExec(self):
+        # 'exec' expr ['in' expr [',' expr]]
+        z = None
+        del z
+        exec 'z=1+1\n'
+        if z != 2: self.fail('exec \'z=1+1\'\\n')
+        del z
+        exec 'z=1+1'
+        if z != 2: self.fail('exec \'z=1+1\'')
+        z = None
+        del z
+        import types
+        if hasattr(types, "UnicodeType"):
+            exec r"""if 1:
+            exec u'z=1+1\n'
+            if z != 2: self.fail('exec u\'z=1+1\'\\n')
+            del z
+            exec u'z=1+1'
+            if z != 2: self.fail('exec u\'z=1+1\'')"""
+        g = {}
+        exec 'z = 1' in g
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
+        g = {}
+        l = {}
+
+        import warnings
+        warnings.filterwarnings("ignore", "global statement", module="<string>")
+        exec 'global a; a = 1; b = 2' in g, l
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if l.has_key('__builtins__'): del l['__builtins__']
+        if (g, l) != ({'a':1}, {'b':2}):
+            self.fail('exec ... in g (%s), l (%s)' %(g,l))
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError, e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr [('as' | ',') expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError, msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError), msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 <> 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort()
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
+        ### dictmaker: test ':' test (',' test ':' test)* [',']
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = `x`
+        x = `1 or 2 or 3`
+        self.assertEqual(`1,2`, '(1, 2)')
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x):
+            x.decorated = True
+            return x
+        @class_decorator
+        class G:
+            pass
+        self.assertEqual(G.decorated, True)
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [None < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(g.next(), [x for x in range(10)])
+        try:
+            g.next()
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            g.next()
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print x
+            return ret
+
+        self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()
+

python/examples/python3-grammar-crlf.py

@@ -0,0 +1,945 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(type(000), type(0))
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0o377, 255)
+        self.assertEquals(2147483647, 0o17777777777)
+        self.assertEquals(0b1001, 9)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxsize
+        if maxsize == 2147483647:
+            self.assertEquals(-2147483647-1, -0o20000000000)
+            # XXX -2147483648
+            self.assert_(0o37777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            self.assert_(0b1111111111111111111111111111111 > 0)
+            for s in ('2147483648', '0o40000000000', '0x100000000',
+                      '0b10000000000000000000000000000000'):
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxsize == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
+            self.assert_(0o1777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
+            for s in '9223372036854775808', '0o2000000000000000000000', \
+                     '0x10000000000000000', \
+                     '0b100000000000000000000000000000000000000000000000000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxsize value %r' % maxsize)
+
+    def testLongIntegers(self):
+        x = 0
+        x = 0xffffffffffffffff
+        x = 0Xffffffffffffffff
+        x = 0o77777777777777777
+        x = 0O77777777777777777
+        x = 123456789012345678901234567890
+        x = 0b100000000000000000000000000000000000000000000000000000000000000000000
+        x = 0B111111111111111111111111111111111111111111111111111111111111111111111
+
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+    def testEllipsis(self):
+        x = ...
+        self.assert_(x is Ellipsis)
+        self.assertRaises(SyntaxError, eval, ".. .")
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorators: decorator+
+        ### parameters: '(' [typedargslist] ')'
+        ### typedargslist: ((tfpdef ['=' test] ',')*
+        ###                ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+        ###                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+        ### tfpdef: NAME [':' test]
+        ### varargslist: ((vfpdef ['=' test] ',')*
+        ###              ('*' [vfpdef] (',' vfpdef ['=' test])*  [',' '**' vfpdef] | '**' vfpdef)
+        ###              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+        ### vfpdef: NAME
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
+        self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+
+        # keyword argument type tests
+        try:
+            str('x', **{b'foo':1 })
+        except TypeError:
+            pass
+        else:
+            self.fail('Bytes should not work as keyword argument names')
+        # keyword only argument tests
+        def pos0key1(*, key): return key
+        pos0key1(key=100)
+        def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
+        pos2key2(1, 2, k1=100)
+        pos2key2(1, 2, k1=100, k2=200)
+        pos2key2(1, 2, k2=100, k1=200)
+        def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
+        pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
+        pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # argument annotation tests
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+        def f(x:int): pass
+        self.assertEquals(f.__annotations__, {'x': int})
+        def f(*x:str): pass
+        self.assertEquals(f.__annotations__, {'x': str})
+        def f(**x:float): pass
+        self.assertEquals(f.__annotations__, {'x': float})
+        def f(x, y:1+2): pass
+        self.assertEquals(f.__annotations__, {'y': 3})
+        def f(a, b:1, c:2, d): pass
+        self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+              **k:11) -> 12: pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+        # Check for SF Bug #1697248 - mixing decorators and a return annotation
+        def null(x): return x
+        @null
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+
+        # test closures with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+        l6 = lambda x, y, *, k=20: x+y+k
+        self.assertEquals(l6(1,2), 1+2+20)
+        self.assertEquals(l6(1,2,k=10), 1+2+10)
+
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo != 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError('just testing')
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testNonlocal(self):
+        # 'nonlocal' NAME (',' NAME)*
+        x = 0
+        y = 0
+        def f():
+            nonlocal x
+            nonlocal x, y
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError as e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr ['as' expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError as msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError) as msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort(key=lambda x: x if isinstance(x, tuple) else ())
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
+        ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = {'one'}
+        x = {'one', 1,}
+        x = {'one', 'two', 'three'}
+        x = {2, 3, 4,}
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x): return x
+        @class_decorator
+        class G: pass
+
+    def testDictcomps(self):
+        # dictorsetmaker: ( (test ':' test (comp_for |
+        #                                   (',' test ':' test)* [','])) |
+        #                   (test (comp_for | (',' test)* [','])) )
+        nums = [1, 2, 3]
+        self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [0 < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(next(g), [x for x in range(10)])
+        try:
+            next(g)
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            next(g)
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print(x)
+            return ret
+
+        # the next line is not allowed anymore
+        #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

python/examples/python3-grammar.py

@@ -0,0 +1,945 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(type(000), type(0))
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0o377, 255)
+        self.assertEquals(2147483647, 0o17777777777)
+        self.assertEquals(0b1001, 9)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxsize
+        if maxsize == 2147483647:
+            self.assertEquals(-2147483647-1, -0o20000000000)
+            # XXX -2147483648
+            self.assert_(0o37777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            self.assert_(0b1111111111111111111111111111111 > 0)
+            for s in ('2147483648', '0o40000000000', '0x100000000',
+                      '0b10000000000000000000000000000000'):
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxsize == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
+            self.assert_(0o1777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
+            for s in '9223372036854775808', '0o2000000000000000000000', \
+                     '0x10000000000000000', \
+                     '0b100000000000000000000000000000000000000000000000000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxsize value %r' % maxsize)
+
+    def testLongIntegers(self):
+        x = 0
+        x = 0xffffffffffffffff
+        x = 0Xffffffffffffffff
+        x = 0o77777777777777777
+        x = 0O77777777777777777
+        x = 123456789012345678901234567890
+        x = 0b100000000000000000000000000000000000000000000000000000000000000000000
+        x = 0B111111111111111111111111111111111111111111111111111111111111111111111
+
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+    def testEllipsis(self):
+        x = ...
+        self.assert_(x is Ellipsis)
+        self.assertRaises(SyntaxError, eval, ".. .")
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorators: decorator+
+        ### parameters: '(' [typedargslist] ')'
+        ### typedargslist: ((tfpdef ['=' test] ',')*
+        ###                ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+        ###                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+        ### tfpdef: NAME [':' test]
+        ### varargslist: ((vfpdef ['=' test] ',')*
+        ###              ('*' [vfpdef] (',' vfpdef ['=' test])*  [',' '**' vfpdef] | '**' vfpdef)
+        ###              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+        ### vfpdef: NAME
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
+        self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+
+        # keyword argument type tests
+        try:
+            str('x', **{b'foo':1 })
+        except TypeError:
+            pass
+        else:
+            self.fail('Bytes should not work as keyword argument names')
+        # keyword only argument tests
+        def pos0key1(*, key): return key
+        pos0key1(key=100)
+        def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
+        pos2key2(1, 2, k1=100)
+        pos2key2(1, 2, k1=100, k2=200)
+        pos2key2(1, 2, k2=100, k1=200)
+        def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
+        pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
+        pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # argument annotation tests
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+        def f(x:int): pass
+        self.assertEquals(f.__annotations__, {'x': int})
+        def f(*x:str): pass
+        self.assertEquals(f.__annotations__, {'x': str})
+        def f(**x:float): pass
+        self.assertEquals(f.__annotations__, {'x': float})
+        def f(x, y:1+2): pass
+        self.assertEquals(f.__annotations__, {'y': 3})
+        def f(a, b:1, c:2, d): pass
+        self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+              **k:11) -> 12: pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+        # Check for SF Bug #1697248 - mixing decorators and a return annotation
+        def null(x): return x
+        @null
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+
+        # test closures with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+        l6 = lambda x, y, *, k=20: x+y+k
+        self.assertEquals(l6(1,2), 1+2+20)
+        self.assertEquals(l6(1,2,k=10), 1+2+10)
+
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo != 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError('just testing')
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testNonlocal(self):
+        # 'nonlocal' NAME (',' NAME)*
+        x = 0
+        y = 0
+        def f():
+            nonlocal x
+            nonlocal x, y
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError as e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr ['as' expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError as msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError) as msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort(key=lambda x: x if isinstance(x, tuple) else ())
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
+        ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = {'one'}
+        x = {'one', 1,}
+        x = {'one', 'two', 'three'}
+        x = {2, 3, 4,}
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x): return x
+        @class_decorator
+        class G: pass
+
+    def testDictcomps(self):
+        # dictorsetmaker: ( (test ':' test (comp_for |
+        #                                   (',' test ':' test)* [','])) |
+        #                   (test (comp_for | (',' test)* [','])) )
+        nums = [1, 2, 3]
+        self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [0 < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(next(g), [x for x in range(10)])
+        try:
+            next(g)
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            next(g)
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print(x)
+            return ret
+
+        # the next line is not allowed anymore
+        #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = f"""
+The "quick"
+brown fo{ok()}x
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

python/examples/simple-statements-without-trailing-newline.py

@@ -0,0 +1 @@
+pass; print "hi"

python/examples/tabs.py

@@ -0,0 +1,32 @@
+def set_password(args):
+	password = args.password
+	while not password  :
+		password1 = getpass("" if args.quiet else "Provide password: ")
+		password_repeat = getpass("" if args.quiet else "Repeat password:  ")
+		if password1 != password_repeat:
+			print("Passwords do not match, try again")
+		elif len(password1) < 4:
+			print("Please provide at least 4 characters")
+		else:
+			password = password1
+
+	password_hash = passwd(password)
+	cfg = BaseJSONConfigManager(config_dir=jupyter_config_dir())
+	cfg.update('jupyter_notebook_config', {
+		'NotebookApp': {
+			'password': password_hash,
+		}
+	})
+	if not args.quiet:
+		print("password stored in config dir: %s" % jupyter_config_dir())
+
+def main(argv):
+	parser = argparse.ArgumentParser(argv[0])
+	subparsers = parser.add_subparsers()
+	parser_password = subparsers.add_parser('password', help='sets a password for your notebook server')
+	parser_password.add_argument("password", help="password to set, if not given, a password will be queried for (NOTE: this may not be safe)",
+			nargs="?")
+	parser_password.add_argument("--quiet", help="suppress messages", action="store_true")
+	parser_password.set_defaults(function=set_password)
+	args = parser.parse_args(argv[1:])
+	args.function(args)

python/examples/trailing-whitespace.py

@@ -0,0 +1,6 @@
+print a    
+
+if b:    
+    if c:    
+        d
+    e     

python/src/scanner.c

@@ -0,0 +1,528 @@
+#include "tree_sitter/parser.h"
+
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#define VEC_RESIZE(vec, _cap)                                                  \
+    void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0]));           \
+    assert(tmp != NULL);                                                       \
+    (vec).data = tmp;                                                          \
+    (vec).cap = (_cap);
+
+#define VEC_GROW(vec, _cap)                                                    \
+    if ((vec).cap < (_cap)) {                                                  \
+        VEC_RESIZE((vec), (_cap));                                             \
+    }
+
+#define VEC_PUSH(vec, el)                                                      \
+    if ((vec).cap == (vec).len) {                                              \
+        VEC_RESIZE((vec), MAX(16, (vec).len * 2));                             \
+    }                                                                          \
+    (vec).data[(vec).len++] = (el);
+
+#define VEC_POP(vec) (vec).len--;
+
+#define VEC_NEW                                                                \
+    { .len = 0, .cap = 0, .data = NULL }
+
+#define VEC_BACK(vec) ((vec).data[(vec).len - 1])
+
+#define VEC_FREE(vec)                                                          \
+    {                                                                          \
+        if ((vec).data != NULL)                                                \
+            free((vec).data);                                                  \
+    }
+
+#define VEC_CLEAR(vec) (vec).len = 0;
+
+enum TokenType {
+    NEWLINE,
+    INDENT,
+    DEDENT,
+    STRING_START,
+    STRING_CONTENT,
+    ESCAPE_INTERPOLATION,
+    STRING_END,
+    COMMENT,
+    CLOSE_PAREN,
+    CLOSE_BRACKET,
+    CLOSE_BRACE,
+    EXCEPT,
+};
+
+typedef enum {
+    SingleQuote = 1 << 0,
+    DoubleQuote = 1 << 1,
+    BackQuote = 1 << 2,
+    Raw = 1 << 3,
+    Format = 1 << 4,
+    Triple = 1 << 5,
+    Bytes = 1 << 6,
+} Flags;
+
+typedef struct {
+    char flags;
+} Delimiter;
+
+static inline Delimiter new_delimiter() { return (Delimiter){0}; }
+
+static inline bool is_format(Delimiter *delimiter) {
+    return delimiter->flags & Format;
+}
+
+static inline bool is_raw(Delimiter *delimiter) {
+    return delimiter->flags & Raw;
+}
+
+static inline bool is_triple(Delimiter *delimiter) {
+    return delimiter->flags & Triple;
+}
+
+static inline bool is_bytes(Delimiter *delimiter) {
+    return delimiter->flags & Bytes;
+}
+
+static inline int32_t end_character(Delimiter *delimiter) {
+    if (delimiter->flags & SingleQuote) {
+        return '\'';
+    }
+    if (delimiter->flags & DoubleQuote) {
+        return '"';
+    }
+    if (delimiter->flags & BackQuote) {
+        return '`';
+    }
+    return 0;
+}
+
+static inline void set_format(Delimiter *delimiter) {
+    delimiter->flags |= Format;
+}
+
+static inline void set_raw(Delimiter *delimiter) { delimiter->flags |= Raw; }
+
+static inline void set_triple(Delimiter *delimiter) {
+    delimiter->flags |= Triple;
+}
+
+static inline void set_bytes(Delimiter *delimiter) {
+    delimiter->flags |= Bytes;
+}
+
+static inline void set_end_character(Delimiter *delimiter, int32_t character) {
+    switch (character) {
+        case '\'':
+            delimiter->flags |= SingleQuote;
+            break;
+        case '"':
+            delimiter->flags |= DoubleQuote;
+            break;
+        case '`':
+            delimiter->flags |= BackQuote;
+            break;
+        default:
+            assert(false);
+    }
+}
+
+typedef struct {
+    uint32_t len;
+    uint32_t cap;
+    uint16_t *data;
+} indent_vec;
+
+static indent_vec indent_vec_new() {
+    indent_vec vec = VEC_NEW;
+    vec.data = calloc(1, sizeof(uint16_t));
+    vec.cap = 1;
+    return vec;
+}
+
+typedef struct {
+    uint32_t len;
+    uint32_t cap;
+    Delimiter *data;
+} delimiter_vec;
+
+static delimiter_vec delimiter_vec_new() {
+    delimiter_vec vec = VEC_NEW;
+    vec.data = calloc(1, sizeof(Delimiter));
+    vec.cap = 1;
+    return vec;
+}
+
+typedef struct {
+    indent_vec indents;
+    delimiter_vec delimiters;
+    bool inside_f_string;
+} Scanner;
+
+static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+
+static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer,
+                                              const bool *valid_symbols) {
+    Scanner *scanner = (Scanner *)payload;
+
+    bool error_recovery_mode =
+        valid_symbols[STRING_CONTENT] && valid_symbols[INDENT];
+    bool within_brackets = valid_symbols[CLOSE_BRACE] ||
+                           valid_symbols[CLOSE_PAREN] ||
+                           valid_symbols[CLOSE_BRACKET];
+
+    bool advanced_once = false;
+    if (valid_symbols[ESCAPE_INTERPOLATION] && scanner->delimiters.len > 0 &&
+        (lexer->lookahead == '{' || lexer->lookahead == '}') &&
+        !error_recovery_mode) {
+        Delimiter delimiter = VEC_BACK(scanner->delimiters);
+        if (is_format(&delimiter)) {
+            lexer->mark_end(lexer);
+            bool is_left_brace = lexer->lookahead == '{';
+            advance(lexer);
+            advanced_once = true;
+            if ((lexer->lookahead == '{' && is_left_brace) ||
+                (lexer->lookahead == '}' && !is_left_brace)) {
+                advance(lexer);
+                lexer->mark_end(lexer);
+                lexer->result_symbol = ESCAPE_INTERPOLATION;
+                return true;
+            }
+            return false;
+        }
+    }
+
+    if (valid_symbols[STRING_CONTENT] && scanner->delimiters.len > 0 &&
+        !error_recovery_mode) {
+        Delimiter delimiter = VEC_BACK(scanner->delimiters);
+        int32_t end_char = end_character(&delimiter);
+        bool has_content = advanced_once;
+        while (lexer->lookahead) {
+            if ((advanced_once || lexer->lookahead == '{' ||
+                 lexer->lookahead == '}') &&
+                is_format(&delimiter)) {
+                lexer->mark_end(lexer);
+                lexer->result_symbol = STRING_CONTENT;
+                return has_content;
+            }
+            if (lexer->lookahead == '\\') {
+                if (is_raw(&delimiter)) {
+                    // Step over the backslash.
+                    advance(lexer);
+                    // Step over any escaped quotes.
+                    if (lexer->lookahead == end_character(&delimiter) ||
+                        lexer->lookahead == '\\') {
+                        advance(lexer);
+                    }
+                    // Step over newlines
+                    if (lexer->lookahead == '\r') {
+                        advance(lexer);
+                        if (lexer->lookahead == '\n') {
+                            advance(lexer);
+                        }
+                    } else if (lexer->lookahead == '\n') {
+                        advance(lexer);
+                    }
+                    continue;
+                }
+                if (is_bytes(&delimiter)) {
+                    lexer->mark_end(lexer);
+                    advance(lexer);
+                    if (lexer->lookahead == 'N' || lexer->lookahead == 'u' ||
+                        lexer->lookahead == 'U') {
+                        // In bytes string, \N{...}, \uXXXX and \UXXXXXXXX are
+                        // not escape sequences
+                        // https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
+                        advance(lexer);
+                    } else {
+                        lexer->result_symbol = STRING_CONTENT;
+                        return has_content;
+                    }
+                } else {
+                    lexer->mark_end(lexer);
+                    lexer->result_symbol = STRING_CONTENT;
+                    return has_content;
+                }
+            } else if (lexer->lookahead == end_char) {
+                if (is_triple(&delimiter)) {
+                    lexer->mark_end(lexer);
+                    advance(lexer);
+                    if (lexer->lookahead == end_char) {
+                        advance(lexer);
+                        if (lexer->lookahead == end_char) {
+                            if (has_content) {
+                                lexer->result_symbol = STRING_CONTENT;
+                            } else {
+                                advance(lexer);
+                                lexer->mark_end(lexer);
+                                VEC_POP(scanner->delimiters);
+                                lexer->result_symbol = STRING_END;
+                                scanner->inside_f_string = false;
+                            }
+                            return true;
+                        }
+                        lexer->mark_end(lexer);
+                        lexer->result_symbol = STRING_CONTENT;
+                        return true;
+                    }
+                    lexer->mark_end(lexer);
+                    lexer->result_symbol = STRING_CONTENT;
+                    return true;
+                }
+                if (has_content) {
+                    lexer->result_symbol = STRING_CONTENT;
+                } else {
+                    advance(lexer);
+                    VEC_POP(scanner->delimiters);
+                    lexer->result_symbol = STRING_END;
+                    scanner->inside_f_string = false;
+                }
+                lexer->mark_end(lexer);
+                return true;
+
+            } else if (lexer->lookahead == '\n' && has_content &&
+                       !is_triple(&delimiter)) {
+                return false;
+            }
+            advance(lexer);
+            has_content = true;
+        }
+    }
+
+    lexer->mark_end(lexer);
+
+    bool found_end_of_line = false;
+    uint32_t indent_length = 0;
+    int32_t first_comment_indent_length = -1;
+    for (;;) {
+        if (lexer->lookahead == '\n') {
+            found_end_of_line = true;
+            indent_length = 0;
+            skip(lexer);
+        } else if (lexer->lookahead == ' ') {
+            indent_length++;
+            skip(lexer);
+        } else if (lexer->lookahead == '\r' || lexer->lookahead == '\f') {
+            indent_length = 0;
+            skip(lexer);
+        } else if (lexer->lookahead == '\t') {
+            indent_length += 8;
+            skip(lexer);
+        } else if (lexer->lookahead == '#' &&
+                   (valid_symbols[INDENT] || valid_symbols[DEDENT] ||
+                    valid_symbols[NEWLINE] || valid_symbols[EXCEPT])) {
+            // If we haven't found an EOL yet,
+            // then this is a comment after an expression:
+            //   foo = bar # comment
+            // Just return, since we don't want to generate an indent/dedent
+            // token.
+            if (!found_end_of_line) {
+                return false;
+            }
+            if (first_comment_indent_length == -1) {
+                first_comment_indent_length = (int32_t)indent_length;
+            }
+            while (lexer->lookahead && lexer->lookahead != '\n') {
+                skip(lexer);
+            }
+            skip(lexer);
+            indent_length = 0;
+        } else if (lexer->lookahead == '\\') {
+            skip(lexer);
+            if (lexer->lookahead == '\r') {
+                skip(lexer);
+            }
+            if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
+                skip(lexer);
+            } else {
+                return false;
+            }
+        } else if (lexer->eof(lexer)) {
+            indent_length = 0;
+            found_end_of_line = true;
+            break;
+        } else {
+            break;
+        }
+    }
+
+    if (found_end_of_line) {
+        if (scanner->indents.len > 0) {
+            uint16_t current_indent_length = VEC_BACK(scanner->indents);
+
+            if (valid_symbols[INDENT] &&
+                indent_length > current_indent_length) {
+                VEC_PUSH(scanner->indents, indent_length);
+                lexer->result_symbol = INDENT;
+                return true;
+            }
+
+            bool next_tok_is_string_start = lexer->lookahead == '\"' ||
+                                            lexer->lookahead == '\'' ||
+                                            lexer->lookahead == '`';
+
+            if ((valid_symbols[DEDENT] ||
+                 (!valid_symbols[NEWLINE] &&
+                  !(valid_symbols[STRING_START] && next_tok_is_string_start) &&
+                  !within_brackets)) &&
+                indent_length < current_indent_length &&
+                !scanner->inside_f_string &&
+
+                // Wait to create a dedent token until we've consumed any
+                // comments
+                // whose indentation matches the current block.
+                first_comment_indent_length < (int32_t)current_indent_length) {
+                VEC_POP(scanner->indents);
+                lexer->result_symbol = DEDENT;
+                return true;
+            }
+        }
+
+        if (valid_symbols[NEWLINE] && !error_recovery_mode) {
+            lexer->result_symbol = NEWLINE;
+            return true;
+        }
+    }
+
+    if (first_comment_indent_length == -1 && valid_symbols[STRING_START]) {
+        Delimiter delimiter = new_delimiter();
+
+        bool has_flags = false;
+        while (lexer->lookahead) {
+            if (lexer->lookahead == 'f' || lexer->lookahead == 'F') {
+                set_format(&delimiter);
+            } else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
+                set_raw(&delimiter);
+            } else if (lexer->lookahead == 'b' || lexer->lookahead == 'B') {
+                set_bytes(&delimiter);
+            } else if (lexer->lookahead != 'u' && lexer->lookahead != 'U') {
+                break;
+            }
+            has_flags = true;
+            advance(lexer);
+        }
+
+        if (lexer->lookahead == '`') {
+            set_end_character(&delimiter, '`');
+            advance(lexer);
+            lexer->mark_end(lexer);
+        } else if (lexer->lookahead == '\'') {
+            set_end_character(&delimiter, '\'');
+            advance(lexer);
+            lexer->mark_end(lexer);
+            if (lexer->lookahead == '\'') {
+                advance(lexer);
+                if (lexer->lookahead == '\'') {
+                    advance(lexer);
+                    lexer->mark_end(lexer);
+                    set_triple(&delimiter);
+                }
+            }
+        } else if (lexer->lookahead == '"') {
+            set_end_character(&delimiter, '"');
+            advance(lexer);
+            lexer->mark_end(lexer);
+            if (lexer->lookahead == '"') {
+                advance(lexer);
+                if (lexer->lookahead == '"') {
+                    advance(lexer);
+                    lexer->mark_end(lexer);
+                    set_triple(&delimiter);
+                }
+            }
+        }
+
+        if (end_character(&delimiter)) {
+            VEC_PUSH(scanner->delimiters, delimiter);
+            lexer->result_symbol = STRING_START;
+            scanner->inside_f_string = is_format(&delimiter);
+            return true;
+        }
+        if (has_flags) {
+            return false;
+        }
+    }
+
+    return false;
+}
+
+unsigned tree_sitter_python_external_scanner_serialize(void *payload,
+                                                       char *buffer) {
+    Scanner *scanner = (Scanner *)payload;
+
+    size_t size = 0;
+
+    buffer[size++] = (char)scanner->inside_f_string;
+
+    size_t delimiter_count = scanner->delimiters.len;
+    if (delimiter_count > UINT8_MAX) {
+        delimiter_count = UINT8_MAX;
+    }
+    buffer[size++] = (char)delimiter_count;
+
+    if (delimiter_count > 0) {
+        memcpy(&buffer[size], scanner->delimiters.data, delimiter_count);
+    }
+    size += delimiter_count;
+
+    int iter = 1;
+    for (; iter < scanner->indents.len &&
+           size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
+         ++iter) {
+        buffer[size++] = (char)scanner->indents.data[iter];
+    }
+
+    return size;
+}
+
+void tree_sitter_python_external_scanner_deserialize(void *payload,
+                                                     const char *buffer,
+                                                     unsigned length) {
+    Scanner *scanner = (Scanner *)payload;
+
+    VEC_CLEAR(scanner->delimiters);
+    VEC_CLEAR(scanner->indents);
+    VEC_PUSH(scanner->indents, 0);
+
+    if (length > 0) {
+        size_t size = 0;
+
+        scanner->inside_f_string = (bool)buffer[size++];
+
+        size_t delimiter_count = (uint8_t)buffer[size++];
+        if (delimiter_count > 0) {
+            VEC_GROW(scanner->delimiters, delimiter_count);
+            scanner->delimiters.len = delimiter_count;
+            memcpy(scanner->delimiters.data, &buffer[size], delimiter_count);
+            size += delimiter_count;
+        }
+
+        for (; size < length; size++) {
+            VEC_PUSH(scanner->indents, (unsigned char)buffer[size]);
+        }
+    }
+}
+
+void *tree_sitter_python_external_scanner_create() {
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+    _Static_assert(sizeof(Delimiter) == sizeof(char), "");
+#else
+    assert(sizeof(Delimiter) == sizeof(char));
+#endif
+    Scanner *scanner = calloc(1, sizeof(Scanner));
+    scanner->indents = indent_vec_new();
+    scanner->delimiters = delimiter_vec_new();
+    tree_sitter_python_external_scanner_deserialize(scanner, NULL, 0);
+    return scanner;
+}
+
+void tree_sitter_python_external_scanner_destroy(void *payload) {
+    Scanner *scanner = (Scanner *)payload;
+    VEC_FREE(scanner->indents);
+    VEC_FREE(scanner->delimiters);
+    free(scanner);
+}

python/test/corpus/errors.txt

@@ -0,0 +1,30 @@
+====================================
+An error before a string literal
+====================================
+
+def a(b):
+    c.
+
+    """
+    d
+    """
+
+    e
+
+---
+
+(module
+  (function_definition
+    (identifier)
+    (parameters
+      (identifier))
+    (ERROR
+      (identifier))
+    (block
+      (expression_statement
+        (string
+          (string_start)
+          (string_content)
+          (string_end)))
+      (expression_statement
+        (identifier)))))

python/test/highlight/keywords.py

@@ -0,0 +1,30 @@
+if foo():
+# <- keyword
+    pass
+    # <- keyword
+elif bar():
+# <- keyword
+    pass
+else:
+# <- keyword
+    foo
+
+return
+# ^ keyword
+raise e
+# ^ keyword
+
+for i in foo():
+# <- keyword
+#   ^ variable
+#     ^ operator
+#        ^ function
+    continue
+    # <- keyword
+    break
+    # <- keyword
+
+a and b or c
+# ^ operator
+#     ^ variable
+#       ^ operator

python/test/highlight/parameters.py

@@ -0,0 +1,4 @@
+def g(h, i, /, j, *, k=100, **kwarg):
+    #       ^ operator
+    #             ^ operator
+    pass

python/test/highlight/pattern_matching.py

@@ -0,0 +1,54 @@
+match command.split():
+# ^ keyword
+    case ["quit"]:
+    # ^ keyword
+        print("Goodbye!")
+        quit_game()
+    case ["look"]:
+    # ^ keyword
+        current_room.describe()
+    case ["get", obj]:
+    # ^ keyword
+        character.get(obj, current_room)
+    case ["go", direction]:
+    # ^ keyword
+        current_room = current_room.neighbor(direction)
+    # The rest of your commands go here
+
+match command.split():
+# ^ keyword
+    case ["drop", *objects]:
+    # ^ keyword
+        for obj in objects:
+            character.drop(obj, current_room)
+
+match command.split():
+# ^ keyword
+    case ["quit"]: ... # Code omitted for brevity
+    case ["go", direction]: pass
+    case ["drop", *objects]: pass
+    case _:
+        print(f"Sorry, I couldn't understand {command!r}")
+
+match command.split():
+# ^ keyword
+    case ["north"] | ["go", "north"]:
+    # ^ keyword
+        current_room = current_room.neighbor("north")
+    case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]:
+    # ^ keyword
+        pass
+
+match = 2
+#   ^ variable
+match, a = 2, 3
+#   ^ variable
+match: int = secret
+#   ^ variable
+x, match: str = 2, "hey, what's up?"
+# <- variable
+#   ^ variable
+
+if match := re.fullmatch(r"(-)?(\d+:)?\d?\d:\d\d(\.\d*)?", time, flags=re.ASCII):
+    # ^ variable
+    return match

python/test/tags/main.py

@@ -0,0 +1,15 @@
+class MyClass:
+  #    ^ definition.class
+  def hello():
+    #  ^ definition.function
+    print "hello from MyClass"
+
+MyClass.hello()
+#        ^ reference.call
+
+def main():
+  #  ^ definition.function
+  print "Hello, world!"
+
+main()
+# <- reference.call

rust/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Maxim Sokolov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rust/examples/weird-exprs.rs

@@ -0,0 +1,260 @@
+// run-pass
+
+#![feature(generators)]
+
+#![allow(non_camel_case_types)]
+#![allow(dead_code)]
+#![allow(redundant_semicolons)]
+#![allow(unreachable_code)]
+#![allow(unused_braces, unused_must_use, unused_parens)]
+#![allow(uncommon_codepoints, confusable_idents)]
+#![allow(unused_imports)]
+#![allow(unreachable_patterns)]
+
+#![recursion_limit = "256"]
+
+extern crate core;
+use std::cell::Cell;
+use std::mem::swap;
+use std::ops::Deref;
+
+// Just a grab bag of stuff that you wouldn't want to actually write.
+
+fn strange() -> bool { let _x: bool = return true; }
+
+fn funny() {
+    fn f(_x: ()) { }
+    f(return);
+}
+
+fn what() {
+    fn the(x: &Cell<bool>) {
+        return while !x.get() { x.set(true); };
+    }
+    let i = &Cell::new(false);
+    let dont = {||the(i)};
+    dont();
+    assert!((i.get()));
+}
+
+fn zombiejesus() {
+    loop {
+        while (return) {
+            if (return) {
+                match (return) {
+                    1 => {
+                        if (return) {
+                            return
+                        } else {
+                            return
+                        }
+                    }
+                    _ => { return }
+                };
+            } else if (return) {
+                return;
+            }
+        }
+        if (return) { break; }
+    }
+}
+
+fn notsure() {
+    let mut _x: isize;
+    let mut _y = (_x = 0) == (_x = 0);
+    let mut _z = (_x = 0) < (_x = 0);
+    let _a = (_x += 0) == (_x = 0);
+    let _b = swap(&mut _y, &mut _z) == swap(&mut _y, &mut _z);
+}
+
+fn canttouchthis() -> usize {
+    fn p() -> bool { true }
+    let _a = (assert!((true)) == (assert!(p())));
+    let _c = (assert!((p())) == ());
+    let _b: bool = (println!("{}", 0) == (return 0));
+}
+
+fn angrydome() {
+    loop { if break { } }
+    let mut i = 0;
+    loop { i += 1; if i == 1 { match (continue) { 1 => { }, _ => panic!("wat") } }
+      break; }
+}
+
+fn evil_lincoln() { let _evil: () = println!("lincoln"); }
+
+fn dots() {
+    assert_eq!(String::from(".................................................."),
+               format!("{:?}", .. .. .. .. .. .. .. .. .. .. .. .. ..
+                               .. .. .. .. .. .. .. .. .. .. .. ..));
+}
+
+fn u8(u8: u8) {
+    if u8 != 0u8 {
+        assert_eq!(8u8, {
+            macro_rules! u8 {
+                (u8) => {
+                    mod u8 {
+                        pub fn u8<'u8: 'u8 + 'u8>(u8: &'u8 u8) -> &'u8 u8 {
+                            "u8";
+                            u8
+                        }
+                    }
+                };
+            }
+
+            u8!(u8);
+            let &u8: &u8 = u8::u8(&8u8);
+            ::u8(0u8);
+            u8
+        });
+    }
+}
+
+fn fishy() {
+    assert_eq!(String::from("><>"),
+               String::<>::from::<>("><>").chars::<>().rev::<>().collect::<String>());
+}
+
+fn union() {
+    union union<'union> { union: &'union union<'union>, }
+}
+
+fn special_characters() {
+    let val = !((|(..):(_,_),(|__@_|__)|__)((&*"\\",'🤔')/**/,{})=={&[..=..][..];})//
+    ;
+    assert!(!val);
+}
+
+fn punch_card() -> impl std::fmt::Debug {
+    ..=..=.. ..    .. .. .. ..    .. .. .. ..    .. ..=.. ..
+    ..=.. ..=..    .. .. .. ..    .. .. .. ..    ..=..=..=..
+    ..=.. ..=..    ..=.. ..=..    .. ..=..=..    .. ..=.. ..
+    ..=..=.. ..    ..=.. ..=..    ..=.. .. ..    .. ..=.. ..
+    ..=.. ..=..    ..=.. ..=..    .. ..=.. ..    .. ..=.. ..
+    ..=.. ..=..    ..=.. ..=..    .. .. ..=..    .. ..=.. ..
+    ..=.. ..=..    .. ..=..=..    ..=..=.. ..    .. ..=.. ..
+}
+
+fn r#match() {
+    let val: () = match match match match match () {
+        () => ()
+    } {
+        () => ()
+    } {
+        () => ()
+    } {
+        () => ()
+    } {
+        () => ()
+    };
+    assert_eq!(val, ());
+}
+
+fn i_yield() {
+    static || {
+        yield yield yield yield yield yield yield yield yield;
+    };
+}
+
+fn match_nested_if() {
+    let val = match () {
+        () if if if if true {true} else {false} {true} else {false} {true} else {false} => true,
+        _ => false,
+    };
+    assert!(val);
+}
+
+fn monkey_barrel() {
+    let val: () = ()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=();
+    assert_eq!(val, ());
+}
+
+fn 𝚌𝚘𝚗𝚝𝚒𝚗𝚞𝚎() {
+    type 𝚕𝚘𝚘𝚙 = i32;
+    fn 𝚋𝚛𝚎𝚊𝚔() -> 𝚕𝚘𝚘𝚙 {
+        let 𝚛𝚎𝚝𝚞𝚛𝚗 = 42;
+        return 𝚛𝚎𝚝𝚞𝚛𝚗;
+    }
+    assert_eq!(loop {
+        break 𝚋𝚛𝚎𝚊𝚔 ();
+    }, 42);
+}
+
+fn function() {
+    struct foo;
+    impl Deref for foo {
+        type Target = fn() -> Self;
+        fn deref(&self) -> &Self::Target {
+            &((|| foo) as _)
+        }
+    }
+    let foo = foo () ()() ()()() ()()()() ()()()()();
+}
+
+fn bathroom_stall() {
+    let mut i = 1;
+    matches!(2, _|_|_|_|_|_ if (i+=1) != (i+=1));
+    assert_eq!(i, 13);
+}
+
+fn closure_matching() {
+    let x = |_| Some(1);
+    let (|x| x) = match x(..) {
+        |_| Some(2) => |_| Some(3),
+        |_| _ => unreachable!(),
+    };
+    assert!(matches!(x(..), |_| Some(4)));
+}
+
+fn semisemisemisemisemi() {
+    ;;;;;;; ;;;;;;; ;;;    ;;; ;;
+    ;;      ;;      ;;;;  ;;;; ;;
+    ;;;;;;; ;;;;;   ;; ;;;; ;; ;;
+         ;; ;;      ;;  ;;  ;; ;;
+    ;;;;;;; ;;;;;;; ;;      ;; ;;
+}
+
+fn useful_syntax() {
+    use {{std::{{collections::{{HashMap}}}}}};
+    use ::{{{{core}, {std}}}};
+    use {{::{{core as core2}}}};
+}
+
+fn infcx() {
+    pub mod cx {
+        pub mod cx {
+            pub use super::cx;
+            pub struct Cx;
+        }
+    }
+    let _cx: cx::cx::Cx = cx::cx::cx::cx::cx::Cx;
+}
+
+pub fn main() {
+    strange();
+    funny();
+    what();
+    zombiejesus();
+    notsure();
+    canttouchthis();
+    angrydome();
+    evil_lincoln();
+    dots();
+    u8(8u8);
+    fishy();
+    union();
+    special_characters();
+    punch_card();
+    r#match();
+    i_yield();
+    match_nested_if();
+    monkey_barrel();
+    𝚌𝚘𝚗𝚝𝚒𝚗𝚞𝚎();
+    function();
+    bathroom_stall();
+    closure_matching();
+    semisemisemisemisemi();
+    useful_syntax();
+    infcx();
+}

rust/src/scanner.c

@@ -0,0 +1,186 @@
+#include <tree_sitter/parser.h>
+#include <wctype.h>
+
+enum TokenType {
+  STRING_CONTENT,
+  RAW_STRING_LITERAL,
+  FLOAT_LITERAL,
+  BLOCK_COMMENT,
+};
+
+void *tree_sitter_rust_external_scanner_create() { return NULL; }
+void tree_sitter_rust_external_scanner_destroy(void *p) {}
+void tree_sitter_rust_external_scanner_reset(void *p) {}
+unsigned tree_sitter_rust_external_scanner_serialize(void *p, char *buffer) { return 0; }
+void tree_sitter_rust_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
+
+static void advance(TSLexer *lexer) {
+  lexer->advance(lexer, false);
+}
+
+static bool is_num_char(int32_t c) {
+  return c == '_' || iswdigit(c);
+}
+
+bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
+                                            const bool *valid_symbols) {
+  if (valid_symbols[STRING_CONTENT] && !valid_symbols[FLOAT_LITERAL]) {
+    bool has_content = false;
+    for (;;) {
+      if (lexer->lookahead == '\"' || lexer->lookahead == '\\') {
+        break;
+      } else if (lexer->lookahead == 0) {
+        return false;
+      }
+      has_content = true;
+      advance(lexer);
+    }
+    lexer->result_symbol = STRING_CONTENT;
+    return has_content;
+  }
+
+  while (iswspace(lexer->lookahead)) lexer->advance(lexer, true);
+
+  if (
+    valid_symbols[RAW_STRING_LITERAL] &&
+    (lexer->lookahead == 'r' || lexer->lookahead == 'b')
+  ) {
+    lexer->result_symbol = RAW_STRING_LITERAL;
+    if (lexer->lookahead == 'b') advance(lexer);
+    if (lexer->lookahead != 'r') return false;
+    advance(lexer);
+
+    unsigned opening_hash_count = 0;
+    while (lexer->lookahead == '#') {
+      advance(lexer);
+      opening_hash_count++;
+    }
+
+    if (lexer->lookahead != '"') return false;
+    advance(lexer);
+
+    for (;;) {
+      if (lexer->lookahead == 0) {
+        return false;
+      } else if (lexer->lookahead == '"') {
+        advance(lexer);
+        unsigned hash_count = 0;
+        while (lexer->lookahead == '#' && hash_count < opening_hash_count) {
+          advance(lexer);
+          hash_count++;
+        }
+        if (hash_count == opening_hash_count) {
+          return true;
+        }
+      } else {
+        advance(lexer);
+      }
+    }
+  }
+
+  if (valid_symbols[FLOAT_LITERAL] && iswdigit(lexer->lookahead)) {
+    lexer->result_symbol = FLOAT_LITERAL;
+
+    advance(lexer);
+    while (is_num_char(lexer->lookahead)) {
+      advance(lexer);
+    }
+
+    bool has_fraction = false, has_exponent = false;
+
+    if (lexer->lookahead == '.') {
+      has_fraction = true;
+      advance(lexer);
+      if (iswalpha(lexer->lookahead)) {
+          // The dot is followed by a letter: 1.max(2) => not a float but an integer
+          return false;
+      }
+
+      if (lexer->lookahead == '.') {
+        return false;
+      }
+      while (is_num_char(lexer->lookahead)) {
+        advance(lexer);
+      }
+    }
+
+    lexer->mark_end(lexer);
+
+    if (lexer->lookahead == 'e' || lexer->lookahead == 'E') {
+      has_exponent = true;
+      advance(lexer);
+      if (lexer->lookahead == '+' || lexer->lookahead == '-') {
+        advance(lexer);
+      }
+      if (!is_num_char(lexer->lookahead)) {
+        return true;
+      }
+      advance(lexer);
+      while (is_num_char(lexer->lookahead)) {
+        advance(lexer);
+      }
+
+      lexer->mark_end(lexer);
+    }
+
+    if (!has_exponent && !has_fraction) return false;
+
+    if (lexer->lookahead != 'u' && lexer->lookahead != 'i' && lexer->lookahead != 'f') {
+      return true;
+    }
+    advance(lexer);
+    if (!iswdigit(lexer->lookahead)) {
+      return true;
+    }
+
+    while (iswdigit(lexer->lookahead)) {
+      advance(lexer);
+    }
+
+    lexer->mark_end(lexer);
+    return true;
+  }
+
+  if (lexer->lookahead == '/') {
+    advance(lexer);
+    if (lexer->lookahead != '*') return false;
+    advance(lexer);
+
+    bool after_star = false;
+    unsigned nesting_depth = 1;
+    for (;;) {
+      switch (lexer->lookahead) {
+        case '\0':
+          return false;
+        case '*':
+          advance(lexer);
+          after_star = true;
+          break;
+        case '/':
+          if (after_star) {
+            advance(lexer);
+            after_star = false;
+            nesting_depth--;
+            if (nesting_depth == 0) {
+              lexer->result_symbol = BLOCK_COMMENT;
+              return true;
+            }
+          } else {
+            advance(lexer);
+            after_star = false;
+            if (lexer->lookahead == '*') {
+              nesting_depth++;
+              advance(lexer);
+            }
+          }
+          break;
+        default:
+          advance(lexer);
+          after_star = false;
+          break;
+      }
+    }
+  }
+
+  return false;
+}

rust/test/corpus/async.txt

@@ -0,0 +1,88 @@
+================================================================================
+Async function
+================================================================================
+
+async fn abc() {}
+
+async fn main() {
+    let x = futures.await?;
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (function_item
+    (function_modifiers)
+    (identifier)
+    (parameters)
+    (block))
+  (function_item
+    (function_modifiers)
+    (identifier)
+    (parameters)
+    (block
+      (let_declaration
+        (identifier)
+        (try_expression
+          (await_expression
+            (identifier)))))))
+
+================================================================================
+Await expression
+================================================================================
+
+futures.await;
+futures.await?;
+futures.await?.await?;
+futures.await?.function().await?;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (await_expression
+      (identifier)))
+  (expression_statement
+    (try_expression
+      (await_expression
+        (identifier))))
+  (expression_statement
+    (try_expression
+      (await_expression
+        (try_expression
+          (await_expression
+            (identifier))))))
+  (expression_statement
+    (try_expression
+      (await_expression
+        (call_expression
+          (field_expression
+            (try_expression
+              (await_expression
+                (identifier)))
+            (field_identifier))
+          (arguments))))))
+
+================================================================================
+Async Block
+================================================================================
+
+async {}
+async { let x = 10; }
+async move {}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (async_block
+      (block)))
+  (expression_statement
+    (async_block
+      (block
+        (let_declaration
+          (identifier)
+          (integer_literal)))))
+  (expression_statement
+    (async_block
+      (block))))

rust/test/corpus/literals.txt

@@ -0,0 +1,193 @@
+================================================================================
+Integer literals
+================================================================================
+
+0;
+0___0;
+123;
+0usize;
+123i32;
+123u32;
+123_u32;
+0xff_u8;
+0o70_i16;
+0b1111_1111_1001_0000_i32;
+1u128;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal))
+  (expression_statement
+    (integer_literal)))
+
+================================================================================
+Floating-point literals
+================================================================================
+
+123.123;
+2.;
+123.0f64;
+0.1f64;
+0.1f32;
+12E+99_f64;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (float_literal))
+  (expression_statement
+    (float_literal))
+  (expression_statement
+    (float_literal))
+  (expression_statement
+    (float_literal))
+  (expression_statement
+    (float_literal))
+  (expression_statement
+    (float_literal)))
+
+================================================================================
+String literals
+================================================================================
+
+"";
+"abc";
+b"foo\nbar";
+"foo\
+    bar";
+"\"foo\"";
+"/* foo bar */ foo bar";
+"foo\x42\x43bar";
+"foo \x42  \x43 bar";
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (string_literal))
+  (expression_statement
+    (string_literal))
+  (expression_statement
+    (string_literal
+      (escape_sequence)))
+  (expression_statement
+    (string_literal
+      (escape_sequence)))
+  (expression_statement
+    (string_literal
+      (escape_sequence)
+      (escape_sequence)))
+  (expression_statement
+    (string_literal))
+  (expression_statement
+    (string_literal
+      (escape_sequence)
+      (escape_sequence)))
+  (expression_statement
+    (string_literal
+      (escape_sequence)
+      (escape_sequence))))
+
+================================================================================
+Raw string literals
+================================================================================
+
+r#"abc"#; r##"ok"##;
+r##"foo #"# bar"##;
+r###"foo ##"## bar"###;
+r######"foo ##### bar"######;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (raw_string_literal))
+  (expression_statement
+    (raw_string_literal))
+  (expression_statement
+    (raw_string_literal))
+  (expression_statement
+    (raw_string_literal))
+  (expression_statement
+    (raw_string_literal)))
+
+================================================================================
+Raw byte string literals
+================================================================================
+
+br#"abc"#;
+br##"abc"##;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (raw_string_literal))
+  (expression_statement
+    (raw_string_literal)))
+
+================================================================================
+Character literals
+================================================================================
+
+'a';
+'\'';
+'\0';
+b'x';
+'\t';
+'\xff';
+'\\';
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (char_literal))
+  (expression_statement
+    (char_literal))
+  (expression_statement
+    (char_literal))
+  (expression_statement
+    (char_literal))
+  (expression_statement
+    (char_literal))
+  (expression_statement
+    (char_literal))
+  (expression_statement
+    (char_literal)))
+
+================================================================================
+Boolean literals
+================================================================================
+
+true;
+false;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (boolean_literal))
+  (expression_statement
+    (boolean_literal)))

rust/test/corpus/macros.txt

@@ -0,0 +1,257 @@
+================================================================================
+Macro invocation - no arguments
+================================================================================
+
+a!();
+b![];
+c!{};
+d::e!();
+f::g::h!{};
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree)))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree)))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree)))
+  (expression_statement
+    (macro_invocation
+      (scoped_identifier
+        (identifier)
+        (identifier))
+      (token_tree)))
+  (expression_statement
+    (macro_invocation
+      (scoped_identifier
+        (scoped_identifier
+          (identifier)
+          (identifier))
+        (identifier))
+      (token_tree))))
+
+================================================================================
+Macro invocation - arbitrary tokens
+================================================================================
+
+a!(* a *);
+a!(& a &);
+a!(- a -);
+a!(b + c + +);
+a!('a'..='z');
+a!('\u{0}'..='\u{2}');
+a!('lifetime)
+default!(a);
+union!(a);
+a!($);
+a!($());
+a!($ a $);
+a!(${$([ a ])});
+a!($a $a:ident $($a);*);
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier)
+        (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (char_literal)
+        (char_literal))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (char_literal)
+        (char_literal))))
+  (macro_invocation
+    (identifier)
+    (token_tree
+      (identifier)))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree)))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree (token_tree))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree (identifier))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree (token_tree (token_tree (token_tree (identifier)))))))
+  (expression_statement
+    (macro_invocation
+      (identifier)
+      (token_tree
+        (identifier)
+        (identifier)
+        (identifier)
+        (token_tree
+          (identifier))))))
+
+================================================================================
+Macro invocation with comments
+================================================================================
+
+ok! {
+  // one
+  /* two */
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (macro_invocation
+    (identifier)
+    (token_tree
+      (line_comment)
+      (block_comment))))
+
+================================================================================
+Macro definition
+================================================================================
+
+macro_rules! say_hello {
+    () => (
+        println!("Hello!");
+    )
+}
+
+macro_rules! four {
+    () => {1 + 3};
+}
+
+macro_rules! foo {
+    (x => $e:expr) => (println!("mode X: {}", $e));
+    (y => $e:expr) => (println!("mode Y: {}", $e))
+}
+
+macro_rules! o_O {
+    (
+      $($x:expr; [ $( $y:expr ),* ]);*
+    ) => {
+      $($($x + $e),*),*
+    }
+}
+
+macro_rules! zero_or_one {
+    ($($e:expr),?) => {
+        $($e),?
+    };
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (macro_definition
+    name: (identifier)
+    (macro_rule
+      left: (token_tree_pattern)
+      right: (token_tree
+        (identifier)
+        (token_tree
+          (string_literal)))))
+  (macro_definition
+    name: (identifier)
+    (macro_rule
+      left: (token_tree_pattern)
+      right: (token_tree
+        (integer_literal)
+        (integer_literal))))
+  (macro_definition
+    name: (identifier)
+    (macro_rule
+      left: (token_tree_pattern
+        (identifier)
+        (token_binding_pattern
+          name: (metavariable)
+          type: (fragment_specifier)))
+      right: (token_tree
+        (identifier)
+        (token_tree
+          (string_literal)
+          (metavariable))))
+    (macro_rule
+      left: (token_tree_pattern
+        (identifier)
+        (token_binding_pattern
+          name: (metavariable)
+          type: (fragment_specifier)))
+      right: (token_tree
+        (identifier)
+        (token_tree
+          (string_literal)
+          (metavariable)))))
+  (macro_definition
+    name: (identifier)
+    (macro_rule
+      left: (token_tree_pattern
+        (token_repetition_pattern
+          (token_binding_pattern
+            name: (metavariable)
+            type: (fragment_specifier))
+          (token_tree_pattern
+            (token_repetition_pattern
+              (token_binding_pattern
+                name: (metavariable)
+                type: (fragment_specifier))))))
+      right: (token_tree
+        (token_repetition
+          (token_repetition
+            (metavariable)
+            (metavariable))))))
+  (macro_definition
+    name: (identifier)
+    (macro_rule
+      left: (token_tree_pattern
+        (token_repetition_pattern
+          (token_binding_pattern
+            name: (metavariable)
+            type: (fragment_specifier))))
+      right: (token_tree
+        (token_repetition
+          (metavariable))))))

rust/test/corpus/patterns.txt

@@ -0,0 +1,469 @@
+================================================================================
+Tuple struct patterns
+================================================================================
+
+match x {
+  Some(x) => "some",
+  std::None() => "none"
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (match_expression
+      (identifier)
+      (match_block
+        (match_arm
+          (match_pattern
+            (tuple_struct_pattern
+              (identifier)
+              (identifier)))
+          (string_literal))
+        (match_arm
+          (match_pattern
+            (tuple_struct_pattern
+              (scoped_identifier
+                (identifier)
+                (identifier))))
+          (string_literal))))))
+
+================================================================================
+Reference patterns
+================================================================================
+
+match x {
+  A(ref x) => x.0,
+  ref mut y => y,
+  & mut  z => z,
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (match_expression
+      (identifier)
+      (match_block
+        (match_arm
+          (match_pattern
+            (tuple_struct_pattern
+              (identifier)
+              (ref_pattern
+                (identifier))))
+          (field_expression
+            (identifier)
+            (integer_literal)))
+        (match_arm
+          (match_pattern
+            (ref_pattern
+              (mut_pattern
+                (mutable_specifier)
+                (identifier))))
+          (identifier))
+        (match_arm
+          (match_pattern
+            (reference_pattern
+              (mutable_specifier)
+              (identifier)))
+          (identifier))))))
+
+================================================================================
+Struct patterns
+================================================================================
+
+match x {
+  Person{name, age} if age < 5 => ("toddler", name),
+  Person{name: adult_name, age: _} => ("adult", adult_name),
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (match_expression
+      (identifier)
+      (match_block
+        (match_arm
+          (match_pattern
+            (struct_pattern
+              (type_identifier)
+              (field_pattern
+                (shorthand_field_identifier))
+              (field_pattern
+                (shorthand_field_identifier)))
+            (binary_expression
+              (identifier)
+              (integer_literal)))
+          (tuple_expression
+            (string_literal)
+            (identifier)))
+        (match_arm
+          (match_pattern
+            (struct_pattern
+              (type_identifier)
+              (field_pattern
+                (field_identifier)
+                (identifier))
+              (field_pattern
+                (field_identifier))))
+          (tuple_expression
+            (string_literal)
+            (identifier)))))))
+
+================================================================================
+Ignored patterns
+================================================================================
+
+match x {
+  (a, ..) => a,
+  B(..) => c,
+  D::E{f: g, ..} => g
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (match_expression
+      (identifier)
+      (match_block
+        (match_arm
+          (match_pattern
+            (tuple_pattern
+              (identifier)
+              (remaining_field_pattern)))
+          (identifier))
+        (match_arm
+          (match_pattern
+            (tuple_struct_pattern
+              (identifier)
+              (remaining_field_pattern)))
+          (identifier))
+        (match_arm
+          (match_pattern
+            (struct_pattern
+              (scoped_type_identifier
+                (identifier)
+                (type_identifier))
+              (field_pattern
+                (field_identifier)
+                (identifier))
+              (remaining_field_pattern)))
+          (identifier))))))
+
+================================================================================
+Captured patterns
+================================================================================
+
+match x {
+  a @ A(_) | b @ B(..) => a,
+  a @ 1 ... 5 => a,
+  Some(1 ... 5) => a,
+  a @ b...c => a,
+  a @ b..=c => a,
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (match_expression
+      value: (identifier)
+      body: (match_block
+        (match_arm
+          pattern: (match_pattern
+            (or_pattern
+              (captured_pattern
+                (identifier)
+                (tuple_struct_pattern
+                  type: (identifier)))
+              (captured_pattern
+                (identifier)
+                (tuple_struct_pattern
+                  type: (identifier)
+                  (remaining_field_pattern)))))
+          value: (identifier))
+        (match_arm
+          pattern: (match_pattern
+            (captured_pattern
+              (identifier)
+              (range_pattern
+                (integer_literal)
+                (integer_literal))))
+          value: (identifier))
+        (match_arm
+          pattern: (match_pattern
+            (tuple_struct_pattern
+              type: (identifier)
+              (range_pattern
+                (integer_literal)
+                (integer_literal))))
+          value: (identifier))
+        (match_arm
+          pattern: (match_pattern
+            (captured_pattern
+              (identifier)
+              (range_pattern
+                (identifier)
+                (identifier))))
+          value: (identifier))
+        (match_arm
+          pattern: (match_pattern
+            (captured_pattern
+              (identifier)
+              (range_pattern
+                (identifier)
+                (identifier))))
+          value: (identifier))))))
+
+================================================================================
+Or patterns
+================================================================================
+
+if let A(x) | B(x) = expr {
+    do_stuff_with(x);
+}
+
+while let A(x) | B(x) = expr {
+    do_stuff_with(x);
+}
+
+let Ok(index) | Err(index) = slice.binary_search(&x);
+
+for ref a | b in c {}
+
+let Ok(x) | Err(x) = binary_search(x);
+
+for A | B | C in c {}
+
+|(Ok(x) | Err(x))| expr();
+
+let ref mut x @ (A | B | C);
+
+fn foo((1 | 2 | 3): u8) {}
+
+if let x!() | y!() = () {}
+
+// Discomment after box pattern land on master
+// let box (A | B | C);
+
+// Not handled cause devs didn't got into agreement if should be acceptd or not
+// |Ok(x) | Err(x)| expr();
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (if_expression
+      condition: (let_condition
+        pattern: (or_pattern
+          (tuple_struct_pattern
+            type: (identifier)
+            (identifier))
+          (tuple_struct_pattern
+            type: (identifier)
+            (identifier)))
+        value: (identifier))
+      consequence: (block
+        (expression_statement
+          (call_expression
+            function: (identifier)
+            arguments: (arguments
+              (identifier)))))))
+  (expression_statement
+    (while_expression
+      condition: (let_condition
+        pattern: (or_pattern
+          (tuple_struct_pattern
+            type: (identifier)
+            (identifier))
+          (tuple_struct_pattern
+            type: (identifier)
+            (identifier)))
+        value: (identifier))
+      body: (block
+        (expression_statement
+          (call_expression
+            function: (identifier)
+            arguments: (arguments
+              (identifier)))))))
+  (let_declaration
+    pattern: (or_pattern
+      (tuple_struct_pattern
+        type: (identifier)
+        (identifier))
+      (tuple_struct_pattern
+        type: (identifier)
+        (identifier)))
+    value: (call_expression
+      function: (field_expression
+        value: (identifier)
+        field: (field_identifier))
+      arguments: (arguments
+        (reference_expression
+          value: (identifier)))))
+  (expression_statement
+    (for_expression
+      pattern: (or_pattern
+        (ref_pattern
+          (identifier))
+        (identifier))
+      value: (identifier)
+      body: (block)))
+  (let_declaration
+    pattern: (or_pattern
+      (tuple_struct_pattern
+        type: (identifier)
+        (identifier))
+      (tuple_struct_pattern
+        type: (identifier)
+        (identifier)))
+    value: (call_expression
+      function: (identifier)
+      arguments: (arguments
+        (identifier))))
+  (expression_statement
+    (for_expression
+      pattern: (or_pattern
+        (or_pattern
+          (identifier)
+          (identifier))
+        (identifier))
+      value: (identifier)
+      body: (block)))
+  (expression_statement
+    (closure_expression
+      parameters: (closure_parameters
+        (tuple_pattern
+          (or_pattern
+            (tuple_struct_pattern
+              type: (identifier)
+              (identifier))
+            (tuple_struct_pattern
+              type: (identifier)
+              (identifier)))))
+      body: (call_expression
+        function: (identifier)
+        arguments: (arguments))))
+  (let_declaration
+    pattern: (ref_pattern
+      (mut_pattern
+        (mutable_specifier)
+        (captured_pattern
+          (identifier)
+          (tuple_pattern
+            (or_pattern
+              (or_pattern
+                (identifier)
+                (identifier))
+              (identifier)))))))
+  (function_item
+    name: (identifier)
+    parameters: (parameters
+      (parameter
+        pattern: (tuple_pattern
+          (or_pattern
+            (or_pattern
+              (integer_literal)
+              (integer_literal))
+            (integer_literal)))
+        type: (primitive_type)))
+    body: (block))
+  (expression_statement
+    (if_expression
+      condition: (let_condition
+        pattern: (or_pattern
+          (macro_invocation
+            macro: (identifier)
+            (token_tree))
+          (macro_invocation
+            macro: (identifier)
+            (token_tree)))
+        value: (unit_expression))
+      consequence: (block)))
+  (line_comment)
+  (line_comment)
+  (line_comment)
+  (line_comment))
+
+================================================================================
+Inline const or Const blocks as pattern
+================================================================================
+
+fn foo(x: i32) {
+    const CUBE: i32 = 3.pow(3);
+    match x {
+        CUBE => println!("three cubed"),
+        _ => {}
+    }
+}
+
+fn foo(x: i32) {
+    match x {
+        const { 3.pow(3) } => println!("three cubed"),
+        _ => {}
+    }
+}
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (function_item
+    name: (identifier)
+    parameters: (parameters
+      (parameter
+        pattern: (identifier)
+        type: (primitive_type)))
+    body: (block
+      (const_item
+        name: (identifier)
+        type: (primitive_type)
+        value: (call_expression
+          function: (field_expression
+            value: (integer_literal)
+            field: (field_identifier))
+          arguments: (arguments
+            (integer_literal))))
+      (expression_statement
+        (match_expression
+          value: (identifier)
+          body: (match_block
+            (match_arm
+              pattern: (match_pattern
+                (identifier))
+              value: (macro_invocation
+                macro: (identifier)
+                (token_tree
+                  (string_literal))))
+            (match_arm
+              pattern: (match_pattern)
+              value: (block)))))))
+  (function_item
+    name: (identifier)
+    parameters: (parameters
+      (parameter
+        pattern: (identifier)
+        type: (primitive_type)))
+    body: (block
+      (expression_statement
+        (match_expression
+          value: (identifier)
+          body: (match_block
+            (match_arm
+              pattern: (match_pattern
+                (const_block
+                  body: (block
+                    (call_expression
+                      function: (field_expression
+                        value: (integer_literal)
+                        field: (field_identifier))
+                      arguments: (arguments
+                        (integer_literal))))))
+              value: (macro_invocation
+                macro: (identifier)
+                (token_tree
+                  (string_literal))))
+            (match_arm
+              pattern: (match_pattern)
+              value: (block))))))))

rust/test/corpus/source_files.txt

@@ -0,0 +1,69 @@
+============================================
+Block comments
+============================================
+
+/*
+ * Block comments
+ */
+
+/* Comment with asterisks **/
+
+----
+
+(source_file
+  (block_comment)
+  (block_comment))
+
+============================================
+Nested block comments
+============================================
+
+/* /* double nested */ */
+
+// ---
+
+/*/*/* triple nested */*/*/
+
+// ---
+
+/****
+  /****
+    nested with extra stars
+  ****/
+****/
+
+// ---
+
+----
+
+(source_file
+  (block_comment)
+  (line_comment)
+  (block_comment)
+  (line_comment)
+  (block_comment)
+  (line_comment))
+
+============================================
+Line comments
+============================================
+
+// Comment
+
+----
+
+(source_file
+  (line_comment))
+
+=====================================
+Greek letters in identifiers
+=====================================
+
+const σ1 : Σ = 0;
+const ψ_2 : Ψ = 1;
+
+---
+
+(source_file
+  (const_item (identifier) (type_identifier) (integer_literal))
+  (const_item (identifier) (type_identifier) (integer_literal)))

rust/test/corpus/types.txt

@@ -0,0 +1,384 @@
+================================================================================
+The unit type
+================================================================================
+
+type A = ();
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (unit_type)))
+
+================================================================================
+Tuple types
+================================================================================
+
+type A = (i32, String);
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (tuple_type
+      (primitive_type)
+      (type_identifier))))
+
+================================================================================
+Reference types
+================================================================================
+
+type A = &B;
+type C = &'a str;
+type D = &'a mut str;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (reference_type
+      (type_identifier)))
+  (type_item
+    (type_identifier)
+    (reference_type
+      (lifetime
+        (identifier))
+      (primitive_type)))
+  (type_item
+    (type_identifier)
+    (reference_type
+      (lifetime
+        (identifier))
+      (mutable_specifier)
+      (primitive_type))))
+
+================================================================================
+Raw pointer types
+================================================================================
+
+type A = *mut B;
+type C = *const str;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (pointer_type
+      (mutable_specifier)
+      (type_identifier)))
+  (type_item
+    (type_identifier)
+    (pointer_type
+      (primitive_type))))
+
+================================================================================
+Generic types
+================================================================================
+
+type A = B<C>;
+type D = E<F, str>;
+type G = H<'a, I>;
+type J = H<K=L>;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (type_identifier))))
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (type_identifier)
+        (primitive_type))))
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (lifetime
+          (identifier))
+        (type_identifier))))
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (type_binding
+          (type_identifier)
+          (type_identifier))))))
+
+================================================================================
+Scoped types
+================================================================================
+
+type A = B::C;
+type D = E::F::G;
+type H = I::J<K>;
+type L = M<N>::O;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (scoped_type_identifier
+      (identifier)
+      (type_identifier)))
+  (type_item
+    (type_identifier)
+    (scoped_type_identifier
+      (scoped_identifier
+        (identifier)
+        (identifier))
+      (type_identifier)))
+  (type_item
+    (type_identifier)
+    (generic_type
+      (scoped_type_identifier
+        (identifier)
+        (type_identifier))
+      (type_arguments
+        (type_identifier))))
+  (type_item
+    (type_identifier)
+    (scoped_type_identifier
+      (generic_type
+        (type_identifier)
+        (type_arguments
+          (type_identifier)))
+      (type_identifier))))
+
+================================================================================
+Array types
+================================================================================
+
+type A = [B; 4];
+type C = &[D];
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (array_type
+      (type_identifier)
+      (integer_literal)))
+  (type_item
+    (type_identifier)
+    (reference_type
+      (array_type
+        (type_identifier)))))
+
+================================================================================
+Function types
+================================================================================
+
+fn high_order1(value: i32, f: fn(i32)) -> i32 {}
+
+fn high_order2(value: i32, f: fn(i32) -> i32) -> i32 {
+    f(value)
+}
+
+fn high_order3(value: i32, f: &FnOnce(i32) -> i32) -> i32 {
+    f(value)
+}
+
+type F = for<'a, 'b> fn(x: &'a A, y: &'a mut B<'i, 't>,) -> C;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (function_item
+    (identifier)
+    (parameters
+      (parameter
+        (identifier)
+        (primitive_type))
+      (parameter
+        (identifier)
+        (function_type
+          (parameters
+            (primitive_type)))))
+    (primitive_type)
+    (block))
+  (function_item
+    (identifier)
+    (parameters
+      (parameter
+        (identifier)
+        (primitive_type))
+      (parameter
+        (identifier)
+        (function_type
+          (parameters
+            (primitive_type))
+          (primitive_type))))
+    (primitive_type)
+    (block
+      (call_expression
+        (identifier)
+        (arguments
+          (identifier)))))
+  (function_item
+    (identifier)
+    (parameters
+      (parameter
+        (identifier)
+        (primitive_type))
+      (parameter
+        (identifier)
+        (reference_type
+          (function_type
+            (type_identifier)
+            (parameters
+              (primitive_type))
+            (primitive_type)))))
+    (primitive_type)
+    (block
+      (call_expression
+        (identifier)
+        (arguments
+          (identifier)))))
+  (type_item
+    (type_identifier)
+    (function_type
+      (for_lifetimes
+        (lifetime
+          (identifier))
+        (lifetime
+          (identifier)))
+      (parameters
+        (parameter
+          (identifier)
+          (reference_type
+            (lifetime
+              (identifier))
+            (type_identifier)))
+        (parameter
+          (identifier)
+          (reference_type
+            (lifetime
+              (identifier))
+            (mutable_specifier)
+            (generic_type
+              (type_identifier)
+              (type_arguments
+                (lifetime
+                  (identifier))
+                (lifetime
+                  (identifier)))))))
+      (type_identifier))))
+
+================================================================================
+Unsafe and extern function types
+================================================================================
+
+type a = extern "C" fn(*mut c_void);
+type b = unsafe extern "C" fn() -> *mut c_void;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (function_type
+      (function_modifiers
+        (extern_modifier
+          (string_literal)))
+      (parameters
+        (pointer_type
+          (mutable_specifier)
+          (type_identifier)))))
+  (type_item
+    (type_identifier)
+    (function_type
+      (function_modifiers
+        (extern_modifier
+          (string_literal)))
+      (parameters)
+      (pointer_type
+        (mutable_specifier)
+        (type_identifier)))))
+
+================================================================================
+Trait objects
+================================================================================
+
+type a = Box<Something + 'a>;
+type b = Rc<dyn Something>;
+type c = A<&dyn Fn(&B) -> C>;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (bounded_type
+          (type_identifier)
+          (lifetime
+            (identifier))))))
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (dynamic_type
+          (type_identifier)))))
+  (type_item
+    (type_identifier)
+    (generic_type
+      (type_identifier)
+      (type_arguments
+        (reference_type
+          (dynamic_type
+            (function_type
+              (type_identifier)
+              (parameters
+                (reference_type
+                  (type_identifier)))
+              (type_identifier))))))))
+
+================================================================================
+Type cast expressions with generics
+================================================================================
+
+a as B<C>;
+d as *mut E<<F as E>::G>;
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (expression_statement
+    (type_cast_expression
+      (identifier)
+      (generic_type
+        (type_identifier)
+        (type_arguments
+          (type_identifier)))))
+  (expression_statement
+    (type_cast_expression
+      (identifier)
+      (pointer_type
+        (mutable_specifier)
+        (generic_type
+          (type_identifier)
+          (type_arguments
+            (scoped_type_identifier
+              (bracketed_type
+                (qualified_type
+                  (type_identifier)
+                  (type_identifier)))
+              (type_identifier))))))))