Add agda, bash, c, nix, python and rust

This commit is contained in:
xenia 2023-11-11 23:27:21 +01:00
commit 7c0c72b8b7
78 changed files with 43816 additions and 0 deletions

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023 xenia
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

9
README.md Normal file
View File

@ -0,0 +1,9 @@
# xenia/tree-sitters — a collection of tree-sitter grammars, built with nix for static linking
Languages:
* c: https://github.com/tree-sitter/tree-sitter-c (MIT)
* agda: https://github.com/tree-sitter/tree-sitter-agda (MIT)
* bash: https://github.com/tree-sitter/tree-sitter-bash (MIT)
* nix: https://github.com/nix-community/tree-sitter-nix (MIT)
* python: https://github.com/tree-sitter/tree-sitter-python (MIT)
* rust: https://github.com/tree-sitter/tree-sitter-rust (MIT)

21
agda/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2018 LUA Ting-Gan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -0,0 +1,8 @@
record RawMonoid c : Set (suc (c ⊔ )) where
infixl 7 _∙_
infix 4 _≈_
field
Carrier : Set c
_≈_ : Rel Carrier
_b_ : Op Carrier
a : Carrier

1084
agda/grammar.js Normal file

File diff suppressed because it is too large Load Diff

291
agda/src/scanner.c Normal file
View File

@ -0,0 +1,291 @@
#include "tree_sitter/parser.h"
#include <assert.h>
#include <stdio.h>
#include <string.h>
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define VEC_RESIZE(vec, _cap) \
void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0])); \
assert(tmp != NULL); \
(vec).data = tmp; \
(vec).cap = (_cap);
#define VEC_GROW(vec, _cap) \
if ((vec).cap < (_cap)) { \
VEC_RESIZE((vec), (_cap)); \
}
#define VEC_PUSH(vec, el) \
if ((vec).cap == (vec).len) { \
VEC_RESIZE((vec), MAX(16, (vec).len * 2)); \
} \
(vec).data[(vec).len++] = (el);
#define VEC_POP(vec) (vec).len--;
#define VEC_NEW \
{ .len = 0, .cap = 0, .data = NULL }
#define VEC_BACK(vec) ((vec).data[(vec).len - 1])
#define VEC_FREE(vec) \
{ \
if ((vec).data != NULL) \
free((vec).data); \
}
#define VEC_CLEAR(vec) (vec).len = 0;
#define QUEUE_RESIZE(queue, _cap) \
do { \
void *tmp = realloc((queue).data, (_cap) * sizeof((queue).data[0])); \
assert(tmp != NULL); \
(queue).data = tmp; \
(queue).cap = (_cap); \
} while (0)
#define QUEUE_GROW(queue, _cap) \
do { \
if ((queue).cap < (_cap)) { \
QUEUE_RESIZE((queue), (_cap)); \
} \
} while (0)
#define QUEUE_PUSH(queue, el) \
do { \
if ((queue).cap == 0) { \
QUEUE_RESIZE((queue), 16); \
} else if ((queue).cap == ((queue).tail - (queue).head)) { \
QUEUE_RESIZE((queue), (queue).cap * 2); \
} \
(queue).data[(queue).tail % (queue).cap] = (el); \
(queue).tail++; \
} while (0)
#define QUEUE_POP(queue) \
do { \
assert((queue).head < (queue).tail); \
(queue).head++; \
} while (0)
#define QUEUE_FRONT(queue) (queue).data[(queue).head % (queue).cap]
#define QUEUE_EMPTY(queue) ((queue).head == (queue).tail)
#define QUEUE_NEW \
{ .head = 0, .tail = 0, .cap = 0, .data = NULL }
#define QUEUE_FREE(queue) \
do { \
if ((queue).data != NULL) \
free((queue).data); \
} while (0)
#define QUEUE_CLEAR(queue) \
do { \
(queue).head = 0; \
(queue).tail = 0; \
} while (0)
enum TokenType {
NEWLINE,
INDENT,
DEDENT,
};
typedef struct {
uint32_t len;
uint32_t cap;
uint16_t *data;
} indent_vec;
static indent_vec indent_vec_new() {
indent_vec vec = VEC_NEW;
vec.data = calloc(1, sizeof(uint16_t));
vec.cap = 1;
return vec;
}
typedef struct {
uint32_t head;
uint32_t tail;
uint32_t cap;
uint16_t *data;
} token_queue;
static token_queue token_queue_new() {
token_queue queue = QUEUE_NEW;
queue.data = calloc(1, sizeof(uint16_t));
queue.cap = 1;
return queue;
}
typedef struct {
indent_vec indents;
uint32_t queued_dedent_count;
token_queue tokens;
} Scanner;
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
bool tree_sitter_agda_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
if (QUEUE_EMPTY(scanner->tokens)) {
if (valid_symbols[DEDENT] && scanner->queued_dedent_count > 0) {
scanner->queued_dedent_count--;
QUEUE_PUSH(scanner->tokens, DEDENT);
QUEUE_PUSH(scanner->tokens, NEWLINE);
} else {
bool skipped_newline = false;
while (lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
lexer->lookahead == '\r' || lexer->lookahead == '\n') {
if (lexer->lookahead == '\n') {
skipped_newline = true;
skip(lexer);
} else {
skip(lexer);
}
}
if (lexer->eof(lexer)) {
if (valid_symbols[DEDENT] && scanner->indents.len > 1) {
VEC_POP(scanner->indents);
QUEUE_PUSH(scanner->tokens, DEDENT);
QUEUE_PUSH(scanner->tokens, NEWLINE);
} else if (valid_symbols[NEWLINE]) {
QUEUE_PUSH(scanner->tokens, NEWLINE);
}
} else {
bool next_token_is_comment = false;
uint16_t indent_length = (uint16_t)lexer->get_column(lexer);
bool indent = indent_length > VEC_BACK(scanner->indents);
bool dedent = indent_length < VEC_BACK(scanner->indents);
if (!next_token_is_comment) {
if (skipped_newline) {
if (indent) {
if (valid_symbols[INDENT]) {
VEC_PUSH(scanner->indents, indent_length);
QUEUE_PUSH(scanner->tokens, INDENT);
}
} else if (dedent) {
if (valid_symbols[NEWLINE]) {
QUEUE_PUSH(scanner->tokens, NEWLINE);
}
} else {
if (valid_symbols[NEWLINE]) {
QUEUE_PUSH(scanner->tokens, NEWLINE);
}
}
} else {
if (indent) {
if (valid_symbols[INDENT]) {
VEC_PUSH(scanner->indents, indent_length);
QUEUE_PUSH(scanner->tokens, INDENT);
}
} else if (dedent) {
VEC_POP(scanner->indents);
while (indent_length < VEC_BACK(scanner->indents)) {
VEC_POP(scanner->indents);
scanner->queued_dedent_count++;
}
if (valid_symbols[DEDENT]) {
QUEUE_PUSH(scanner->tokens, DEDENT);
QUEUE_PUSH(scanner->tokens, NEWLINE);
} else {
scanner->queued_dedent_count++;
}
}
}
}
}
}
}
if (QUEUE_EMPTY(scanner->tokens)) {
return false;
}
lexer->result_symbol = QUEUE_FRONT(scanner->tokens);
QUEUE_POP(scanner->tokens);
return true;
}
unsigned tree_sitter_agda_external_scanner_serialize(void *payload,
char *buffer) {
Scanner *scanner = (Scanner *)payload;
if (scanner->indents.len * sizeof(uint16_t) + 1 >
TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
return 0;
}
unsigned size = 0;
buffer[size++] = (char)scanner->queued_dedent_count;
memcpy(&buffer[size], scanner->indents.data,
scanner->indents.len * sizeof(uint16_t));
size += (unsigned)(scanner->indents.len * sizeof(uint16_t));
return size;
}
void tree_sitter_agda_external_scanner_deserialize(void *payload,
const char *buffer,
unsigned length) {
Scanner *scanner = (Scanner *)payload;
scanner->queued_dedent_count = 0;
VEC_CLEAR(scanner->indents);
if (length == 0) {
if (buffer == NULL) {
VEC_PUSH(scanner->indents, 0);
}
return;
}
scanner->queued_dedent_count = (uint8_t)buffer[0];
unsigned size = 1;
if (length > size) {
VEC_GROW(scanner->indents,
(uint32_t)(length - size) / sizeof(uint16_t));
scanner->indents.len = (length - size) / sizeof(uint16_t);
memcpy(scanner->indents.data, &buffer[size],
scanner->indents.len * sizeof(uint16_t));
size += (unsigned)(scanner->indents.len * sizeof(uint16_t));
}
if (scanner->indents.len == 0) {
VEC_PUSH(scanner->indents, 0);
return;
}
assert(size == length);
}
void *tree_sitter_agda_external_scanner_create() {
Scanner *scanner = calloc(1, sizeof(Scanner));
scanner->indents = indent_vec_new();
scanner->tokens = token_queue_new();
tree_sitter_agda_external_scanner_deserialize(scanner, NULL, 0);
return scanner;
}
void tree_sitter_agda_external_scanner_destroy(void *payload) {
Scanner *scanner = (Scanner *)payload;
VEC_FREE(scanner->indents);
QUEUE_FREE(scanner->tokens);
free(scanner);
}

21
bash/LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Max Brunsfeld
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

143
bash/examples/atom.sh Executable file
View File

@ -0,0 +1,143 @@
#!/bin/bash
if [ "$(uname)" == 'Darwin' ]; then
OS='Mac'
elif [ "$(expr substr $(uname -s) 1 5)" == 'Linux' ]; then
OS='Linux'
else
echo "Your platform ($(uname -a)) is not supported."
exit 1
fi
if [ "$(basename $0)" == 'atom-beta' ]; then
BETA_VERSION=true
else
BETA_VERSION=
fi
export ATOM_DISABLE_SHELLING_OUT_FOR_ENVIRONMENT=true
while getopts ":wtfvh-:" opt; do
case "$opt" in
-)
case "${OPTARG}" in
wait)
WAIT=1
;;
help|version)
REDIRECT_STDERR=1
EXPECT_OUTPUT=1
;;
foreground|benchmark|benchmark-test|test)
EXPECT_OUTPUT=1
;;
esac
;;
w)
WAIT=1
;;
h|v)
REDIRECT_STDERR=1
EXPECT_OUTPUT=1
;;
f|t)
EXPECT_OUTPUT=1
;;
esac
done
if [ $REDIRECT_STDERR ]; then
exec 2> /dev/null
fi
if [ $EXPECT_OUTPUT ]; then
export ELECTRON_ENABLE_LOGGING=1
fi
if [ $OS == 'Mac' ]; then
if [ -L "$0" ]; then
SCRIPT="$(readlink "$0")"
else
SCRIPT="$0"
fi
ATOM_APP="$(dirname "$(dirname "$(dirname "$(dirname "$SCRIPT")")")")"
if [ "$ATOM_APP" == . ]; then
unset ATOM_APP
else
ATOM_PATH="$(dirname "$ATOM_APP")"
ATOM_APP_NAME="$(basename "$ATOM_APP")"
fi
if [ -n "$BETA_VERSION" ]; then
ATOM_EXECUTABLE_NAME="Atom Beta"
else
ATOM_EXECUTABLE_NAME="Atom"
fi
if [ -z "${ATOM_PATH}" ]; then
# If ATOM_PATH isn't set, check /Applications and then ~/Applications for Atom.app
if [ -x "/Applications/$ATOM_APP_NAME" ]; then
ATOM_PATH="/Applications"
elif [ -x "$HOME/Applications/$ATOM_APP_NAME" ]; then
ATOM_PATH="$HOME/Applications"
else
# We haven't found an Atom.app, use spotlight to search for Atom
ATOM_PATH="$(mdfind "kMDItemCFBundleIdentifier == 'com.github.atom'" | grep -v ShipIt | head -1 | xargs -0 dirname)"
# Exit if Atom can't be found
if [ ! -x "$ATOM_PATH/$ATOM_APP_NAME" ]; then
echo "Cannot locate ${ATOM_APP_NAME}, it is usually located in /Applications. Set the ATOM_PATH environment variable to the directory containing ${ATOM_APP_NAME}."
exit 1
fi
fi
fi
if [ $EXPECT_OUTPUT ]; then
"$ATOM_PATH/$ATOM_APP_NAME/Contents/MacOS/$ATOM_EXECUTABLE_NAME" --executed-from="$(pwd)" --pid=$$ "$@"
exit $?
else
open -a "$ATOM_PATH/$ATOM_APP_NAME" -n --args --executed-from="$(pwd)" --pid=$$ --path-environment="$PATH" "$@"
fi
elif [ $OS == 'Linux' ]; then
SCRIPT=$(readlink -f "$0")
USR_DIRECTORY=$(readlink -f $(dirname $SCRIPT)/..)
if [ -n "$BETA_VERSION" ]; then
ATOM_PATH="$USR_DIRECTORY/share/atom-beta/atom"
else
ATOM_PATH="$USR_DIRECTORY/share/atom/atom"
fi
ATOM_HOME="${ATOM_HOME:-$HOME/.atom}"
mkdir -p "$ATOM_HOME"
: ${TMPDIR:=/tmp}
[ -x "$ATOM_PATH" ] || ATOM_PATH="$TMPDIR/atom-build/Atom/atom"
if [ $EXPECT_OUTPUT ]; then
"$ATOM_PATH" --executed-from="$(pwd)" --pid=$$ "$@"
exit $?
else
(
nohup "$ATOM_PATH" --executed-from="$(pwd)" --pid=$$ "$@" > "$ATOM_HOME/nohup.out" 2>&1
if [ $? -ne 0 ]; then
cat "$ATOM_HOME/nohup.out"
exit $?
fi
) &
fi
fi
# Exits this process when Atom is used as $EDITOR
on_die() {
exit 0
}
trap 'on_die' SIGQUIT SIGTERM
# If the wait flag is set, don't exit this process until Atom tells it to.
if [ $WAIT ]; then
while true; do
sleep 1
done
fi

165
bash/examples/clean-old.sh Executable file
View File

@ -0,0 +1,165 @@
#!/bin/bash
# look for old 0.x cruft, and get rid of it.
# Should already be sitting in the npm folder.
# This doesn't have to be quite as cross-platform as install.sh.
# There are some bash-isms, because maintaining *two*
# fully-portable posix/bourne sh scripts is too much for
# one project with a sane maintainer.
# If readlink isn't available, then this is just too tricky.
# However, greadlink is fine, so Solaris can join the party, too.
readlink="readlink"
which $readlink >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
readlink="greadlink"
which $readlink >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
echo "Can't find the readlink or greadlink command. Aborting."
exit 1
fi
fi
if [ "x$npm_config_prefix" != "x" ]; then
PREFIXES=$npm_config_prefix
else
node="$NODE"
if [ "x$node" = "x" ]; then
node=`which node`
fi
if [ "x$node" = "x" ]; then
echo "Can't find node to determine prefix. Aborting."
exit 1
fi
PREFIX=`dirname $node`
PREFIX=`dirname $PREFIX`
echo "cleanup prefix=$PREFIX"
PREFIXES=$PREFIX
altprefix=`"$node" -e process.installPrefix`
if [ "x$altprefix" != "x" ] && [ "x$altprefix" != "x$PREFIX" ]; then
echo "altprefix=$altprefix"
PREFIXES="$PREFIX $altprefix"
fi
fi
# now prefix is where npm would be rooted by default
# go hunting.
packages=
for prefix in $PREFIXES; do
packages="$packages
"`ls "$prefix"/lib/node/.npm 2>/dev/null | grep -v .cache`
done
packages=`echo $packages`
filelist=()
fid=0
for prefix in $PREFIXES; do
# remove any links into the .npm dir, or links to
# version-named shims/symlinks.
for folder in share/man bin lib/node; do
find $prefix/$folder -type l | while read file; do
target=`$readlink $file | grep '/\.npm/'`
if [ "x$target" != "x" ]; then
# found one!
filelist[$fid]="$file"
let 'fid++'
# also remove any symlinks to this file.
base=`basename "$file"`
base=`echo "$base" | awk -F@ '{print $1}'`
if [ "x$base" != "x" ]; then
find "`dirname $file`" -type l -name "$base"'*' \
| while read l; do
target=`$readlink "$l" | grep "$base"`
if [ "x$target" != "x" ]; then
filelist[$fid]="$1"
let 'fid++'
fi
done
fi
fi
done
# Scour for shim files. These are relics of 0.2 npm installs.
# note: grep -r is not portable.
find $prefix/$folder -type f \
| xargs grep -sl '// generated by npm' \
| while read file; do
filelist[$fid]="$file"
let 'fid++'
done
done
# now remove the package modules, and the .npm folder itself.
if [ "x$packages" != "x" ]; then
for pkg in $packages; do
filelist[$fid]="$prefix/lib/node/$pkg"
let 'fid++'
for i in $prefix/lib/node/$pkg\@*; do
filelist[$fid]="$i"
let 'fid++'
done
done
fi
for folder in lib/node/.npm lib/npm share/npm; do
if [ -d $prefix/$folder ]; then
filelist[$fid]="$prefix/$folder"
let 'fid++'
fi
done
done
# now actually clean, but only if there's anything TO clean
if [ "${#filelist[@]}" -gt 0 ]; then
echo ""
echo "This script will find and eliminate any shims, symbolic"
echo "links, and other cruft that was installed by npm 0.x."
echo ""
if [ "x$packages" != "x" ]; then
echo "The following packages appear to have been installed with"
echo "an old version of npm, and will be removed forcibly:"
for pkg in $packages; do
echo " $pkg"
done
echo "Make a note of these. You may want to install them"
echo "with npm 1.0 when this process is completed."
echo ""
fi
OK=
if [ "x$1" = "x-y" ]; then
OK="yes"
fi
while [ "$OK" != "y" ] && [ "$OK" != "yes" ] && [ "$OK" != "no" ]; do
echo "Is this OK?"
echo " enter 'yes' or 'no'"
echo " or 'show' to see a list of files "
read OK
if [ "x$OK" = "xshow" ] || [ "x$OK" = "xs" ]; then
for i in "${filelist[@]}"; do
echo "$i"
done
fi
done
if [ "$OK" = "no" ]; then
echo "Aborting"
exit 1
fi
for i in "${filelist[@]}"; do
rm -rf "$i"
done
fi
echo ""
echo 'All clean!'
exit 0

119
bash/examples/doc-build.sh Executable file
View File

@ -0,0 +1,119 @@
#!/usr/bin/env bash
if [[ $DEBUG != "" ]]; then
set -x
fi
set -o errexit
set -o pipefail
if ! [ -x node_modules/.bin/marked-man ]; then
ps=0
if [ -f .building_marked-man ]; then
pid=$(cat .building_marked-man)
ps=$(ps -p $pid | grep $pid | wc -l) || true
fi
if [ -f .building_marked-man ] && [ $ps != 0 ]; then
while [ -f .building_marked-man ]; do
sleep 1
done
else
# a race to see which make process will be the one to install marked-man
echo $$ > .building_marked-man
sleep 1
if [ $(cat .building_marked-man) == $$ ]; then
make node_modules/.bin/marked-man
rm .building_marked-man
else
while [ -f .building_marked-man ]; do
sleep 1
done
fi
fi
fi
if ! [ -x node_modules/.bin/marked ]; then
ps=0
if [ -f .building_marked ]; then
pid=$(cat .building_marked)
ps=$(ps -p $pid | grep $pid | wc -l) || true
fi
if [ -f .building_marked ] && [ $ps != 0 ]; then
while [ -f .building_marked ]; do
sleep 1
done
else
# a race to see which make process will be the one to install marked
echo $$ > .building_marked
sleep 1
if [ $(cat .building_marked) == $$ ]; then
make node_modules/.bin/marked
rm .building_marked
else
while [ -f .building_marked ]; do
sleep 1
done
fi
fi
fi
src=$1
dest=$2
name=$(basename ${src%.*})
date=$(date -u +'%Y-%m-%d %H:%M:%S')
version=$(node cli.js -v)
mkdir -p $(dirname $dest)
html_replace_tokens () {
local url=$1
sed "s|@NAME@|$name|g" \
| sed "s|@DATE@|$date|g" \
| sed "s|@URL@|$url|g" \
| sed "s|@VERSION@|$version|g" \
| perl -p -e 's/<h1([^>]*)>([^\(]*\([0-9]\)) -- (.*?)<\/h1>/<h1>\2<\/h1> <p>\3<\/p>/g' \
| perl -p -e 's/npm-npm/npm/g' \
| perl -p -e 's/([^"-])(npm-)?README(?!\.html)(\(1\))?/\1<a href="..\/..\/doc\/README.html">README<\/a>/g' \
| perl -p -e 's/<title><a href="[^"]+README.html">README<\/a><\/title>/<title>README<\/title>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(1\))/\1<a href="..\/cli\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(3\))/\1<a href="..\/api\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(5\))/\1<a href="..\/files\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(7\))/\1<a href="..\/misc\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/\([1357]\)<\/a><\/h1>/<\/a><\/h1>/g' \
| (if [ $(basename $(dirname $dest)) == "doc" ]; then
perl -p -e 's/ href="\.\.\// href="/g'
else
cat
fi)
}
man_replace_tokens () {
sed "s|@VERSION@|$version|g" \
| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(1\)/npm help \2/g' \
| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(([57])\)/npm help \3 \2/g' \
| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(3\)/npm apihelp \2/g' \
| perl -p -e 's/npm\(1\)/npm help npm/g' \
| perl -p -e 's/npm\(3\)/npm apihelp npm/g'
}
case $dest in
*.[1357])
./node_modules/.bin/marked-man --roff $src \
| man_replace_tokens > $dest
exit $?
;;
*.html)
url=${dest/html\//}
(cat html/dochead.html && \
cat $src | ./node_modules/.bin/marked &&
cat html/docfoot.html)\
| html_replace_tokens $url \
> $dest
exit $?
;;
*)
echo "Invalid destination type: $dest" >&2
exit 1
;;
esac

270
bash/examples/install.sh Executable file
View File

@ -0,0 +1,270 @@
#!/bin/sh
# A word about this shell script:
#
# It must work everywhere, including on systems that lack
# a /bin/bash, map 'sh' to ksh, ksh97, bash, ash, or zsh,
# and potentially have either a posix shell or bourne
# shell living at /bin/sh.
#
# See this helpful document on writing portable shell scripts:
# http://www.gnu.org/s/hello/manual/autoconf/Portable-Shell.html
#
# The only shell it won't ever work on is cmd.exe.
if [ "x$0" = "xsh" ]; then
# run as curl | sh
# on some systems, you can just do cat>npm-install.sh
# which is a bit cuter. But on others, &1 is already closed,
# so catting to another script file won't do anything.
# Follow Location: headers, and fail on errors
curl -f -L -s https://www.npmjs.org/install.sh > npm-install-$$.sh
ret=$?
if [ $ret -eq 0 ]; then
(exit 0)
else
rm npm-install-$$.sh
echo "Failed to download script" >&2
exit $ret
fi
sh npm-install-$$.sh
ret=$?
rm npm-install-$$.sh
exit $ret
fi
# See what "npm_config_*" things there are in the env,
# and make them permanent.
# If this fails, it's not such a big deal.
configures="`env | grep 'npm_config_' | sed -e 's|^npm_config_||g'`"
npm_config_loglevel="error"
if [ "x$npm_debug" = "x" ]; then
(exit 0)
else
echo "Running in debug mode."
echo "Note that this requires bash or zsh."
set -o xtrace
set -o pipefail
npm_config_loglevel="verbose"
fi
export npm_config_loglevel
# make sure that node exists
node=`which node 2>&1`
ret=$?
if [ $ret -eq 0 ] && [ -x "$node" ]; then
(exit 0)
else
echo "npm cannot be installed without node.js." >&2
echo "Install node first, and then try again." >&2
echo "" >&2
echo "Maybe node is installed, but not in the PATH?" >&2
echo "Note that running as sudo can change envs." >&2
echo ""
echo "PATH=$PATH" >&2
exit $ret
fi
# set the temp dir
TMP="${TMPDIR}"
if [ "x$TMP" = "x" ]; then
TMP="/tmp"
fi
TMP="${TMP}/npm.$$"
rm -rf "$TMP" || true
mkdir "$TMP"
if [ $? -ne 0 ]; then
echo "failed to mkdir $TMP" >&2
exit 1
fi
BACK="$PWD"
ret=0
tar="${TAR}"
if [ -z "$tar" ]; then
tar="${npm_config_tar}"
fi
if [ -z "$tar" ]; then
tar=`which tar 2>&1`
ret=$?
fi
if [ $ret -eq 0 ] && [ -x "$tar" ]; then
echo "tar=$tar"
echo "version:"
$tar --version
ret=$?
fi
if [ $ret -eq 0 ]; then
(exit 0)
else
echo "No suitable tar program found."
exit 1
fi
# Try to find a suitable make
# If the MAKE environment var is set, use that.
# otherwise, try to find gmake, and then make.
# If no make is found, then just execute the necessary commands.
# XXX For some reason, make is building all the docs every time. This
# is an annoying source of bugs. Figure out why this happens.
MAKE=NOMAKE
if [ "x$MAKE" = "x" ]; then
make=`which gmake 2>&1`
if [ $? -eq 0 ] && [ -x "$make" ]; then
(exit 0)
else
make=`which make 2>&1`
if [ $? -eq 0 ] && [ -x "$make" ]; then
(exit 0)
else
make=NOMAKE
fi
fi
else
make="$MAKE"
fi
if [ -x "$make" ]; then
(exit 0)
else
# echo "Installing without make. This may fail." >&2
make=NOMAKE
fi
# If there's no bash, then don't even try to clean
if [ -x "/bin/bash" ]; then
(exit 0)
else
clean="no"
fi
node_version=`"$node" --version 2>&1`
ret=$?
if [ $ret -ne 0 ]; then
echo "You need node to run this program." >&2
echo "node --version reports: $node_version" >&2
echo "with exit code = $ret" >&2
echo "Please install node before continuing." >&2
exit $ret
fi
t="${npm_install}"
if [ -z "$t" ]; then
# switch based on node version.
# note that we can only use strict sh-compatible patterns here.
case $node_version in
0.[01234567].* | v0.[01234567].*)
echo "You are using an outdated and unsupported version of" >&2
echo "node ($node_version). Please update node and try again." >&2
exit 99
;;
*)
echo "install npm@latest"
t="latest"
;;
esac
fi
# need to echo "" after, because Posix sed doesn't treat EOF
# as an implied end of line.
url=`(curl -SsL https://registry.npmjs.org/npm/$t; echo "") \
| sed -e 's/^.*tarball":"//' \
| sed -e 's/".*$//'`
ret=$?
if [ "x$url" = "x" ]; then
ret=125
# try without the -e arg to sed.
url=`(curl -SsL https://registry.npmjs.org/npm/$t; echo "") \
| sed 's/^.*tarball":"//' \
| sed 's/".*$//'`
ret=$?
if [ "x$url" = "x" ]; then
ret=125
fi
fi
if [ $ret -ne 0 ]; then
echo "Failed to get tarball url for npm/$t" >&2
exit $ret
fi
echo "fetching: $url" >&2
cd "$TMP" \
&& curl -SsL "$url" \
| $tar -xzf - \
&& cd "$TMP"/* \
&& (ver=`"$node" bin/read-package-json.js package.json version`
isnpm10=0
if [ $ret -eq 0 ]; then
if [ -d node_modules ]; then
if "$node" node_modules/semver/bin/semver -v "$ver" -r "1"
then
isnpm10=1
fi
else
if "$node" bin/semver -v "$ver" -r ">=1.0"; then
isnpm10=1
fi
fi
fi
ret=0
if [ $isnpm10 -eq 1 ] && [ -f "scripts/clean-old.sh" ]; then
if [ "x$skipclean" = "x" ]; then
(exit 0)
else
clean=no
fi
if [ "x$clean" = "xno" ] \
|| [ "x$clean" = "xn" ]; then
echo "Skipping 0.x cruft clean" >&2
ret=0
elif [ "x$clean" = "xy" ] || [ "x$clean" = "xyes" ]; then
NODE="$node" /bin/bash "scripts/clean-old.sh" "-y"
ret=$?
else
NODE="$node" /bin/bash "scripts/clean-old.sh" </dev/tty
ret=$?
fi
fi
if [ $ret -ne 0 ]; then
echo "Aborted 0.x cleanup. Exiting." >&2
exit $ret
fi) \
&& (if [ "x$configures" = "x" ]; then
(exit 0)
else
echo "./configure $configures"
echo "$configures" > npmrc
fi) \
&& (if [ "$make" = "NOMAKE" ]; then
(exit 0)
elif "$make" uninstall install; then
(exit 0)
else
make="NOMAKE"
fi
if [ "$make" = "NOMAKE" ]; then
"$node" cli.js rm npm -gf
"$node" cli.js install -gf
fi) \
&& cd "$BACK" \
&& rm -rf "$TMP" \
&& echo "It worked"
ret=$?
if [ $ret -ne 0 ]; then
echo "It failed" >&2
fi
exit $ret

36
bash/examples/release.sh Normal file
View File

@ -0,0 +1,36 @@
#!/bin/bash
# script for creating a zip and tarball for inclusion in node
unset CDPATH
set -e
rm -rf release *.tgz || true
mkdir release
node ./cli.js pack --loglevel error >/dev/null
mv *.tgz release
cd release
tar xzf *.tgz
mkdir node_modules
mv package node_modules/npm
# make the zip for windows users
cp node_modules/npm/bin/*.cmd .
zipname=npm-$(node ../cli.js -v).zip
zip -q -9 -r -X "$zipname" *.cmd node_modules
# make the tar for node's deps
cd node_modules
tarname=npm-$(node ../../cli.js -v).tgz
tar czf "$tarname" npm
cd ..
mv "node_modules/$tarname" .
rm -rf *.cmd
rm -rf node_modules
echo "release/$tarname"
echo "release/$zipname"

26
bash/examples/relocate.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/bash
# Change the cli shebang to point at the specified node
# Useful for when the program is moved around after install.
# Also used by the default 'make install' in node to point
# npm at the newly installed node, rather than the first one
# in the PATH, which would be the default otherwise.
# bash /path/to/npm/scripts/relocate.sh $nodepath
# If $nodepath is blank, then it'll use /usr/bin/env
dir="$(dirname "$(dirname "$0")")"
cli="$dir"/bin/npm-cli.js
tmp="$cli".tmp
node="$1"
if [ "x$node" = "x" ]; then
node="/usr/bin/env node"
fi
node="#!$node"
sed -e 1d "$cli" > "$tmp"
echo "$node" > "$cli"
cat "$tmp" >> "$cli"
rm "$tmp"
chmod ogu+x $cli

138
bash/examples/test.sh Executable file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env bash
set -e
function usage {
cat <<-EOF
USAGE
$0 [-dgGhv] [-f focus-string] [-s seed]
OPTIONS
-h print this message
-b run make under scan-build static analyzer
-d run tests in a debugger (either lldb or gdb)
-g run tests with valgrind's memcheck tool
-G run tests with valgrind's memcheck tool, including a full leak check
-v run tests with verbose output
-f run only tests whose description contain the given string
-s set the seed used to control random behavior
-z pipe tests' stderr to \`dot(1)\` to render an SVG log
EOF
}
profile=
leak_check=no
mode=normal
verbose=
args=()
target=tests
export BUILDTYPE=Test
cmd="out/${BUILDTYPE}/${target}"
run_scan_build=
if [ "$(uname -s)" == "Darwin" ]; then
export LINK="clang++ -fsanitize=address"
fi
while getopts "bdf:s:gGhpvS" option; do
case ${option} in
h)
usage
exit
;;
d)
mode=debug
;;
g)
mode=valgrind
;;
G)
mode=valgrind
leak_check=full
;;
p)
profile=true
;;
f)
args+=("--only=${OPTARG}")
;;
v)
verbose=true
;;
s)
export TREE_SITTER_SEED=${OPTARG}
;;
S)
mode=SVG
;;
b)
run_scan_build=true
;;
esac
done
if [[ -n $verbose ]]; then
args+=("--reporter=spec")
else
args+=("--reporter=singleline")
fi
if [[ -n "$run_scan_build" ]]; then
. script/util/scan-build.sh
scan_build make -j2 $target
else
make -j2 $target
fi
args=${args:-""}
if [[ -n $profile ]]; then
export CPUPROFILE=/tmp/${target}-$(date '+%s').prof
fi
case ${mode} in
valgrind)
valgrind \
--suppressions=./script/util/valgrind.supp \
--dsymutil=yes \
--leak-check=${leak_check} \
$cmd "${args[@]}" 2>&1 | \
grep --color -E '\w+_tests?.cc:\d+|$'
;;
debug)
if which -s lldb; then
lldb $cmd -- "${args[@]}"
elif which -s gdb; then
gdb $cmd -- "${args[@]}"
else
echo "No debugger found"
exit 1
fi
;;
SVG)
echo "<!DOCTYPE html><style>svg { width: 100%; margin-bottom: 20px; }</style>" > index.html
$cmd "${args[@]}" 2> >(grep -v 'Assertion failed' | dot -Tsvg >> index.html)
echo "Wrote index.html"
;;
normal)
time $cmd "${args[@]}"
;;
esac
if [[ -n $profile ]]; then
pprof $cmd $CPUPROFILE
fi

View File

@ -0,0 +1,9 @@
#!/bin/sh
git log --reverse --format='%aN <%aE>' | perl -wnE '
BEGIN {
say "# Authors sorted by whether or not they\x27re me";
}
print $seen{$_} = $_ unless $seen{$_}
' > AUTHORS

1164
bash/grammar.js Normal file

File diff suppressed because it is too large Load Diff

1271
bash/src/scanner.c Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,708 @@
===============================
Commands
===============================
whoami
---
(program
(command (command_name (word))))
===============================
Commands with arguments
===============================
cat file1.txt
git diff --word-diff=color -- file1.txt file2.txt
echo $sing\
levar
---
(program
(command (command_name (word)) (word))
(command (command_name (word)) (word) (word) (word) (word) (word))
(command (command_name (word)) (simple_expansion (variable_name)) (word)))
===============================
Quoted command names
===============================
"$a/$b" c
---
(program
(command
(command_name (string (simple_expansion (variable_name)) (string_content) (simple_expansion (variable_name))))
(word)))
===============================
Commands with numeric arguments
===============================
exit 1
---
(program
(command (command_name (word)) (number)))
===================================
Commands with environment variables
===================================
VAR1=1 ./script/test
VAR1=a VAR2="ok" git diff --word-diff=color
---
(program
(command
(variable_assignment (variable_name) (number))
(command_name (word)))
(command
(variable_assignment (variable_name) (word))
(variable_assignment (variable_name) (string (string_content)))
(command_name (word))
(word)
(word)))
===================================
Empty environment variables
===================================
VAR1=
VAR2= echo
---
(program
(variable_assignment (variable_name))
(command (variable_assignment (variable_name)) (command_name (word))))
===============================
File redirects
===============================
whoami > /dev/null
cat a b > /dev/null
2>&1 whoami
echo "foobar" >&2
[ ! command -v go &>/dev/null ] && return
if [ ]; then
>aa >bb
fi
exec {VIRTWL[0]} {VIRTWL[1]} <&- >&-
exec {VIRTWL[0]}<&- {VIRTWL[1]}>&-
grep 2>/dev/null -q "^/usr/bin/scponly$" /etc/shells
---
(program
(redirected_statement
(command (command_name (word)))
(file_redirect (word)))
(redirected_statement
(command (command_name (word)) (word) (word))
(file_redirect (word)))
(command
(file_redirect (file_descriptor) (number))
(command_name (word)))
(redirected_statement
(command (command_name (word)) (string (string_content)))
(file_redirect (number)))
(list
(test_command
(redirected_statement
(negated_command
(command (command_name (word)) (word) (word)))
(file_redirect (word))))
(command (command_name (word))))
(if_statement
(test_command)
(redirected_statement
(file_redirect (word))
(file_redirect (word))))
(redirected_statement
(command
(command_name (word))
(concatenation (word) (word) (word) (number) (word) (word))
(concatenation (word) (word) (word) (number) (word) (word)))
(file_redirect)
(file_redirect))
(redirected_statement
(command
(command_name (word))
(concatenation (word) (word) (word) (number) (word) (word)))
(file_redirect
(concatenation (word) (word) (word) (number) (word) (word)))
(file_redirect))
(redirected_statement
(command (command_name (word)))
(file_redirect (file_descriptor) (word) (word) (string (string_content)) (word))))
===============================
File redirects (noclobber override)
===============================
whoami >| /dev/null
cat a b >| /dev/null
---
(program
(redirected_statement
(command (command_name (word)))
(file_redirect (word)))
(redirected_statement
(command (command_name (word)) (word) (word))
(file_redirect (word))))
===============================
Heredoc redirects
===============================
node <<JS
console.log("hi")
JS
bash -c <<JS
echo hi
JS
newins <<-EOF - org.freedesktop.Notifications.service
[D-BUS Service]
Name=org.freedesktop.Notifications
Exec=/usr/libexec/notification-daemon
EOF
---
(program
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end)))
(redirected_statement
(command (command_name (word)) (word))
(heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end)))
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(word)
(word)
(heredoc_body)
(heredoc_end))))
===============================
Heredocs with variables
===============================
node <<JS
a $B ${C}
JS
exit
---
(program
(redirected_statement
(command
(command_name
(word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body
(simple_expansion
(variable_name))
(heredoc_content)
(expansion
(variable_name))
(heredoc_content))
(heredoc_end)))
(command
(command_name
(word))))
=================================
Heredocs with file redirects
=================================
cat <<EOF > $tmpfile
a $B ${C}
EOF
wc -l $tmpfile
---
(program
(redirected_statement
(command
(command_name
(word)))
(heredoc_redirect
(heredoc_start)
(file_redirect
(simple_expansion
(variable_name)))
(heredoc_body
(simple_expansion
(variable_name))
(heredoc_content)
(expansion
(variable_name))
(heredoc_content))
(heredoc_end)))
(command
(command_name
(word))
(word)
(simple_expansion
(variable_name))))
=================================
Heredocs with many file redirects
=================================
FOO=bar echo <<EOF 2> err.txt > hello.txt
hello
EOF
---
(program
(redirected_statement
body: (command
(variable_assignment
name: (variable_name)
value: (word))
name: (command_name
(word)))
redirect: (heredoc_redirect
(heredoc_start)
redirect: (file_redirect
descriptor: (file_descriptor)
destination: (word))
redirect: (file_redirect
destination: (word))
(heredoc_body)
(heredoc_end))))
=================================
Heredocs with pipes
=================================
one <<EOF | grep two
three
EOF
---
(program
(redirected_statement
(command
(command_name
(word)))
(heredoc_redirect
(heredoc_start)
(pipeline
(command
(command_name
(word))
(word)))
(heredoc_body)
(heredoc_end))))
======================================
Heredocs with escaped expansions
======================================
cat << EOF
DEV_NAME=\$(lsblk)
EOF
---
(program (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end))))
======================================
Quoted Heredocs
======================================
cat << 'EOF'
a=$b
EOF
cat << "EOF"
a=$b
EOF
cat <<"END OF FILE"
hello,
world
END OF FILE
cat << \EOF
EOF
---
(program
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end))))
==========================================
Heredocs with indented closing delimiters
==========================================
usage() {
cat <<-EOF
Usage: ${0##*/} FOO BAR
EOF
}
---
(program
(function_definition
(word)
(compound_statement
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body (expansion (special_variable_name) (regex)) (heredoc_content))
(heredoc_end))))))
==========================================
Heredocs with empty bodies
==========================================
node <<JS
JS
node << 'SJ'
SJ
usage() {
cat <<-EOF
EOF
}
node << 'EOF' > temp
EOF
---
(program
(redirected_statement
body: (command
name: (command_name
(word)))
redirect: (heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end)))
(redirected_statement
body: (command
name: (command_name
(word)))
redirect: (heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end)))
(function_definition
name: (word)
body: (compound_statement
(redirected_statement
body: (command
name: (command_name
(word)))
redirect: (heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end)))))
(redirected_statement
body: (command
name: (command_name
(word)))
redirect: (heredoc_redirect
(heredoc_start)
redirect: (file_redirect
destination: (word))
(heredoc_body)
(heredoc_end))))
==========================================
Heredocs with weird characters
==========================================
node <<_DELIMITER_WITH_UNDERSCORES_
Hello.
_DELIMITER_WITH_UNDERSCORES_
node <<'```'
Hello.
```
node <<!HEREDOC!
Hello.
!HEREDOC!
node <<\'
Hello.
'
node <<\\
Hello.
\
---
(program
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end)))
(redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start) (heredoc_body) (heredoc_end))))
==========================================
Heredocs with a rhs statement
==========================================
cat <<-_EOF_ || die "cat EOF failed"
#!/bin/sh
echo hello
_EOF_
---
(program
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(command (command_name (word)) (string (string_content)))
(heredoc_body)
(heredoc_end))))
==========================================
Heredocs with a $ that is not an expansion
==========================================
cat <<EOF
# check out this regex '^EOF$'
EOF
---
(program
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end))))
==========================================
Nested Heredocs
==========================================
cat <<OUTER
Outer Heredoc Start
$(cat <<INNER
Inner Heredoc Content
$(cat <<INNERMOST
Innermost Heredoc Content
INNERMOST
)
INNER)
Outer Heredoc End
OUTER
---
(program
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body
(command_substitution
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body
(command_substitution
(redirected_statement
(command (command_name (word)))
(heredoc_redirect
(heredoc_start)
(heredoc_body)
(heredoc_end))))
(heredoc_content))
(heredoc_end))))
(heredoc_content))
(heredoc_end))))
==========================================
Herestrings
==========================================
node <<< foo
while read -u 3 entry; do
echo $entry
done 3<<<"$ENTRIES"
$(tc-getCC) -Werror -Wl,-l:libobjc.so.${ver} -x objective-c \
- <<<$'int main() {}' -o /dev/null 2> /dev/null;
<<<string cmd arg
cmd arg <<<string
cmd <<<string arg
<<<string
---
(program
(command (command_name (word)) (herestring_redirect (word)))
(redirected_statement
(while_statement
(command (command_name (word)) (word) (number) (word))
(do_group
(command (command_name (word)) (simple_expansion (variable_name)))))
(herestring_redirect
(file_descriptor)
(string (simple_expansion (variable_name)))))
(redirected_statement
(command
(command_name (command_substitution (command (command_name (word)))))
(word)
(concatenation (word) (expansion (variable_name)))
(word)
(word)
(word)
(herestring_redirect (ansi_c_string))
(word)
(word))
(file_redirect (file_descriptor) (word)))
(command (herestring_redirect (word)) (command_name (word)) (word))
(command (command_name (word)) (word) (herestring_redirect (word)))
(command (command_name (word)) (herestring_redirect (word)) (word))
(redirected_statement (herestring_redirect (word))))
==========================================
Subscripts
==========================================
echo ${a[1 + 2]}
echo ${b[1234 % 2]}
${words[++counter]}
${array[(($number+1))]}
${array[((number+1))]}
---
(program
(command
(command_name (word))
(expansion
(subscript (variable_name) (binary_expression (number) (number)))))
(command
(command_name (word))
(expansion
(subscript (variable_name) (binary_expression (number) (number)))))
(command
(command_name
(expansion
(subscript (variable_name) (unary_expression (word))))))
(command
(command_name
(expansion
(subscript
(variable_name)
(arithmetic_expansion (binary_expression (simple_expansion (variable_name)) (number)))))))
(command
(command_name
(expansion
(subscript
(variable_name)
(arithmetic_expansion (binary_expression (variable_name) (number))))))))
==========================================
Bare $
==========================================
echo $
echo "${module}"$
echo $$
---
(program
(command (command_name (word)))
(command
(command_name (word))
(concatenation (string (expansion (variable_name)))))
(command
(command_name (word))
(simple_expansion (special_variable_name))))
==========================================
Arithmetic with command substitution
==========================================
$(( $( ver_cut 2 ) - 1 ))
---
(program
(command
(command_name
(arithmetic_expansion
(binary_expression
(command_substitution
(command
(command_name
(word))
(number)))
(number))))))
==========================================
Ralative path without dots
==========================================
bin/ls /usr/bin
---
(program
(command
(command_name
(word))
(word)))

13
bash/test/corpus/crlf.txt Normal file
View File

@ -0,0 +1,13 @@
================================
Variables with CRLF line endings
================================
A=one
B=two
---
(program
(variable_assignment (variable_name) (word))
(variable_assignment (variable_name) (word)))

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,108 @@
===============================
Comments
===============================
#!/bin/bash
# hi
---
(program
(comment)
(comment))
===============================
Escaped newlines
===============================
abc \
d \
e
f=g \
h=i \
j \
--k
---
(program
(command
(command_name
(word))
(word)
(word))
(command
(variable_assignment
(variable_name)
(word))
(variable_assignment
(variable_name)
(word))
(command_name
(word))
(word)))
=============================
escaped newline immediately after a char
=============================
echo a \
b
echo a\
b
echo a\
b\
c
-----------------------------
(program
(command
(command_name
(word))
(word)
(word))
(command
(command_name
(word))
(word)
(word))
(command
(command_name
(word))
(word)
(word)
(word)))
=============================
Escaped whitespace
=============================
echo 1 \ 2 \ 3
---
(program
(command
(command_name
(word))
(number)
(number)
(number)))
====================================
Files without trailing terminators
====================================
echo hi
---
(program
(command
(command_name
(word))
(word)))

File diff suppressed because it is too large Load Diff

21
c/LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014 Max Brunsfeld
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

5446
c/examples/cluster.c Normal file

File diff suppressed because it is too large Load Diff

532
c/examples/malloc.c Normal file
View File

@ -0,0 +1,532 @@
#define _GNU_SOURCE
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <stdint.h>
#include <errno.h>
#include <sys/mman.h>
#include "libc.h"
#include "atomic.h"
#include "pthread_impl.h"
#if defined(__GNUC__) && defined(__PIC__)
#define inline inline __attribute__((always_inline))
#endif
void *__mmap(void *, size_t, int, int, int, off_t);
int __munmap(void *, size_t);
void *__mremap(void *, size_t, size_t, int, ...);
int __madvise(void *, size_t, int);
struct chunk {
size_t psize, csize;
struct chunk *next, *prev;
};
struct bin {
volatile int lock[2];
struct chunk *head;
struct chunk *tail;
};
static struct {
volatile uint64_t binmap;
struct bin bins[64];
volatile int free_lock[2];
} mal;
#define SIZE_ALIGN (4*sizeof(size_t))
#define SIZE_MASK (-SIZE_ALIGN)
#define OVERHEAD (2*sizeof(size_t))
#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN)
#define DONTCARE 16
#define RECLAIM 163840
#define CHUNK_SIZE(c) ((c)->csize & -2)
#define CHUNK_PSIZE(c) ((c)->psize & -2)
#define PREV_CHUNK(c) ((struct chunk *)((char *)(c) - CHUNK_PSIZE(c)))
#define NEXT_CHUNK(c) ((struct chunk *)((char *)(c) + CHUNK_SIZE(c)))
#define MEM_TO_CHUNK(p) (struct chunk *)((char *)(p) - OVERHEAD)
#define CHUNK_TO_MEM(c) (void *)((char *)(c) + OVERHEAD)
#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head))
#define C_INUSE ((size_t)1)
#define IS_MMAPPED(c) !((c)->csize & (C_INUSE))
/* Synchronization tools */
static inline void lock(volatile int *lk)
{
if (libc.threads_minus_1)
while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
}
static inline void unlock(volatile int *lk)
{
if (lk[0]) {
a_store(lk, 0);
if (lk[1]) __wake(lk, 1, 1);
}
}
static inline void lock_bin(int i)
{
lock(mal.bins[i].lock);
if (!mal.bins[i].head)
mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i);
}
static inline void unlock_bin(int i)
{
unlock(mal.bins[i].lock);
}
static int first_set(uint64_t x)
{
#if 1
return a_ctz_64(x);
#else
static const char debruijn64[64] = {
0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
};
static const char debruijn32[32] = {
0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
};
if (sizeof(long) < 8) {
uint32_t y = x;
if (!y) {
y = x>>32;
return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
}
return debruijn32[(y&-y)*0x076be629 >> 27];
}
return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
#endif
}
static const unsigned char bin_tab[60] = {
32,33,34,35,36,36,37,37,38,38,39,39,
40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43,
44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,
46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47,
};
static int bin_index(size_t x)
{
x = x / SIZE_ALIGN - 1;
if (x <= 32) return x;
if (x < 512) return bin_tab[x/8-4];
if (x > 0x1c00) return 63;
return bin_tab[x/128-4] + 16;
}
static int bin_index_up(size_t x)
{
x = x / SIZE_ALIGN - 1;
if (x <= 32) return x;
x--;
if (x < 512) return bin_tab[x/8-4] + 1;
return bin_tab[x/128-4] + 17;
}
#if 0
void __dump_heap(int x)
{
struct chunk *c;
int i;
for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c))
fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n",
c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)),
c->csize & 15,
NEXT_CHUNK(c)->psize & 15);
for (i=0; i<64; i++) {
if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) {
fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head);
if (!(mal.binmap & 1ULL<<i))
fprintf(stderr, "missing from binmap!\n");
} else if (mal.binmap & 1ULL<<i)
fprintf(stderr, "binmap wrongly contains %d!\n", i);
}
}
#endif
void *__expand_heap(size_t *);
static struct chunk *expand_heap(size_t n)
{
static int heap_lock[2];
static void *end;
void *p;
struct chunk *w;
/* The argument n already accounts for the caller's chunk
* overhead needs, but if the heap can't be extended in-place,
* we need room for an extra zero-sized sentinel chunk. */
n += SIZE_ALIGN;
lock(heap_lock);
p = __expand_heap(&n);
if (!p) {
unlock(heap_lock);
return 0;
}
/* If not just expanding existing space, we need to make a
* new sentinel chunk below the allocated space. */
if (p != end) {
/* Valid/safe because of the prologue increment. */
n -= SIZE_ALIGN;
p = (char *)p + SIZE_ALIGN;
w = MEM_TO_CHUNK(p);
w->psize = 0 | C_INUSE;
}
/* Record new heap end and fill in footer. */
end = (char *)p + n;
w = MEM_TO_CHUNK(end);
w->psize = n | C_INUSE;
w->csize = 0 | C_INUSE;
/* Fill in header, which may be new or may be replacing a
* zero-size sentinel header at the old end-of-heap. */
w = MEM_TO_CHUNK(p);
w->csize = n | C_INUSE;
unlock(heap_lock);
return w;
}
static int adjust_size(size_t *n)
{
/* Result of pointer difference must fit in ptrdiff_t. */
if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) {
if (*n) {
errno = ENOMEM;
return -1;
} else {
*n = SIZE_ALIGN;
return 0;
}
}
*n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK;
return 0;
}
static void unbin(struct chunk *c, int i)
{
if (c->prev == c->next)
a_and_64(&mal.binmap, ~(1ULL<<i));
c->prev->next = c->next;
c->next->prev = c->prev;
c->csize |= C_INUSE;
NEXT_CHUNK(c)->psize |= C_INUSE;
}
static int alloc_fwd(struct chunk *c)
{
int i;
size_t k;
while (!((k=c->csize) & C_INUSE)) {
i = bin_index(k);
lock_bin(i);
if (c->csize == k) {
unbin(c, i);
unlock_bin(i);
return 1;
}
unlock_bin(i);
}
return 0;
}
static int alloc_rev(struct chunk *c)
{
int i;
size_t k;
while (!((k=c->psize) & C_INUSE)) {
i = bin_index(k);
lock_bin(i);
if (c->psize == k) {
unbin(PREV_CHUNK(c), i);
unlock_bin(i);
return 1;
}
unlock_bin(i);
}
return 0;
}
/* pretrim - trims a chunk _prior_ to removing it from its bin.
* Must be called with i as the ideal bin for size n, j the bin
* for the _free_ chunk self, and bin j locked. */
static int pretrim(struct chunk *self, size_t n, int i, int j)
{
size_t n1;
struct chunk *next, *split;
/* We cannot pretrim if it would require re-binning. */
if (j < 40) return 0;
if (j < i+3) {
if (j != 63) return 0;
n1 = CHUNK_SIZE(self);
if (n1-n <= MMAP_THRESHOLD) return 0;
} else {
n1 = CHUNK_SIZE(self);
}
if (bin_index(n1-n) != j) return 0;
next = NEXT_CHUNK(self);
split = (void *)((char *)self + n);
split->prev = self->prev;
split->next = self->next;
split->prev->next = split;
split->next->prev = split;
split->psize = n | C_INUSE;
split->csize = n1-n;
next->psize = n1-n;
self->csize = n | C_INUSE;
return 1;
}
static void trim(struct chunk *self, size_t n)
{
size_t n1 = CHUNK_SIZE(self);
struct chunk *next, *split;
if (n >= n1 - DONTCARE) return;
next = NEXT_CHUNK(self);
split = (void *)((char *)self + n);
split->psize = n | C_INUSE;
split->csize = n1-n | C_INUSE;
next->psize = n1-n | C_INUSE;
self->csize = n | C_INUSE;
free(CHUNK_TO_MEM(split));
}
void *malloc(size_t n)
{
struct chunk *c;
int i, j;
if (adjust_size(&n) < 0) return 0;
if (n > MMAP_THRESHOLD) {
size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE;
char *base = __mmap(0, len, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (base == (void *)-1) return 0;
c = (void *)(base + SIZE_ALIGN - OVERHEAD);
c->csize = len - (SIZE_ALIGN - OVERHEAD);
c->psize = SIZE_ALIGN - OVERHEAD;
return CHUNK_TO_MEM(c);
}
i = bin_index_up(n);
for (;;) {
uint64_t mask = mal.binmap & -(1ULL<<i);
if (!mask) {
c = expand_heap(n);
if (!c) return 0;
if (alloc_rev(c)) {
struct chunk *x = c;
c = PREV_CHUNK(c);
NEXT_CHUNK(x)->psize = c->csize =
x->csize + CHUNK_SIZE(c);
}
break;
}
j = first_set(mask);
lock_bin(j);
c = mal.bins[j].head;
if (c != BIN_TO_CHUNK(j)) {
if (!pretrim(c, n, i, j)) unbin(c, j);
unlock_bin(j);
break;
}
unlock_bin(j);
}
/* Now patch up in case we over-allocated */
trim(c, n);
return CHUNK_TO_MEM(c);
}
void *__malloc0(size_t n)
{
void *p = malloc(n);
if (p && !IS_MMAPPED(MEM_TO_CHUNK(p))) {
size_t *z;
n = (n + sizeof *z - 1)/sizeof *z;
for (z=p; n; n--, z++) if (*z) *z=0;
}
return p;
}
void *realloc(void *p, size_t n)
{
struct chunk *self, *next;
size_t n0, n1;
void *new;
if (!p) return malloc(n);
if (adjust_size(&n) < 0) return 0;
self = MEM_TO_CHUNK(p);
n1 = n0 = CHUNK_SIZE(self);
if (IS_MMAPPED(self)) {
size_t extra = self->psize;
char *base = (char *)self - extra;
size_t oldlen = n0 + extra;
size_t newlen = n + extra;
/* Crash on realloc of freed chunk */
if (extra & 1) a_crash();
if (newlen < PAGE_SIZE && (new = malloc(n))) {
memcpy(new, p, n-OVERHEAD);
free(p);
return new;
}
newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE;
if (oldlen == newlen) return p;
base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE);
if (base == (void *)-1)
goto copy_realloc;
self = (void *)(base + extra);
self->csize = newlen - extra;
return CHUNK_TO_MEM(self);
}
next = NEXT_CHUNK(self);
/* Crash on corrupted footer (likely from buffer overflow) */
if (next->psize != self->csize) a_crash();
/* Merge adjacent chunks if we need more space. This is not
* a waste of time even if we fail to get enough space, because our
* subsequent call to free would otherwise have to do the merge. */
if (n > n1 && alloc_fwd(next)) {
n1 += CHUNK_SIZE(next);
next = NEXT_CHUNK(next);
}
/* FIXME: find what's wrong here and reenable it..? */
if (0 && n > n1 && alloc_rev(self)) {
self = PREV_CHUNK(self);
n1 += CHUNK_SIZE(self);
}
self->csize = n1 | C_INUSE;
next->psize = n1 | C_INUSE;
/* If we got enough space, split off the excess and return */
if (n <= n1) {
//memmove(CHUNK_TO_MEM(self), p, n0-OVERHEAD);
trim(self, n);
return CHUNK_TO_MEM(self);
}
copy_realloc:
/* As a last resort, allocate a new chunk and copy to it. */
new = malloc(n-OVERHEAD);
if (!new) return 0;
memcpy(new, p, n0-OVERHEAD);
free(CHUNK_TO_MEM(self));
return new;
}
void free(void *p)
{
struct chunk *self = MEM_TO_CHUNK(p);
struct chunk *next;
size_t final_size, new_size, size;
int reclaim=0;
int i;
if (!p) return;
if (IS_MMAPPED(self)) {
size_t extra = self->psize;
char *base = (char *)self - extra;
size_t len = CHUNK_SIZE(self) + extra;
/* Crash on double free */
if (extra & 1) a_crash();
__munmap(base, len);
return;
}
final_size = new_size = CHUNK_SIZE(self);
next = NEXT_CHUNK(self);
/* Crash on corrupted footer (likely from buffer overflow) */
if (next->psize != self->csize) a_crash();
for (;;) {
if (self->psize & next->csize & C_INUSE) {
self->csize = final_size | C_INUSE;
next->psize = final_size | C_INUSE;
i = bin_index(final_size);
lock_bin(i);
lock(mal.free_lock);
if (self->psize & next->csize & C_INUSE)
break;
unlock(mal.free_lock);
unlock_bin(i);
}
if (alloc_rev(self)) {
self = PREV_CHUNK(self);
size = CHUNK_SIZE(self);
final_size += size;
if (new_size+size > RECLAIM && (new_size+size^size) > size)
reclaim = 1;
}
if (alloc_fwd(next)) {
size = CHUNK_SIZE(next);
final_size += size;
if (new_size+size > RECLAIM && (new_size+size^size) > size)
reclaim = 1;
next = NEXT_CHUNK(next);
}
}
if (!(mal.binmap & 1ULL<<i))
a_or_64(&mal.binmap, 1ULL<<i);
self->csize = final_size;
next->psize = final_size;
unlock(mal.free_lock);
self->next = BIN_TO_CHUNK(i);
self->prev = mal.bins[i].tail;
self->next->prev = self;
self->prev->next = self;
/* Replace middle of large chunks with fresh zero pages */
if (reclaim) {
uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE;
uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE;
#if 1
__madvise((void *)a, b-a, MADV_DONTNEED);
#else
__mmap((void *)a, b-a, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
#endif
}
unlock_bin(i);
}

1283
c/examples/parser.c Normal file

File diff suppressed because it is too large Load Diff

1360
c/grammar.js Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,274 @@
================================================================================
pointer declarations vs expressions
================================================================================
TSLanguage *(*lang_parser)(void);
char (*ptr_to_array)[];
int main() {
// declare a function pointer
T1 * b(T2 a);
// evaluate expressions
c * d(5);
e(f * g);
}
--------------------------------------------------------------------------------
(translation_unit
(declaration
(type_identifier)
(pointer_declarator
(function_declarator
(parenthesized_declarator
(pointer_declarator
(identifier)))
(parameter_list
(parameter_declaration
(primitive_type))))))
(declaration
(primitive_type)
(array_declarator
(parenthesized_declarator
(pointer_declarator
(identifier)))))
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(comment)
(declaration
(type_identifier)
(pointer_declarator
(function_declarator
(identifier)
(parameter_list
(parameter_declaration
(type_identifier)
(identifier))))))
(comment)
(expression_statement
(binary_expression
(identifier)
(call_expression
(identifier)
(argument_list
(number_literal)))))
(expression_statement
(call_expression
(identifier)
(argument_list
(binary_expression
(identifier)
(identifier))))))))
================================================================================
casts vs multiplications
================================================================================
/*
* ambiguities
*/
int main() {
// cast
a((B *)c);
// parenthesized product
d((e * f));
}
--------------------------------------------------------------------------------
(translation_unit
(comment)
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(comment)
(expression_statement
(call_expression
(identifier)
(argument_list
(cast_expression
(type_descriptor
(type_identifier)
(abstract_pointer_declarator))
(identifier)))))
(comment)
(expression_statement
(call_expression
(identifier)
(argument_list
(parenthesized_expression
(binary_expression
(identifier)
(identifier)))))))))
================================================================================
function-like type macros vs function calls
================================================================================
// this is a macro
GIT_INLINE(int *) x = 5;
--------------------------------------------------------------------------------
(translation_unit
(comment)
(declaration
(macro_type_specifier
(identifier)
(type_descriptor
(primitive_type)
(abstract_pointer_declarator)))
(init_declarator
(identifier)
(number_literal))))
================================================================================
function calls vs parenthesized declarators vs macro types
================================================================================
int main() {
/*
* Could be either:
* - function call
* - declaration w/ parenthesized declarator
* - declaration w/ macro type, no declarator
*/
ABC(d);
/*
* Normal declaration
*/
efg hij;
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(comment)
(expression_statement
(call_expression
(identifier)
(argument_list
(identifier))))
(comment)
(declaration
(type_identifier)
(identifier)))))
================================================================================
Call expressions vs empty declarations w/ macros as types
================================================================================
int main() {
int a = 1;
b(a);
A(A *);
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(declaration
(primitive_type)
(init_declarator
(identifier)
(number_literal)))
(expression_statement
(call_expression
(identifier)
(argument_list
(identifier))))
(macro_type_specifier
(identifier)
(type_descriptor
(type_identifier)
(abstract_pointer_declarator))))))
================================================================================
Comments after for loops with ambiguities
================================================================================
int main() {
for (a *b = c; d; e) {
aff;
}
// a-comment
g;
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(for_statement
(declaration
(type_identifier)
(init_declarator
(pointer_declarator
(identifier))
(identifier)))
(identifier)
(identifier)
(compound_statement
(expression_statement
(identifier))))
(comment)
(expression_statement
(identifier)))))
================================================================================
Top-level macro invocations
================================================================================
DEFINE_SOMETHING(THING_A, "this is a thing a");
DEFINE_SOMETHING(THING_B, "this is a thing b", "thanks");
--------------------------------------------------------------------------------
(translation_unit
(expression_statement
(call_expression
(identifier)
(argument_list
(identifier)
(string_literal
(string_content)))))
(expression_statement
(call_expression
(identifier)
(argument_list
(identifier)
(string_literal
(string_content))
(string_literal
(string_content))))))

13
c/test/corpus/crlf.txt Normal file
View File

@ -0,0 +1,13 @@
============================================
Line comments with escaped CRLF line endings
============================================
// hello \
this is still a comment
this_is_not a_comment;
---
(translation_unit
(comment)
(declaration (type_identifier) (identifier)))

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

253
c/test/corpus/microsoft.txt Normal file
View File

@ -0,0 +1,253 @@
================================
declaration specs
================================
struct __declspec(dllexport) s2
{
};
union __declspec(noinline) u2 {
};
---
(translation_unit
(struct_specifier
(ms_declspec_modifier
(identifier))
name: (type_identifier)
body: (field_declaration_list))
(union_specifier
(ms_declspec_modifier
(identifier))
name: (type_identifier)
body: (field_declaration_list)))
================================
pointers
================================
struct s2
{
int * __restrict x;
int * __sptr psp;
int * __uptr pup;
int * __unaligned pup;
};
void sum2(int n, int * __restrict a, int * __restrict b,
int * c, int * d) {
int i;
for (i = 0; i < n; i++) {
a[i] = b[i] + c[i];
c[i] = b[i] + d[i];
}
}
void MyFunction(char * __uptr myValue);
---
(translation_unit
(struct_specifier
name: (type_identifier)
body: (field_declaration_list
(field_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_restrict_modifier))
declarator: (field_identifier)))
(field_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_signed_ptr_modifier))
declarator: (field_identifier)))
(field_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_unsigned_ptr_modifier))
declarator: (field_identifier)))
(field_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_unaligned_ptr_modifier))
declarator: (field_identifier)))))
(function_definition
type: (primitive_type)
declarator: (function_declarator
declarator: (identifier)
parameters: (parameter_list
(parameter_declaration
type: (primitive_type)
declarator: (identifier))
(parameter_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_restrict_modifier))
declarator: (identifier)))
(parameter_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_restrict_modifier))
declarator: (identifier)))
(parameter_declaration
type: (primitive_type)
declarator: (pointer_declarator
declarator: (identifier)))
(parameter_declaration
type: (primitive_type)
declarator: (pointer_declarator
declarator: (identifier)))))
body: (compound_statement
(declaration
type: (primitive_type)
declarator: (identifier))
(for_statement
initializer: (assignment_expression
left: (identifier)
right: (number_literal))
condition: (binary_expression
left: (identifier)
right: (identifier))
update: (update_expression
argument: (identifier))
body: (compound_statement
(expression_statement
(assignment_expression
left: (subscript_expression
argument: (identifier)
index: (identifier))
right: (binary_expression
left: (subscript_expression
argument: (identifier)
index: (identifier))
right: (subscript_expression
argument: (identifier)
index: (identifier)))))
(expression_statement
(assignment_expression
left: (subscript_expression
argument: (identifier)
index: (identifier))
right: (binary_expression
left: (subscript_expression
argument: (identifier)
index: (identifier))
right: (subscript_expression
argument: (identifier)
index: (identifier)))))))))
(declaration
type: (primitive_type)
declarator: (function_declarator
declarator: (identifier)
parameters: (parameter_list
(parameter_declaration
type: (primitive_type)
declarator: (pointer_declarator
(ms_pointer_modifier
(ms_unsigned_ptr_modifier))
declarator: (identifier)))))))
================================
call modifiers
================================
__cdecl void mymethod(){
return;
}
__fastcall void mymethod(){
return;
}
---
(translation_unit
(function_definition
(ms_call_modifier)
type: (primitive_type)
declarator: (function_declarator
declarator: (identifier)
parameters: (parameter_list))
body: (compound_statement
(return_statement)))
(function_definition
(ms_call_modifier)
type: (primitive_type)
declarator: (function_declarator
declarator: (identifier)
parameters: (parameter_list))
body: (compound_statement
(return_statement))))
================================
SEH exception handling
================================
int main() {
int arg;
__try {
__try {
arg = 1;
__leave;
} __except (-1) {
arg = 2;
}
__leave;
arg = 3;
} __finally {
printf("arg: %d\n", arg);
}
}
---
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(declaration
(primitive_type)
(identifier))
(seh_try_statement
(compound_statement
(seh_try_statement
(compound_statement
(expression_statement
(assignment_expression
(identifier)
(number_literal)))
(seh_leave_statement))
(seh_except_clause
(parenthesized_expression
(number_literal))
(compound_statement
(expression_statement
(assignment_expression
(identifier)
(number_literal))))))
(seh_leave_statement)
(expression_statement
(assignment_expression
(identifier)
(number_literal))))
(seh_finally_clause
(compound_statement
(expression_statement
(call_expression
(identifier)
(argument_list
(string_literal
(string_content)
(escape_sequence))
(identifier))))))))))

View File

@ -0,0 +1,401 @@
================================================================================
Include directives
================================================================================
#include "some/path.h"
#include <stdint.h>
#include MACRO
#include MACRO(arg1, arg2)
--------------------------------------------------------------------------------
(translation_unit
(preproc_include
path: (string_literal
(string_content)))
(preproc_include
path: (system_lib_string))
(preproc_include
path: (identifier))
(preproc_include
path: (call_expression
function: (identifier)
arguments: (argument_list
(identifier)
(identifier)))))
================================================================================
Object-like macro definitions
================================================================================
#define ONE
#define TWO int a = b;
#define THREE \
c == d ? \
e : \
f
#define FOUR (mno * pq)
#define FIVE(a,b) x \
+ y
#define SIX(a, \
b) x \
+ y
#define SEVEN 7/* seven has an
* annoying comment */
#define EIGHT(x) do { \
x = x + 1; \
x = x / 2; \
} while (x > 0);
--------------------------------------------------------------------------------
(translation_unit
(preproc_def
name: (identifier))
(preproc_def
name: (identifier)
value: (preproc_arg))
(preproc_def
name: (identifier)
value: (preproc_arg))
(preproc_def
name: (identifier)
value: (preproc_arg))
(preproc_function_def
name: (identifier)
parameters: (preproc_params
(identifier)
(identifier))
value: (preproc_arg))
(preproc_function_def
name: (identifier)
parameters: (preproc_params
(identifier)
(identifier))
value: (preproc_arg))
(preproc_def
name: (identifier)
value: (preproc_arg)
(comment))
(preproc_function_def
name: (identifier)
parameters: (preproc_params
(identifier))
value: (preproc_arg)))
================================================================================
Function-like macro definitions
================================================================================
#define ONE() a
#define TWO(b) c
#define THREE(d, e) f
#define FOUR(...) g
#define FIVE(h, i, ...) j
--------------------------------------------------------------------------------
(translation_unit
(preproc_function_def
name: (identifier)
parameters: (preproc_params)
value: (preproc_arg))
(preproc_function_def
name: (identifier)
parameters: (preproc_params
(identifier))
value: (preproc_arg))
(preproc_function_def
name: (identifier)
parameters: (preproc_params
(identifier)
(identifier))
value: (preproc_arg))
(preproc_function_def
name: (identifier)
parameters: (preproc_params)
value: (preproc_arg))
(preproc_function_def
name: (identifier)
parameters: (preproc_params
(identifier)
(identifier))
value: (preproc_arg)))
================================================================================
Ifdefs
================================================================================
#ifndef DEFINE1
int j;
#endif
#ifdef DEFINE2
ssize_t b;
#define c 32
#elif defined DEFINE3
#else
int b;
#define c 16
#endif
#ifdef DEFINE2
#else
# ifdef DEFINE3
# else
# endif
#endif
--------------------------------------------------------------------------------
(translation_unit
(preproc_ifdef
name: (identifier)
(declaration
type: (primitive_type)
declarator: (identifier)))
(preproc_ifdef
name: (identifier)
(declaration
type: (primitive_type)
declarator: (identifier))
(preproc_def
name: (identifier)
value: (preproc_arg))
alternative: (preproc_elif
condition: (preproc_defined
(identifier))
alternative: (preproc_else
(declaration
type: (primitive_type)
declarator: (identifier))
(preproc_def
name: (identifier)
value: (preproc_arg)))))
(preproc_ifdef
name: (identifier)
alternative: (preproc_else
(preproc_ifdef
name: (identifier)
alternative: (preproc_else)))))
================================================================================
Elifdefs
================================================================================
#ifndef DEFINE1
int j;
#elifndef DEFINE2
int k;
#endif
#ifdef DEFINE2
ssize_t b;
#elifdef DEFINE3
ssize_t c;
#else
int b;
#endif
--------------------------------------------------------------------------------
(translation_unit
(preproc_ifdef
(identifier)
(declaration
(primitive_type)
(identifier))
(preproc_elifdef
(identifier)
(declaration
(primitive_type)
(identifier))))
(preproc_ifdef
(identifier)
(declaration
(primitive_type)
(identifier))
(preproc_elifdef
(identifier)
(declaration
(primitive_type)
(identifier))
(preproc_else
(declaration
(primitive_type)
(identifier))))))
================================================================================
General if blocks
================================================================================
#if defined(__GNUC__) && defined(__PIC__)
#define inline inline __attribute__((always_inline))
#elif defined(_WIN32)
#define something
#elif !defined(SOMETHING_ELSE)
#define SOMETHING_ELSE
#else
#include <something>
#endif
--------------------------------------------------------------------------------
(translation_unit
(preproc_if
condition: (binary_expression
left: (preproc_defined
(identifier))
right: (preproc_defined
(identifier)))
(preproc_def
name: (identifier)
value: (preproc_arg))
alternative: (preproc_elif
condition: (preproc_defined
(identifier))
(preproc_def
name: (identifier))
alternative: (preproc_elif
condition: (unary_expression
argument: (preproc_defined
(identifier)))
(preproc_def
name: (identifier))
alternative: (preproc_else
(preproc_include
path: (system_lib_string)))))))
================================================================================
Preprocessor conditionals in functions
================================================================================
int main() {
#if d
puts("1");
#else
puts("2");
#endif
#if a
return 0;
#elif b
return 1;
#elif c
return 2;
#else
return 3;
#endif
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(preproc_if
(identifier)
(expression_statement
(call_expression
(identifier)
(argument_list
(string_literal
(string_content)))))
(preproc_else
(expression_statement
(call_expression
(identifier)
(argument_list
(string_literal
(string_content)))))))
(preproc_if
(identifier)
(return_statement
(number_literal))
(preproc_elif
(identifier)
(return_statement
(number_literal))
(preproc_elif
(identifier)
(return_statement
(number_literal))
(preproc_else
(return_statement
(number_literal)))))))))
================================================================================
Preprocessor conditionals in struct/union bodies
================================================================================
struct S {
#ifdef _WIN32
LONG f2;
#else
uint32_t f2;
#endif
};
--------------------------------------------------------------------------------
(translation_unit
(struct_specifier
(type_identifier)
(field_declaration_list
(preproc_ifdef
(identifier)
(field_declaration
(type_identifier)
(field_identifier))
(preproc_else
(field_declaration
(primitive_type)
(field_identifier)))))))
================================================================================
Unknown preprocessor directives
================================================================================
#pragma mark - UIViewController
--------------------------------------------------------------------------------
(translation_unit
(preproc_call
directive: (preproc_directive)
argument: (preproc_arg)))
================================================================================
Preprocessor expressions
================================================================================
#if A(B || C) && \
!D(F)
uint32_t a;
#endif
--------------------------------------------------------------------------------
(translation_unit
(preproc_if
(binary_expression
(call_expression
(identifier)
(argument_list
(binary_expression
(identifier)
(identifier))))
(unary_expression
(call_expression
(identifier)
(argument_list
(identifier)))))
(declaration
(primitive_type)
(identifier))))

View File

@ -0,0 +1,535 @@
================================================================================
If statements
================================================================================
int main() {
if (a)
1;
if (!a) {
2;
} else {
3;
}
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(if_statement
(parenthesized_expression
(identifier))
(expression_statement
(number_literal)))
(if_statement
(parenthesized_expression
(unary_expression
(identifier)))
(compound_statement
(expression_statement
(number_literal)))
(else_clause
(compound_statement
(expression_statement
(number_literal))))))))
================================================================================
For loops
================================================================================
int main() {
for (;;)
1;
for (int i = 0; i < 5; next(), i++) {
2;
}
for (start(); check(); step())
3;
for (i = 0, j = 0, k = 0, l = 0; i < 1, j < 1; i++, j++, k++, l++)
1;
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(for_statement
(expression_statement
(number_literal)))
(for_statement
(declaration
(primitive_type)
(init_declarator
(identifier)
(number_literal)))
(binary_expression
(identifier)
(number_literal))
(comma_expression
(call_expression
(identifier)
(argument_list))
(update_expression
(identifier)))
(compound_statement
(expression_statement
(number_literal))))
(for_statement
(call_expression
(identifier)
(argument_list))
(call_expression
(identifier)
(argument_list))
(call_expression
(identifier)
(argument_list))
(expression_statement
(number_literal)))
(for_statement
(comma_expression
(assignment_expression
(identifier)
(number_literal))
(comma_expression
(assignment_expression
(identifier)
(number_literal))
(comma_expression
(assignment_expression
(identifier)
(number_literal))
(assignment_expression
(identifier)
(number_literal)))))
(comma_expression
(binary_expression
(identifier)
(number_literal))
(binary_expression
(identifier)
(number_literal)))
(comma_expression
(update_expression
(identifier))
(comma_expression
(update_expression
(identifier))
(comma_expression
(update_expression
(identifier))
(update_expression
(identifier)))))
(expression_statement
(number_literal))))))
================================================================================
While loops
================================================================================
int main() {
while (x)
printf("hi");
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(while_statement
(parenthesized_expression
(identifier))
(expression_statement
(call_expression
(identifier)
(argument_list
(string_literal
(string_content)))))))))
================================================================================
Labeled statements
================================================================================
void foo(T *t) {
recur:
t = t->next();
if (t) goto recur;
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list
(parameter_declaration
(type_identifier)
(pointer_declarator
(identifier)))))
(compound_statement
(labeled_statement
(statement_identifier)
(expression_statement
(assignment_expression
(identifier)
(call_expression
(field_expression
(identifier)
(field_identifier))
(argument_list)))))
(if_statement
(parenthesized_expression
(identifier))
(goto_statement
(statement_identifier))))))
================================================================================
Switch statements
================================================================================
void foo(int a) {
switch (a) {
puts("entered switch!");
case 3:
case 5:
if (b) {
c();
}
break;
default:
c();
break;
}
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list
(parameter_declaration
(primitive_type)
(identifier))))
(compound_statement
(switch_statement
(parenthesized_expression
(identifier))
(compound_statement
(expression_statement
(call_expression
(identifier)
(argument_list
(string_literal
(string_content)))))
(case_statement
(number_literal))
(case_statement
(number_literal)
(if_statement
(parenthesized_expression
(identifier))
(compound_statement
(expression_statement
(call_expression
(identifier)
(argument_list)))))
(break_statement))
(case_statement
(expression_statement
(call_expression
(identifier)
(argument_list)))
(break_statement)))))))
================================================================================
Case statements separate from switch statements
================================================================================
int main() {
switch (count % 8) {
case 0:
do {
*to = *from++;
case 2: *to = *from++;
case 1: *to = *from++;
} while (--n > 0);
}
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(switch_statement
(parenthesized_expression
(binary_expression
(identifier)
(number_literal)))
(compound_statement
(case_statement
(number_literal)
(do_statement
(compound_statement
(expression_statement
(assignment_expression
(pointer_expression
(identifier))
(pointer_expression
(update_expression
(identifier)))))
(case_statement
(number_literal)
(expression_statement
(assignment_expression
(pointer_expression
(identifier))
(pointer_expression
(update_expression
(identifier))))))
(case_statement
(number_literal)
(expression_statement
(assignment_expression
(pointer_expression
(identifier))
(pointer_expression
(update_expression
(identifier)))))))
(parenthesized_expression
(binary_expression
(update_expression
(identifier))
(number_literal))))))))))
================================================================================
Return statements
================================================================================
void foo() {
return;
return a;
return a, b;
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(return_statement)
(return_statement
(identifier))
(return_statement
(comma_expression
(identifier)
(identifier))))))
================================================================================
Comments with asterisks
================================================================================
/*************************
* odd number of asterisks
*************************/
int a;
/**************************
* even number of asterisks
**************************/
int b;
--------------------------------------------------------------------------------
(translation_unit
(comment)
(declaration
(primitive_type)
(identifier))
(comment)
(declaration
(primitive_type)
(identifier)))
================================================================================
Comment with multiple backslashes
================================================================================
int a = 3; // Hello \\
World
--------------------------------------------------------------------------------
(translation_unit
(declaration
(primitive_type)
(init_declarator
(identifier)
(number_literal)))
(comment))
================================================================================
Attributes
================================================================================
void f() {
[[a]] switch (b) {
[[c]] case 1: {}
case 2:
[[fallthrough]];
default:
}
[[a]] while (true) {}
[[a]] if (true) {}
[[a]] for (;;) {}
[[a]] return;
[[a]] a;
[[a]];
[[a]] label: {}
[[a]] goto label;
// these are c++ specific, but their bind locations should be c-compatible
if (true) [[likely]] {} else [[unlikely]] {}
do [[likely]] {} while (true);
}
--------------------------------------------------------------------------------
(translation_unit
(function_definition
(primitive_type)
(function_declarator
(identifier)
(parameter_list))
(compound_statement
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(switch_statement
(parenthesized_expression
(identifier))
(compound_statement
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(case_statement
(number_literal)
(compound_statement)))
(case_statement
(number_literal)
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(expression_statement)))
(case_statement))))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(while_statement
(parenthesized_expression
(true))
(compound_statement)))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(if_statement
(parenthesized_expression
(true))
(compound_statement)))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(for_statement
(compound_statement)))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(return_statement))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(expression_statement
(identifier)))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(expression_statement))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(labeled_statement
(statement_identifier)
(compound_statement)))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(goto_statement
(statement_identifier)))
(comment)
(if_statement
(parenthesized_expression
(true))
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(compound_statement))
(else_clause
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(compound_statement))))
(do_statement
(attributed_statement
(attribute_declaration
(attribute
(identifier)))
(compound_statement))
(parenthesized_expression
(true))))))

80
c/test/corpus/types.txt Normal file
View File

@ -0,0 +1,80 @@
========================================
Primitive types
========================================
int a;
uint8_t a;
uint16_t a;
uint32_t a;
uint64_t a;
uintptr_t a;
int8_t a;
int16_t a;
int32_t a;
int64_t a;
intptr_t a;
char16_t a;
char32_t a;
size_t a;
ssize_t a;
---
(translation_unit
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier))
(declaration (primitive_type) (identifier)))
========================================
Type modifiers
========================================
void f(unsigned);
void f(unsigned int);
void f(signed long int);
void f(unsigned v1);
void f(unsigned long v2);
---
(translation_unit
(declaration
(primitive_type)
(function_declarator
(identifier)
(parameter_list (parameter_declaration (sized_type_specifier)))))
(declaration
(primitive_type)
(function_declarator
(identifier)
(parameter_list (parameter_declaration (sized_type_specifier (primitive_type))))))
(declaration
(primitive_type)
(function_declarator
(identifier)
(parameter_list (parameter_declaration (sized_type_specifier (primitive_type))))))
(declaration
(primitive_type)
(function_declarator
(identifier)
(parameter_list (parameter_declaration (sized_type_specifier) (identifier)))))
(declaration
(primitive_type)
(function_declarator
(identifier)
(parameter_list (parameter_declaration (sized_type_specifier) (identifier))))))

View File

@ -0,0 +1,6 @@
#include <stdlib.h>
// ^ keyword
// ^ string
#include "something.h"
// ^ string

33
c/test/highlight/names.c Normal file
View File

@ -0,0 +1,33 @@
typedef struct {
// ^ keyword
// ^ keyword
a_t b;
// <- type
// ^ property
unsigned c_t (*d)[2];
// ^ type
// ^ type
// ^ property
}, T, V;
// ^ type
// ^ type
int main(const char string[SIZE]) {
// <- type
// ^ function
// ^ keyword
// ^ type
// ^ variable
// ^ constant
return foo.bar + foo.baz();
// ^ keyword
// ^ variable
// ^ property
// ^ function
error:
// <- label
return 0;
}

21
nix/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Charles Strahan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

411
nix/grammar.js Normal file
View File

@ -0,0 +1,411 @@
const PREC = {
impl: 1,
or: 2,
and: 3,
eq: 4,
neq: 4,
"<": 5,
">": 5,
leq: 5,
geq: 5,
update: 6,
not: 7,
"+": 8,
"-": 8,
"*": 9,
"/": 9,
concat: 10,
"?": 11,
negate: 12,
};
module.exports = grammar({
name: "nix",
extras: ($) => [/\s/, $.comment],
supertypes: ($) => [$._expression],
inline: ($) => [],
externals: ($) => [
$.string_fragment,
$._indented_string_fragment,
$._path_start,
$.path_fragment,
$.dollar_escape,
$._indented_dollar_escape,
],
word: ($) => $.keyword,
conflicts: ($) => [],
rules: {
source_code: ($) => optional(field("expression", $._expression)),
_expression: ($) => $._expr_function_expression,
// Keywords go before identifiers to let them take precedence when both are expected.
// Workaround before https://github.com/tree-sitter/tree-sitter/pull/246
keyword: ($) => /if|then|else|let|inherit|in|rec|with|assert/,
identifier: ($) => /[a-zA-Z_][a-zA-Z0-9_\'\-]*/,
variable_expression: ($) => field("name", $.identifier),
integer_expression: ($) => /[0-9]+/,
float_expression: ($) =>
/(([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?/,
path_expression: ($) =>
seq(
alias($._path_start, $.path_fragment),
repeat(
choice(
$.path_fragment,
alias($._immediate_interpolation, $.interpolation)
)
)
),
_hpath_start: ($) => /\~\/[a-zA-Z0-9\._\-\+\/]+/,
hpath_expression: ($) =>
seq(
alias($._hpath_start, $.path_fragment),
repeat(
choice(
$.path_fragment,
alias($._immediate_interpolation, $.interpolation)
)
)
),
spath_expression: ($) => /<[a-zA-Z0-9\._\-\+]+(\/[a-zA-Z0-9\._\-\+]+)*>/,
uri_expression: ($) =>
/[a-zA-Z][a-zA-Z0-9\+\-\.]*:[a-zA-Z0-9%\/\?:@\&=\+\$,\-_\.\!\~\*\']+/,
_expr_function_expression: ($) =>
choice(
$.function_expression,
$.assert_expression,
$.with_expression,
$.let_expression,
$._expr_if
),
function_expression: ($) =>
choice(
seq(
field("universal", $.identifier),
":",
field("body", $._expr_function_expression)
),
seq(
field("formals", $.formals),
":",
field("body", $._expr_function_expression)
),
seq(
field("formals", $.formals),
"@",
field("universal", $.identifier),
":",
field("body", $._expr_function_expression)
),
seq(
field("universal", $.identifier),
"@",
field("formals", $.formals),
":",
field("body", $._expr_function_expression)
)
),
formals: ($) =>
choice(
seq("{", "}"),
seq("{", commaSep1(field("formal", $.formal)), "}"),
seq(
"{",
commaSep1(field("formal", $.formal)),
",",
field("ellipses", $.ellipses),
"}"
),
seq("{", field("ellipses", $.ellipses), "}")
),
formal: ($) =>
seq(
field("name", $.identifier),
optional(seq("?", field("default", $._expression)))
),
ellipses: ($) => "...",
assert_expression: ($) =>
seq(
"assert",
field("condition", $._expression),
";",
field("body", $._expr_function_expression)
),
with_expression: ($) =>
seq(
"with",
field("environment", $._expression),
";",
field("body", $._expr_function_expression)
),
let_expression: ($) =>
seq(
"let",
optional($.binding_set),
"in",
field("body", $._expr_function_expression)
),
_expr_if: ($) => choice($.if_expression, $._expr_op),
if_expression: ($) =>
seq(
"if",
field("condition", $._expression),
"then",
field("consequence", $._expression),
"else",
field("alternative", $._expression)
),
_expr_op: ($) =>
choice(
$.has_attr_expression,
$.unary_expression,
$.binary_expression,
$._expr_apply_expression
),
// I choose to *not* have this among the binary operators because
// this is the sole exception that takes an attrpath (instead of expression)
// as its right operand.
// My gut feeling is that this is:
// 1) better in theory, and
// 2) will be easier to work with in practice.
has_attr_expression: ($) =>
prec(
PREC["?"],
seq(
field("expression", $._expr_op),
field("operator", "?"),
field("attrpath", $.attrpath)
)
),
unary_expression: ($) =>
choice(
...[
["!", PREC.not],
["-", PREC.negate],
].map(([operator, precedence]) =>
prec(
precedence,
seq(field("operator", operator), field("argument", $._expr_op))
)
)
),
binary_expression: ($) =>
choice(
// left assoc.
...[
["==", PREC.eq],
["!=", PREC.neq],
["<", PREC["<"]],
["<=", PREC.leq],
[">", PREC[">"]],
[">=", PREC.geq],
["&&", PREC.and],
["||", PREC.or],
["+", PREC["+"]],
["-", PREC["-"]],
["*", PREC["*"]],
["/", PREC["/"]],
].map(([operator, precedence]) =>
prec.left(
precedence,
seq(
field("left", $._expr_op),
field("operator", operator),
field("right", $._expr_op)
)
)
),
// right assoc.
...[
["->", PREC.impl],
["//", PREC.update],
["++", PREC.concat],
].map(([operator, precedence]) =>
prec.right(
precedence,
seq(
field("left", $._expr_op),
field("operator", operator),
field("right", $._expr_op)
)
)
)
),
_expr_apply_expression: ($) =>
choice($.apply_expression, $._expr_select_expression),
apply_expression: ($) =>
seq(
field("function", $._expr_apply_expression),
field("argument", $._expr_select_expression)
),
_expr_select_expression: ($) => choice($.select_expression, $._expr_simple),
select_expression: ($) =>
choice(
seq(
field("expression", $._expr_simple),
".",
field("attrpath", $.attrpath)
),
seq(
field("expression", $._expr_simple),
".",
field("attrpath", $.attrpath),
"or",
field("default", $._expr_select_expression)
)
),
_expr_simple: ($) =>
choice(
$.variable_expression,
$.integer_expression,
$.float_expression,
$.string_expression,
$.indented_string_expression,
$.path_expression,
$.hpath_expression,
$.spath_expression,
$.uri_expression,
$.parenthesized_expression,
$.attrset_expression,
$.let_attrset_expression,
$.rec_attrset_expression,
$.list_expression
),
parenthesized_expression: ($) =>
seq("(", field("expression", $._expression), ")"),
attrset_expression: ($) => seq("{", optional($.binding_set), "}"),
let_attrset_expression: ($) =>
seq("let", "{", optional($.binding_set), "}"),
rec_attrset_expression: ($) =>
seq("rec", "{", optional($.binding_set), "}"),
string_expression: ($) =>
seq(
'"',
repeat(
choice(
$.string_fragment,
$.interpolation,
choice(
$.escape_sequence,
seq($.dollar_escape, alias("$", $.string_fragment))
)
)
),
'"'
),
escape_sequence: ($) => token.immediate(/\\([^$]|\s)/), // Can also escape newline.
indented_string_expression: ($) =>
seq(
"''",
repeat(
choice(
alias($._indented_string_fragment, $.string_fragment),
$.interpolation,
choice(
alias($._indented_escape_sequence, $.escape_sequence),
seq(
alias($._indented_dollar_escape, $.dollar_escape),
alias("$", $.string_fragment)
)
)
)
),
"''"
),
_indented_escape_sequence: ($) => token.immediate(/'''|''\\([^$]|\s)/), // Can also escape newline.
binding_set: ($) =>
repeat1(field("binding", choice($.binding, $.inherit, $.inherit_from))),
binding: ($) =>
seq(
field("attrpath", $.attrpath),
"=",
field("expression", $._expression),
";"
),
inherit: ($) => seq("inherit", field("attrs", $.inherited_attrs), ";"),
inherit_from: ($) =>
seq(
"inherit",
"(",
field("expression", $._expression),
")",
field("attrs", $.inherited_attrs),
";"
),
attrpath: ($) =>
sep1(
field(
"attr",
choice($.identifier, $.string_expression, $.interpolation)
),
"."
),
inherited_attrs: ($) =>
repeat1(
field(
"attr",
choice($.identifier, $.string_expression, $.interpolation)
)
),
_immediate_interpolation: ($) =>
seq(token.immediate("${"), field("expression", $._expression), "}"),
interpolation: ($) => seq("${", field("expression", $._expression), "}"),
list_expression: ($) =>
seq("[", repeat(field("element", $._expr_select_expression)), "]"),
comment: ($) =>
token(choice(seq("#", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/"))),
},
});
function sep(rule, separator) {
return optional(sep1(rule, separator));
}
function sep1(rule, separator) {
return seq(rule, repeat(seq(separator, rule)));
}
function commaSep1(rule) {
return sep1(rule, ",");
}
function commaSep(rule) {
return optional(commaSep1(rule));
}

238
nix/src/scanner.c Normal file
View File

@ -0,0 +1,238 @@
#include <tree_sitter/parser.h>
enum TokenType {
STRING_FRAGMENT,
INDENTED_STRING_FRAGMENT,
PATH_START,
PATH_FRAGMENT,
DOLLAR_ESCAPE,
INDENTED_DOLLAR_ESCAPE,
};
static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
static bool scan_dollar_escape(TSLexer *lexer) {
lexer->result_symbol = DOLLAR_ESCAPE;
advance(lexer);
lexer->mark_end(lexer);
if (lexer->lookahead == '$') {
return true;
} else {
return false;
}
}
static bool scan_indented_dollar_escape(TSLexer *lexer) {
lexer->result_symbol = INDENTED_DOLLAR_ESCAPE;
advance(lexer);
lexer->mark_end(lexer);
if (lexer->lookahead == '$') {
return true;
} else {
if (lexer->lookahead == '\\') {
advance(lexer);
if (lexer->lookahead == '$') {
lexer->mark_end(lexer);
return true;
}
}
return false;
}
}
// Here we only parse literal fragment inside a string.
// Delimiter, interpolation and escape sequence are handled by the parser and we
// simply stop at them.
//
// The implementation is inspired by tree-sitter-javascript:
// https://github.com/tree-sitter/tree-sitter-javascript/blob/fdeb68ac8d2bd5a78b943528bb68ceda3aade2eb/src/scanner.c#L19
static bool scan_string_fragment(TSLexer *lexer) {
lexer->result_symbol = STRING_FRAGMENT;
for (bool has_content = false;; has_content = true) {
lexer->mark_end(lexer);
switch (lexer->lookahead) {
case '"':
case '\\':
return has_content;
case '$':
advance(lexer);
if (lexer->lookahead == '{') {
return has_content;
} else if (lexer->lookahead != '"' && lexer->lookahead != '\\') {
// Any char following '$' other than '"', '\\' and '{' (which was
// handled above) should be consumed as additional string content. This
// means `$${` doesn't start an interpolation, but `$$${` does.
advance(lexer);
}
break;
// Simply give up on EOF or '\0'.
case '\0':
return false;
default:
advance(lexer);
}
}
}
// See comments of scan_string_fragment.
static bool scan_indented_string_fragment(TSLexer *lexer) {
lexer->result_symbol = INDENTED_STRING_FRAGMENT;
for (bool has_content = false;; has_content = true) {
lexer->mark_end(lexer);
switch (lexer->lookahead) {
case '$':
advance(lexer);
if (lexer->lookahead == '{') {
return has_content;
} else if (lexer->lookahead != '\'') {
// Any char following '$' other than '\'' and '{' (which was handled
// above) should be consumed as additional string content. This means
// `$${` doesn't start an interpolation, but `$$${` does.
advance(lexer);
}
break;
case '\'':
advance(lexer);
if (lexer->lookahead == '\'') {
// Two single quotes always stop current string fragment.
// It can be either an end delimiter '', or escape sequences ''', ''$,
// ''\<any>
return has_content;
}
break;
// Simply give up on EOF or '\0'.
case '\0':
return false;
default:
advance(lexer);
}
}
}
static bool is_path_char(int32_t c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') || c == '-' || c == '+' || c == '_' ||
c == '.' || c == '/';
}
static bool scan_path_start(TSLexer *lexer) {
lexer->result_symbol = PATH_START;
bool have_sep = false;
bool have_after_sep = false;
int32_t c = lexer->lookahead;
// unlike string_fragments which which are preceded by initial token (i.e.
// '"') and thus will have all leading external whitespace consumed, we have
// no such luxury with the path_start token.
//
// so we must skip over any leading whitespace here.
while (c == ' ' || c == '\n' || c == '\r' || c == '\t') {
skip(lexer);
c = lexer->lookahead;
}
while (true) {
lexer->mark_end(lexer);
c = lexer->lookahead;
if (c == '/') {
have_sep = true;
} else if (is_path_char(c)) {
if (have_sep) {
have_after_sep = true;
}
} else if (c == '$') {
// starting a interpolation,
// so we have a valid token as long as we've seen a separator.
// example: a/${x}
return have_sep;
} else {
// we have a valid token if we've consumed anything after a separator.
// example: a/b
return have_after_sep;
}
advance(lexer);
}
}
static bool scan_path_fragment(TSLexer *lexer) {
lexer->result_symbol = PATH_FRAGMENT;
for (bool has_content = false;; has_content = true) {
lexer->mark_end(lexer);
if (!is_path_char(lexer->lookahead)) {
return has_content;
}
advance(lexer);
}
}
void *tree_sitter_nix_external_scanner_create() { return NULL; }
bool tree_sitter_nix_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
// This never happens in valid grammar. Only during error recovery, everything
// becomes valid. See: https://github.com/tree-sitter/tree-sitter/issues/1259
//
// We should not consume any content as string fragment during error recovery,
// or we'll break more valid grammar below. The test 'attrset typing field
// following string' covers this.
if (valid_symbols[STRING_FRAGMENT] &&
valid_symbols[INDENTED_STRING_FRAGMENT] && valid_symbols[PATH_START] &&
valid_symbols[PATH_FRAGMENT] && valid_symbols[DOLLAR_ESCAPE] &&
valid_symbols[INDENTED_DOLLAR_ESCAPE]) {
return false;
} else if (valid_symbols[STRING_FRAGMENT]) {
if (lexer->lookahead == '\\') {
return scan_dollar_escape(lexer);
}
return scan_string_fragment(lexer);
} else if (valid_symbols[INDENTED_STRING_FRAGMENT]) {
if (lexer->lookahead == '\'') {
lexer->mark_end(lexer);
advance(lexer);
if (lexer->lookahead == '\'') {
return scan_indented_dollar_escape(lexer);
}
}
return scan_indented_string_fragment(lexer);
} else if (valid_symbols[PATH_FRAGMENT] && is_path_char(lexer->lookahead)) {
// path_fragments should be scanned as immediate tokens, with no preceding
// extras. so we assert that the very first token is a path character, and
// otherwise we fall through to the case below. example:
// a/b${c} d/e${f}
// ^--- note that scanning for the path_fragment will start here.
// this *should* be parsed as a function application.
// so we want to fall through to the path_start case below,
// which will skip the whitespace and correctly scan the
// following path_start.
//
// also, we want this above path_start, because wherever there's ambiguity
// we want to parse another fragment instead of starting a new path.
// example:
// a/b${c}d/e${f}
// if we swap the precedence, we'd effectively parse the above as the
// following function application:
// (a/b${c}) (d/e${f})
return scan_path_fragment(lexer);
} else if (valid_symbols[PATH_START]) {
return scan_path_start(lexer);
}
return false;
}
unsigned tree_sitter_nix_external_scanner_serialize(void *payload,
char *buffer) {
return 0;
}
void tree_sitter_nix_external_scanner_deserialize(void *payload,
const char *buffer,
unsigned length) {}
void tree_sitter_nix_external_scanner_destroy(void *payload) {}

View File

@ -0,0 +1,80 @@
{
or = { or = 1; }.or or 42;
# <- property
# ^ punctuation.delimiter
# ^ property
# ^ property
# ^ keyword
the-question = if builtins.true then "to be" else "not to be";
# <- property
# ^ property
# ^ property
# ^ keyword
# ^ variable.builtin
# ^ property
# ^ keyword
# ^ string
# ^ keyword
# ^ string
null = if null then true else false;
# <- property
# ^ variable.builtin
# ^ variable.builtin
# ^ variable.builtin
pkgs' = { inherit (pkgs) stdenv lib; };
# <- property
# ^ property
# ^ keyword
# ^ variable
# ^ property
# ^ property
thing' =
# <- property
let inherit (pkgs) stdenv lib;
# <- keyword
# ^ keyword
# ^ variable
# ^ property
# ^ property
in derivation rec {
# <- keyword
# ^ function.builtin
# ^ keyword
pname = "thing";
# <- property
# ^ string
version = "v1.2.3";
name = "${pname}-${version}";
# <- property
# ^ string
# ^ punctuation.special
# ^ variable
# ^ punctuation.special
# ^ string
# ^ variable
# ^ string
buildInputs = with pkgs; [ thing_a thing_b ];
# <- property
# ^ keyword
# ^ variable
# ^ variable
# ^ variable
};
assert_bool = bool: assert lib.isBool bool; bool;
# <- property
# ^ variable.parameter
# ^ keyword
# ^ variable
# ^ function
# ^ variable
# ^ variable
import = import ./overlays.nix { inherit pkgs; };
# <- property
# ^ function.builtin
# ^ string.special.path
# ^ keyword
# ^ property
uri = https://github.com;
# ^ string.special.uri
# ^ string.special.uri
}

21
python/LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Max Brunsfeld
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -0,0 +1,3 @@
class Foo:
def bar():
print "hi"

View File

@ -0,0 +1,6 @@
print a
if b:
if c:
d
e

View File

@ -0,0 +1,4 @@
def main():
print "hello"
# 1 tab = 8 spaces in Python 2
return

View File

@ -0,0 +1,25 @@
def hi():
print "hi"
def bye():
print "bye"

View File

@ -0,0 +1,973 @@
# Python test set -- part 1, grammar.
# This just tests whether the parser accepts them all.
# NOTE: When you run this test as a script from the command line, you
# get warnings about certain hex/oct constants. Since those are
# issued by the parser, you can't suppress them by adding a
# filterwarnings() call to this module. Therefore, to shut up the
# regression test, the filterwarnings() call has been added to
# regrtest.py.
from test.test_support import run_unittest, check_syntax_error
import unittest
import sys
# testing import *
from sys import *
class TokenTests(unittest.TestCase):
def testBackslash(self):
# Backslash means line continuation:
x = 1 \
+ 1
self.assertEquals(x, 2, 'backslash for line continuation')
# Backslash does not means continuation in comments :\
x = 0
self.assertEquals(x, 0, 'backslash ending comment')
def testPlainIntegers(self):
self.assertEquals(0xff, 255)
self.assertEquals(0377, 255)
self.assertEquals(2147483647, 017777777777)
# "0x" is not a valid literal
self.assertRaises(SyntaxError, eval, "0x")
from sys import maxint
if maxint == 2147483647:
self.assertEquals(-2147483647-1, -020000000000)
# XXX -2147483648
self.assert_(037777777777 > 0)
self.assert_(0xffffffff > 0)
for s in '2147483648', '040000000000', '0x100000000':
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
elif maxint == 9223372036854775807:
self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
self.assert_(01777777777777777777777 > 0)
self.assert_(0xffffffffffffffff > 0)
for s in '9223372036854775808', '02000000000000000000000','0x10000000000000000':
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
else:
self.fail('Weird maxint value %r' % maxint)
def testLongIntegers(self):
x = 0L
x = 0l
x = 0xffffffffffffffffL
x = 0xffffffffffffffffl
x = 077777777777777777L
x = 077777777777777777l
x = 123456789012345678901234567890L
x = 123456789012345678901234567890l
def testFloats(self):
x = 3.14
x = 314.
x = 0.314
# XXX x = 000.314
x = .314
x = 3e14
x = 3E14
x = 3e-14
x = 3e+14
x = 3.e14
x = .3e14
x = 3.1e4
class GrammarTests(unittest.TestCase):
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
# XXX can't test in a script -- this rule is only used when interactive
# file_input: (NEWLINE | stmt)* ENDMARKER
# Being tested as this very moment this very module
# expr_input: testlist NEWLINE
# XXX Hard to test -- used only in calls to input()
def testEvalInput(self):
# testlist ENDMARKER
x = eval('1, 0 or 1')
def testFuncdef(self):
### 'def' NAME parameters ':' suite
### parameters: '(' [varargslist] ')'
### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
### | ('**'|'*' '*') NAME)
### | fpdef ['=' test] (',' fpdef ['=' test])* [',']
### fpdef: NAME | '(' fplist ')'
### fplist: fpdef (',' fpdef)* [',']
### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
### argument: [test '='] test # Really [keyword '='] test
def f1(): pass
f1()
f1(*())
f1(*(), **{})
def f2(one_argument): pass
def f3(two, arguments): pass
def f4(two, (compound, (argument, list))): pass
def f5((compound, first), two): pass
self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
if sys.platform.startswith('java'):
self.assertEquals(f4.func_code.co_varnames,
('two', '(compound, (argument, list))', 'compound', 'argument',
'list',))
self.assertEquals(f5.func_code.co_varnames,
('(compound, first)', 'two', 'compound', 'first'))
else:
self.assertEquals(f4.func_code.co_varnames,
('two', '.1', 'compound', 'argument', 'list'))
self.assertEquals(f5.func_code.co_varnames,
('.0', 'two', 'compound', 'first'))
def a1(one_arg,): pass
def a2(two, args,): pass
def v0(*rest): pass
def v1(a, *rest): pass
def v2(a, b, *rest): pass
def v3(a, (b, c), *rest): return a, b, c, rest
f1()
f2(1)
f2(1,)
f3(1, 2)
f3(1, 2,)
f4(1, (2, (3, 4)))
v0()
v0(1)
v0(1,)
v0(1,2)
v0(1,2,3,4,5,6,7,8,9,0)
v1(1)
v1(1,)
v1(1,2)
v1(1,2,3)
v1(1,2,3,4,5,6,7,8,9,0)
v2(1,2)
v2(1,2,3)
v2(1,2,3,4)
v2(1,2,3,4,5,6,7,8,9,0)
v3(1,(2,3))
v3(1,(2,3),4)
v3(1,(2,3),4,5,6,7,8,9,0)
# ceval unpacks the formal arguments into the first argcount names;
# thus, the names nested inside tuples must appear after these names.
if sys.platform.startswith('java'):
self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
else:
self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
def d01(a=1): pass
d01()
d01(1)
d01(*(1,))
d01(**{'a':2})
def d11(a, b=1): pass
d11(1)
d11(1, 2)
d11(1, **{'b':2})
def d21(a, b, c=1): pass
d21(1, 2)
d21(1, 2, 3)
d21(*(1, 2, 3))
d21(1, *(2, 3))
d21(1, 2, *(3,))
d21(1, 2, **{'c':3})
def d02(a=1, b=2): pass
d02()
d02(1)
d02(1, 2)
d02(*(1, 2))
d02(1, *(2,))
d02(1, **{'b':2})
d02(**{'a': 1, 'b': 2})
def d12(a, b=1, c=2): pass
d12(1)
d12(1, 2)
d12(1, 2, 3)
def d22(a, b, c=1, d=2): pass
d22(1, 2)
d22(1, 2, 3)
d22(1, 2, 3, 4)
def d01v(a=1, *rest): pass
d01v()
d01v(1)
d01v(1, 2)
d01v(*(1, 2, 3, 4))
d01v(*(1,))
d01v(**{'a':2})
def d11v(a, b=1, *rest): pass
d11v(1)
d11v(1, 2)
d11v(1, 2, 3)
def d21v(a, b, c=1, *rest): pass
d21v(1, 2)
d21v(1, 2, 3)
d21v(1, 2, 3, 4)
d21v(*(1, 2, 3, 4))
d21v(1, 2, **{'c': 3})
def d02v(a=1, b=2, *rest): pass
d02v()
d02v(1)
d02v(1, 2)
d02v(1, 2, 3)
d02v(1, *(2, 3, 4))
d02v(**{'a': 1, 'b': 2})
def d12v(a, b=1, c=2, *rest): pass
d12v(1)
d12v(1, 2)
d12v(1, 2, 3)
d12v(1, 2, 3, 4)
d12v(*(1, 2, 3, 4))
d12v(1, 2, *(3, 4, 5))
d12v(1, *(2,), **{'c': 3})
def d22v(a, b, c=1, d=2, *rest): pass
d22v(1, 2)
d22v(1, 2, 3)
d22v(1, 2, 3, 4)
d22v(1, 2, 3, 4, 5)
d22v(*(1, 2, 3, 4))
d22v(1, 2, *(3, 4, 5))
d22v(1, *(2, 3), **{'d': 4})
def d31v((x)): pass
d31v(1)
def d32v((x,)): pass
d32v((1,))
# keyword arguments after *arglist
def f(*args, **kwargs):
return args, kwargs
self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
{'x':2, 'y':5}))
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
# Check ast errors in *args and *kwargs
check_syntax_error(self, "f(*g(1=2))")
check_syntax_error(self, "f(**g(1=2))")
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
self.assertEquals(l1(), 0)
l2 = lambda : a[d] # XXX just testing the expression
l3 = lambda : [2 < x for x in [-1, 3, 0L]]
self.assertEquals(l3(), [0, 1, 0])
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
self.assertEquals(l4(), 1)
l5 = lambda x, y, z=2: x + y + z
self.assertEquals(l5(1, 2), 5)
self.assertEquals(l5(1, 2, 3), 6)
check_syntax_error(self, "lambda x: x = 2")
check_syntax_error(self, "lambda (None,): None")
### stmt: simple_stmt | compound_stmt
# Tested below
def testSimpleStmt(self):
### simple_stmt: small_stmt (';' small_stmt)* [';']
x = 1; pass; del x
def foo():
# verify statements that end with semi-colons
x = 1; pass; del x;
foo()
### small_stmt: expr_stmt | print_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
# Tested below
def testExprStmt(self):
# (exprlist '=')* exprlist
1
1, 2, 3
x = 1
x = 1, 2, 3
x = y = z = 1, 2, 3
x, y, z = 1, 2, 3
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
check_syntax_error(self, "x + 1 = 1")
check_syntax_error(self, "a + 1 = b + 2")
def testPrintStmt(self):
# 'print' (test ',')* [test]
import StringIO
# Can't test printing to real stdout without comparing output
# which is not available in unittest.
save_stdout = sys.stdout
sys.stdout = StringIO.StringIO()
print 1, 2, 3
print 1, 2, 3,
print
print 0 or 1, 0 or 1,
print 0 or 1
# 'print' '>>' test ','
print >> sys.stdout, 1, 2, 3
print >> sys.stdout, 1, 2, 3,
print >> sys.stdout
print >> sys.stdout, 0 or 1, 0 or 1,
print >> sys.stdout, 0 or 1
# test printing to an instance
class Gulp:
def write(self, msg): pass
gulp = Gulp()
print >> gulp, 1, 2, 3
print >> gulp, 1, 2, 3,
print >> gulp
print >> gulp, 0 or 1, 0 or 1,
print >> gulp, 0 or 1
# test print >> None
def driver():
oldstdout = sys.stdout
sys.stdout = Gulp()
try:
tellme(Gulp())
tellme()
finally:
sys.stdout = oldstdout
# we should see this once
def tellme(file=sys.stdout):
print >> file, 'hello world'
driver()
# we should not see this at all
def tellme(file=None):
print >> file, 'goodbye universe'
driver()
self.assertEqual(sys.stdout.getvalue(), '''\
1 2 3
1 2 3
1 1 1
1 2 3
1 2 3
1 1 1
hello world
''')
sys.stdout = save_stdout
# syntax errors
check_syntax_error(self, 'print ,')
check_syntax_error(self, 'print >> x,')
def testDelStmt(self):
# 'del' exprlist
abc = [1,2,3]
x, y, z = abc
xyz = x, y, z
del abc
del x, y, (z, xyz)
def testPassStmt(self):
# 'pass'
pass
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
# Tested below
def testBreakStmt(self):
# 'break'
while 1: break
def testContinueStmt(self):
# 'continue'
i = 1
while i: i = 0; continue
msg = ""
while not msg:
msg = "ok"
try:
continue
msg = "continue failed to continue inside try"
except:
msg = "continue inside try called except block"
if msg != "ok":
self.fail(msg)
msg = ""
while not msg:
msg = "finally block not called"
try:
continue
finally:
msg = "ok"
if msg != "ok":
self.fail(msg)
def test_break_continue_loop(self):
# This test warrants an explanation. It is a test specifically for SF bugs
# #463359 and #462937. The bug is that a 'break' statement executed or
# exception raised inside a try/except inside a loop, *after* a continue
# statement has been executed in that loop, will cause the wrong number of
# arguments to be popped off the stack and the instruction pointer reset to
# a very small number (usually 0.) Because of this, the following test
# *must* written as a function, and the tracking vars *must* be function
# arguments with default values. Otherwise, the test will loop and loop.
def test_inner(extra_burning_oil = 1, count=0):
big_hippo = 2
while big_hippo:
count += 1
try:
if extra_burning_oil and big_hippo == 1:
extra_burning_oil -= 1
break
big_hippo -= 1
continue
except:
raise
if count > 2 or big_hippo <> 1:
self.fail("continue then break in try/except in loop broken!")
test_inner()
def testReturn(self):
# 'return' [testlist]
def g1(): return
def g2(): return 1
g1()
x = g2()
check_syntax_error(self, "class foo:return 1")
def testYield(self):
check_syntax_error(self, "class foo:yield 1")
def testRaise(self):
# 'raise' test [',' test]
try: raise RuntimeError, 'just testing'
except RuntimeError: pass
try: raise KeyboardInterrupt
except KeyboardInterrupt: pass
def testImport(self):
# 'import' dotted_as_names
import sys
import time, sys
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
from time import time
from time import (time)
# not testable inside a function, but already done at top of the module
# from sys import *
from sys import path, argv
from sys import (path, argv)
from sys import (path, argv,)
def testGlobal(self):
# 'global' NAME (',' NAME)*
global a
global a, b
global one, two, three, four, five, six, seven, eight, nine, ten
def testExec(self):
# 'exec' expr ['in' expr [',' expr]]
z = None
del z
exec 'z=1+1\n'
if z != 2: self.fail('exec \'z=1+1\'\\n')
del z
exec 'z=1+1'
if z != 2: self.fail('exec \'z=1+1\'')
z = None
del z
import types
if hasattr(types, "UnicodeType"):
exec r"""if 1:
exec u'z=1+1\n'
if z != 2: self.fail('exec u\'z=1+1\'\\n')
del z
exec u'z=1+1'
if z != 2: self.fail('exec u\'z=1+1\'')"""
g = {}
exec 'z = 1' in g
if g.has_key('__builtins__'): del g['__builtins__']
if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
g = {}
l = {}
import warnings
warnings.filterwarnings("ignore", "global statement", module="<string>")
exec 'global a; a = 1; b = 2' in g, l
if g.has_key('__builtins__'): del g['__builtins__']
if l.has_key('__builtins__'): del l['__builtins__']
if (g, l) != ({'a':1}, {'b':2}):
self.fail('exec ... in g (%s), l (%s)' %(g,l))
def testAssert(self):
# assert_stmt: 'assert' test [',' test]
assert 1
assert 1, 1
assert lambda x:x
assert 1, lambda x:x+1
try:
assert 0, "msg"
except AssertionError, e:
self.assertEquals(e.args[0], "msg")
else:
if __debug__:
self.fail("AssertionError not raised by assert 0")
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
# Tested below
def testIf(self):
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
if 1: pass
if 1: pass
else: pass
if 0: pass
elif 0: pass
if 0: pass
elif 0: pass
elif 0: pass
elif 0: pass
else: pass
def testWhile(self):
# 'while' test ':' suite ['else' ':' suite]
while 0: pass
while 0: pass
else: pass
# Issue1920: "while 0" is optimized away,
# ensure that the "else" clause is still present.
x = 0
while 0:
x = 1
else:
x = 2
self.assertEquals(x, 2)
def testFor(self):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
for i in 1, 2, 3: pass
for i, j, k in (): pass
else: pass
class Squares:
def __init__(self, max):
self.max = max
self.sofar = []
def __len__(self): return len(self.sofar)
def __getitem__(self, i):
if not 0 <= i < self.max: raise IndexError
n = len(self.sofar)
while n <= i:
self.sofar.append(n*n)
n = n+1
return self.sofar[i]
n = 0
for x in Squares(10): n = n+x
if n != 285:
self.fail('for over growing sequence')
result = []
for x, in [(1,), (2,), (3,)]:
result.append(x)
self.assertEqual(result, [1, 2, 3])
def testTry(self):
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
### | 'try' ':' suite 'finally' ':' suite
### except_clause: 'except' [expr [('as' | ',') expr]]
try:
1/0
except ZeroDivisionError:
pass
else:
pass
try: 1/0
except EOFError: pass
except TypeError as msg: pass
except RuntimeError, msg: pass
except: pass
else: pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError): pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError), msg: pass
try: pass
finally: pass
def testSuite(self):
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
if 1: pass
if 1:
pass
if 1:
#
#
#
pass
pass
#
pass
#
def testTest(self):
### and_test ('or' and_test)*
### and_test: not_test ('and' not_test)*
### not_test: 'not' not_test | comparison
if not 1: pass
if 1 and 1: pass
if 1 or 1: pass
if not not not 1: pass
if not 1 and 1 and 1: pass
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
def testComparison(self):
### comparison: expr (comp_op expr)*
### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
if 1: pass
x = (1 == 1)
if 1 == 1: pass
if 1 != 1: pass
if 1 <> 1: pass
if 1 < 1: pass
if 1 > 1: pass
if 1 <= 1: pass
if 1 >= 1: pass
if 1 is 1: pass
if 1 is not 1: pass
if 1 in (): pass
if 1 not in (): pass
if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
def testBinaryMaskOps(self):
x = 1 & 1
x = 1 ^ 1
x = 1 | 1
def testShiftOps(self):
x = 1 << 1
x = 1 >> 1
x = 1 << 1 >> 1
def testAdditiveOps(self):
x = 1
x = 1 + 1
x = 1 - 1 - 1
x = 1 - 1 + 1 - 1 + 1
def testMultiplicativeOps(self):
x = 1 * 1
x = 1 / 1
x = 1 % 1
x = 1 / 1 * 1 % 1
def testUnaryOps(self):
x = +1
x = -1
x = ~1
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
x = -1*1/1 + 1*1 - ---1*1
def testSelectors(self):
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
### subscript: expr | [expr] ':' [expr]
import sys, time
c = sys.path[0]
x = time.time()
x = sys.modules['time'].time()
a = '01234'
c = a[0]
c = a[-1]
s = a[0:5]
s = a[:5]
s = a[0:]
s = a[:]
s = a[-5:]
s = a[:-1]
s = a[-4:-3]
# A rough test of SF bug 1333982. http://python.org/sf/1333982
# The testing here is fairly incomplete.
# Test cases should include: commas with 1 and 2 colons
d = {}
d[1] = 1
d[1,] = 2
d[1,2] = 3
d[1,2,3] = 4
L = list(d)
L.sort()
self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
def testAtoms(self):
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
### dictmaker: test ':' test (',' test ':' test)* [',']
x = (1)
x = (1 or 2 or 3)
x = (1 or 2 or 3, 2, 3)
x = []
x = [1]
x = [1 or 2 or 3]
x = [1 or 2 or 3, 2, 3]
x = []
x = {}
x = {'one': 1}
x = {'one': 1,}
x = {'one' or 'two': 1 or 2}
x = {'one': 1, 'two': 2}
x = {'one': 1, 'two': 2,}
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
x = `x`
x = `1 or 2 or 3`
self.assertEqual(`1,2`, '(1, 2)')
x = x
x = 'x'
x = 123
### exprlist: expr (',' expr)* [',']
### testlist: test (',' test)* [',']
# These have been exercised enough above
def testClassdef(self):
# 'class' NAME ['(' [testlist] ')'] ':' suite
class B: pass
class B2(): pass
class C1(B): pass
class C2(B): pass
class D(C1, C2, B): pass
class C:
def meth1(self): pass
def meth2(self, arg): pass
def meth3(self, a1, a2): pass
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
# decorators: decorator+
# decorated: decorators (classdef | funcdef)
def class_decorator(x):
x.decorated = True
return x
@class_decorator
class G:
pass
self.assertEqual(G.decorated, True)
def testListcomps(self):
# list comprehension tests
nums = [1, 2, 3, 4, 5]
strs = ["Apple", "Banana", "Coconut"]
spcs = [" Apple", " Banana ", "Coco nut "]
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
self.assertEqual([(i, s) for i in nums for s in strs],
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
def test_in_func(l):
return [None < x < 3 for x in l if x > 2]
self.assertEqual(test_in_func(nums), [False, False, False])
def test_nested_front():
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
[[1, 2], [3, 4], [5, 6]])
test_nested_front()
check_syntax_error(self, "[i, s for i in nums for s in strs]")
check_syntax_error(self, "[x if y]")
suppliers = [
(1, "Boeing"),
(2, "Ford"),
(3, "Macdonalds")
]
parts = [
(10, "Airliner"),
(20, "Engine"),
(30, "Cheeseburger")
]
suppart = [
(1, 10), (1, 20), (2, 20), (3, 30)
]
x = [
(sname, pname)
for (sno, sname) in suppliers
for (pno, pname) in parts
for (sp_sno, sp_pno) in suppart
if sno == sp_sno and pno == sp_pno
]
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
('Macdonalds', 'Cheeseburger')])
def testGenexps(self):
# generator expression tests
g = ([x for x in range(10)] for x in range(1))
self.assertEqual(g.next(), [x for x in range(10)])
try:
g.next()
self.fail('should produce StopIteration exception')
except StopIteration:
pass
a = 1
try:
g = (a for d in a)
g.next()
self.fail('should produce TypeError')
except TypeError:
pass
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
a = [x for x in range(10)]
b = (x for x in (y for y in a))
self.assertEqual(sum(b), sum([x for x in range(10)]))
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
check_syntax_error(self, "foo(x for x in range(10), 100)")
check_syntax_error(self, "foo(100, x for x in range(10))")
def testComprehensionSpecials(self):
# test for outmost iterable precomputation
x = 10; g = (i for i in range(x)); x = 5
self.assertEqual(len(list(g)), 10)
# This should hold, since we're only precomputing outmost iterable.
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
x = 5; t = True;
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
# even though it's silly. Make sure it works (ifelse broke this.)
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
# verify unpacking single element tuples in listcomp/genexp.
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
def test_with_statement(self):
class manager(object):
def __enter__(self):
return (1, 2)
def __exit__(self, *args):
pass
with manager():
pass
with manager() as x:
pass
with manager() as (x, y):
pass
with manager(), manager():
pass
with manager() as x, manager() as y:
pass
with manager() as x, manager():
pass
def testIfElseExpr(self):
# Test ifelse expressions in various cases
def _checkeval(msg, ret):
"helper to check that evaluation of expressions is done correctly"
print x
return ret
self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
self.assertEqual((5 and 6 if 0 else 1), 1)
self.assertEqual(((5 and 6) if 0 else 1), 1)
self.assertEqual((5 and (6 if 1 else 1)), 6)
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
self.assertEqual((not 5 if 1 else 1), False)
self.assertEqual((not 5 if 0 else 1), 1)
self.assertEqual((6 + 1 if 1 else 2), 7)
self.assertEqual((6 - 1 if 1 else 2), 5)
self.assertEqual((6 * 2 if 1 else 4), 12)
self.assertEqual((6 / 2 if 1 else 3), 3)
self.assertEqual((6 < 4 if 0 else 2), 2)
def testStringLiterals(self):
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
x = "doesn't \"shrink\" does it"
y = 'doesn\'t "shrink" does it'
self.assert_(len(x) == 24 and x == y)
x = "does \"shrink\" doesn't it"
y = 'does "shrink" doesn\'t it'
self.assert_(len(x) == 24 and x == y)
x = """
The "quick"
brown fox
jumps over
the 'lazy' dog.
"""
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
self.assertEquals(x, y)
y = '''
The "quick"
brown fox
jumps over
the 'lazy' dog.
'''
self.assertEquals(x, y)
y = "\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the 'lazy' dog.\n\
"
self.assertEquals(x, y)
y = '\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the \'lazy\' dog.\n\
'
self.assertEquals(x, y)
def test_main():
run_unittest(TokenTests, GrammarTests)
if __name__ == '__main__':
test_main()

View File

@ -0,0 +1,975 @@
# Python test set -- part 1, grammar.
# This just tests whether the parser accepts them all.
# NOTE: When you run this test as a script from the command line, you
# get warnings about certain hex/oct constants. Since those are
# issued by the parser, you can't suppress them by adding a
# filterwarnings() call to this module. Therefore, to shut up the
# regression test, the filterwarnings() call has been added to
# regrtest.py.
from test.test_support import run_unittest, check_syntax_error
import unittest
import sys
# testing import *
from sys import *
class TokenTests(unittest.TestCase):
def testBackslash(self):
# Backslash means line continuation:
x = 1 \
+ 1
self.assertEquals(x, 2, 'backslash for line continuation')
# Backslash does not means continuation in comments :\
x = 0
self.assertEquals(x, 0, 'backslash ending comment')
def testPlainIntegers(self):
self.assertEquals(0xff, 255)
self.assertEquals(0377, 255)
self.assertEquals(2147483647, 017777777777)
# "0x" is not a valid literal
self.assertRaises(SyntaxError, eval, "0x")
from sys import maxint
if maxint == 2147483647:
self.assertEquals(-2147483647-1, -020000000000)
# XXX -2147483648
self.assert_(037777777777 > 0)
self.assert_(0xffffffff > 0)
for s in '2147483648', '040000000000', '0x100000000':
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
elif maxint == 9223372036854775807:
self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
self.assert_(01777777777777777777777 > 0)
self.assert_(0xffffffffffffffff > 0)
for s in '9223372036854775808', '02000000000000000000000', \
'0x10000000000000000':
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
else:
self.fail('Weird maxint value %r' % maxint)
def testLongIntegers(self):
x = 0L
x = 0l
x = 0xffffffffffffffffL
x = 0xffffffffffffffffl
x = 077777777777777777L
x = 077777777777777777l
x = 123456789012345678901234567890L
x = 123456789012345678901234567890l
def testFloats(self):
x = 3.14
x = 314.
x = 0.314
# XXX x = 000.314
x = .314
x = 3e14
x = 3E14
x = 3e-14
x = 3e+14
x = 3.e14
x = .3e14
x = 3.1e4
class GrammarTests(unittest.TestCase):
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
# XXX can't test in a script -- this rule is only used when interactive
# file_input: (NEWLINE | stmt)* ENDMARKER
# Being tested as this very moment this very module
# expr_input: testlist NEWLINE
# XXX Hard to test -- used only in calls to input()
def testEvalInput(self):
# testlist ENDMARKER
x = eval('1, 0 or 1')
def testFuncdef(self):
### 'def' NAME parameters ':' suite
### parameters: '(' [varargslist] ')'
### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
### | ('**'|'*' '*') NAME)
### | fpdef ['=' test] (',' fpdef ['=' test])* [',']
### fpdef: NAME | '(' fplist ')'
### fplist: fpdef (',' fpdef)* [',']
### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
### argument: [test '='] test # Really [keyword '='] test
def f1(): pass
f1()
f1(*())
f1(*(), **{})
def f2(one_argument): pass
def f3(two, arguments): pass
def f4(two, (compound, (argument, list))): pass
def f5((compound, first), two): pass
self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
if sys.platform.startswith('java'):
self.assertEquals(f4.func_code.co_varnames,
('two', '(compound, (argument, list))', 'compound', 'argument',
'list',))
self.assertEquals(f5.func_code.co_varnames,
('(compound, first)', 'two', 'compound', 'first'))
else:
self.assertEquals(f4.func_code.co_varnames,
('two', '.1', 'compound', 'argument', 'list'))
self.assertEquals(f5.func_code.co_varnames,
('.0', 'two', 'compound', 'first'))
def a1(one_arg,): pass
def a2(two, args,): pass
def v0(*rest): pass
def v1(a, *rest): pass
def v2(a, b, *rest): pass
def v3(a, (b, c), *rest): return a, b, c, rest
f1()
f2(1)
f2(1,)
f3(1, 2)
f3(1, 2,)
f4(1, (2, (3, 4)))
v0()
v0(1)
v0(1,)
v0(1,2)
v0(1,2,3,4,5,6,7,8,9,0)
v1(1)
v1(1,)
v1(1,2)
v1(1,2,3)
v1(1,2,3,4,5,6,7,8,9,0)
v2(1,2)
v2(1,2,3)
v2(1,2,3,4)
v2(1,2,3,4,5,6,7,8,9,0)
v3(1,(2,3))
v3(1,(2,3),4)
v3(1,(2,3),4,5,6,7,8,9,0)
# ceval unpacks the formal arguments into the first argcount names;
# thus, the names nested inside tuples must appear after these names.
if sys.platform.startswith('java'):
self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
else:
self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
def d01(a=1): pass
d01()
d01(1)
d01(*(1,))
d01(**{'a':2})
def d11(a, b=1): pass
d11(1)
d11(1, 2)
d11(1, **{'b':2})
def d21(a, b, c=1): pass
d21(1, 2)
d21(1, 2, 3)
d21(*(1, 2, 3))
d21(1, *(2, 3))
d21(1, 2, *(3,))
d21(1, 2, **{'c':3})
def d02(a=1, b=2): pass
d02()
d02(1)
d02(1, 2)
d02(*(1, 2))
d02(1, *(2,))
d02(1, **{'b':2})
d02(**{'a': 1, 'b': 2})
def d12(a, b=1, c=2): pass
d12(1)
d12(1, 2)
d12(1, 2, 3)
def d22(a, b, c=1, d=2): pass
d22(1, 2)
d22(1, 2, 3)
d22(1, 2, 3, 4)
def d01v(a=1, *rest): pass
d01v()
d01v(1)
d01v(1, 2)
d01v(*(1, 2, 3, 4))
d01v(*(1,))
d01v(**{'a':2})
def d11v(a, b=1, *rest): pass
d11v(1)
d11v(1, 2)
d11v(1, 2, 3)
def d21v(a, b, c=1, *rest): pass
d21v(1, 2)
d21v(1, 2, 3)
d21v(1, 2, 3, 4)
d21v(*(1, 2, 3, 4))
d21v(1, 2, **{'c': 3})
def d02v(a=1, b=2, *rest): pass
d02v()
d02v(1)
d02v(1, 2)
d02v(1, 2, 3)
d02v(1, *(2, 3, 4))
d02v(**{'a': 1, 'b': 2})
def d12v(a, b=1, c=2, *rest): pass
d12v(1)
d12v(1, 2)
d12v(1, 2, 3)
d12v(1, 2, 3, 4)
d12v(*(1, 2, 3, 4))
d12v(1, 2, *(3, 4, 5))
d12v(1, *(2,), **{'c': 3})
def d22v(a, b, c=1, d=2, *rest): pass
d22v(1, 2)
d22v(1, 2, 3)
d22v(1, 2, 3, 4)
d22v(1, 2, 3, 4, 5)
d22v(*(1, 2, 3, 4))
d22v(1, 2, *(3, 4, 5))
d22v(1, *(2, 3), **{'d': 4})
def d31v((x)): pass
d31v(1)
def d32v((x,)): pass
d32v((1,))
# keyword arguments after *arglist
def f(*args, **kwargs):
return args, kwargs
self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
{'x':2, 'y':5}))
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
# Check ast errors in *args and *kwargs
check_syntax_error(self, "f(*g(1=2))")
check_syntax_error(self, "f(**g(1=2))")
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
self.assertEquals(l1(), 0)
l2 = lambda : a[d] # XXX just testing the expression
l3 = lambda : [2 < x for x in [-1, 3, 0L]]
self.assertEquals(l3(), [0, 1, 0])
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
self.assertEquals(l4(), 1)
l5 = lambda x, y, z=2: x + y + z
self.assertEquals(l5(1, 2), 5)
self.assertEquals(l5(1, 2, 3), 6)
check_syntax_error(self, "lambda x: x = 2")
check_syntax_error(self, "lambda (None,): None")
### stmt: simple_stmt | compound_stmt
# Tested below
def testSimpleStmt(self):
### simple_stmt: small_stmt (';' small_stmt)* [';']
x = 1; pass; del x
def foo():
# verify statements that end with semi-colons
x = 1; pass; del x;
foo()
### small_stmt: expr_stmt | print_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
# Tested below
def testExprStmt(self):
# (exprlist '=')* exprlist
1
1, 2, 3
x = 1
x = 1, 2, 3
x = y = z = 1, 2, 3
x, y, z = 1, 2, 3
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
check_syntax_error(self, "x + 1 = 1")
check_syntax_error(self, "a + 1 = b + 2")
def testPrintStmt(self):
# 'print' (test ',')* [test]
import StringIO
# Can't test printing to real stdout without comparing output
# which is not available in unittest.
save_stdout = sys.stdout
sys.stdout = StringIO.StringIO()
print 1, 2, 3
print 1, 2, 3,
print
print 0 or 1, 0 or 1,
print 0 or 1
# 'print' '>>' test ','
print >> sys.stdout, 1, 2, 3
print >> sys.stdout, 1, 2, 3,
print >> sys.stdout
print >> sys.stdout, 0 or 1, 0 or 1,
print >> sys.stdout, 0 or 1
# test printing to an instance
class Gulp:
def write(self, msg): pass
gulp = Gulp()
print >> gulp, 1, 2, 3
print >> gulp, 1, 2, 3,
print >> gulp
print >> gulp, 0 or 1, 0 or 1,
print >> gulp, 0 or 1
# test print >> None
def driver():
oldstdout = sys.stdout
sys.stdout = Gulp()
try:
tellme(Gulp())
tellme()
finally:
sys.stdout = oldstdout
# we should see this once
def tellme(file=sys.stdout):
print >> file, 'hello world'
driver()
# we should not see this at all
def tellme(file=None):
print >> file, 'goodbye universe'
driver()
self.assertEqual(sys.stdout.getvalue(), '''\
1 2 3
1 2 3
1 1 1
1 2 3
1 2 3
1 1 1
hello world
''')
sys.stdout = save_stdout
# syntax errors
check_syntax_error(self, 'print ,')
check_syntax_error(self, 'print >> x,')
def testDelStmt(self):
# 'del' exprlist
abc = [1,2,3]
x, y, z = abc
xyz = x, y, z
del abc
del x, y, (z, xyz)
def testPassStmt(self):
# 'pass'
pass
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
# Tested below
def testBreakStmt(self):
# 'break'
while 1: break
def testContinueStmt(self):
# 'continue'
i = 1
while i: i = 0; continue
msg = ""
while not msg:
msg = "ok"
try:
continue
msg = "continue failed to continue inside try"
except:
msg = "continue inside try called except block"
if msg != "ok":
self.fail(msg)
msg = ""
while not msg:
msg = "finally block not called"
try:
continue
finally:
msg = "ok"
if msg != "ok":
self.fail(msg)
def test_break_continue_loop(self):
# This test warrants an explanation. It is a test specifically for SF bugs
# #463359 and #462937. The bug is that a 'break' statement executed or
# exception raised inside a try/except inside a loop, *after* a continue
# statement has been executed in that loop, will cause the wrong number of
# arguments to be popped off the stack and the instruction pointer reset to
# a very small number (usually 0.) Because of this, the following test
# *must* written as a function, and the tracking vars *must* be function
# arguments with default values. Otherwise, the test will loop and loop.
def test_inner(extra_burning_oil = 1, count=0):
big_hippo = 2
while big_hippo:
count += 1
try:
if extra_burning_oil and big_hippo == 1:
extra_burning_oil -= 1
break
big_hippo -= 1
continue
except:
raise
if count > 2 or big_hippo <> 1:
self.fail("continue then break in try/except in loop broken!")
test_inner()
def testReturn(self):
# 'return' [testlist]
def g1(): return
def g2(): return 1
g1()
x = g2()
check_syntax_error(self, "class foo:return 1")
def testYield(self):
check_syntax_error(self, "class foo:yield 1")
def testRaise(self):
# 'raise' test [',' test]
try: raise RuntimeError, 'just testing'
except RuntimeError: pass
try: raise KeyboardInterrupt
except KeyboardInterrupt: pass
def testImport(self):
# 'import' dotted_as_names
import sys
import time, sys
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
from time import time
from time import (time)
# not testable inside a function, but already done at top of the module
# from sys import *
from sys import path, argv
from sys import (path, argv)
from sys import (path, argv,)
def testGlobal(self):
# 'global' NAME (',' NAME)*
global a
global a, b
global one, two, three, four, five, six, seven, eight, nine, ten
def testExec(self):
# 'exec' expr ['in' expr [',' expr]]
z = None
del z
exec 'z=1+1\n'
if z != 2: self.fail('exec \'z=1+1\'\\n')
del z
exec 'z=1+1'
if z != 2: self.fail('exec \'z=1+1\'')
z = None
del z
import types
if hasattr(types, "UnicodeType"):
exec r"""if 1:
exec u'z=1+1\n'
if z != 2: self.fail('exec u\'z=1+1\'\\n')
del z
exec u'z=1+1'
if z != 2: self.fail('exec u\'z=1+1\'')"""
g = {}
exec 'z = 1' in g
if g.has_key('__builtins__'): del g['__builtins__']
if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
g = {}
l = {}
import warnings
warnings.filterwarnings("ignore", "global statement", module="<string>")
exec 'global a; a = 1; b = 2' in g, l
if g.has_key('__builtins__'): del g['__builtins__']
if l.has_key('__builtins__'): del l['__builtins__']
if (g, l) != ({'a':1}, {'b':2}):
self.fail('exec ... in g (%s), l (%s)' %(g,l))
def testAssert(self):
# assert_stmt: 'assert' test [',' test]
assert 1
assert 1, 1
assert lambda x:x
assert 1, lambda x:x+1
try:
assert 0, "msg"
except AssertionError, e:
self.assertEquals(e.args[0], "msg")
else:
if __debug__:
self.fail("AssertionError not raised by assert 0")
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
# Tested below
def testIf(self):
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
if 1: pass
if 1: pass
else: pass
if 0: pass
elif 0: pass
if 0: pass
elif 0: pass
elif 0: pass
elif 0: pass
else: pass
def testWhile(self):
# 'while' test ':' suite ['else' ':' suite]
while 0: pass
while 0: pass
else: pass
# Issue1920: "while 0" is optimized away,
# ensure that the "else" clause is still present.
x = 0
while 0:
x = 1
else:
x = 2
self.assertEquals(x, 2)
def testFor(self):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
for i in 1, 2, 3: pass
for i, j, k in (): pass
else: pass
class Squares:
def __init__(self, max):
self.max = max
self.sofar = []
def __len__(self): return len(self.sofar)
def __getitem__(self, i):
if not 0 <= i < self.max: raise IndexError
n = len(self.sofar)
while n <= i:
self.sofar.append(n*n)
n = n+1
return self.sofar[i]
n = 0
for x in Squares(10): n = n+x
if n != 285:
self.fail('for over growing sequence')
result = []
for x, in [(1,), (2,), (3,)]:
result.append(x)
self.assertEqual(result, [1, 2, 3])
def testTry(self):
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
### | 'try' ':' suite 'finally' ':' suite
### except_clause: 'except' [expr [('as' | ',') expr]]
try:
1/0
except ZeroDivisionError:
pass
else:
pass
try: 1/0
except EOFError: pass
except TypeError as msg: pass
except RuntimeError, msg: pass
except: pass
else: pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError): pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError), msg: pass
try: pass
finally: pass
def testSuite(self):
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
if 1: pass
if 1:
pass
if 1:
#
#
#
pass
pass
#
pass
#
def testTest(self):
### and_test ('or' and_test)*
### and_test: not_test ('and' not_test)*
### not_test: 'not' not_test | comparison
if not 1: pass
if 1 and 1: pass
if 1 or 1: pass
if not not not 1: pass
if not 1 and 1 and 1: pass
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
def testComparison(self):
### comparison: expr (comp_op expr)*
### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
if 1: pass
x = (1 == 1)
if 1 == 1: pass
if 1 != 1: pass
if 1 <> 1: pass
if 1 < 1: pass
if 1 > 1: pass
if 1 <= 1: pass
if 1 >= 1: pass
if 1 is 1: pass
if 1 is not 1: pass
if 1 in (): pass
if 1 not in (): pass
if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
def testBinaryMaskOps(self):
x = 1 & 1
x = 1 ^ 1
x = 1 | 1
def testShiftOps(self):
x = 1 << 1
x = 1 >> 1
x = 1 << 1 >> 1
def testAdditiveOps(self):
x = 1
x = 1 + 1
x = 1 - 1 - 1
x = 1 - 1 + 1 - 1 + 1
def testMultiplicativeOps(self):
x = 1 * 1
x = 1 / 1
x = 1 % 1
x = 1 / 1 * 1 % 1
def testUnaryOps(self):
x = +1
x = -1
x = ~1
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
x = -1*1/1 + 1*1 - ---1*1
def testSelectors(self):
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
### subscript: expr | [expr] ':' [expr]
import sys, time
c = sys.path[0]
x = time.time()
x = sys.modules['time'].time()
a = '01234'
c = a[0]
c = a[-1]
s = a[0:5]
s = a[:5]
s = a[0:]
s = a[:]
s = a[-5:]
s = a[:-1]
s = a[-4:-3]
# A rough test of SF bug 1333982. http://python.org/sf/1333982
# The testing here is fairly incomplete.
# Test cases should include: commas with 1 and 2 colons
d = {}
d[1] = 1
d[1,] = 2
d[1,2] = 3
d[1,2,3] = 4
L = list(d)
L.sort()
self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
def testAtoms(self):
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
### dictmaker: test ':' test (',' test ':' test)* [',']
x = (1)
x = (1 or 2 or 3)
x = (1 or 2 or 3, 2, 3)
x = []
x = [1]
x = [1 or 2 or 3]
x = [1 or 2 or 3, 2, 3]
x = []
x = {}
x = {'one': 1}
x = {'one': 1,}
x = {'one' or 'two': 1 or 2}
x = {'one': 1, 'two': 2}
x = {'one': 1, 'two': 2,}
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
x = `x`
x = `1 or 2 or 3`
self.assertEqual(`1,2`, '(1, 2)')
x = x
x = 'x'
x = 123
### exprlist: expr (',' expr)* [',']
### testlist: test (',' test)* [',']
# These have been exercised enough above
def testClassdef(self):
# 'class' NAME ['(' [testlist] ')'] ':' suite
class B: pass
class B2(): pass
class C1(B): pass
class C2(B): pass
class D(C1, C2, B): pass
class C:
def meth1(self): pass
def meth2(self, arg): pass
def meth3(self, a1, a2): pass
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
# decorators: decorator+
# decorated: decorators (classdef | funcdef)
def class_decorator(x):
x.decorated = True
return x
@class_decorator
class G:
pass
self.assertEqual(G.decorated, True)
def testListcomps(self):
# list comprehension tests
nums = [1, 2, 3, 4, 5]
strs = ["Apple", "Banana", "Coconut"]
spcs = [" Apple", " Banana ", "Coco nut "]
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
self.assertEqual([(i, s) for i in nums for s in strs],
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
def test_in_func(l):
return [None < x < 3 for x in l if x > 2]
self.assertEqual(test_in_func(nums), [False, False, False])
def test_nested_front():
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
[[1, 2], [3, 4], [5, 6]])
test_nested_front()
check_syntax_error(self, "[i, s for i in nums for s in strs]")
check_syntax_error(self, "[x if y]")
suppliers = [
(1, "Boeing"),
(2, "Ford"),
(3, "Macdonalds")
]
parts = [
(10, "Airliner"),
(20, "Engine"),
(30, "Cheeseburger")
]
suppart = [
(1, 10), (1, 20), (2, 20), (3, 30)
]
x = [
(sname, pname)
for (sno, sname) in suppliers
for (pno, pname) in parts
for (sp_sno, sp_pno) in suppart
if sno == sp_sno and pno == sp_pno
]
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
('Macdonalds', 'Cheeseburger')])
def testGenexps(self):
# generator expression tests
g = ([x for x in range(10)] for x in range(1))
self.assertEqual(g.next(), [x for x in range(10)])
try:
g.next()
self.fail('should produce StopIteration exception')
except StopIteration:
pass
a = 1
try:
g = (a for d in a)
g.next()
self.fail('should produce TypeError')
except TypeError:
pass
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
a = [x for x in range(10)]
b = (x for x in (y for y in a))
self.assertEqual(sum(b), sum([x for x in range(10)]))
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
check_syntax_error(self, "foo(x for x in range(10), 100)")
check_syntax_error(self, "foo(100, x for x in range(10))")
def testComprehensionSpecials(self):
# test for outmost iterable precomputation
x = 10; g = (i for i in range(x)); x = 5
self.assertEqual(len(list(g)), 10)
# This should hold, since we're only precomputing outmost iterable.
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
x = 5; t = True;
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
# even though it's silly. Make sure it works (ifelse broke this.)
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
# verify unpacking single element tuples in listcomp/genexp.
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
def test_with_statement(self):
class manager(object):
def __enter__(self):
return (1, 2)
def __exit__(self, *args):
pass
with manager():
pass
with manager() as x:
pass
with manager() as (x, y):
pass
with manager(), manager():
pass
with manager() as x, manager() as y:
pass
with manager() as x, manager():
pass
def testIfElseExpr(self):
# Test ifelse expressions in various cases
def _checkeval(msg, ret):
"helper to check that evaluation of expressions is done correctly"
print x
return ret
self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
self.assertEqual((5 and 6 if 0 else 1), 1)
self.assertEqual(((5 and 6) if 0 else 1), 1)
self.assertEqual((5 and (6 if 1 else 1)), 6)
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
self.assertEqual((not 5 if 1 else 1), False)
self.assertEqual((not 5 if 0 else 1), 1)
self.assertEqual((6 + 1 if 1 else 2), 7)
self.assertEqual((6 - 1 if 1 else 2), 5)
self.assertEqual((6 * 2 if 1 else 4), 12)
self.assertEqual((6 / 2 if 1 else 3), 3)
self.assertEqual((6 < 4 if 0 else 2), 2)
def testStringLiterals(self):
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
x = "doesn't \"shrink\" does it"
y = 'doesn\'t "shrink" does it'
self.assert_(len(x) == 24 and x == y)
x = "does \"shrink\" doesn't it"
y = 'does "shrink" doesn\'t it'
self.assert_(len(x) == 24 and x == y)
x = """
The "quick"
brown fox
jumps over
the 'lazy' dog.
"""
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
self.assertEquals(x, y)
y = '''
The "quick"
brown fox
jumps over
the 'lazy' dog.
'''
self.assertEquals(x, y)
y = "\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the 'lazy' dog.\n\
"
self.assertEquals(x, y)
y = '\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the \'lazy\' dog.\n\
'
self.assertEquals(x, y)
def test_main():
run_unittest(TokenTests, GrammarTests)
if __name__ == '__main__':
test_main()

View File

@ -0,0 +1,945 @@
# Python test set -- part 1, grammar.
# This just tests whether the parser accepts them all.
# NOTE: When you run this test as a script from the command line, you
# get warnings about certain hex/oct constants. Since those are
# issued by the parser, you can't suppress them by adding a
# filterwarnings() call to this module. Therefore, to shut up the
# regression test, the filterwarnings() call has been added to
# regrtest.py.
from test.support import run_unittest, check_syntax_error
import unittest
import sys
# testing import *
from sys import *
class TokenTests(unittest.TestCase):
def testBackslash(self):
# Backslash means line continuation:
x = 1 \
+ 1
self.assertEquals(x, 2, 'backslash for line continuation')
# Backslash does not means continuation in comments :\
x = 0
self.assertEquals(x, 0, 'backslash ending comment')
def testPlainIntegers(self):
self.assertEquals(type(000), type(0))
self.assertEquals(0xff, 255)
self.assertEquals(0o377, 255)
self.assertEquals(2147483647, 0o17777777777)
self.assertEquals(0b1001, 9)
# "0x" is not a valid literal
self.assertRaises(SyntaxError, eval, "0x")
from sys import maxsize
if maxsize == 2147483647:
self.assertEquals(-2147483647-1, -0o20000000000)
# XXX -2147483648
self.assert_(0o37777777777 > 0)
self.assert_(0xffffffff > 0)
self.assert_(0b1111111111111111111111111111111 > 0)
for s in ('2147483648', '0o40000000000', '0x100000000',
'0b10000000000000000000000000000000'):
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
elif maxsize == 9223372036854775807:
self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
self.assert_(0o1777777777777777777777 > 0)
self.assert_(0xffffffffffffffff > 0)
self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
for s in '9223372036854775808', '0o2000000000000000000000', \
'0x10000000000000000', \
'0b100000000000000000000000000000000000000000000000000000000000000':
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
else:
self.fail('Weird maxsize value %r' % maxsize)
def testLongIntegers(self):
x = 0
x = 0xffffffffffffffff
x = 0Xffffffffffffffff
x = 0o77777777777777777
x = 0O77777777777777777
x = 123456789012345678901234567890
x = 0b100000000000000000000000000000000000000000000000000000000000000000000
x = 0B111111111111111111111111111111111111111111111111111111111111111111111
def testUnderscoresInNumbers(self):
# Integers
x = 1_0
x = 123_456_7_89
x = 0xabc_123_4_5
x = 0X_abc_123
x = 0B11_01
x = 0b_11_01
x = 0o45_67
x = 0O_45_67
# Floats
x = 3_1.4
x = 03_1.4
x = 3_1.
x = .3_1
x = 3.1_4
x = 0_3.1_4
x = 3e1_4
x = 3_1e+4_1
x = 3_1E-4_1
def testFloats(self):
x = 3.14
x = 314.
x = 0.314
# XXX x = 000.314
x = .314
x = 3e14
x = 3E14
x = 3e-14
x = 3e+14
x = 3.e14
x = .3e14
x = 3.1e4
def testEllipsis(self):
x = ...
self.assert_(x is Ellipsis)
self.assertRaises(SyntaxError, eval, ".. .")
class GrammarTests(unittest.TestCase):
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
# XXX can't test in a script -- this rule is only used when interactive
# file_input: (NEWLINE | stmt)* ENDMARKER
# Being tested as this very moment this very module
# expr_input: testlist NEWLINE
# XXX Hard to test -- used only in calls to input()
def testEvalInput(self):
# testlist ENDMARKER
x = eval('1, 0 or 1')
def testFuncdef(self):
### [decorators] 'def' NAME parameters ['->' test] ':' suite
### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
### decorators: decorator+
### parameters: '(' [typedargslist] ')'
### typedargslist: ((tfpdef ['=' test] ',')*
### ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
### | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
### tfpdef: NAME [':' test]
### varargslist: ((vfpdef ['=' test] ',')*
### ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
### | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
### vfpdef: NAME
def f1(): pass
f1()
f1(*())
f1(*(), **{})
def f2(one_argument): pass
def f3(two, arguments): pass
self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
def a1(one_arg,): pass
def a2(two, args,): pass
def v0(*rest): pass
def v1(a, *rest): pass
def v2(a, b, *rest): pass
f1()
f2(1)
f2(1,)
f3(1, 2)
f3(1, 2,)
v0()
v0(1)
v0(1,)
v0(1,2)
v0(1,2,3,4,5,6,7,8,9,0)
v1(1)
v1(1,)
v1(1,2)
v1(1,2,3)
v1(1,2,3,4,5,6,7,8,9,0)
v2(1,2)
v2(1,2,3)
v2(1,2,3,4)
v2(1,2,3,4,5,6,7,8,9,0)
def d01(a=1): pass
d01()
d01(1)
d01(*(1,))
d01(**{'a':2})
def d11(a, b=1): pass
d11(1)
d11(1, 2)
d11(1, **{'b':2})
def d21(a, b, c=1): pass
d21(1, 2)
d21(1, 2, 3)
d21(*(1, 2, 3))
d21(1, *(2, 3))
d21(1, 2, *(3,))
d21(1, 2, **{'c':3})
def d02(a=1, b=2): pass
d02()
d02(1)
d02(1, 2)
d02(*(1, 2))
d02(1, *(2,))
d02(1, **{'b':2})
d02(**{'a': 1, 'b': 2})
def d12(a, b=1, c=2): pass
d12(1)
d12(1, 2)
d12(1, 2, 3)
def d22(a, b, c=1, d=2): pass
d22(1, 2)
d22(1, 2, 3)
d22(1, 2, 3, 4)
def d01v(a=1, *rest): pass
d01v()
d01v(1)
d01v(1, 2)
d01v(*(1, 2, 3, 4))
d01v(*(1,))
d01v(**{'a':2})
def d11v(a, b=1, *rest): pass
d11v(1)
d11v(1, 2)
d11v(1, 2, 3)
def d21v(a, b, c=1, *rest): pass
d21v(1, 2)
d21v(1, 2, 3)
d21v(1, 2, 3, 4)
d21v(*(1, 2, 3, 4))
d21v(1, 2, **{'c': 3})
def d02v(a=1, b=2, *rest): pass
d02v()
d02v(1)
d02v(1, 2)
d02v(1, 2, 3)
d02v(1, *(2, 3, 4))
d02v(**{'a': 1, 'b': 2})
def d12v(a, b=1, c=2, *rest): pass
d12v(1)
d12v(1, 2)
d12v(1, 2, 3)
d12v(1, 2, 3, 4)
d12v(*(1, 2, 3, 4))
d12v(1, 2, *(3, 4, 5))
d12v(1, *(2,), **{'c': 3})
def d22v(a, b, c=1, d=2, *rest): pass
d22v(1, 2)
d22v(1, 2, 3)
d22v(1, 2, 3, 4)
d22v(1, 2, 3, 4, 5)
d22v(*(1, 2, 3, 4))
d22v(1, 2, *(3, 4, 5))
d22v(1, *(2, 3), **{'d': 4})
# keyword argument type tests
try:
str('x', **{b'foo':1 })
except TypeError:
pass
else:
self.fail('Bytes should not work as keyword argument names')
# keyword only argument tests
def pos0key1(*, key): return key
pos0key1(key=100)
def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
pos2key2(1, 2, k1=100)
pos2key2(1, 2, k1=100, k2=200)
pos2key2(1, 2, k2=100, k1=200)
def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
# keyword arguments after *arglist
def f(*args, **kwargs):
return args, kwargs
self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
{'x':2, 'y':5}))
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
# argument annotation tests
def f(x) -> list: pass
self.assertEquals(f.__annotations__, {'return': list})
def f(x:int): pass
self.assertEquals(f.__annotations__, {'x': int})
def f(*x:str): pass
self.assertEquals(f.__annotations__, {'x': str})
def f(**x:float): pass
self.assertEquals(f.__annotations__, {'x': float})
def f(x, y:1+2): pass
self.assertEquals(f.__annotations__, {'y': 3})
def f(a, b:1, c:2, d): pass
self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
self.assertEquals(f.__annotations__,
{'b': 1, 'c': 2, 'e': 3, 'g': 6})
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
**k:11) -> 12: pass
self.assertEquals(f.__annotations__,
{'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
'k': 11, 'return': 12})
# Check for SF Bug #1697248 - mixing decorators and a return annotation
def null(x): return x
@null
def f(x) -> list: pass
self.assertEquals(f.__annotations__, {'return': list})
# test closures with a variety of oparg's
closure = 1
def f(): return closure
def f(x=1): return closure
def f(*, k=1): return closure
def f() -> int: return closure
# Check ast errors in *args and *kwargs
check_syntax_error(self, "f(*g(1=2))")
check_syntax_error(self, "f(**g(1=2))")
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
self.assertEquals(l1(), 0)
l2 = lambda : a[d] # XXX just testing the expression
l3 = lambda : [2 < x for x in [-1, 3, 0]]
self.assertEquals(l3(), [0, 1, 0])
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
self.assertEquals(l4(), 1)
l5 = lambda x, y, z=2: x + y + z
self.assertEquals(l5(1, 2), 5)
self.assertEquals(l5(1, 2, 3), 6)
check_syntax_error(self, "lambda x: x = 2")
check_syntax_error(self, "lambda (None,): None")
l6 = lambda x, y, *, k=20: x+y+k
self.assertEquals(l6(1,2), 1+2+20)
self.assertEquals(l6(1,2,k=10), 1+2+10)
### stmt: simple_stmt | compound_stmt
# Tested below
def testSimpleStmt(self):
### simple_stmt: small_stmt (';' small_stmt)* [';']
x = 1; pass; del x
def foo():
# verify statements that end with semi-colons
x = 1; pass; del x;
foo()
### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
# Tested below
def testExprStmt(self):
# (exprlist '=')* exprlist
1
1, 2, 3
x = 1
x = 1, 2, 3
x = y = z = 1, 2, 3
x, y, z = 1, 2, 3
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
check_syntax_error(self, "x + 1 = 1")
check_syntax_error(self, "a + 1 = b + 2")
def testDelStmt(self):
# 'del' exprlist
abc = [1,2,3]
x, y, z = abc
xyz = x, y, z
del abc
del x, y, (z, xyz)
def testPassStmt(self):
# 'pass'
pass
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
# Tested below
def testBreakStmt(self):
# 'break'
while 1: break
def testContinueStmt(self):
# 'continue'
i = 1
while i: i = 0; continue
msg = ""
while not msg:
msg = "ok"
try:
continue
msg = "continue failed to continue inside try"
except:
msg = "continue inside try called except block"
if msg != "ok":
self.fail(msg)
msg = ""
while not msg:
msg = "finally block not called"
try:
continue
finally:
msg = "ok"
if msg != "ok":
self.fail(msg)
def test_break_continue_loop(self):
# This test warrants an explanation. It is a test specifically for SF bugs
# #463359 and #462937. The bug is that a 'break' statement executed or
# exception raised inside a try/except inside a loop, *after* a continue
# statement has been executed in that loop, will cause the wrong number of
# arguments to be popped off the stack and the instruction pointer reset to
# a very small number (usually 0.) Because of this, the following test
# *must* written as a function, and the tracking vars *must* be function
# arguments with default values. Otherwise, the test will loop and loop.
def test_inner(extra_burning_oil = 1, count=0):
big_hippo = 2
while big_hippo:
count += 1
try:
if extra_burning_oil and big_hippo == 1:
extra_burning_oil -= 1
break
big_hippo -= 1
continue
except:
raise
if count > 2 or big_hippo != 1:
self.fail("continue then break in try/except in loop broken!")
test_inner()
def testReturn(self):
# 'return' [testlist]
def g1(): return
def g2(): return 1
g1()
x = g2()
check_syntax_error(self, "class foo:return 1")
def testYield(self):
check_syntax_error(self, "class foo:yield 1")
def testRaise(self):
# 'raise' test [',' test]
try: raise RuntimeError('just testing')
except RuntimeError: pass
try: raise KeyboardInterrupt
except KeyboardInterrupt: pass
def testImport(self):
# 'import' dotted_as_names
import sys
import time, sys
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
from time import time
from time import (time)
# not testable inside a function, but already done at top of the module
# from sys import *
from sys import path, argv
from sys import (path, argv)
from sys import (path, argv,)
def testGlobal(self):
# 'global' NAME (',' NAME)*
global a
global a, b
global one, two, three, four, five, six, seven, eight, nine, ten
def testNonlocal(self):
# 'nonlocal' NAME (',' NAME)*
x = 0
y = 0
def f():
nonlocal x
nonlocal x, y
def testAssert(self):
# assert_stmt: 'assert' test [',' test]
assert 1
assert 1, 1
assert lambda x:x
assert 1, lambda x:x+1
try:
assert 0, "msg"
except AssertionError as e:
self.assertEquals(e.args[0], "msg")
else:
if __debug__:
self.fail("AssertionError not raised by assert 0")
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
# Tested below
def testIf(self):
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
if 1: pass
if 1: pass
else: pass
if 0: pass
elif 0: pass
if 0: pass
elif 0: pass
elif 0: pass
elif 0: pass
else: pass
def testWhile(self):
# 'while' test ':' suite ['else' ':' suite]
while 0: pass
while 0: pass
else: pass
# Issue1920: "while 0" is optimized away,
# ensure that the "else" clause is still present.
x = 0
while 0:
x = 1
else:
x = 2
self.assertEquals(x, 2)
def testFor(self):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
for i in 1, 2, 3: pass
for i, j, k in (): pass
else: pass
class Squares:
def __init__(self, max):
self.max = max
self.sofar = []
def __len__(self): return len(self.sofar)
def __getitem__(self, i):
if not 0 <= i < self.max: raise IndexError
n = len(self.sofar)
while n <= i:
self.sofar.append(n*n)
n = n+1
return self.sofar[i]
n = 0
for x in Squares(10): n = n+x
if n != 285:
self.fail('for over growing sequence')
result = []
for x, in [(1,), (2,), (3,)]:
result.append(x)
self.assertEqual(result, [1, 2, 3])
def testTry(self):
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
### | 'try' ':' suite 'finally' ':' suite
### except_clause: 'except' [expr ['as' expr]]
try:
1/0
except ZeroDivisionError:
pass
else:
pass
try: 1/0
except EOFError: pass
except TypeError as msg: pass
except RuntimeError as msg: pass
except: pass
else: pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError): pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError) as msg: pass
try: pass
finally: pass
def testSuite(self):
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
if 1: pass
if 1:
pass
if 1:
#
#
#
pass
pass
#
pass
#
def testTest(self):
### and_test ('or' and_test)*
### and_test: not_test ('and' not_test)*
### not_test: 'not' not_test | comparison
if not 1: pass
if 1 and 1: pass
if 1 or 1: pass
if not not not 1: pass
if not 1 and 1 and 1: pass
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
def testComparison(self):
### comparison: expr (comp_op expr)*
### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
if 1: pass
x = (1 == 1)
if 1 == 1: pass
if 1 != 1: pass
if 1 < 1: pass
if 1 > 1: pass
if 1 <= 1: pass
if 1 >= 1: pass
if 1 is 1: pass
if 1 is not 1: pass
if 1 in (): pass
if 1 not in (): pass
if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
def testBinaryMaskOps(self):
x = 1 & 1
x = 1 ^ 1
x = 1 | 1
def testShiftOps(self):
x = 1 << 1
x = 1 >> 1
x = 1 << 1 >> 1
def testAdditiveOps(self):
x = 1
x = 1 + 1
x = 1 - 1 - 1
x = 1 - 1 + 1 - 1 + 1
def testMultiplicativeOps(self):
x = 1 * 1
x = 1 / 1
x = 1 % 1
x = 1 / 1 * 1 % 1
def testUnaryOps(self):
x = +1
x = -1
x = ~1
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
x = -1*1/1 + 1*1 - ---1*1
def testSelectors(self):
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
### subscript: expr | [expr] ':' [expr]
import sys, time
c = sys.path[0]
x = time.time()
x = sys.modules['time'].time()
a = '01234'
c = a[0]
c = a[-1]
s = a[0:5]
s = a[:5]
s = a[0:]
s = a[:]
s = a[-5:]
s = a[:-1]
s = a[-4:-3]
# A rough test of SF bug 1333982. http://python.org/sf/1333982
# The testing here is fairly incomplete.
# Test cases should include: commas with 1 and 2 colons
d = {}
d[1] = 1
d[1,] = 2
d[1,2] = 3
d[1,2,3] = 4
L = list(d)
L.sort(key=lambda x: x if isinstance(x, tuple) else ())
self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
def testAtoms(self):
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
x = (1)
x = (1 or 2 or 3)
x = (1 or 2 or 3, 2, 3)
x = []
x = [1]
x = [1 or 2 or 3]
x = [1 or 2 or 3, 2, 3]
x = []
x = {}
x = {'one': 1}
x = {'one': 1,}
x = {'one' or 'two': 1 or 2}
x = {'one': 1, 'two': 2}
x = {'one': 1, 'two': 2,}
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
x = {'one'}
x = {'one', 1,}
x = {'one', 'two', 'three'}
x = {2, 3, 4,}
x = x
x = 'x'
x = 123
### exprlist: expr (',' expr)* [',']
### testlist: test (',' test)* [',']
# These have been exercised enough above
def testClassdef(self):
# 'class' NAME ['(' [testlist] ')'] ':' suite
class B: pass
class B2(): pass
class C1(B): pass
class C2(B): pass
class D(C1, C2, B): pass
class C:
def meth1(self): pass
def meth2(self, arg): pass
def meth3(self, a1, a2): pass
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
# decorators: decorator+
# decorated: decorators (classdef | funcdef)
def class_decorator(x): return x
@class_decorator
class G: pass
def testDictcomps(self):
# dictorsetmaker: ( (test ':' test (comp_for |
# (',' test ':' test)* [','])) |
# (test (comp_for | (',' test)* [','])) )
nums = [1, 2, 3]
self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
def testListcomps(self):
# list comprehension tests
nums = [1, 2, 3, 4, 5]
strs = ["Apple", "Banana", "Coconut"]
spcs = [" Apple", " Banana ", "Coco nut "]
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
self.assertEqual([(i, s) for i in nums for s in strs],
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
def test_in_func(l):
return [0 < x < 3 for x in l if x > 2]
self.assertEqual(test_in_func(nums), [False, False, False])
def test_nested_front():
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
[[1, 2], [3, 4], [5, 6]])
test_nested_front()
check_syntax_error(self, "[i, s for i in nums for s in strs]")
check_syntax_error(self, "[x if y]")
suppliers = [
(1, "Boeing"),
(2, "Ford"),
(3, "Macdonalds")
]
parts = [
(10, "Airliner"),
(20, "Engine"),
(30, "Cheeseburger")
]
suppart = [
(1, 10), (1, 20), (2, 20), (3, 30)
]
x = [
(sname, pname)
for (sno, sname) in suppliers
for (pno, pname) in parts
for (sp_sno, sp_pno) in suppart
if sno == sp_sno and pno == sp_pno
]
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
('Macdonalds', 'Cheeseburger')])
def testGenexps(self):
# generator expression tests
g = ([x for x in range(10)] for x in range(1))
self.assertEqual(next(g), [x for x in range(10)])
try:
next(g)
self.fail('should produce StopIteration exception')
except StopIteration:
pass
a = 1
try:
g = (a for d in a)
next(g)
self.fail('should produce TypeError')
except TypeError:
pass
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
a = [x for x in range(10)]
b = (x for x in (y for y in a))
self.assertEqual(sum(b), sum([x for x in range(10)]))
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
check_syntax_error(self, "foo(x for x in range(10), 100)")
check_syntax_error(self, "foo(100, x for x in range(10))")
def testComprehensionSpecials(self):
# test for outmost iterable precomputation
x = 10; g = (i for i in range(x)); x = 5
self.assertEqual(len(list(g)), 10)
# This should hold, since we're only precomputing outmost iterable.
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
x = 5; t = True;
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
# even though it's silly. Make sure it works (ifelse broke this.)
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
# verify unpacking single element tuples in listcomp/genexp.
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
def test_with_statement(self):
class manager(object):
def __enter__(self):
return (1, 2)
def __exit__(self, *args):
pass
with manager():
pass
with manager() as x:
pass
with manager() as (x, y):
pass
with manager(), manager():
pass
with manager() as x, manager() as y:
pass
with manager() as x, manager():
pass
def testIfElseExpr(self):
# Test ifelse expressions in various cases
def _checkeval(msg, ret):
"helper to check that evaluation of expressions is done correctly"
print(x)
return ret
# the next line is not allowed anymore
#self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
self.assertEqual((5 and 6 if 0 else 1), 1)
self.assertEqual(((5 and 6) if 0 else 1), 1)
self.assertEqual((5 and (6 if 1 else 1)), 6)
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
self.assertEqual((not 5 if 1 else 1), False)
self.assertEqual((not 5 if 0 else 1), 1)
self.assertEqual((6 + 1 if 1 else 2), 7)
self.assertEqual((6 - 1 if 1 else 2), 5)
self.assertEqual((6 * 2 if 1 else 4), 12)
self.assertEqual((6 / 2 if 1 else 3), 3)
self.assertEqual((6 < 4 if 0 else 2), 2)
def testStringLiterals(self):
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
x = "doesn't \"shrink\" does it"
y = 'doesn\'t "shrink" does it'
self.assert_(len(x) == 24 and x == y)
x = "does \"shrink\" doesn't it"
y = 'does "shrink" doesn\'t it'
self.assert_(len(x) == 24 and x == y)
x = """
The "quick"
brown fox
jumps over
the 'lazy' dog.
"""
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
self.assertEquals(x, y)
y = '''
The "quick"
brown fox
jumps over
the 'lazy' dog.
'''
self.assertEquals(x, y)
y = "\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the 'lazy' dog.\n\
"
self.assertEquals(x, y)
y = '\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the \'lazy\' dog.\n\
'
self.assertEquals(x, y)
def test_main():
run_unittest(TokenTests, GrammarTests)
if __name__ == '__main__':
test_main()

View File

@ -0,0 +1,945 @@
# Python test set -- part 1, grammar.
# This just tests whether the parser accepts them all.
# NOTE: When you run this test as a script from the command line, you
# get warnings about certain hex/oct constants. Since those are
# issued by the parser, you can't suppress them by adding a
# filterwarnings() call to this module. Therefore, to shut up the
# regression test, the filterwarnings() call has been added to
# regrtest.py.
from test.support import run_unittest, check_syntax_error
import unittest
import sys
# testing import *
from sys import *
class TokenTests(unittest.TestCase):
def testBackslash(self):
# Backslash means line continuation:
x = 1 \
+ 1
self.assertEquals(x, 2, 'backslash for line continuation')
# Backslash does not means continuation in comments :\
x = 0
self.assertEquals(x, 0, 'backslash ending comment')
def testPlainIntegers(self):
self.assertEquals(type(000), type(0))
self.assertEquals(0xff, 255)
self.assertEquals(0o377, 255)
self.assertEquals(2147483647, 0o17777777777)
self.assertEquals(0b1001, 9)
# "0x" is not a valid literal
self.assertRaises(SyntaxError, eval, "0x")
from sys import maxsize
if maxsize == 2147483647:
self.assertEquals(-2147483647-1, -0o20000000000)
# XXX -2147483648
self.assert_(0o37777777777 > 0)
self.assert_(0xffffffff > 0)
self.assert_(0b1111111111111111111111111111111 > 0)
for s in ('2147483648', '0o40000000000', '0x100000000',
'0b10000000000000000000000000000000'):
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
elif maxsize == 9223372036854775807:
self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
self.assert_(0o1777777777777777777777 > 0)
self.assert_(0xffffffffffffffff > 0)
self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
for s in '9223372036854775808', '0o2000000000000000000000', \
'0x10000000000000000', \
'0b100000000000000000000000000000000000000000000000000000000000000':
try:
x = eval(s)
except OverflowError:
self.fail("OverflowError on huge integer literal %r" % s)
else:
self.fail('Weird maxsize value %r' % maxsize)
def testLongIntegers(self):
x = 0
x = 0xffffffffffffffff
x = 0Xffffffffffffffff
x = 0o77777777777777777
x = 0O77777777777777777
x = 123456789012345678901234567890
x = 0b100000000000000000000000000000000000000000000000000000000000000000000
x = 0B111111111111111111111111111111111111111111111111111111111111111111111
def testUnderscoresInNumbers(self):
# Integers
x = 1_0
x = 123_456_7_89
x = 0xabc_123_4_5
x = 0X_abc_123
x = 0B11_01
x = 0b_11_01
x = 0o45_67
x = 0O_45_67
# Floats
x = 3_1.4
x = 03_1.4
x = 3_1.
x = .3_1
x = 3.1_4
x = 0_3.1_4
x = 3e1_4
x = 3_1e+4_1
x = 3_1E-4_1
def testFloats(self):
x = 3.14
x = 314.
x = 0.314
# XXX x = 000.314
x = .314
x = 3e14
x = 3E14
x = 3e-14
x = 3e+14
x = 3.e14
x = .3e14
x = 3.1e4
def testEllipsis(self):
x = ...
self.assert_(x is Ellipsis)
self.assertRaises(SyntaxError, eval, ".. .")
class GrammarTests(unittest.TestCase):
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
# XXX can't test in a script -- this rule is only used when interactive
# file_input: (NEWLINE | stmt)* ENDMARKER
# Being tested as this very moment this very module
# expr_input: testlist NEWLINE
# XXX Hard to test -- used only in calls to input()
def testEvalInput(self):
# testlist ENDMARKER
x = eval('1, 0 or 1')
def testFuncdef(self):
### [decorators] 'def' NAME parameters ['->' test] ':' suite
### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
### decorators: decorator+
### parameters: '(' [typedargslist] ')'
### typedargslist: ((tfpdef ['=' test] ',')*
### ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
### | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
### tfpdef: NAME [':' test]
### varargslist: ((vfpdef ['=' test] ',')*
### ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
### | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
### vfpdef: NAME
def f1(): pass
f1()
f1(*())
f1(*(), **{})
def f2(one_argument): pass
def f3(two, arguments): pass
self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
def a1(one_arg,): pass
def a2(two, args,): pass
def v0(*rest): pass
def v1(a, *rest): pass
def v2(a, b, *rest): pass
f1()
f2(1)
f2(1,)
f3(1, 2)
f3(1, 2,)
v0()
v0(1)
v0(1,)
v0(1,2)
v0(1,2,3,4,5,6,7,8,9,0)
v1(1)
v1(1,)
v1(1,2)
v1(1,2,3)
v1(1,2,3,4,5,6,7,8,9,0)
v2(1,2)
v2(1,2,3)
v2(1,2,3,4)
v2(1,2,3,4,5,6,7,8,9,0)
def d01(a=1): pass
d01()
d01(1)
d01(*(1,))
d01(**{'a':2})
def d11(a, b=1): pass
d11(1)
d11(1, 2)
d11(1, **{'b':2})
def d21(a, b, c=1): pass
d21(1, 2)
d21(1, 2, 3)
d21(*(1, 2, 3))
d21(1, *(2, 3))
d21(1, 2, *(3,))
d21(1, 2, **{'c':3})
def d02(a=1, b=2): pass
d02()
d02(1)
d02(1, 2)
d02(*(1, 2))
d02(1, *(2,))
d02(1, **{'b':2})
d02(**{'a': 1, 'b': 2})
def d12(a, b=1, c=2): pass
d12(1)
d12(1, 2)
d12(1, 2, 3)
def d22(a, b, c=1, d=2): pass
d22(1, 2)
d22(1, 2, 3)
d22(1, 2, 3, 4)
def d01v(a=1, *rest): pass
d01v()
d01v(1)
d01v(1, 2)
d01v(*(1, 2, 3, 4))
d01v(*(1,))
d01v(**{'a':2})
def d11v(a, b=1, *rest): pass
d11v(1)
d11v(1, 2)
d11v(1, 2, 3)
def d21v(a, b, c=1, *rest): pass
d21v(1, 2)
d21v(1, 2, 3)
d21v(1, 2, 3, 4)
d21v(*(1, 2, 3, 4))
d21v(1, 2, **{'c': 3})
def d02v(a=1, b=2, *rest): pass
d02v()
d02v(1)
d02v(1, 2)
d02v(1, 2, 3)
d02v(1, *(2, 3, 4))
d02v(**{'a': 1, 'b': 2})
def d12v(a, b=1, c=2, *rest): pass
d12v(1)
d12v(1, 2)
d12v(1, 2, 3)
d12v(1, 2, 3, 4)
d12v(*(1, 2, 3, 4))
d12v(1, 2, *(3, 4, 5))
d12v(1, *(2,), **{'c': 3})
def d22v(a, b, c=1, d=2, *rest): pass
d22v(1, 2)
d22v(1, 2, 3)
d22v(1, 2, 3, 4)
d22v(1, 2, 3, 4, 5)
d22v(*(1, 2, 3, 4))
d22v(1, 2, *(3, 4, 5))
d22v(1, *(2, 3), **{'d': 4})
# keyword argument type tests
try:
str('x', **{b'foo':1 })
except TypeError:
pass
else:
self.fail('Bytes should not work as keyword argument names')
# keyword only argument tests
def pos0key1(*, key): return key
pos0key1(key=100)
def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
pos2key2(1, 2, k1=100)
pos2key2(1, 2, k1=100, k2=200)
pos2key2(1, 2, k2=100, k1=200)
def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
# keyword arguments after *arglist
def f(*args, **kwargs):
return args, kwargs
self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
{'x':2, 'y':5}))
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
# argument annotation tests
def f(x) -> list: pass
self.assertEquals(f.__annotations__, {'return': list})
def f(x:int): pass
self.assertEquals(f.__annotations__, {'x': int})
def f(*x:str): pass
self.assertEquals(f.__annotations__, {'x': str})
def f(**x:float): pass
self.assertEquals(f.__annotations__, {'x': float})
def f(x, y:1+2): pass
self.assertEquals(f.__annotations__, {'y': 3})
def f(a, b:1, c:2, d): pass
self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
self.assertEquals(f.__annotations__,
{'b': 1, 'c': 2, 'e': 3, 'g': 6})
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
**k:11) -> 12: pass
self.assertEquals(f.__annotations__,
{'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
'k': 11, 'return': 12})
# Check for SF Bug #1697248 - mixing decorators and a return annotation
def null(x): return x
@null
def f(x) -> list: pass
self.assertEquals(f.__annotations__, {'return': list})
# test closures with a variety of oparg's
closure = 1
def f(): return closure
def f(x=1): return closure
def f(*, k=1): return closure
def f() -> int: return closure
# Check ast errors in *args and *kwargs
check_syntax_error(self, "f(*g(1=2))")
check_syntax_error(self, "f(**g(1=2))")
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
self.assertEquals(l1(), 0)
l2 = lambda : a[d] # XXX just testing the expression
l3 = lambda : [2 < x for x in [-1, 3, 0]]
self.assertEquals(l3(), [0, 1, 0])
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
self.assertEquals(l4(), 1)
l5 = lambda x, y, z=2: x + y + z
self.assertEquals(l5(1, 2), 5)
self.assertEquals(l5(1, 2, 3), 6)
check_syntax_error(self, "lambda x: x = 2")
check_syntax_error(self, "lambda (None,): None")
l6 = lambda x, y, *, k=20: x+y+k
self.assertEquals(l6(1,2), 1+2+20)
self.assertEquals(l6(1,2,k=10), 1+2+10)
### stmt: simple_stmt | compound_stmt
# Tested below
def testSimpleStmt(self):
### simple_stmt: small_stmt (';' small_stmt)* [';']
x = 1; pass; del x
def foo():
# verify statements that end with semi-colons
x = 1; pass; del x;
foo()
### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
# Tested below
def testExprStmt(self):
# (exprlist '=')* exprlist
1
1, 2, 3
x = 1
x = 1, 2, 3
x = y = z = 1, 2, 3
x, y, z = 1, 2, 3
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
check_syntax_error(self, "x + 1 = 1")
check_syntax_error(self, "a + 1 = b + 2")
def testDelStmt(self):
# 'del' exprlist
abc = [1,2,3]
x, y, z = abc
xyz = x, y, z
del abc
del x, y, (z, xyz)
def testPassStmt(self):
# 'pass'
pass
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
# Tested below
def testBreakStmt(self):
# 'break'
while 1: break
def testContinueStmt(self):
# 'continue'
i = 1
while i: i = 0; continue
msg = ""
while not msg:
msg = "ok"
try:
continue
msg = "continue failed to continue inside try"
except:
msg = "continue inside try called except block"
if msg != "ok":
self.fail(msg)
msg = ""
while not msg:
msg = "finally block not called"
try:
continue
finally:
msg = "ok"
if msg != "ok":
self.fail(msg)
def test_break_continue_loop(self):
# This test warrants an explanation. It is a test specifically for SF bugs
# #463359 and #462937. The bug is that a 'break' statement executed or
# exception raised inside a try/except inside a loop, *after* a continue
# statement has been executed in that loop, will cause the wrong number of
# arguments to be popped off the stack and the instruction pointer reset to
# a very small number (usually 0.) Because of this, the following test
# *must* written as a function, and the tracking vars *must* be function
# arguments with default values. Otherwise, the test will loop and loop.
def test_inner(extra_burning_oil = 1, count=0):
big_hippo = 2
while big_hippo:
count += 1
try:
if extra_burning_oil and big_hippo == 1:
extra_burning_oil -= 1
break
big_hippo -= 1
continue
except:
raise
if count > 2 or big_hippo != 1:
self.fail("continue then break in try/except in loop broken!")
test_inner()
def testReturn(self):
# 'return' [testlist]
def g1(): return
def g2(): return 1
g1()
x = g2()
check_syntax_error(self, "class foo:return 1")
def testYield(self):
check_syntax_error(self, "class foo:yield 1")
def testRaise(self):
# 'raise' test [',' test]
try: raise RuntimeError('just testing')
except RuntimeError: pass
try: raise KeyboardInterrupt
except KeyboardInterrupt: pass
def testImport(self):
# 'import' dotted_as_names
import sys
import time, sys
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
from time import time
from time import (time)
# not testable inside a function, but already done at top of the module
# from sys import *
from sys import path, argv
from sys import (path, argv)
from sys import (path, argv,)
def testGlobal(self):
# 'global' NAME (',' NAME)*
global a
global a, b
global one, two, three, four, five, six, seven, eight, nine, ten
def testNonlocal(self):
# 'nonlocal' NAME (',' NAME)*
x = 0
y = 0
def f():
nonlocal x
nonlocal x, y
def testAssert(self):
# assert_stmt: 'assert' test [',' test]
assert 1
assert 1, 1
assert lambda x:x
assert 1, lambda x:x+1
try:
assert 0, "msg"
except AssertionError as e:
self.assertEquals(e.args[0], "msg")
else:
if __debug__:
self.fail("AssertionError not raised by assert 0")
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
# Tested below
def testIf(self):
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
if 1: pass
if 1: pass
else: pass
if 0: pass
elif 0: pass
if 0: pass
elif 0: pass
elif 0: pass
elif 0: pass
else: pass
def testWhile(self):
# 'while' test ':' suite ['else' ':' suite]
while 0: pass
while 0: pass
else: pass
# Issue1920: "while 0" is optimized away,
# ensure that the "else" clause is still present.
x = 0
while 0:
x = 1
else:
x = 2
self.assertEquals(x, 2)
def testFor(self):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
for i in 1, 2, 3: pass
for i, j, k in (): pass
else: pass
class Squares:
def __init__(self, max):
self.max = max
self.sofar = []
def __len__(self): return len(self.sofar)
def __getitem__(self, i):
if not 0 <= i < self.max: raise IndexError
n = len(self.sofar)
while n <= i:
self.sofar.append(n*n)
n = n+1
return self.sofar[i]
n = 0
for x in Squares(10): n = n+x
if n != 285:
self.fail('for over growing sequence')
result = []
for x, in [(1,), (2,), (3,)]:
result.append(x)
self.assertEqual(result, [1, 2, 3])
def testTry(self):
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
### | 'try' ':' suite 'finally' ':' suite
### except_clause: 'except' [expr ['as' expr]]
try:
1/0
except ZeroDivisionError:
pass
else:
pass
try: 1/0
except EOFError: pass
except TypeError as msg: pass
except RuntimeError as msg: pass
except: pass
else: pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError): pass
try: 1/0
except (EOFError, TypeError, ZeroDivisionError) as msg: pass
try: pass
finally: pass
def testSuite(self):
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
if 1: pass
if 1:
pass
if 1:
#
#
#
pass
pass
#
pass
#
def testTest(self):
### and_test ('or' and_test)*
### and_test: not_test ('and' not_test)*
### not_test: 'not' not_test | comparison
if not 1: pass
if 1 and 1: pass
if 1 or 1: pass
if not not not 1: pass
if not 1 and 1 and 1: pass
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
def testComparison(self):
### comparison: expr (comp_op expr)*
### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
if 1: pass
x = (1 == 1)
if 1 == 1: pass
if 1 != 1: pass
if 1 < 1: pass
if 1 > 1: pass
if 1 <= 1: pass
if 1 >= 1: pass
if 1 is 1: pass
if 1 is not 1: pass
if 1 in (): pass
if 1 not in (): pass
if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
def testBinaryMaskOps(self):
x = 1 & 1
x = 1 ^ 1
x = 1 | 1
def testShiftOps(self):
x = 1 << 1
x = 1 >> 1
x = 1 << 1 >> 1
def testAdditiveOps(self):
x = 1
x = 1 + 1
x = 1 - 1 - 1
x = 1 - 1 + 1 - 1 + 1
def testMultiplicativeOps(self):
x = 1 * 1
x = 1 / 1
x = 1 % 1
x = 1 / 1 * 1 % 1
def testUnaryOps(self):
x = +1
x = -1
x = ~1
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
x = -1*1/1 + 1*1 - ---1*1
def testSelectors(self):
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
### subscript: expr | [expr] ':' [expr]
import sys, time
c = sys.path[0]
x = time.time()
x = sys.modules['time'].time()
a = '01234'
c = a[0]
c = a[-1]
s = a[0:5]
s = a[:5]
s = a[0:]
s = a[:]
s = a[-5:]
s = a[:-1]
s = a[-4:-3]
# A rough test of SF bug 1333982. http://python.org/sf/1333982
# The testing here is fairly incomplete.
# Test cases should include: commas with 1 and 2 colons
d = {}
d[1] = 1
d[1,] = 2
d[1,2] = 3
d[1,2,3] = 4
L = list(d)
L.sort(key=lambda x: x if isinstance(x, tuple) else ())
self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
def testAtoms(self):
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
x = (1)
x = (1 or 2 or 3)
x = (1 or 2 or 3, 2, 3)
x = []
x = [1]
x = [1 or 2 or 3]
x = [1 or 2 or 3, 2, 3]
x = []
x = {}
x = {'one': 1}
x = {'one': 1,}
x = {'one' or 'two': 1 or 2}
x = {'one': 1, 'two': 2}
x = {'one': 1, 'two': 2,}
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
x = {'one'}
x = {'one', 1,}
x = {'one', 'two', 'three'}
x = {2, 3, 4,}
x = x
x = 'x'
x = 123
### exprlist: expr (',' expr)* [',']
### testlist: test (',' test)* [',']
# These have been exercised enough above
def testClassdef(self):
# 'class' NAME ['(' [testlist] ')'] ':' suite
class B: pass
class B2(): pass
class C1(B): pass
class C2(B): pass
class D(C1, C2, B): pass
class C:
def meth1(self): pass
def meth2(self, arg): pass
def meth3(self, a1, a2): pass
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
# decorators: decorator+
# decorated: decorators (classdef | funcdef)
def class_decorator(x): return x
@class_decorator
class G: pass
def testDictcomps(self):
# dictorsetmaker: ( (test ':' test (comp_for |
# (',' test ':' test)* [','])) |
# (test (comp_for | (',' test)* [','])) )
nums = [1, 2, 3]
self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
def testListcomps(self):
# list comprehension tests
nums = [1, 2, 3, 4, 5]
strs = ["Apple", "Banana", "Coconut"]
spcs = [" Apple", " Banana ", "Coco nut "]
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
self.assertEqual([(i, s) for i in nums for s in strs],
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
(5, 'Banana'), (5, 'Coconut')])
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
def test_in_func(l):
return [0 < x < 3 for x in l if x > 2]
self.assertEqual(test_in_func(nums), [False, False, False])
def test_nested_front():
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
[[1, 2], [3, 4], [5, 6]])
test_nested_front()
check_syntax_error(self, "[i, s for i in nums for s in strs]")
check_syntax_error(self, "[x if y]")
suppliers = [
(1, "Boeing"),
(2, "Ford"),
(3, "Macdonalds")
]
parts = [
(10, "Airliner"),
(20, "Engine"),
(30, "Cheeseburger")
]
suppart = [
(1, 10), (1, 20), (2, 20), (3, 30)
]
x = [
(sname, pname)
for (sno, sname) in suppliers
for (pno, pname) in parts
for (sp_sno, sp_pno) in suppart
if sno == sp_sno and pno == sp_pno
]
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
('Macdonalds', 'Cheeseburger')])
def testGenexps(self):
# generator expression tests
g = ([x for x in range(10)] for x in range(1))
self.assertEqual(next(g), [x for x in range(10)])
try:
next(g)
self.fail('should produce StopIteration exception')
except StopIteration:
pass
a = 1
try:
g = (a for d in a)
next(g)
self.fail('should produce TypeError')
except TypeError:
pass
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
a = [x for x in range(10)]
b = (x for x in (y for y in a))
self.assertEqual(sum(b), sum([x for x in range(10)]))
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
check_syntax_error(self, "foo(x for x in range(10), 100)")
check_syntax_error(self, "foo(100, x for x in range(10))")
def testComprehensionSpecials(self):
# test for outmost iterable precomputation
x = 10; g = (i for i in range(x)); x = 5
self.assertEqual(len(list(g)), 10)
# This should hold, since we're only precomputing outmost iterable.
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
x = 5; t = True;
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
# even though it's silly. Make sure it works (ifelse broke this.)
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
# verify unpacking single element tuples in listcomp/genexp.
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
def test_with_statement(self):
class manager(object):
def __enter__(self):
return (1, 2)
def __exit__(self, *args):
pass
with manager():
pass
with manager() as x:
pass
with manager() as (x, y):
pass
with manager(), manager():
pass
with manager() as x, manager() as y:
pass
with manager() as x, manager():
pass
def testIfElseExpr(self):
# Test ifelse expressions in various cases
def _checkeval(msg, ret):
"helper to check that evaluation of expressions is done correctly"
print(x)
return ret
# the next line is not allowed anymore
#self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
self.assertEqual((5 and 6 if 0 else 1), 1)
self.assertEqual(((5 and 6) if 0 else 1), 1)
self.assertEqual((5 and (6 if 1 else 1)), 6)
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
self.assertEqual((not 5 if 1 else 1), False)
self.assertEqual((not 5 if 0 else 1), 1)
self.assertEqual((6 + 1 if 1 else 2), 7)
self.assertEqual((6 - 1 if 1 else 2), 5)
self.assertEqual((6 * 2 if 1 else 4), 12)
self.assertEqual((6 / 2 if 1 else 3), 3)
self.assertEqual((6 < 4 if 0 else 2), 2)
def testStringLiterals(self):
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
x = "doesn't \"shrink\" does it"
y = 'doesn\'t "shrink" does it'
self.assert_(len(x) == 24 and x == y)
x = "does \"shrink\" doesn't it"
y = 'does "shrink" doesn\'t it'
self.assert_(len(x) == 24 and x == y)
x = f"""
The "quick"
brown fo{ok()}x
jumps over
the 'lazy' dog.
"""
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
self.assertEquals(x, y)
y = '''
The "quick"
brown fox
jumps over
the 'lazy' dog.
'''
self.assertEquals(x, y)
y = "\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the 'lazy' dog.\n\
"
self.assertEquals(x, y)
y = '\n\
The \"quick\"\n\
brown fox\n\
jumps over\n\
the \'lazy\' dog.\n\
'
self.assertEquals(x, y)
def test_main():
run_unittest(TokenTests, GrammarTests)
if __name__ == '__main__':
test_main()

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1 @@
pass; print "hi"

32
python/examples/tabs.py Normal file
View File

@ -0,0 +1,32 @@
def set_password(args):
password = args.password
while not password :
password1 = getpass("" if args.quiet else "Provide password: ")
password_repeat = getpass("" if args.quiet else "Repeat password: ")
if password1 != password_repeat:
print("Passwords do not match, try again")
elif len(password1) < 4:
print("Please provide at least 4 characters")
else:
password = password1
password_hash = passwd(password)
cfg = BaseJSONConfigManager(config_dir=jupyter_config_dir())
cfg.update('jupyter_notebook_config', {
'NotebookApp': {
'password': password_hash,
}
})
if not args.quiet:
print("password stored in config dir: %s" % jupyter_config_dir())
def main(argv):
parser = argparse.ArgumentParser(argv[0])
subparsers = parser.add_subparsers()
parser_password = subparsers.add_parser('password', help='sets a password for your notebook server')
parser_password.add_argument("password", help="password to set, if not given, a password will be queried for (NOTE: this may not be safe)",
nargs="?")
parser_password.add_argument("--quiet", help="suppress messages", action="store_true")
parser_password.set_defaults(function=set_password)
args = parser.parse_args(argv[1:])
args.function(args)

View File

@ -0,0 +1,6 @@
print a
if b:
if c:
d
e

1224
python/grammar.js Normal file

File diff suppressed because it is too large Load Diff

528
python/src/scanner.c Normal file
View File

@ -0,0 +1,528 @@
#include "tree_sitter/parser.h"
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define VEC_RESIZE(vec, _cap) \
void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0])); \
assert(tmp != NULL); \
(vec).data = tmp; \
(vec).cap = (_cap);
#define VEC_GROW(vec, _cap) \
if ((vec).cap < (_cap)) { \
VEC_RESIZE((vec), (_cap)); \
}
#define VEC_PUSH(vec, el) \
if ((vec).cap == (vec).len) { \
VEC_RESIZE((vec), MAX(16, (vec).len * 2)); \
} \
(vec).data[(vec).len++] = (el);
#define VEC_POP(vec) (vec).len--;
#define VEC_NEW \
{ .len = 0, .cap = 0, .data = NULL }
#define VEC_BACK(vec) ((vec).data[(vec).len - 1])
#define VEC_FREE(vec) \
{ \
if ((vec).data != NULL) \
free((vec).data); \
}
#define VEC_CLEAR(vec) (vec).len = 0;
enum TokenType {
NEWLINE,
INDENT,
DEDENT,
STRING_START,
STRING_CONTENT,
ESCAPE_INTERPOLATION,
STRING_END,
COMMENT,
CLOSE_PAREN,
CLOSE_BRACKET,
CLOSE_BRACE,
EXCEPT,
};
typedef enum {
SingleQuote = 1 << 0,
DoubleQuote = 1 << 1,
BackQuote = 1 << 2,
Raw = 1 << 3,
Format = 1 << 4,
Triple = 1 << 5,
Bytes = 1 << 6,
} Flags;
typedef struct {
char flags;
} Delimiter;
static inline Delimiter new_delimiter() { return (Delimiter){0}; }
static inline bool is_format(Delimiter *delimiter) {
return delimiter->flags & Format;
}
static inline bool is_raw(Delimiter *delimiter) {
return delimiter->flags & Raw;
}
static inline bool is_triple(Delimiter *delimiter) {
return delimiter->flags & Triple;
}
static inline bool is_bytes(Delimiter *delimiter) {
return delimiter->flags & Bytes;
}
static inline int32_t end_character(Delimiter *delimiter) {
if (delimiter->flags & SingleQuote) {
return '\'';
}
if (delimiter->flags & DoubleQuote) {
return '"';
}
if (delimiter->flags & BackQuote) {
return '`';
}
return 0;
}
static inline void set_format(Delimiter *delimiter) {
delimiter->flags |= Format;
}
static inline void set_raw(Delimiter *delimiter) { delimiter->flags |= Raw; }
static inline void set_triple(Delimiter *delimiter) {
delimiter->flags |= Triple;
}
static inline void set_bytes(Delimiter *delimiter) {
delimiter->flags |= Bytes;
}
static inline void set_end_character(Delimiter *delimiter, int32_t character) {
switch (character) {
case '\'':
delimiter->flags |= SingleQuote;
break;
case '"':
delimiter->flags |= DoubleQuote;
break;
case '`':
delimiter->flags |= BackQuote;
break;
default:
assert(false);
}
}
typedef struct {
uint32_t len;
uint32_t cap;
uint16_t *data;
} indent_vec;
static indent_vec indent_vec_new() {
indent_vec vec = VEC_NEW;
vec.data = calloc(1, sizeof(uint16_t));
vec.cap = 1;
return vec;
}
typedef struct {
uint32_t len;
uint32_t cap;
Delimiter *data;
} delimiter_vec;
static delimiter_vec delimiter_vec_new() {
delimiter_vec vec = VEC_NEW;
vec.data = calloc(1, sizeof(Delimiter));
vec.cap = 1;
return vec;
}
typedef struct {
indent_vec indents;
delimiter_vec delimiters;
bool inside_f_string;
} Scanner;
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
bool error_recovery_mode =
valid_symbols[STRING_CONTENT] && valid_symbols[INDENT];
bool within_brackets = valid_symbols[CLOSE_BRACE] ||
valid_symbols[CLOSE_PAREN] ||
valid_symbols[CLOSE_BRACKET];
bool advanced_once = false;
if (valid_symbols[ESCAPE_INTERPOLATION] && scanner->delimiters.len > 0 &&
(lexer->lookahead == '{' || lexer->lookahead == '}') &&
!error_recovery_mode) {
Delimiter delimiter = VEC_BACK(scanner->delimiters);
if (is_format(&delimiter)) {
lexer->mark_end(lexer);
bool is_left_brace = lexer->lookahead == '{';
advance(lexer);
advanced_once = true;
if ((lexer->lookahead == '{' && is_left_brace) ||
(lexer->lookahead == '}' && !is_left_brace)) {
advance(lexer);
lexer->mark_end(lexer);
lexer->result_symbol = ESCAPE_INTERPOLATION;
return true;
}
return false;
}
}
if (valid_symbols[STRING_CONTENT] && scanner->delimiters.len > 0 &&
!error_recovery_mode) {
Delimiter delimiter = VEC_BACK(scanner->delimiters);
int32_t end_char = end_character(&delimiter);
bool has_content = advanced_once;
while (lexer->lookahead) {
if ((advanced_once || lexer->lookahead == '{' ||
lexer->lookahead == '}') &&
is_format(&delimiter)) {
lexer->mark_end(lexer);
lexer->result_symbol = STRING_CONTENT;
return has_content;
}
if (lexer->lookahead == '\\') {
if (is_raw(&delimiter)) {
// Step over the backslash.
advance(lexer);
// Step over any escaped quotes.
if (lexer->lookahead == end_character(&delimiter) ||
lexer->lookahead == '\\') {
advance(lexer);
}
// Step over newlines
if (lexer->lookahead == '\r') {
advance(lexer);
if (lexer->lookahead == '\n') {
advance(lexer);
}
} else if (lexer->lookahead == '\n') {
advance(lexer);
}
continue;
}
if (is_bytes(&delimiter)) {
lexer->mark_end(lexer);
advance(lexer);
if (lexer->lookahead == 'N' || lexer->lookahead == 'u' ||
lexer->lookahead == 'U') {
// In bytes string, \N{...}, \uXXXX and \UXXXXXXXX are
// not escape sequences
// https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
advance(lexer);
} else {
lexer->result_symbol = STRING_CONTENT;
return has_content;
}
} else {
lexer->mark_end(lexer);
lexer->result_symbol = STRING_CONTENT;
return has_content;
}
} else if (lexer->lookahead == end_char) {
if (is_triple(&delimiter)) {
lexer->mark_end(lexer);
advance(lexer);
if (lexer->lookahead == end_char) {
advance(lexer);
if (lexer->lookahead == end_char) {
if (has_content) {
lexer->result_symbol = STRING_CONTENT;
} else {
advance(lexer);
lexer->mark_end(lexer);
VEC_POP(scanner->delimiters);
lexer->result_symbol = STRING_END;
scanner->inside_f_string = false;
}
return true;
}
lexer->mark_end(lexer);
lexer->result_symbol = STRING_CONTENT;
return true;
}
lexer->mark_end(lexer);
lexer->result_symbol = STRING_CONTENT;
return true;
}
if (has_content) {
lexer->result_symbol = STRING_CONTENT;
} else {
advance(lexer);
VEC_POP(scanner->delimiters);
lexer->result_symbol = STRING_END;
scanner->inside_f_string = false;
}
lexer->mark_end(lexer);
return true;
} else if (lexer->lookahead == '\n' && has_content &&
!is_triple(&delimiter)) {
return false;
}
advance(lexer);
has_content = true;
}
}
lexer->mark_end(lexer);
bool found_end_of_line = false;
uint32_t indent_length = 0;
int32_t first_comment_indent_length = -1;
for (;;) {
if (lexer->lookahead == '\n') {
found_end_of_line = true;
indent_length = 0;
skip(lexer);
} else if (lexer->lookahead == ' ') {
indent_length++;
skip(lexer);
} else if (lexer->lookahead == '\r' || lexer->lookahead == '\f') {
indent_length = 0;
skip(lexer);
} else if (lexer->lookahead == '\t') {
indent_length += 8;
skip(lexer);
} else if (lexer->lookahead == '#' &&
(valid_symbols[INDENT] || valid_symbols[DEDENT] ||
valid_symbols[NEWLINE] || valid_symbols[EXCEPT])) {
// If we haven't found an EOL yet,
// then this is a comment after an expression:
// foo = bar # comment
// Just return, since we don't want to generate an indent/dedent
// token.
if (!found_end_of_line) {
return false;
}
if (first_comment_indent_length == -1) {
first_comment_indent_length = (int32_t)indent_length;
}
while (lexer->lookahead && lexer->lookahead != '\n') {
skip(lexer);
}
skip(lexer);
indent_length = 0;
} else if (lexer->lookahead == '\\') {
skip(lexer);
if (lexer->lookahead == '\r') {
skip(lexer);
}
if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
skip(lexer);
} else {
return false;
}
} else if (lexer->eof(lexer)) {
indent_length = 0;
found_end_of_line = true;
break;
} else {
break;
}
}
if (found_end_of_line) {
if (scanner->indents.len > 0) {
uint16_t current_indent_length = VEC_BACK(scanner->indents);
if (valid_symbols[INDENT] &&
indent_length > current_indent_length) {
VEC_PUSH(scanner->indents, indent_length);
lexer->result_symbol = INDENT;
return true;
}
bool next_tok_is_string_start = lexer->lookahead == '\"' ||
lexer->lookahead == '\'' ||
lexer->lookahead == '`';
if ((valid_symbols[DEDENT] ||
(!valid_symbols[NEWLINE] &&
!(valid_symbols[STRING_START] && next_tok_is_string_start) &&
!within_brackets)) &&
indent_length < current_indent_length &&
!scanner->inside_f_string &&
// Wait to create a dedent token until we've consumed any
// comments
// whose indentation matches the current block.
first_comment_indent_length < (int32_t)current_indent_length) {
VEC_POP(scanner->indents);
lexer->result_symbol = DEDENT;
return true;
}
}
if (valid_symbols[NEWLINE] && !error_recovery_mode) {
lexer->result_symbol = NEWLINE;
return true;
}
}
if (first_comment_indent_length == -1 && valid_symbols[STRING_START]) {
Delimiter delimiter = new_delimiter();
bool has_flags = false;
while (lexer->lookahead) {
if (lexer->lookahead == 'f' || lexer->lookahead == 'F') {
set_format(&delimiter);
} else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
set_raw(&delimiter);
} else if (lexer->lookahead == 'b' || lexer->lookahead == 'B') {
set_bytes(&delimiter);
} else if (lexer->lookahead != 'u' && lexer->lookahead != 'U') {
break;
}
has_flags = true;
advance(lexer);
}
if (lexer->lookahead == '`') {
set_end_character(&delimiter, '`');
advance(lexer);
lexer->mark_end(lexer);
} else if (lexer->lookahead == '\'') {
set_end_character(&delimiter, '\'');
advance(lexer);
lexer->mark_end(lexer);
if (lexer->lookahead == '\'') {
advance(lexer);
if (lexer->lookahead == '\'') {
advance(lexer);
lexer->mark_end(lexer);
set_triple(&delimiter);
}
}
} else if (lexer->lookahead == '"') {
set_end_character(&delimiter, '"');
advance(lexer);
lexer->mark_end(lexer);
if (lexer->lookahead == '"') {
advance(lexer);
if (lexer->lookahead == '"') {
advance(lexer);
lexer->mark_end(lexer);
set_triple(&delimiter);
}
}
}
if (end_character(&delimiter)) {
VEC_PUSH(scanner->delimiters, delimiter);
lexer->result_symbol = STRING_START;
scanner->inside_f_string = is_format(&delimiter);
return true;
}
if (has_flags) {
return false;
}
}
return false;
}
unsigned tree_sitter_python_external_scanner_serialize(void *payload,
char *buffer) {
Scanner *scanner = (Scanner *)payload;
size_t size = 0;
buffer[size++] = (char)scanner->inside_f_string;
size_t delimiter_count = scanner->delimiters.len;
if (delimiter_count > UINT8_MAX) {
delimiter_count = UINT8_MAX;
}
buffer[size++] = (char)delimiter_count;
if (delimiter_count > 0) {
memcpy(&buffer[size], scanner->delimiters.data, delimiter_count);
}
size += delimiter_count;
int iter = 1;
for (; iter < scanner->indents.len &&
size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
++iter) {
buffer[size++] = (char)scanner->indents.data[iter];
}
return size;
}
void tree_sitter_python_external_scanner_deserialize(void *payload,
const char *buffer,
unsigned length) {
Scanner *scanner = (Scanner *)payload;
VEC_CLEAR(scanner->delimiters);
VEC_CLEAR(scanner->indents);
VEC_PUSH(scanner->indents, 0);
if (length > 0) {
size_t size = 0;
scanner->inside_f_string = (bool)buffer[size++];
size_t delimiter_count = (uint8_t)buffer[size++];
if (delimiter_count > 0) {
VEC_GROW(scanner->delimiters, delimiter_count);
scanner->delimiters.len = delimiter_count;
memcpy(scanner->delimiters.data, &buffer[size], delimiter_count);
size += delimiter_count;
}
for (; size < length; size++) {
VEC_PUSH(scanner->indents, (unsigned char)buffer[size]);
}
}
}
void *tree_sitter_python_external_scanner_create() {
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
_Static_assert(sizeof(Delimiter) == sizeof(char), "");
#else
assert(sizeof(Delimiter) == sizeof(char));
#endif
Scanner *scanner = calloc(1, sizeof(Scanner));
scanner->indents = indent_vec_new();
scanner->delimiters = delimiter_vec_new();
tree_sitter_python_external_scanner_deserialize(scanner, NULL, 0);
return scanner;
}
void tree_sitter_python_external_scanner_destroy(void *payload) {
Scanner *scanner = (Scanner *)payload;
VEC_FREE(scanner->indents);
VEC_FREE(scanner->delimiters);
free(scanner);
}

View File

@ -0,0 +1,30 @@
====================================
An error before a string literal
====================================
def a(b):
c.
"""
d
"""
e
---
(module
(function_definition
(identifier)
(parameters
(identifier))
(ERROR
(identifier))
(block
(expression_statement
(string
(string_start)
(string_content)
(string_end)))
(expression_statement
(identifier)))))

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,30 @@
if foo():
# <- keyword
pass
# <- keyword
elif bar():
# <- keyword
pass
else:
# <- keyword
foo
return
# ^ keyword
raise e
# ^ keyword
for i in foo():
# <- keyword
# ^ variable
# ^ operator
# ^ function
continue
# <- keyword
break
# <- keyword
a and b or c
# ^ operator
# ^ variable
# ^ operator

View File

@ -0,0 +1,4 @@
def g(h, i, /, j, *, k=100, **kwarg):
# ^ operator
# ^ operator
pass

View File

@ -0,0 +1,54 @@
match command.split():
# ^ keyword
case ["quit"]:
# ^ keyword
print("Goodbye!")
quit_game()
case ["look"]:
# ^ keyword
current_room.describe()
case ["get", obj]:
# ^ keyword
character.get(obj, current_room)
case ["go", direction]:
# ^ keyword
current_room = current_room.neighbor(direction)
# The rest of your commands go here
match command.split():
# ^ keyword
case ["drop", *objects]:
# ^ keyword
for obj in objects:
character.drop(obj, current_room)
match command.split():
# ^ keyword
case ["quit"]: ... # Code omitted for brevity
case ["go", direction]: pass
case ["drop", *objects]: pass
case _:
print(f"Sorry, I couldn't understand {command!r}")
match command.split():
# ^ keyword
case ["north"] | ["go", "north"]:
# ^ keyword
current_room = current_room.neighbor("north")
case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]:
# ^ keyword
pass
match = 2
# ^ variable
match, a = 2, 3
# ^ variable
match: int = secret
# ^ variable
x, match: str = 2, "hey, what's up?"
# <- variable
# ^ variable
if match := re.fullmatch(r"(-)?(\d+:)?\d?\d:\d\d(\.\d*)?", time, flags=re.ASCII):
# ^ variable
return match

15
python/test/tags/main.py Normal file
View File

@ -0,0 +1,15 @@
class MyClass:
# ^ definition.class
def hello():
# ^ definition.function
print "hello from MyClass"
MyClass.hello()
# ^ reference.call
def main():
# ^ definition.function
print "Hello, world!"
main()
# <- reference.call

21
rust/LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Maxim Sokolov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

2157
rust/examples/ast.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,260 @@
// run-pass
#![feature(generators)]
#![allow(non_camel_case_types)]
#![allow(dead_code)]
#![allow(redundant_semicolons)]
#![allow(unreachable_code)]
#![allow(unused_braces, unused_must_use, unused_parens)]
#![allow(uncommon_codepoints, confusable_idents)]
#![allow(unused_imports)]
#![allow(unreachable_patterns)]
#![recursion_limit = "256"]
extern crate core;
use std::cell::Cell;
use std::mem::swap;
use std::ops::Deref;
// Just a grab bag of stuff that you wouldn't want to actually write.
fn strange() -> bool { let _x: bool = return true; }
fn funny() {
fn f(_x: ()) { }
f(return);
}
fn what() {
fn the(x: &Cell<bool>) {
return while !x.get() { x.set(true); };
}
let i = &Cell::new(false);
let dont = {||the(i)};
dont();
assert!((i.get()));
}
fn zombiejesus() {
loop {
while (return) {
if (return) {
match (return) {
1 => {
if (return) {
return
} else {
return
}
}
_ => { return }
};
} else if (return) {
return;
}
}
if (return) { break; }
}
}
fn notsure() {
let mut _x: isize;
let mut _y = (_x = 0) == (_x = 0);
let mut _z = (_x = 0) < (_x = 0);
let _a = (_x += 0) == (_x = 0);
let _b = swap(&mut _y, &mut _z) == swap(&mut _y, &mut _z);
}
fn canttouchthis() -> usize {
fn p() -> bool { true }
let _a = (assert!((true)) == (assert!(p())));
let _c = (assert!((p())) == ());
let _b: bool = (println!("{}", 0) == (return 0));
}
fn angrydome() {
loop { if break { } }
let mut i = 0;
loop { i += 1; if i == 1 { match (continue) { 1 => { }, _ => panic!("wat") } }
break; }
}
fn evil_lincoln() { let _evil: () = println!("lincoln"); }
fn dots() {
assert_eq!(String::from(".................................................."),
format!("{:?}", .. .. .. .. .. .. .. .. .. .. .. .. ..
.. .. .. .. .. .. .. .. .. .. .. ..));
}
fn u8(u8: u8) {
if u8 != 0u8 {
assert_eq!(8u8, {
macro_rules! u8 {
(u8) => {
mod u8 {
pub fn u8<'u8: 'u8 + 'u8>(u8: &'u8 u8) -> &'u8 u8 {
"u8";
u8
}
}
};
}
u8!(u8);
let &u8: &u8 = u8::u8(&8u8);
::u8(0u8);
u8
});
}
}
fn fishy() {
assert_eq!(String::from("><>"),
String::<>::from::<>("><>").chars::<>().rev::<>().collect::<String>());
}
fn union() {
union union<'union> { union: &'union union<'union>, }
}
fn special_characters() {
let val = !((|(..):(_,_),(|__@_|__)|__)((&*"\\",'🤔')/**/,{})=={&[..=..][..];})//
;
assert!(!val);
}
fn punch_card() -> impl std::fmt::Debug {
..=..=.. .. .. .. .. .. .. .. .. .. .. ..=.. ..
..=.. ..=.. .. .. .. .. .. .. .. .. ..=..=..=..
..=.. ..=.. ..=.. ..=.. .. ..=..=.. .. ..=.. ..
..=..=.. .. ..=.. ..=.. ..=.. .. .. .. ..=.. ..
..=.. ..=.. ..=.. ..=.. .. ..=.. .. .. ..=.. ..
..=.. ..=.. ..=.. ..=.. .. .. ..=.. .. ..=.. ..
..=.. ..=.. .. ..=..=.. ..=..=.. .. .. ..=.. ..
}
fn r#match() {
let val: () = match match match match match () {
() => ()
} {
() => ()
} {
() => ()
} {
() => ()
} {
() => ()
};
assert_eq!(val, ());
}
fn i_yield() {
static || {
yield yield yield yield yield yield yield yield yield;
};
}
fn match_nested_if() {
let val = match () {
() if if if if true {true} else {false} {true} else {false} {true} else {false} => true,
_ => false,
};
assert!(val);
}
fn monkey_barrel() {
let val: () = ()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=()=();
assert_eq!(val, ());
}
fn 𝚌𝚘𝚗𝚝𝚒𝚗𝚞𝚎() {
type 𝚕𝚘𝚘𝚙 = i32;
fn 𝚋𝚛𝚎𝚊𝚔() -> 𝚕𝚘𝚘𝚙 {
let 𝚛𝚎𝚝𝚞𝚛𝚗 = 42;
return 𝚛𝚎𝚝𝚞𝚛𝚗;
}
assert_eq!(loop {
break 𝚋𝚛𝚎𝚊𝚔 ();
}, 42);
}
fn function() {
struct foo;
impl Deref for foo {
type Target = fn() -> Self;
fn deref(&self) -> &Self::Target {
&((|| foo) as _)
}
}
let foo = foo () ()() ()()() ()()()() ()()()()();
}
fn bathroom_stall() {
let mut i = 1;
matches!(2, _|_|_|_|_|_ if (i+=1) != (i+=1));
assert_eq!(i, 13);
}
fn closure_matching() {
let x = |_| Some(1);
let (|x| x) = match x(..) {
|_| Some(2) => |_| Some(3),
|_| _ => unreachable!(),
};
assert!(matches!(x(..), |_| Some(4)));
}
fn semisemisemisemisemi() {
;;;;;;; ;;;;;;; ;;; ;;; ;;
;; ;; ;;;; ;;;; ;;
;;;;;;; ;;;;; ;; ;;;; ;; ;;
;; ;; ;; ;; ;; ;;
;;;;;;; ;;;;;;; ;; ;; ;;
}
fn useful_syntax() {
use {{std::{{collections::{{HashMap}}}}}};
use ::{{{{core}, {std}}}};
use {{::{{core as core2}}}};
}
fn infcx() {
pub mod cx {
pub mod cx {
pub use super::cx;
pub struct Cx;
}
}
let _cx: cx::cx::Cx = cx::cx::cx::cx::cx::Cx;
}
pub fn main() {
strange();
funny();
what();
zombiejesus();
notsure();
canttouchthis();
angrydome();
evil_lincoln();
dots();
u8(8u8);
fishy();
union();
special_characters();
punch_card();
r#match();
i_yield();
match_nested_if();
monkey_barrel();
𝚌𝚘𝚗𝚝𝚒𝚗𝚞𝚎();
function();
bathroom_stall();
closure_matching();
semisemisemisemisemi();
useful_syntax();
infcx();
}

1542
rust/grammar.js Normal file

File diff suppressed because it is too large Load Diff

186
rust/src/scanner.c Normal file
View File

@ -0,0 +1,186 @@
#include <tree_sitter/parser.h>
#include <wctype.h>
enum TokenType {
STRING_CONTENT,
RAW_STRING_LITERAL,
FLOAT_LITERAL,
BLOCK_COMMENT,
};
void *tree_sitter_rust_external_scanner_create() { return NULL; }
void tree_sitter_rust_external_scanner_destroy(void *p) {}
void tree_sitter_rust_external_scanner_reset(void *p) {}
unsigned tree_sitter_rust_external_scanner_serialize(void *p, char *buffer) { return 0; }
void tree_sitter_rust_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
static void advance(TSLexer *lexer) {
lexer->advance(lexer, false);
}
static bool is_num_char(int32_t c) {
return c == '_' || iswdigit(c);
}
bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
if (valid_symbols[STRING_CONTENT] && !valid_symbols[FLOAT_LITERAL]) {
bool has_content = false;
for (;;) {
if (lexer->lookahead == '\"' || lexer->lookahead == '\\') {
break;
} else if (lexer->lookahead == 0) {
return false;
}
has_content = true;
advance(lexer);
}
lexer->result_symbol = STRING_CONTENT;
return has_content;
}
while (iswspace(lexer->lookahead)) lexer->advance(lexer, true);
if (
valid_symbols[RAW_STRING_LITERAL] &&
(lexer->lookahead == 'r' || lexer->lookahead == 'b')
) {
lexer->result_symbol = RAW_STRING_LITERAL;
if (lexer->lookahead == 'b') advance(lexer);
if (lexer->lookahead != 'r') return false;
advance(lexer);
unsigned opening_hash_count = 0;
while (lexer->lookahead == '#') {
advance(lexer);
opening_hash_count++;
}
if (lexer->lookahead != '"') return false;
advance(lexer);
for (;;) {
if (lexer->lookahead == 0) {
return false;
} else if (lexer->lookahead == '"') {
advance(lexer);
unsigned hash_count = 0;
while (lexer->lookahead == '#' && hash_count < opening_hash_count) {
advance(lexer);
hash_count++;
}
if (hash_count == opening_hash_count) {
return true;
}
} else {
advance(lexer);
}
}
}
if (valid_symbols[FLOAT_LITERAL] && iswdigit(lexer->lookahead)) {
lexer->result_symbol = FLOAT_LITERAL;
advance(lexer);
while (is_num_char(lexer->lookahead)) {
advance(lexer);
}
bool has_fraction = false, has_exponent = false;
if (lexer->lookahead == '.') {
has_fraction = true;
advance(lexer);
if (iswalpha(lexer->lookahead)) {
// The dot is followed by a letter: 1.max(2) => not a float but an integer
return false;
}
if (lexer->lookahead == '.') {
return false;
}
while (is_num_char(lexer->lookahead)) {
advance(lexer);
}
}
lexer->mark_end(lexer);
if (lexer->lookahead == 'e' || lexer->lookahead == 'E') {
has_exponent = true;
advance(lexer);
if (lexer->lookahead == '+' || lexer->lookahead == '-') {
advance(lexer);
}
if (!is_num_char(lexer->lookahead)) {
return true;
}
advance(lexer);
while (is_num_char(lexer->lookahead)) {
advance(lexer);
}
lexer->mark_end(lexer);
}
if (!has_exponent && !has_fraction) return false;
if (lexer->lookahead != 'u' && lexer->lookahead != 'i' && lexer->lookahead != 'f') {
return true;
}
advance(lexer);
if (!iswdigit(lexer->lookahead)) {
return true;
}
while (iswdigit(lexer->lookahead)) {
advance(lexer);
}
lexer->mark_end(lexer);
return true;
}
if (lexer->lookahead == '/') {
advance(lexer);
if (lexer->lookahead != '*') return false;
advance(lexer);
bool after_star = false;
unsigned nesting_depth = 1;
for (;;) {
switch (lexer->lookahead) {
case '\0':
return false;
case '*':
advance(lexer);
after_star = true;
break;
case '/':
if (after_star) {
advance(lexer);
after_star = false;
nesting_depth--;
if (nesting_depth == 0) {
lexer->result_symbol = BLOCK_COMMENT;
return true;
}
} else {
advance(lexer);
after_star = false;
if (lexer->lookahead == '*') {
nesting_depth++;
advance(lexer);
}
}
break;
default:
advance(lexer);
after_star = false;
break;
}
}
}
return false;
}

View File

@ -0,0 +1,88 @@
================================================================================
Async function
================================================================================
async fn abc() {}
async fn main() {
let x = futures.await?;
}
--------------------------------------------------------------------------------
(source_file
(function_item
(function_modifiers)
(identifier)
(parameters)
(block))
(function_item
(function_modifiers)
(identifier)
(parameters)
(block
(let_declaration
(identifier)
(try_expression
(await_expression
(identifier)))))))
================================================================================
Await expression
================================================================================
futures.await;
futures.await?;
futures.await?.await?;
futures.await?.function().await?;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(await_expression
(identifier)))
(expression_statement
(try_expression
(await_expression
(identifier))))
(expression_statement
(try_expression
(await_expression
(try_expression
(await_expression
(identifier))))))
(expression_statement
(try_expression
(await_expression
(call_expression
(field_expression
(try_expression
(await_expression
(identifier)))
(field_identifier))
(arguments))))))
================================================================================
Async Block
================================================================================
async {}
async { let x = 10; }
async move {}
--------------------------------------------------------------------------------
(source_file
(expression_statement
(async_block
(block)))
(expression_statement
(async_block
(block
(let_declaration
(identifier)
(integer_literal)))))
(expression_statement
(async_block
(block))))

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,193 @@
================================================================================
Integer literals
================================================================================
0;
0___0;
123;
0usize;
123i32;
123u32;
123_u32;
0xff_u8;
0o70_i16;
0b1111_1111_1001_0000_i32;
1u128;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal))
(expression_statement
(integer_literal)))
================================================================================
Floating-point literals
================================================================================
123.123;
2.;
123.0f64;
0.1f64;
0.1f32;
12E+99_f64;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(float_literal))
(expression_statement
(float_literal))
(expression_statement
(float_literal))
(expression_statement
(float_literal))
(expression_statement
(float_literal))
(expression_statement
(float_literal)))
================================================================================
String literals
================================================================================
"";
"abc";
b"foo\nbar";
"foo\
bar";
"\"foo\"";
"/* foo bar */ foo bar";
"foo\x42\x43bar";
"foo \x42 \x43 bar";
--------------------------------------------------------------------------------
(source_file
(expression_statement
(string_literal))
(expression_statement
(string_literal))
(expression_statement
(string_literal
(escape_sequence)))
(expression_statement
(string_literal
(escape_sequence)))
(expression_statement
(string_literal
(escape_sequence)
(escape_sequence)))
(expression_statement
(string_literal))
(expression_statement
(string_literal
(escape_sequence)
(escape_sequence)))
(expression_statement
(string_literal
(escape_sequence)
(escape_sequence))))
================================================================================
Raw string literals
================================================================================
r#"abc"#; r##"ok"##;
r##"foo #"# bar"##;
r###"foo ##"## bar"###;
r######"foo ##### bar"######;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(raw_string_literal))
(expression_statement
(raw_string_literal))
(expression_statement
(raw_string_literal))
(expression_statement
(raw_string_literal))
(expression_statement
(raw_string_literal)))
================================================================================
Raw byte string literals
================================================================================
br#"abc"#;
br##"abc"##;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(raw_string_literal))
(expression_statement
(raw_string_literal)))
================================================================================
Character literals
================================================================================
'a';
'\'';
'\0';
b'x';
'\t';
'\xff';
'\\';
--------------------------------------------------------------------------------
(source_file
(expression_statement
(char_literal))
(expression_statement
(char_literal))
(expression_statement
(char_literal))
(expression_statement
(char_literal))
(expression_statement
(char_literal))
(expression_statement
(char_literal))
(expression_statement
(char_literal)))
================================================================================
Boolean literals
================================================================================
true;
false;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(boolean_literal))
(expression_statement
(boolean_literal)))

257
rust/test/corpus/macros.txt Normal file
View File

@ -0,0 +1,257 @@
================================================================================
Macro invocation - no arguments
================================================================================
a!();
b![];
c!{};
d::e!();
f::g::h!{};
--------------------------------------------------------------------------------
(source_file
(expression_statement
(macro_invocation
(identifier)
(token_tree)))
(expression_statement
(macro_invocation
(identifier)
(token_tree)))
(expression_statement
(macro_invocation
(identifier)
(token_tree)))
(expression_statement
(macro_invocation
(scoped_identifier
(identifier)
(identifier))
(token_tree)))
(expression_statement
(macro_invocation
(scoped_identifier
(scoped_identifier
(identifier)
(identifier))
(identifier))
(token_tree))))
================================================================================
Macro invocation - arbitrary tokens
================================================================================
a!(* a *);
a!(& a &);
a!(- a -);
a!(b + c + +);
a!('a'..='z');
a!('\u{0}'..='\u{2}');
a!('lifetime)
default!(a);
union!(a);
a!($);
a!($());
a!($ a $);
a!(${$([ a ])});
a!($a $a:ident $($a);*);
--------------------------------------------------------------------------------
(source_file
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier)
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(char_literal)
(char_literal))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(char_literal)
(char_literal))))
(macro_invocation
(identifier)
(token_tree
(identifier)))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree)))
(expression_statement
(macro_invocation
(identifier)
(token_tree (token_tree))))
(expression_statement
(macro_invocation
(identifier)
(token_tree (identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree (token_tree (token_tree (token_tree (identifier)))))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier)
(identifier)
(identifier)
(token_tree
(identifier))))))
================================================================================
Macro invocation with comments
================================================================================
ok! {
// one
/* two */
}
--------------------------------------------------------------------------------
(source_file
(macro_invocation
(identifier)
(token_tree
(line_comment)
(block_comment))))
================================================================================
Macro definition
================================================================================
macro_rules! say_hello {
() => (
println!("Hello!");
)
}
macro_rules! four {
() => {1 + 3};
}
macro_rules! foo {
(x => $e:expr) => (println!("mode X: {}", $e));
(y => $e:expr) => (println!("mode Y: {}", $e))
}
macro_rules! o_O {
(
$($x:expr; [ $( $y:expr ),* ]);*
) => {
$($($x + $e),*),*
}
}
macro_rules! zero_or_one {
($($e:expr),?) => {
$($e),?
};
}
--------------------------------------------------------------------------------
(source_file
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern)
right: (token_tree
(identifier)
(token_tree
(string_literal)))))
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern)
right: (token_tree
(integer_literal)
(integer_literal))))
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern
(identifier)
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier)))
right: (token_tree
(identifier)
(token_tree
(string_literal)
(metavariable))))
(macro_rule
left: (token_tree_pattern
(identifier)
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier)))
right: (token_tree
(identifier)
(token_tree
(string_literal)
(metavariable)))))
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern
(token_repetition_pattern
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier))
(token_tree_pattern
(token_repetition_pattern
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier))))))
right: (token_tree
(token_repetition
(token_repetition
(metavariable)
(metavariable))))))
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern
(token_repetition_pattern
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier))))
right: (token_tree
(token_repetition
(metavariable))))))

View File

@ -0,0 +1,469 @@
================================================================================
Tuple struct patterns
================================================================================
match x {
Some(x) => "some",
std::None() => "none"
}
--------------------------------------------------------------------------------
(source_file
(expression_statement
(match_expression
(identifier)
(match_block
(match_arm
(match_pattern
(tuple_struct_pattern
(identifier)
(identifier)))
(string_literal))
(match_arm
(match_pattern
(tuple_struct_pattern
(scoped_identifier
(identifier)
(identifier))))
(string_literal))))))
================================================================================
Reference patterns
================================================================================
match x {
A(ref x) => x.0,
ref mut y => y,
& mut z => z,
}
--------------------------------------------------------------------------------
(source_file
(expression_statement
(match_expression
(identifier)
(match_block
(match_arm
(match_pattern
(tuple_struct_pattern
(identifier)
(ref_pattern
(identifier))))
(field_expression
(identifier)
(integer_literal)))
(match_arm
(match_pattern
(ref_pattern
(mut_pattern
(mutable_specifier)
(identifier))))
(identifier))
(match_arm
(match_pattern
(reference_pattern
(mutable_specifier)
(identifier)))
(identifier))))))
================================================================================
Struct patterns
================================================================================
match x {
Person{name, age} if age < 5 => ("toddler", name),
Person{name: adult_name, age: _} => ("adult", adult_name),
}
--------------------------------------------------------------------------------
(source_file
(expression_statement
(match_expression
(identifier)
(match_block
(match_arm
(match_pattern
(struct_pattern
(type_identifier)
(field_pattern
(shorthand_field_identifier))
(field_pattern
(shorthand_field_identifier)))
(binary_expression
(identifier)
(integer_literal)))
(tuple_expression
(string_literal)
(identifier)))
(match_arm
(match_pattern
(struct_pattern
(type_identifier)
(field_pattern
(field_identifier)
(identifier))
(field_pattern
(field_identifier))))
(tuple_expression
(string_literal)
(identifier)))))))
================================================================================
Ignored patterns
================================================================================
match x {
(a, ..) => a,
B(..) => c,
D::E{f: g, ..} => g
}
--------------------------------------------------------------------------------
(source_file
(expression_statement
(match_expression
(identifier)
(match_block
(match_arm
(match_pattern
(tuple_pattern
(identifier)
(remaining_field_pattern)))
(identifier))
(match_arm
(match_pattern
(tuple_struct_pattern
(identifier)
(remaining_field_pattern)))
(identifier))
(match_arm
(match_pattern
(struct_pattern
(scoped_type_identifier
(identifier)
(type_identifier))
(field_pattern
(field_identifier)
(identifier))
(remaining_field_pattern)))
(identifier))))))
================================================================================
Captured patterns
================================================================================
match x {
a @ A(_) | b @ B(..) => a,
a @ 1 ... 5 => a,
Some(1 ... 5) => a,
a @ b...c => a,
a @ b..=c => a,
}
--------------------------------------------------------------------------------
(source_file
(expression_statement
(match_expression
value: (identifier)
body: (match_block
(match_arm
pattern: (match_pattern
(or_pattern
(captured_pattern
(identifier)
(tuple_struct_pattern
type: (identifier)))
(captured_pattern
(identifier)
(tuple_struct_pattern
type: (identifier)
(remaining_field_pattern)))))
value: (identifier))
(match_arm
pattern: (match_pattern
(captured_pattern
(identifier)
(range_pattern
(integer_literal)
(integer_literal))))
value: (identifier))
(match_arm
pattern: (match_pattern
(tuple_struct_pattern
type: (identifier)
(range_pattern
(integer_literal)
(integer_literal))))
value: (identifier))
(match_arm
pattern: (match_pattern
(captured_pattern
(identifier)
(range_pattern
(identifier)
(identifier))))
value: (identifier))
(match_arm
pattern: (match_pattern
(captured_pattern
(identifier)
(range_pattern
(identifier)
(identifier))))
value: (identifier))))))
================================================================================
Or patterns
================================================================================
if let A(x) | B(x) = expr {
do_stuff_with(x);
}
while let A(x) | B(x) = expr {
do_stuff_with(x);
}
let Ok(index) | Err(index) = slice.binary_search(&x);
for ref a | b in c {}
let Ok(x) | Err(x) = binary_search(x);
for A | B | C in c {}
|(Ok(x) | Err(x))| expr();
let ref mut x @ (A | B | C);
fn foo((1 | 2 | 3): u8) {}
if let x!() | y!() = () {}
// Discomment after box pattern land on master
// let box (A | B | C);
// Not handled cause devs didn't got into agreement if should be acceptd or not
// |Ok(x) | Err(x)| expr();
--------------------------------------------------------------------------------
(source_file
(expression_statement
(if_expression
condition: (let_condition
pattern: (or_pattern
(tuple_struct_pattern
type: (identifier)
(identifier))
(tuple_struct_pattern
type: (identifier)
(identifier)))
value: (identifier))
consequence: (block
(expression_statement
(call_expression
function: (identifier)
arguments: (arguments
(identifier)))))))
(expression_statement
(while_expression
condition: (let_condition
pattern: (or_pattern
(tuple_struct_pattern
type: (identifier)
(identifier))
(tuple_struct_pattern
type: (identifier)
(identifier)))
value: (identifier))
body: (block
(expression_statement
(call_expression
function: (identifier)
arguments: (arguments
(identifier)))))))
(let_declaration
pattern: (or_pattern
(tuple_struct_pattern
type: (identifier)
(identifier))
(tuple_struct_pattern
type: (identifier)
(identifier)))
value: (call_expression
function: (field_expression
value: (identifier)
field: (field_identifier))
arguments: (arguments
(reference_expression
value: (identifier)))))
(expression_statement
(for_expression
pattern: (or_pattern
(ref_pattern
(identifier))
(identifier))
value: (identifier)
body: (block)))
(let_declaration
pattern: (or_pattern
(tuple_struct_pattern
type: (identifier)
(identifier))
(tuple_struct_pattern
type: (identifier)
(identifier)))
value: (call_expression
function: (identifier)
arguments: (arguments
(identifier))))
(expression_statement
(for_expression
pattern: (or_pattern
(or_pattern
(identifier)
(identifier))
(identifier))
value: (identifier)
body: (block)))
(expression_statement
(closure_expression
parameters: (closure_parameters
(tuple_pattern
(or_pattern
(tuple_struct_pattern
type: (identifier)
(identifier))
(tuple_struct_pattern
type: (identifier)
(identifier)))))
body: (call_expression
function: (identifier)
arguments: (arguments))))
(let_declaration
pattern: (ref_pattern
(mut_pattern
(mutable_specifier)
(captured_pattern
(identifier)
(tuple_pattern
(or_pattern
(or_pattern
(identifier)
(identifier))
(identifier)))))))
(function_item
name: (identifier)
parameters: (parameters
(parameter
pattern: (tuple_pattern
(or_pattern
(or_pattern
(integer_literal)
(integer_literal))
(integer_literal)))
type: (primitive_type)))
body: (block))
(expression_statement
(if_expression
condition: (let_condition
pattern: (or_pattern
(macro_invocation
macro: (identifier)
(token_tree))
(macro_invocation
macro: (identifier)
(token_tree)))
value: (unit_expression))
consequence: (block)))
(line_comment)
(line_comment)
(line_comment)
(line_comment))
================================================================================
Inline const or Const blocks as pattern
================================================================================
fn foo(x: i32) {
const CUBE: i32 = 3.pow(3);
match x {
CUBE => println!("three cubed"),
_ => {}
}
}
fn foo(x: i32) {
match x {
const { 3.pow(3) } => println!("three cubed"),
_ => {}
}
}
--------------------------------------------------------------------------------
(source_file
(function_item
name: (identifier)
parameters: (parameters
(parameter
pattern: (identifier)
type: (primitive_type)))
body: (block
(const_item
name: (identifier)
type: (primitive_type)
value: (call_expression
function: (field_expression
value: (integer_literal)
field: (field_identifier))
arguments: (arguments
(integer_literal))))
(expression_statement
(match_expression
value: (identifier)
body: (match_block
(match_arm
pattern: (match_pattern
(identifier))
value: (macro_invocation
macro: (identifier)
(token_tree
(string_literal))))
(match_arm
pattern: (match_pattern)
value: (block)))))))
(function_item
name: (identifier)
parameters: (parameters
(parameter
pattern: (identifier)
type: (primitive_type)))
body: (block
(expression_statement
(match_expression
value: (identifier)
body: (match_block
(match_arm
pattern: (match_pattern
(const_block
body: (block
(call_expression
function: (field_expression
value: (integer_literal)
field: (field_identifier))
arguments: (arguments
(integer_literal))))))
value: (macro_invocation
macro: (identifier)
(token_tree
(string_literal))))
(match_arm
pattern: (match_pattern)
value: (block))))))))

View File

@ -0,0 +1,69 @@
============================================
Block comments
============================================
/*
* Block comments
*/
/* Comment with asterisks **/
----
(source_file
(block_comment)
(block_comment))
============================================
Nested block comments
============================================
/* /* double nested */ */
// ---
/*/*/* triple nested */*/*/
// ---
/****
/****
nested with extra stars
****/
****/
// ---
----
(source_file
(block_comment)
(line_comment)
(block_comment)
(line_comment)
(block_comment)
(line_comment))
============================================
Line comments
============================================
// Comment
----
(source_file
(line_comment))
=====================================
Greek letters in identifiers
=====================================
const σ1 : Σ = 0;
const ψ_2 : Ψ = 1;
---
(source_file
(const_item (identifier) (type_identifier) (integer_literal))
(const_item (identifier) (type_identifier) (integer_literal)))

384
rust/test/corpus/types.txt Normal file
View File

@ -0,0 +1,384 @@
================================================================================
The unit type
================================================================================
type A = ();
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(unit_type)))
================================================================================
Tuple types
================================================================================
type A = (i32, String);
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(tuple_type
(primitive_type)
(type_identifier))))
================================================================================
Reference types
================================================================================
type A = &B;
type C = &'a str;
type D = &'a mut str;
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(reference_type
(type_identifier)))
(type_item
(type_identifier)
(reference_type
(lifetime
(identifier))
(primitive_type)))
(type_item
(type_identifier)
(reference_type
(lifetime
(identifier))
(mutable_specifier)
(primitive_type))))
================================================================================
Raw pointer types
================================================================================
type A = *mut B;
type C = *const str;
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(pointer_type
(mutable_specifier)
(type_identifier)))
(type_item
(type_identifier)
(pointer_type
(primitive_type))))
================================================================================
Generic types
================================================================================
type A = B<C>;
type D = E<F, str>;
type G = H<'a, I>;
type J = H<K=L>;
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(type_identifier))))
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(type_identifier)
(primitive_type))))
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(lifetime
(identifier))
(type_identifier))))
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(type_binding
(type_identifier)
(type_identifier))))))
================================================================================
Scoped types
================================================================================
type A = B::C;
type D = E::F::G;
type H = I::J<K>;
type L = M<N>::O;
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(scoped_type_identifier
(identifier)
(type_identifier)))
(type_item
(type_identifier)
(scoped_type_identifier
(scoped_identifier
(identifier)
(identifier))
(type_identifier)))
(type_item
(type_identifier)
(generic_type
(scoped_type_identifier
(identifier)
(type_identifier))
(type_arguments
(type_identifier))))
(type_item
(type_identifier)
(scoped_type_identifier
(generic_type
(type_identifier)
(type_arguments
(type_identifier)))
(type_identifier))))
================================================================================
Array types
================================================================================
type A = [B; 4];
type C = &[D];
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(array_type
(type_identifier)
(integer_literal)))
(type_item
(type_identifier)
(reference_type
(array_type
(type_identifier)))))
================================================================================
Function types
================================================================================
fn high_order1(value: i32, f: fn(i32)) -> i32 {}
fn high_order2(value: i32, f: fn(i32) -> i32) -> i32 {
f(value)
}
fn high_order3(value: i32, f: &FnOnce(i32) -> i32) -> i32 {
f(value)
}
type F = for<'a, 'b> fn(x: &'a A, y: &'a mut B<'i, 't>,) -> C;
--------------------------------------------------------------------------------
(source_file
(function_item
(identifier)
(parameters
(parameter
(identifier)
(primitive_type))
(parameter
(identifier)
(function_type
(parameters
(primitive_type)))))
(primitive_type)
(block))
(function_item
(identifier)
(parameters
(parameter
(identifier)
(primitive_type))
(parameter
(identifier)
(function_type
(parameters
(primitive_type))
(primitive_type))))
(primitive_type)
(block
(call_expression
(identifier)
(arguments
(identifier)))))
(function_item
(identifier)
(parameters
(parameter
(identifier)
(primitive_type))
(parameter
(identifier)
(reference_type
(function_type
(type_identifier)
(parameters
(primitive_type))
(primitive_type)))))
(primitive_type)
(block
(call_expression
(identifier)
(arguments
(identifier)))))
(type_item
(type_identifier)
(function_type
(for_lifetimes
(lifetime
(identifier))
(lifetime
(identifier)))
(parameters
(parameter
(identifier)
(reference_type
(lifetime
(identifier))
(type_identifier)))
(parameter
(identifier)
(reference_type
(lifetime
(identifier))
(mutable_specifier)
(generic_type
(type_identifier)
(type_arguments
(lifetime
(identifier))
(lifetime
(identifier)))))))
(type_identifier))))
================================================================================
Unsafe and extern function types
================================================================================
type a = extern "C" fn(*mut c_void);
type b = unsafe extern "C" fn() -> *mut c_void;
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(function_type
(function_modifiers
(extern_modifier
(string_literal)))
(parameters
(pointer_type
(mutable_specifier)
(type_identifier)))))
(type_item
(type_identifier)
(function_type
(function_modifiers
(extern_modifier
(string_literal)))
(parameters)
(pointer_type
(mutable_specifier)
(type_identifier)))))
================================================================================
Trait objects
================================================================================
type a = Box<Something + 'a>;
type b = Rc<dyn Something>;
type c = A<&dyn Fn(&B) -> C>;
--------------------------------------------------------------------------------
(source_file
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(bounded_type
(type_identifier)
(lifetime
(identifier))))))
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(dynamic_type
(type_identifier)))))
(type_item
(type_identifier)
(generic_type
(type_identifier)
(type_arguments
(reference_type
(dynamic_type
(function_type
(type_identifier)
(parameters
(reference_type
(type_identifier)))
(type_identifier))))))))
================================================================================
Type cast expressions with generics
================================================================================
a as B<C>;
d as *mut E<<F as E>::G>;
--------------------------------------------------------------------------------
(source_file
(expression_statement
(type_cast_expression
(identifier)
(generic_type
(type_identifier)
(type_arguments
(type_identifier)))))
(expression_statement
(type_cast_expression
(identifier)
(pointer_type
(mutable_specifier)
(generic_type
(type_identifier)
(type_arguments
(scoped_type_identifier
(bracketed_type
(qualified_type
(type_identifier)
(type_identifier)))
(type_identifier))))))))