Make charset support working with C++ scanners too

Extend test-charset to check C++ scanner too
Add implementation for charset support
2014-11-30 19:22:43 -05:00 · 2014-11-30 19:22:43 -05:00 · 2014-11-30 19:22:43 -05:00 · 2014-11-30 19:21:31 -05:00 · 2014-11-30 19:18:21 -05:00 · 2014-11-26 15:31:53 -05:00
300 changed files with 2769 additions and 4379 deletions
@@ -1,7 +1,11 @@
+*.orig
+*.rej
 *.tar.bz2
 *.tar.gz
-.bootstrap
+*.tar.xz
+*~
 .deps
+.libs
 ABOUT-NLS
 ChangeLog
 INSTALL
@@ -9,34 +13,11 @@ Makefile
 Makefile.in
 aclocal.m4
 autom4te.cache
-compile
-conf.in
-config.cache
-config.guess
-config.h
 config.log
-config.status*
-config.sub
+config.status
 configure
-depcomp
-flex
-flex-*
-install-sh
-missing
-mkinstalldirs
-parse.c
-parse.h
-scan.c
-skel.c
-stamp-*
-config.rpath
-ylwrap
-*.o
-.libs
-*.lo
-*.la
+flex-*/
 libtool
-ltmain.sh
-*~
-*.orig
-*.rej
+stamp-*
+m4/
+build-aux/
@@ -33,128 +33,28 @@
 # For MS-DOS, add "-DMS_DOS" to DEFS.  See the directory MISC/MSDOS for
 # additional info.

-AM_YFLAGS = -d
 ACLOCAL_AMFLAGS = -I m4
-m4 = @M4@
 indent = @INDENT@

-bin_PROGRAMS = flex
-lib_LTLIBRARIES = \
-	libfl.la \
-	libfl_pic.la
-
-flex_SOURCES = \
-	ccl.c \
-	dfa.c \
-	ecs.c \
-	scanflags.c \
-	gen.c \
-	main.c \
-	misc.c \
-	nfa.c \
-	parse.y \
-	scan.l \
-	skel.c \
-	sym.c \
-	tblcmp.c \
-	yylex.c \
-	options.c \
-	scanopt.c \
-	buf.c \
-	tables.c \
-	tables_shared.c \
-	filter.c \
-	regex.c
-
-
-LDADD = lib/libcompat.la
-
-libfl_la_SOURCES = \
-	libmain.c \
-	libyywrap.c
-
-libfl_la_LDFLAGS = -no-undefined -version-info @SHARED_VERSION_INFO@
-
-libfl_pic_la_SOURCES = \
-	libmain.c \
-	libyywrap.c
-
-libfl_pic_la_LDFLAGS = -no-undefined -version-info @SHARED_VERSION_INFO@
-
-noinst_HEADERS = \
-	flexdef.h \
-	flexint.h \
-	version.h \
-	options.h \
-	scanopt.h \
-	tables.h \
-	tables_shared.h
-
-include_HEADERS = \
-	FlexLexer.h
-
 dist_doc_DATA = \
 	AUTHORS \
 	COPYING \
 	NEWS \
 	ONEWS \
-	README \
-	TODO
+	README

 EXTRA_DIST = \
 	.indent.pro \
-	ABOUT-NLS \
-	INSTALL \
-	autogen.sh \
-	flex.skl \
-	mkskel.sh \
-	config.rpath \
-	gettext.h
-
-BUILT_SOURCES = \
-	skel.c
+	autogen.sh

 SUBDIRS = \
 	lib \
-	. \
+	src \
 	doc \
 	examples \
 	po \
-	tests
-
-localedir = $(datadir)/locale
-AM_CPPFLAGS = -DLOCALEDIR=\"$(localedir)\" -I$(top_srcdir)/intl
-LIBS = @LIBINTL@ @LIBS@
-
-skel.c: flex.skl mkskel.sh flexint.h tables_shared.h
-	sed 's/m4_/m4postproc_/g; s/m4preproc_/m4_/g' $(srcdir)/flex.skl | $(m4) -P -DFLEX_MAJOR_VERSION=`echo $(VERSION)|cut -f 1 -d .` -DFLEX_MINOR_VERSION=`echo $(VERSION)|cut -f 2 -d .` -DFLEX_SUBMINOR_VERSION=`echo $(VERSION)|cut -f 3 -d .` | sed 's/m4postproc_/m4_/g' | $(SHELL) $(srcdir)/mkskel.sh  >skel.c
-
-# Explicitly describe dependencies.
-# You can recreate this with `gcc -I. -MM *.c'
-buf.o: buf.c flexdef.h flexint.h
-ccl.o: ccl.c flexdef.h flexint.h
-dfa.o: dfa.c flexdef.h flexint.h tables.h tables_shared.h
-ecs.o: ecs.c flexdef.h flexint.h
-scanflags.o: scanflags.c flexdef.h flexint.h
-gen.o: gen.c flexdef.h flexint.h tables.h tables_shared.h
-libmain.o: libmain.c
-libyywrap.o: libyywrap.c
-main.o: main.c flexdef.h flexint.h version.h options.h scanopt.h \
- tables.h tables_shared.h
-misc.o: misc.c flexdef.h flexint.h tables.h tables_shared.h
-nfa.o: nfa.c flexdef.h flexint.h
-options.o: options.c options.h scanopt.h flexdef.h flexint.h
-parse.o: parse.c flexdef.h flexint.h tables.h tables_shared.h
-scan.o: scan.c flexdef.h flexint.h parse.h
-scanopt.o: scanopt.c flexdef.h flexint.h scanopt.h
-skel.o: skel.c flexdef.h flexint.h
-sym.o: sym.c flexdef.h flexint.h
-tables.o: tables.c flexdef.h flexint.h tables.h tables_shared.h
-tables_shared.o: tables_shared.c flexdef.h flexint.h tables.h \
- tables_shared.h
-tblcmp.o: tblcmp.c flexdef.h flexint.h
-yylex.o: yylex.c flexdef.h flexint.h parse.h
-filter.o: filter.c flexdef.h flexint.h
+	tests \
+	tools

 # Create the ChangeLog, but only if we're inside a git working directory

@@ -163,50 +63,6 @@ ChangeLog: $(srcdir)/tools/git2cl
 		$(srcdir)/tools/git2cl > $@ \
 	; fi

-# Run GNU indent on sources. Don't run this unless all the sources compile cleanly.
-#
-# Whole idea:
-#   1. Check for .indent.pro, otherwise indent will use unknown
-#      settings, or worse, the GNU defaults.)
-#   2. Check that this is GNU indent.
-#   3. Make sure to process only the NON-generated .c and .h files.
-#   4. Run indent twice per file. The first time is a test.
-#      Otherwise, indent overwrites your file even if it fails!
-indentfiles = \
-	buf.c \
-	ccl.c \
-	dfa.c \
-	ecs.c \
-	scanflags.c \
-	filter.c \
-	flexdef.h \
-	gen.c \
-	libmain.c \
-	libyywrap.c \
-	main.c \
-	misc.c \
-	nfa.c \
-	options.c \
-	options.h \
-	regex.c \
-	scanopt.c \
-	scanopt.h \
-	sym.c \
-	tables.c \
-	tables.h \
-	tables_shared.c \
-	tables_shared.h \
-	tblcmp.c
-
-indent:
-	if [ -f .indent.pro ] ; then \
-	for f in $(indentfiles);\
-	do\
-		echo indenting $$f ;\
-		$(indent) < $$f >/dev/null && indent $$f || echo $$f FAILED to indent ;\
-	done \
-	fi
-
 install-exec-hook:
 	cd $(DESTDIR)$(bindir) && \
 		$(LN_S) -f flex$(EXEEXT) flex++$(EXEEXT)
@@ -3,6 +3,8 @@ changes between releases of flex.

 See the file COPYING for copying conditions.

+* version 2.6.0-pre
+
 * flex version  2.5.39 released 2014-03-26

 ** no user visible changes in this release
@@ -24,27 +24,28 @@

 # autoconf requirements and initialization

-AC_INIT([the fast lexical analyser generator], [2.5.39],
+AC_INIT([the fast lexical analyser generator], [2.6.0-pre],
 [flex-help@lists.sourceforge.net], [flex])
+AC_CONFIG_SRCDIR([src/scan.l])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE([gnu check-news std-options dist-bzip2 dist-xz parallel-tests 1.14.1])
+AC_CONFIG_HEADER([src/config.h])
+AC_CONFIG_LIBOBJ_DIR([lib])
+AC_CONFIG_MACRO_DIR([m4])
+LT_INIT
 SHARED_VERSION_INFO="2:0:0"
 AC_SUBST(SHARED_VERSION_INFO)

-AC_CONFIG_SRCDIR([scan.l])
-AM_INIT_AUTOMAKE([gnu check-news std-options dist-bzip2 dist-xz 1.10])
-AC_CONFIG_HEADER([config.h:conf.in])
-AC_CONFIG_LIBOBJ_DIR([lib])
-
 # checks for programs

 AM_GNU_GETTEXT([external])
-AM_GNU_GETTEXT_VERSION([0.18.1])
+AM_GNU_GETTEXT_VERSION([0.19])
 AC_PROG_YACC
 AM_PROG_LEX
 AC_PROG_CC
 AC_PROG_CXX
 AM_PROG_CC_C_O
 AC_PROG_LN_S
-LT_INIT
 AC_PROG_AWK
 AC_PROG_INSTALL

@@ -93,6 +94,7 @@ AC_DEFINE([HAVE_LIBPTHREAD], 1, [pthread library] ),
 AC_DEFINE([HAVE_LIBPTHREAD], 0, [pthread library] )
 )
 AC_CHECK_HEADERS([pthread.h])
+AM_CONDITIONAL([want_pthread], [test x$ac_cv_lib_pthread_pthread_mutex_lock = xyes])

 AC_CHECK_LIB(m, log10)

@@ -118,56 +120,9 @@ examples/fastwc/Makefile
 examples/manual/Makefile
 lib/Makefile
 po/Makefile.in
+src/Makefile
+tools/Makefile
 tests/Makefile
-tests/TEMPLATE/Makefile
-tests/test-array-nr/Makefile
-tests/test-array-r/Makefile
-tests/test-basic-nr/Makefile
-tests/test-basic-r/Makefile
-tests/test-bison-yylloc/Makefile
-tests/test-bison-yylval/Makefile
-tests/test-c-cpp-nr/Makefile
-tests/test-c-cpp-r/Makefile
-tests/test-header-nr/Makefile
-tests/test-header-r/Makefile
-tests/test-include-by-buffer/Makefile
-tests/test-include-by-push/Makefile
-tests/test-include-by-reentrant/Makefile
-tests/test-multiple-scanners-nr/Makefile
-tests/test-multiple-scanners-r/Makefile
-tests/test-noansi-nr/Makefile
-tests/test-noansi-r/Makefile
-tests/test-prefix-nr/Makefile
-tests/test-prefix-r/Makefile
-tests/test-pthread/Makefile
-tests/test-string-nr/Makefile
-tests/test-string-r/Makefile
-tests/test-yyextra/Makefile
-tests/test-alloc-extra/Makefile
-tests/test-lineno-nr/Makefile
-tests/test-lineno-trailing/Makefile
-tests/test-lineno-r/Makefile
-tests/test-linedir-r/Makefile
-tests/test-debug-r/Makefile
-tests/test-debug-nr/Makefile
-tests/test-mem-nr/Makefile
-tests/test-mem-r/Makefile
-tests/test-posix/Makefile
-tests/test-posixly-correct/Makefile
-tests/test-table-opts/Makefile
-tests/test-c++-basic/Makefile
-tests/test-bison-nr/Makefile
-tests/test-reject/Makefile
-tests/test-c++-multiple-scanners/Makefile
-tests/test-top/Makefile
-tests/test-rescan-nr/Makefile
-tests/test-rescan-r/Makefile
-tests/test-quotes/Makefile
-tests/test-ccl/Makefile
-tests/test-extended/Makefile
-tests/test-c++-yywrap/Makefile
-tests/test-concatenated-options/Makefile
-dnl --new-test-here-- This line is processed by tests/create-test.
 )

 AC_OUTPUT
@@ -0,0 +1 @@
+XBS-Destinations: sourceforge
@@ -1,57 +0,0 @@
-# extract all table types from various scanners. We try to get the ones
-# that are declared outside '{'.
-# hopefully we'll hit them all.
-OUTFILE=/tmp/EXTRACT-FLEX-SYMS.$$
-echo > $OUTFILE
-
-for s in  \
-	yy_trans_info \
-	yy_NUL_trans \
-	yy_accept \
-	yy_base \
-	yy_chk \
-	yy_def \
-	yy_ec \
-	yy_meta \
-	yy_nxt \
-	yy_rule_can_match_eol \
-	yy_rule_linenum \
-	yy_start_state_list \
-	yy_state_type \
-	yy_transition
-do
-	perl -ne \
-	'
-		BEGIN{
-		   	$s = qq('$s');
-		}
-		s/\b(short|long)\s+int\b/$1/;
-		s/\b(static|(yy)?const)\b\s+//g;
-		next unless m/((?:struct\s*)?\w+)\W+$s\s*((?:\[\s*\d*\s*\])+)\s*=/;
-		$type=$1;
-		$arr = $2;
-		$arr =~ s/\d+//g;
-		$a{$s}->{$type . $arr}= $_;
-		
-		END{
-			for(sort keys %a){
-				print values %{$a{$_}}
-			}
-		}
-	' \
-	tests/test-*/*.c  >> $OUTFILE
-done
-
-grep -E 'typedef.*yy_state_type'  tests/test-*/*.c | 
-		gawk -F ':' '{print $2}' |
-		sort |
-		uniq >> $OUTFILE
-
-pcregrep  '(\w+)[\s*]*yy_(verify|nxt)\s*;'  tests/test-*/*.c |
-		perl -pe 's/^.*?\.c?[\-\:]//;' |
-		sort |
-		uniq >> $OUTFILE
-
-cat $OUTFILE
-rm -f $OUTFILE
-
@@ -1,2 +0,0 @@
-This directory contains flex developer junk.
-It should not be considered part of the official distribution.
@@ -1,22 +0,0 @@
-#! /usr/bin/perl -w
-# vim:set ft=perl ai si et ts=4 sts=4 sw=4 tw=0:
-# USAGE: dump-tables.pl  FILE [max-data]
-use strict;
-
-my $dir = $0;
-$dir =~ s:/[^/]+$::;
-push @INC, $dir;
-require "tables.pl";
-
-@ARGV == 1 || @ARGV == 2 || die "\nUSAGE: dump-tables.pl FILE [max-data]\n";
-open FD, $ARGV[0] or die "$ARGV[0]: $!";
-
-dump_hdr(read_hdr());
-while(!eof FD){
-   dump_table(read_table(),$ARGV[1]);
-}
-close FD;
-
-1
-__END__
-
@@ -1,165 +0,0 @@
-#! /usr/bin/perl -w
-# vim:set ft=perl ai si et ts=4 sts=4 sw=4 tw=0:
-# THIS FILE IS A LIBRARY, AND MEANT TO BE 'required' BY OTHER SCRIPTS
-use strict;
-use Class::Struct yytbl_hdr => [ 
-                    th_magic => '$', th_magic_offset => '$',
-                    th_hsize => '$', th_hsize_offset => '$',
-                    th_ssize => '$', th_ssize_offset => '$',
-                    th_flags => '$', th_flags_offset => '$',
-                    th_version => '$', th_version_offset => '$',
-                    th_name => '$', th_name_offset => '$'
-                 ];
-use Class::Struct yytbl_data => [
-                    td_id => '$', td_id_offset => '$',
-                    td_flags => '$', td_flags_offset => '$',
-                    td_hilen => '$', td_hilen_offset => '$',
-                    td_lolen => '$', td_lolen_offset => '$',
-                    td_data => '@', td_data_offset => '$'
-                  ];
-
-sub read_hdr {
-    my $th = new yytbl_hdr;
-    $th->th_magic_offset(tell FD);
-    $th->th_magic(read32());
-    $th->th_hsize_offset(tell FD);
-    $th->th_hsize(read32());
-    $th->th_ssize_offset(tell FD);
-    my @a = unpack 'Nna*',readn($th->th_hsize - 8);
-    $th->th_ssize($a[0]);
-    $th->th_flags_offset($th->th_ssize_offset + 4);
-    $th->th_flags($a[1]);
-    $th->th_version_offset($th->th_flags_offset + 2);
-    $th->th_version($a[2]);
-    $th->th_name((split /\0/, $th->th_version)[1]);
-    $th->th_version((split /\0/, $th->th_version)[0]);
-    $th->th_name_offset($th->th_version_offset + length($th->th_version) + 1);
-
-    return $th;
-}
-sub dump_hdr {
-    my $th = shift;
-    printf "%04X th_magic:   %08X\n",$th->th_magic_offset, $th->th_magic;
-    printf "%04X th_hsize:   \%d\n",$th->th_hsize_offset, $th->th_hsize;
-    printf "%04X th_ssize:   \%d\n",$th->th_ssize_offset, $th->th_ssize;
-    printf "%04X th_flags:   \%d\n",$th->th_flags_offset, $th->th_flags;
-    printf "%04X th_version: \%s\n",$th->th_version_offset, $th->th_version;
-    printf "%04X th_name:    \%s\n",$th->th_name_offset, $th->th_name;
-}
-
-
-sub readn { my $s; read FD, $s, shift; return $s }
-sub read32 { my $n; read FD, $n, 4; return unpack 'i', pack 'i', unpack 'N',$n }
-sub read16 { my $n; read FD, $n, 2; return unpack 's', pack 's', unpack 'n',$n }
-sub read8 { my $n; read FD, $n, 1;  return unpack 'c',$n }
-sub readstruct {
-    my $td=shift;
-    my $r = $::TFLAGS{$td->td_flags & 0x07}->[1];
-    return ($r->(), $r->());
-}
-sub getreader {
-    my $td = shift;
-    return \&readstruct if ($td->td_flags & 0x10);
-    return  $::TFLAGS{$td->td_flags & 0x07}->[1];
-}
-
-sub read_table {
-    my $td = new yytbl_data;
-    printf "HERE: %04X\n", tell FD;
-    $td->td_id_offset(tell FD);
-    $td->td_id(read16());
-    $td->td_flags_offset(tell FD);
-    $td->td_flags(read16());
-    $td->td_hilen_offset(tell FD);
-    $td->td_hilen(read32());
-    $td->td_lolen_offset(tell FD);
-    $td->td_lolen(read32());
-    $td->td_data_offset(tell FD);
-
-    my $read = getreader($td);
-    
-    my $tot = $td->td_lolen * ($td->td_hilen?$td->td_hilen:1);
-    $tot *=2 if $::TID{$td->td_id}eq 'YYTH_ID_TRANSITION';
-
-    for(1..$tot){
-        my @v = $read->($td);
-        push @{$td->td_data}, @v;
-    }
-    my $pad = pad64(tell FD);
-    while($pad-- > 0){ read8() }
-    return $td;
-}
-
-sub dump_table {
-    my $td = shift;
-    my $max = shift;
-    defined($max) or $max = 2**15;
-    printf "%04X td_id:    \%d (\%s)\n", $td->td_id_offset, $td->td_id, $::TID{$td->td_id};
-    printf "%04X td_flags: \%0x (\%s)\n",$td->td_flags_offset, $td->td_flags, "@{[TFLAGS($td->td_flags)]}";
-    printf "%04X td_hilen: \%d\n", $td->td_hilen_offset, $td->td_hilen;
-    printf "%04X td_lolen: \%d\n", $td->td_lolen_offset, $td->td_lolen;
-    
-    printf "%04X td_data: ", $td->td_data_offset;
-    foreach( @{$td->td_data} ){
-        last if $max-- == 0;
-        print "$_, ";
-    }
-    print "\n";
-    return $td;
-}
-
-
-sub TFLAGS {
-    my @s;
-    my $f = shift;
-    foreach(keys %::TFLAGS){
-        if ($f & $_){
-            push @s, $::TFLAGS{$_}->[0]
-        }
-    }
-    return join '|', @s;
-}
-
-sub pad64{ return ((8-((shift)%8))%8) }
-
-BEGIN {
-    %::TID = ( 
-	0x01 => 'YYTD_ID_ACCEPT' ,
-	0x02 => 'YYTD_ID_BASE' ,
-	0x03 => 'YYTD_ID_CHK' ,
-	0x04 => 'YYTD_ID_DEF' ,
-	0x05 => 'YYTD_ID_EC' ,
-	0x06 => 'YYTD_ID_META' ,
-	0x07 => 'YYTD_ID_NUL_TRANS' ,
-	0x08 => 'YYTD_ID_NXT' ,
-	0x09 => 'YYTD_ID_RULE_CAN_MATCH_EOL' ,
-	0x0A => 'YYTD_ID_START_STATE_LIST' ,
-	0x0B => 'YYTD_ID_TRANSITION',
-	0x0C => 'YYTD_ID_ACCLIST');
-
-    %::TFLAGS = (
-	0x01 => ['YYTD_DATA8',\&read8] ,
-	0x02 => ['YYTD_DATA16',\&read16] ,
-	0x04 => ['YYTD_DATA32',\&read32] ,
-	0x08 => ['YYTD_PTRANS',undef],
-	0x10 => ['YYTD_STRUCT',\&readstruct]);
-}
-
-# CODE TO EXTRACT TABLES FROM GENERATED C CODE
-sub extract_from_code {
-    my $nms = qr/yy_(?:accept|base|chk|
-                 def|ec|meta|NUL_trans|
-                 nxt|rule_can_match_eol|
-                 start_state_list|transition)/x;
-
-    while(<FD>){
-        next unless m/static\s+yyconst\s+.+yy_($nms)(\[\])?\[(\d+)\]\s*=/;
-        my $n = $1;
-        my $dim = defined($2)?2:1;
-        # TODO 
-    }
-}
-
-1
-__END__
-
@@ -1,29 +1,27 @@
-*.aux
-*.cp
-*.cps
-*.dvi
-*.fn
-*.fns
-*.hk
-*.hks
-*.info*
-*.ky
-*.log
-*.op
-*.ops
-*.pg
-*.toc
-*.tp
-*.tps
-*.vr
-*.vrs
+flex.aux
+flex.cp
+flex.cps
+flex.dvi
+flex.fn
+flex.fns
+flex.hk
+flex.hks
+flex.info*
+flex.ky
+flex.log
+flex.op
+flex.ops
+flex.pg
+flex.toc
+flex.tp
+flex.tps
+flex.vr
+flex.vrs
 Makefile
 Makefile.in
 flex.1
 flex.pdf
 flex.ps
-mdate-sh
-stamp-*
 version.texi
 flex.html
-texinfo.tex
+flex.t2p
@@ -23,9 +23,9 @@ CLEANFILES = \
 	flex.vr \
 	flex.vrs

-$(dist_man_MANS): $(top_srcdir)/main.c
+$(dist_man_MANS): $(top_srcdir)/src/main.c
 	for i in $(dist_man_MANS) ; do \
 	$(help2man) --name='$(PACKAGE_NAME)' \
 	--section=`echo $$i | sed -e 's/.*\.\([^.]*\)$$/\1/'` \
-	 ../flex$(EXEEXT) > $$i || rm -f $$i ; \
+	 $(top_srcdir)/src/flex$(EXEEXT) > $$i || rm -f $$i ; \
 	done
@@ -895,7 +895,7 @@ And to match a sequence of zero or more repetitions of @samp{foo} and
@cindex character classes in patterns
 In addition to characters and ranges of characters, character classes
 can also contain @dfn{character class expressions}.  These are
-expressions enclosed inside @samp{[}: and @samp{:]} delimiters (which
+expressions enclosed inside @samp{[:} and @samp{:]} delimiters (which
 themselves must appear between the @samp{[} and @samp{]} of the
 character class. Other elements may occur inside the character class,
 too).  The valid expressions are:
@@ -1404,7 +1404,7 @@ example, the following is one way to eat up C comments:
@verbatim
    %%
    "/*"        {
-                register int c;
+                int c;

                for ( ; ; )
                    {
@@ -1601,7 +1601,6 @@ Any rule whose pattern is prefixed with @samp{<sc>} will only be active
 when the scanner is in the @dfn{start condition} named @code{sc}.  For
 example,

-@c proofread edit stopped here
@example
@verbatim
    <STRING>[^"]*        { /* eat up the string body ... */
@@ -1934,9 +1933,9 @@ condition @dfn{scope}.  A start condition scope is begun with:
@end verbatim
@end example

-where @code{SCs} is a list of one or more start conditions.  Inside the
+where @code{<SCs>} is a list of one or more start conditions.  Inside the
 start condition scope, every rule automatically has the prefix
-@code{SCs>} applied to it, until a @samp{@}} which matches the initial
+@code{<SCs>} applied to it, until a @samp{@}} which matches the initial
@samp{@{}.  So, for example,

@cindex extended scope of start conditions
@@ -5,7 +5,7 @@

 main()
 {
-	register int c, cc = 0, wc = 0, lc = 0;
+	int c, cc = 0, wc = 0, lc = 0;
 	FILE *f = stdin;

 	while ((c = getc(f)) != EOF) {
@@ -0,0 +1,3 @@
+*.la
+*.lo
+*.o
@@ -1,4 +1,4 @@
 noinst_LTLIBRARIES = libcompat.la
 libcompat_la_SOURCES = lib.c
-libcompat_la_LIBADD = $(LIBOBJS)
+libcompat_la_LIBADD = $(LTLIBOBJS)

@@ -1,17 +1,17 @@
-     #include <config.h>
-     #undef malloc
-     
-     #include <sys/types.h>
-     
-     void *malloc ();
-     
-     /* Allocate an N-byte block of memory from the heap.
-        If N is zero, allocate a 1-byte block.  */
-     
-     void *
-     rpl_malloc (size_t n)
-     {
-       if (n == 0)
-         n = 1;
-       return malloc (n);
-     }
+     #include <config.h>
+     #undef malloc
+     
+     #include <sys/types.h>
+     
+     void *malloc ();
+     
+     /* Allocate an N-byte block of memory from the heap.
+        If N is zero, allocate a 1-byte block.  */
+     
+     void *
+     rpl_malloc (size_t n)
+     {
+       if (n == 0)
+         n = 1;
+       return malloc (n);
+     }
@@ -1,4 +1,6 @@
 #include <config.h>
+#undef realloc
+#undef malloc

 #include <stdlib.h>

@@ -1,3 +0,0 @@
-*.m4
-Makefile
-Makefile.in
@@ -1 +0,0 @@
-EXTRA_DIST = codeset.m4 gettext.m4 glibc21.m4 iconv.m4 intdiv0.m4 inttypes-pri.m4 inttypes.m4 inttypes_h.m4 isc-posix.m4 lcmessage.m4 lib-ld.m4 lib-link.m4 lib-prefix.m4 progtest.m4 stdint_h.m4 uintmax_t.m4 ulonglong.m4 nls.m4 po.m4
@@ -1,18 +1,18 @@
-buf.c
-ccl.c
-dfa.c
-ecs.c
-gen.c
-libmain.c
-libyywrap.c
-main.c
-misc.c
-nfa.c
-options.c
-parse.y
-scan.l
-scanopt.c
-skel.c
-sym.c
-tblcmp.c
-yylex.c
+src/buf.c
+src/ccl.c
+src/dfa.c
+src/ecs.c
+src/gen.c
+src/libmain.c
+src/libyywrap.c
+src/main.c
+src/misc.c
+src/nfa.c
+src/options.c
+src/parse.y
+src/scan.l
+src/scanopt.c
+src/skel.c
+src/sym.c
+src/tblcmp.c
+src/yylex.c
@@ -0,0 +1,9 @@
+# Rules to fetch the translation project's po files for a domain
+
+# Just rsync the *.po files for a particular textual domain into the
+# po/ subdirectory of the project's source tree
+
+getpo:
+	rsync -Ltvz  translationproject.org::tp/latest/flex/*.po $(top_srcdir)/po
+
+.PHONY: getpo
@@ -0,0 +1,10 @@
+*.la
+*.lo
+*.o
+config.h
+config.h.in
+flex
+parse.c
+parse.h
+scan.c
+skel.c
@@ -53,6 +53,11 @@
 #    define FLEX_STD std::
 #  endif

+#ifndef YY_CHAR_DEFINED
+#define YY_CHAR_DEFINED
+typedef unsigned char YY_CHAR;
+#endif
+
 extern "C++" {

 struct yy_buffer_state;
@@ -62,7 +67,7 @@ class FlexLexer {
 public:
 	virtual ~FlexLexer()	{ }

-	const char* YYText() const	{ return yytext; }
+	const YY_CHAR* YYText() const	{ return yytext; }
 	int YYLeng()	const	{ return yyleng; }

 	virtual void
@@ -91,8 +96,16 @@ public:
 	int debug() const		{ return yy_flex_debug; }
 	void set_debug( int flag )	{ yy_flex_debug = flag; }

+#ifdef YY_CHARSET
+	void set_charset(char *charset);
+	char* get_charset();
+#endif
+
 protected:
-	char* yytext;
+	YY_CHAR* yytext;
+#ifdef YY_CHARSET
+	char *yycharset;
+#endif
 	int yyleng;
 	int yylineno;		// only maintained if you use %option yylineno
 	int yy_flex_debug;	// only has effect with -d or "%option debug"
@@ -134,7 +147,7 @@ protected:
 	virtual void LexerOutput( const char* buf, int size );
 	virtual void LexerError( const char* msg );

-	void yyunput( int c, char* buf_ptr );
+	void yyunput( int c, YY_CHAR* buf_ptr );
 	int yyinput();

 	void yy_load_buffer_state();
@@ -153,17 +166,26 @@ protected:
 	yy_state_type yy_try_NUL_trans( yy_state_type current_state );
 	int yy_get_next_buffer();

+#ifdef YY_CHARSET
+	size_t yycharset_convert(char* source, size_t source_bytes, YY_CHAR* target,
+		size_t target_length, size_t* converted_bytes);
+	virtual size_t yycharset_handler(char *charset,
+		char *source, size_t source_bytes,
+		YY_CHAR *target, size_t target_length,
+		size_t *converted_bytes);
+#endif
+
 	FLEX_STD istream* yyin;	// input source for default LexerInput
 	FLEX_STD ostream* yyout;	// output sink for default LexerOutput

 	// yy_hold_char holds the character lost when yytext is formed.
-	char yy_hold_char;
+	YY_CHAR yy_hold_char;

 	// Number of characters read into yy_ch_buf.
 	int yy_n_chars;

 	// Points to current character in buffer.
-	char* yy_c_buf_p;
+	YY_CHAR* yy_c_buf_p;

 	int yy_init;		// whether we need to initialize
 	int yy_start;		// start state number
@@ -182,12 +204,12 @@ protected:
 	// on use of certain flex features (like REJECT or yymore()).

 	yy_state_type yy_last_accepting_state;
-	char* yy_last_accepting_cpos;
+	YY_CHAR* yy_last_accepting_cpos;

 	yy_state_type* yy_state_buf;
 	yy_state_type* yy_state_ptr;

-	char* yy_full_match;
+	YY_CHAR* yy_full_match;
 	int* yy_full_state;
 	int yy_full_lp;

@@ -0,0 +1,142 @@
+AM_YFLAGS = -d
+localedir = $(datadir)/locale
+AM_CPPFLAGS = -DLOCALEDIR=\"$(localedir)\" -I$(top_srcdir)/intl
+LIBS = @LIBINTL@ @LIBS@
+
+m4 = @M4@
+
+bin_PROGRAMS = flex
+lib_LTLIBRARIES = \
+	libfl.la \
+	libfl_pic.la
+
+flex_SOURCES = \
+	ccl.c \
+	dfa.c \
+	ecs.c \
+	scanflags.c \
+	gen.c \
+	main.c \
+	misc.c \
+	nfa.c \
+	parse.y \
+	scan.l \
+	skel.c \
+	sym.c \
+	tblcmp.c \
+	yylex.c \
+	options.c \
+	scanopt.c \
+	buf.c \
+	tables.c \
+	tables_shared.c \
+	filter.c \
+	regex.c
+
+LDADD = ../lib/libcompat.la
+
+libfl_la_SOURCES = \
+	libmain.c \
+	libyywrap.c
+
+libfl_la_LDFLAGS = -no-undefined -version-info @SHARED_VERSION_INFO@
+
+libfl_pic_la_SOURCES = \
+	libmain.c \
+	libyywrap.c
+
+libfl_pic_la_LDFLAGS = -no-undefined -version-info @SHARED_VERSION_INFO@
+
+noinst_HEADERS = \
+	flexdef.h \
+	flexint.h \
+	version.h \
+	options.h \
+	scanopt.h \
+	tables.h \
+	tables_shared.h
+
+include_HEADERS = \
+	FlexLexer.h
+
+EXTRA_DIST = \
+	flex.skl \
+	mkskel.sh \
+	gettext.h
+
+BUILT_SOURCES = \
+	skel.c
+
+skel.c: flex.skl mkskel.sh flexint.h tables_shared.h
+	sed 's/m4_/m4postproc_/g; s/m4preproc_/m4_/g' $(srcdir)/flex.skl | $(m4) -P -DFLEX_MAJOR_VERSION=`echo $(VERSION)|cut -f 1 -d .` -DFLEX_MINOR_VERSION=`echo $(VERSION)|cut -f 2 -d .` -DFLEX_SUBMINOR_VERSION=`echo $(VERSION)|cut -f 3 -d .` | sed 's/m4postproc_/m4_/g' | $(SHELL) $(srcdir)/mkskel.sh  >skel.c
+
+# Explicitly describe dependencies.
+# You can recreate this with `gcc -I. -MM *.c'
+buf.o: buf.c flexdef.h flexint.h
+ccl.o: ccl.c flexdef.h flexint.h
+dfa.o: dfa.c flexdef.h flexint.h tables.h tables_shared.h
+ecs.o: ecs.c flexdef.h flexint.h
+scanflags.o: scanflags.c flexdef.h flexint.h
+gen.o: gen.c flexdef.h flexint.h tables.h tables_shared.h
+libmain.o: libmain.c
+libyywrap.o: libyywrap.c
+main.o: main.c flexdef.h flexint.h version.h options.h scanopt.h \
+ tables.h tables_shared.h
+misc.o: misc.c flexdef.h flexint.h tables.h tables_shared.h
+nfa.o: nfa.c flexdef.h flexint.h
+options.o: options.c options.h scanopt.h flexdef.h flexint.h
+parse.o: parse.c flexdef.h flexint.h tables.h tables_shared.h
+scan.o: scan.c flexdef.h flexint.h parse.h
+scanopt.o: scanopt.c flexdef.h flexint.h scanopt.h
+skel.o: skel.c flexdef.h flexint.h
+sym.o: sym.c flexdef.h flexint.h
+tables.o: tables.c flexdef.h flexint.h tables.h tables_shared.h
+tables_shared.o: tables_shared.c flexdef.h flexint.h tables.h \
+ tables_shared.h
+tblcmp.o: tblcmp.c flexdef.h flexint.h
+yylex.o: yylex.c flexdef.h flexint.h parse.h
+filter.o: filter.c flexdef.h flexint.h
+
+# Run GNU indent on sources. Don't run this unless all the sources compile cleanly.
+#
+# Whole idea:
+#   1. Check for .indent.pro, otherwise indent will use unknown
+#      settings, or worse, the GNU defaults.)
+#   2. Check that this is GNU indent.
+#   3. Make sure to process only the NON-generated .c and .h files.
+#   4. Run indent twice per file. The first time is a test.
+#      Otherwise, indent overwrites your file even if it fails!
+indentfiles = \
+	buf.c \
+	ccl.c \
+	dfa.c \
+	ecs.c \
+	scanflags.c \
+	filter.c \
+	flexdef.h \
+	gen.c \
+	libmain.c \
+	libyywrap.c \
+	main.c \
+	misc.c \
+	nfa.c \
+	options.c \
+	options.h \
+	regex.c \
+	scanopt.c \
+	scanopt.h \
+	sym.c \
+	tables.c \
+	tables.h \
+	tables_shared.c \
+	tables_shared.h \
+	tblcmp.c
+
+indent:
+	if [ -f .indent.pro ] ; then \
+	for f in $(indentfiles);\
+	do\
+		echo indenting $$f ;\
+		$(indent) < $$f >/dev/null && indent $$f || echo $$f FAILED to indent ;\
+	done \
+	fi
@@ -90,7 +90,8 @@ struct Buf *buf_prints (struct Buf *buf, const char *fmt, const char *s)
 */
 struct Buf *buf_linedir (struct Buf *buf, const char* filename, int lineno)
 {
-    char *dst, *src, *t;
+    char *dst, *t;
+    const char *src;

    t = flex_alloc (strlen ("#line \"\"\n")          +   /* constant parts */
                    2 * strlen (filename)            +   /* filename with possibly all backslashes escaped */
@@ -92,13 +92,13 @@ void    ccladd (cclp, ch)

 static void    dump_cclp (FILE* file, int cclp)
 {
-	register int i;
+	int i;

 	putc ('[', file);

 	for (i = 0; i < csize; ++i) {
 		if (ccl_contains(cclp, i)){
-			register int start_char = i;
+			int start_char = i;

 			putc (' ', file);

@@ -244,13 +244,13 @@ void    list_character_set (file, cset)
     FILE   *file;
     int     cset[];
 {
-	register int i;
+	int i;

 	putc ('[', file);

 	for (i = 0; i < csize; ++i) {
 		if (cset[i]) {
-			register int start_char = i;
+			int start_char = i;

 			putc (' ', file);

@@ -101,12 +101,12 @@ void check_trailing_context (nfa_states, num_states, accset, nacc)
     int    *accset;
     int nacc;
 {
-	register int i, j;
+	int i, j;

 	for (i = 1; i <= num_states; ++i) {
 		int     ns = nfa_states[i];
-		register int type = state_type[ns];
-		register int ar = assoc_rule[ns];
+		int type = state_type[ns];
+		int ar = assoc_rule[ns];

 		if (type == STATE_NORMAL || rule_type[ar] != RULE_VARIABLE) {	/* do nothing */
 		}
@@ -141,14 +141,14 @@ void dump_associated_rules (file, ds)
     FILE   *file;
     int ds;
 {
-	register int i, j;
-	register int num_associated_rules = 0;
-	int     rule_set[MAX_ASSOC_RULES + 1];
-	int    *dset = dss[ds];
-	int     size = dfasiz[ds];
+	int i, j;
+	int num_associated_rules = 0;
+	int rule_set[MAX_ASSOC_RULES + 1];
+	int *dset = dss[ds];
+	int size = dfasiz[ds];

 	for (i = 1; i <= size; ++i) {
-		register int rule_num = rule_linenum[assoc_rule[dset[i]]];
+		int rule_num = rule_linenum[assoc_rule[dset[i]]];

 		for (j = 1; j <= num_associated_rules; ++j)
 			if (rule_num == rule_set[j])
@@ -191,8 +191,8 @@ void dump_transitions (file, state)
     FILE   *file;
     int state[];
 {
-	register int i, ec;
-	int     out_char_set[CSIZE];
+	int i, ec;
+	int out_char_set[CSIZE];

 	for (i = 0; i < csize; ++i) {
 		ec = ABS (ecgroup[i]);
@@ -238,7 +238,7 @@ void dump_transitions (file, state)
 int    *epsclosure (t, ns_addr, accset, nacc_addr, hv_addr)
     int    *t, *ns_addr, accset[], *nacc_addr, *hv_addr;
 {
-	register int stkpos, ns, tsp;
+	int     stkpos, ns, tsp;
 	int     numstates = *ns_addr, nacc, hashval, transsym, nfaccnum;
 	int     stkend, nstate;
 	static int did_stk_init = false, *stk;
@@ -823,9 +823,9 @@ void ntod ()
 int snstods (sns, numstates, accset, nacc, hashval, newds_addr)
     int sns[], numstates, accset[], nacc, hashval, *newds_addr;
 {
-	int     didsort = 0;
-	register int i, j;
-	int     newds, *oldsns;
+	int didsort = 0;
+	int i, j;
+	int newds, *oldsns;

 	for (i = 1; i <= lastdfa; ++i)
 		if (hashval == dhash[i]) {
@@ -139,7 +139,7 @@ void    mkeccl (ccls, lenccl, fwd, bck, llsiz, NUL_mapping)

 		for (i = fwd[cclm]; i != NIL && i <= llsiz; i = fwd[i]) {	/* look for the symbol in the character class */
 			for (; j < lenccl; ++j) {
-				register int ccl_char;
+				int ccl_char;

 				if (NUL_mapping && ccls[j] == 0)
 					ccl_char = NUL_mapping;
@@ -191,7 +191,7 @@ void    mkeccl (ccls, lenccl, fwd, bck, llsiz, NUL_mapping)

 		/* Find next ccl member to process. */

-		for (++cclp; cclflags[cclp] && cclp < lenccl; ++cclp) {
+		for (++cclp; cclp < lenccl && cclflags[cclp]; ++cclp) {
 			/* Reset "doesn't need processing" flag. */
 			cclflags[cclp] = 0;
 		}
@@ -135,9 +135,6 @@ struct filter *filter_create_int (struct filter *chain,
 bool filter_apply_chain (struct filter * chain)
 {
 	int     pid, pipes[2];
-	int     r;
-	const int readsz = 512;
-	char   *buf;


 	/* Tricky recursion, since we want to begin the chain
@@ -189,7 +186,7 @@ clearerr(stdin);
 		else {
 			execvp (chain->argv[0],
 				(char **const) (chain->argv));
-            lerrsf_fatal ( _("exec of %s failed"),
+            lerr_fatal ( _("exec of %s failed"),
                    chain->argv[0]);
 		}

@@ -312,21 +309,21 @@ int filter_tee_header (struct filter *chain)

 		fflush (to_h);
 		if (ferror (to_h))
-			lerrsf (_("error writing output file %s"),
+			lerr (_("error writing output file %s"),
 				(char *) chain->extra);

 		else if (fclose (to_h))
-			lerrsf (_("error closing output file %s"),
+			lerr (_("error closing output file %s"),
 				(char *) chain->extra);
 	}

 	fflush (to_c);
 	if (ferror (to_c))
-		lerrsf (_("error writing output file %s"),
+		lerr (_("error writing output file %s"),
 			outfilename ? outfilename : "<stdout>");

 	else if (fclose (to_c))
-		lerrsf (_("error closing output file %s"),
+		lerr (_("error closing output file %s"),
 			outfilename ? outfilename : "<stdout>");

 	while (wait (0) > 0) ;
@@ -364,11 +361,9 @@ int filter_fix_linedirs (struct filter *chain)
 		if (buf[0] == '#'
 			&& regexec (&regex_linedir, buf, 3, m, 0) == 0) {

-			int     num;
 			char   *fname;

 			/* extract the line number and filename */
-			num = regmatch_strtol (&m[1], buf, NULL, 0);
 			fname = regmatch_dup (&m[2], buf);

 			if (strcmp (fname,
@@ -431,11 +426,11 @@ int filter_fix_linedirs (struct filter *chain)
 	}
 	fflush (stdout);
 	if (ferror (stdout))
-		lerrsf (_("error writing output file %s"),
+		lerr (_("error writing output file %s"),
 			outfilename ? outfilename : "<stdout>");

 	else if (fclose (stdout))
-		lerrsf (_("error closing output file %s"),
+		lerr (_("error closing output file %s"),
 			outfilename ? outfilename : "<stdout>");

 	return 0;
@@ -77,6 +77,10 @@ m4_ifelse(M4_YY_PREFIX,yy,,
 #define yyalloc M4_YY_PREFIX[[alloc]]
 #define yyrealloc M4_YY_PREFIX[[realloc]]
 #define yyfree M4_YY_PREFIX[[free]]
+m4_ifdef( [[M4_YY_CHARSET]], [[
+#define yycharset M4_YY_PREFIX[[charset]]
+#define yycharset_handler M4_YY_PREFIX[[charset_handler]]
+]])
 )
 %endif
 %endif
@@ -152,6 +156,12 @@ m4preproc_define(`M4_GEN_PREFIX',
    [[
        M4_GEN_PREFIX(`get_column')
        M4_GEN_PREFIX(`set_column')
+        m4_ifdef( [[M4_YY_CHARSET]], [[
+            M4_GEN_PREFIX(`get_charset')
+            M4_GEN_PREFIX(`set_charset')
+            M4_GEN_PREFIX(`get_charset_handler')
+            M4_GEN_PREFIX(`set_charset_handler')
+        ]])
    ]])
    M4_GEN_PREFIX(`wrap')
 %endif
@@ -274,7 +284,7 @@ m4_ifdef( [[M4_YY_NO_ANSI_FUNC_PROTOS]],
 * we want to instead treat it as an 8-bit unsigned char, hence the
 * double cast.
 */
-#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c)
+#define YY_SC_TO_UI(c) ((unsigned int) (YY_CHAR)c)
 %ok-for-header


@@ -326,6 +336,10 @@ m4_define( [[M4_YY_DOC_PARAM]], [[@param yyscanner The scanner object.]])
 #define yytext YY_G(yytext_r)
 #define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)
 #define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
+m4_ifdef( [[M4_YY_CHARSET]], [[
+#define yycharset YY_G(yycharset_r)
+#define yycharset_handler YY_G(yycharset_handler_r)
+]])
 #define yy_flex_debug YY_G(yy_flex_debug_r)

 m4_define( [[M4_YY_INCR_LINENO]],
@@ -400,6 +414,19 @@ m4_ifdef( [[M4_YY_NO_ANSI_FUNC_DEFS]],
        $3 $4; [[\]]
        $5 $6; [[\]]
        M4_YY_DECL_LAST_ARG]])
+    m4_define( [[YYFARGS4]], [[($2,$4,$6,$8 M4_YY_DEF_LAST_ARG) [[\]]
+        $1 $2; [[\]]
+        $3 $4; [[\]]
+        $5 $6; [[\]]
+        $7 $8; [[\]]
+        M4_YY_DECL_LAST_ARG]])
+    m4_define( [[YYFARGS5]], [[($2,$4,$6,$8,$10 M4_YY_DEF_LAST_ARG) [[\]]
+        $1 $2; [[\]]
+        $3 $4; [[\]]
+        $5 $6; [[\]]
+        $7 $8; [[\]]
+        $9 $10; [[\]]
+        M4_YY_DECL_LAST_ARG]])
 ]],
 [[
 %# Generate C99 function defs.
@@ -407,6 +434,10 @@ m4_ifdef( [[M4_YY_NO_ANSI_FUNC_DEFS]],
    m4_define( [[YYFARGS1]], [[($1 $2 M4_YY_DEF_LAST_ARG)]])
    m4_define( [[YYFARGS2]], [[($1 $2, $3 $4 M4_YY_DEF_LAST_ARG)]])
    m4_define( [[YYFARGS3]], [[($1 $2, $3 $4, $5 $6 M4_YY_DEF_LAST_ARG)]])
+    m4_define( [[YYFARGS4]], [[($1 $2, $3 $4, $5 $6, $7 $8
+            M4_YY_DEF_LAST_ARG)]])
+    m4_define( [[YYFARGS5]], [[($1 $2, $3 $4, $5 $6, $7 $8, $9 $10
+            M4_YY_DEF_LAST_ARG)]])
 ]])

 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
@@ -447,7 +478,15 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],

 /* Size of default input buffer. */
 #ifndef YY_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k.
+ * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
+ * Ditto for the __ia64__ case accordingly.
+ */
+#define YY_BUF_SIZE 32768
+#else
 #define YY_BUF_SIZE 16384
+#endif /* __ia64__ */
 #endif

 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
@@ -475,6 +514,9 @@ extern yy_size_t yyleng;
 %if-c-only
 %if-not-reentrant
 extern FILE *yyin, *yyout;
+m4_ifdef( [[M4_YY_CHARSET]], [[
+extern char *yycharset;
+]])
 %endif
 %endif

@@ -505,7 +547,7 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
            }while(0)
    #define YY_LINENO_REWIND_TO(dst) \
            do {\
-                const char *p;\
+                const YY_CHAR *p;\
                for ( p = yy_cp-1; p >= (dst); --p)\
                    if ( *p == '\n' )\
                        --yylineno;\
@@ -539,6 +581,8 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 #define unput(c) yyunput( c, YY_G(yytext_ptr) M4_YY_CALL_LAST_ARG )
 ]])

+%% [1.0] yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here
+
 #ifndef YY_STRUCT_YY_BUFFER_STATE
 #define YY_STRUCT_YY_BUFFER_STATE
 struct yy_buffer_state
@@ -552,8 +596,8 @@ struct yy_buffer_state
 %endif


-	char *yy_ch_buf;		/* input buffer */
-	char *yy_buf_pos;		/* current position in input buffer */
+	YY_CHAR *yy_ch_buf;     /* input buffer */
+	YY_CHAR *yy_buf_pos;    /* current position in input buffer */

 	/* Size of input buffer in bytes, not including room for EOB
 	 * characters.
@@ -594,6 +638,12 @@ struct yy_buffer_state
 	int yy_fill_buffer;

 	int yy_buffer_status;
+
+m4_ifdef( [[M4_YY_CHARSET]],[[
+	char *yy_input_buffer;
+	size_t yy_input_buffer_length;
+]])
+
 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
 #define YY_BUFFER_NEW 0
@@ -651,12 +701,12 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 %if-not-reentrant
 %not-for-header
 /* yy_hold_char holds the character lost when yytext is formed. */
-static char yy_hold_char;
+static YY_CHAR yy_hold_char;
 static yy_size_t yy_n_chars;		/* number of characters read into yy_ch_buf */
 yy_size_t yyleng;

 /* Points to current character in buffer. */
-static char *yy_c_buf_p = (char *) 0;
+static YY_CHAR *yy_c_buf_p = (YY_CHAR *) 0;
 static int yy_init = 0;		/* whether we need to initialize */
 static int yy_start = 0;	/* start state number */

@@ -687,7 +737,7 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 #define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER M4_YY_CALL_LAST_ARG)
 ]])

-YY_BUFFER_STATE yy_scan_buffer M4_YY_PARAMS( char *base, yy_size_t size M4_YY_PROTO_LAST_ARG );
+YY_BUFFER_STATE yy_scan_buffer M4_YY_PARAMS( YY_CHAR *base, yy_size_t size M4_YY_PROTO_LAST_ARG );
 YY_BUFFER_STATE yy_scan_string M4_YY_PARAMS( yyconst char *yy_str M4_YY_PROTO_LAST_ARG );
 YY_BUFFER_STATE yy_scan_bytes M4_YY_PARAMS( yyconst char *bytes, yy_size_t len M4_YY_PROTO_LAST_ARG );

@@ -733,8 +783,6 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 #define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol)
 ]])

-%% [1.0] yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here
-
 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
 %% [1.5] DFA
@@ -747,6 +795,9 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 static yy_state_type yy_get_previous_state M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
 static yy_state_type yy_try_NUL_trans M4_YY_PARAMS( yy_state_type current_state  M4_YY_PROTO_LAST_ARG);
 static int yy_get_next_buffer M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
+#if defined(__GNUC__) && __GNUC__ >= 3
+__attribute__((__noreturn__))
+#endif
 static void yy_fatal_error M4_YY_PARAMS( yyconst char msg[] M4_YY_PROTO_LAST_ARG );
 ]])

@@ -819,10 +870,10 @@ struct yyguts_t
    size_t yy_buffer_stack_top; /**< index of top of stack. */
    size_t yy_buffer_stack_max; /**< capacity of stack. */
    YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */
-    char yy_hold_char;
+    YY_CHAR yy_hold_char;
    yy_size_t yy_n_chars;
    yy_size_t yyleng_r;
-    char *yy_c_buf_p;
+    YY_CHAR *yy_c_buf_p;
    int yy_init;
    int yy_start;
    int yy_did_buffer_switch_on_eof;
@@ -830,7 +881,7 @@ struct yyguts_t
    int yy_start_stack_depth;
    int *yy_start_stack;
    yy_state_type yy_last_accepting_state;
-    char* yy_last_accepting_cpos;
+    YY_CHAR* yy_last_accepting_cpos;

    int yylineno_r;
    int yy_flex_debug_r;
@@ -839,7 +890,7 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 [[
    yy_state_type *yy_state_buf;
    yy_state_type *yy_state_ptr;
-    char *yy_full_match;
+    YY_CHAR *yy_full_match;
    int yy_lp;

    /* These are only needed for trailing context rules,
@@ -851,13 +902,13 @@ m4_ifdef( [[M4_YY_USES_REJECT]],

 m4_ifdef( [[M4_YY_TEXT_IS_ARRAY]],
 [[
-    char yytext_r[YYLMAX];
-    char *yytext_ptr;
+    YY_CHAR yytext_r[YYLMAX];
+    YY_CHAR *yytext_ptr;
    int yy_more_offset;
    int yy_prev_more_offset;
 ]],
 [[
-    char *yytext_r;
+    YY_CHAR *yytext_r;
    int yy_more_flag;
    int yy_more_len;
 ]])
@@ -872,6 +923,16 @@ m4_ifdef( [[<M4_YY_BISON_LLOC>]],
    YYLTYPE * yylloc_r;
 ]])

+m4_ifdef( [[M4_YY_CHARSET]], [[
+    char *yycharset_r; /** current charset name */
+]])
+
+m4_ifdef( [[M4_YY_CXX]],,[[
+m4_ifdef( [[M4_YY_CHARSET]],[[
+    yycharset_handler_t yycharset_handler_r; /** charset handle function */
+]])
+]])
+
    }; /* end struct yyguts_t */
 ]])

@@ -943,7 +1004,7 @@ FILE *yyget_in M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );

 m4_ifdef( [[M4_YY_NO_SET_IN]],,
 [[
-void yyset_in  M4_YY_PARAMS( FILE * in_str M4_YY_PROTO_LAST_ARG );
+void yyset_in  M4_YY_PARAMS( FILE * _in_str M4_YY_PROTO_LAST_ARG );
 ]])

 m4_ifdef( [[M4_YY_NO_GET_OUT]],,
@@ -953,7 +1014,7 @@ FILE *yyget_out M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );

 m4_ifdef( [[M4_YY_NO_SET_OUT]],,
 [[
-void yyset_out  M4_YY_PARAMS( FILE * out_str M4_YY_PROTO_LAST_ARG );
+void yyset_out  M4_YY_PARAMS( FILE * _out_str M4_YY_PROTO_LAST_ARG );
 ]])

 m4_ifdef( [[M4_YY_NO_GET_LENG]],,
@@ -963,7 +1024,7 @@ yy_size_t yyget_leng M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );

 m4_ifdef( [[M4_YY_NO_GET_TEXT]],,
 [[
-char *yyget_text M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
+YY_CHAR *yyget_text M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
 ]])

 m4_ifdef( [[M4_YY_NO_GET_LINENO]],,
@@ -973,7 +1034,7 @@ int yyget_lineno M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );

 m4_ifdef( [[M4_YY_NO_SET_LINENO]],,
 [[
-void yyset_lineno M4_YY_PARAMS( int line_number M4_YY_PROTO_LAST_ARG );
+void yyset_lineno M4_YY_PARAMS( int _line_number M4_YY_PROTO_LAST_ARG );
 ]])

 m4_ifdef( [[M4_YY_REENTRANT]],
@@ -988,7 +1049,29 @@ m4_ifdef( [[M4_YY_REENTRANT]],
 [[
 m4_ifdef( [[M4_YY_NO_SET_COLUMN]],,
 [[
-void yyset_column M4_YY_PARAMS( int column_no M4_YY_PROTO_LAST_ARG );
+void yyset_column M4_YY_PARAMS( int _column_no M4_YY_PROTO_LAST_ARG );
+]])
+]])
+
+m4_ifdef( [[M4_YY_REENTRANT]],[[
+m4_ifdef( [[M4_YY_CHARSET]],[[
+m4_ifdef( [[M4_YY_NO_GET_CHARSET]],,[[
+char *yyget_charset M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
+%if-c-only
+yycharset_handler_t yyget_charset_handler M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
+%endif
+]])
+]])
+]])
+
+m4_ifdef( [[M4_YY_REENTRANT]],[[
+m4_ifdef( [[M4_YY_CHARSET]], [[
+m4_ifdef( [[M4_YY_NO_SET_CHARSET]],,[[
+void yyset_charset M4_YY_PARAMS( char *charset M4_YY_PROTO_LAST_ARG );
+%if-c-only
+void yyset_charset_handler M4_YY_PARAMS( yycharset_handler_t charset_handler M4_YY_PROTO_LAST_ARG );
+%endif
+]])
 ]])
 ]])

@@ -1027,15 +1110,17 @@ extern int yywrap M4_YY_PARAMS( M4_YY_PROTO_ONLY_ARG );
 #endif

 %not-for-header
+#ifndef YY_NO_UNPUT
    m4_ifdef( [[M4_YY_NO_UNPUT]],,
    [[
    static void yyunput M4_YY_PARAMS( int c, char *buf_ptr  M4_YY_PROTO_LAST_ARG);
    ]])
+#endif
 %ok-for-header
 %endif

 #ifndef yytext_ptr
-static void yy_flex_strncpy M4_YY_PARAMS( char *, yyconst char *, int M4_YY_PROTO_LAST_ARG);
+static void yy_flex_strncpy M4_YY_PARAMS( YY_CHAR *, yyconst YY_CHAR *, int M4_YY_PROTO_LAST_ARG);
 #endif

 #ifdef YY_NEED_STRLEN
@@ -1074,7 +1159,7 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
    m4_ifdef( [[M4_YY_NO_PUSH_STATE]],,
    [[
-    static void yy_push_state M4_YY_PARAMS( int new_state M4_YY_PROTO_LAST_ARG);
+    static void yy_push_state M4_YY_PARAMS( int _new_state M4_YY_PROTO_LAST_ARG);
    ]])
    m4_ifdef( [[M4_YY_NO_POP_STATE]],,
    [[
@@ -1096,7 +1181,12 @@ m4_define( [[M4_YY_NO_TOP_STATE]])

 /* Amount of stuff to slurp up with each read. */
 #ifndef YY_READ_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k */
+#define YY_READ_BUF_SIZE 16384
+#else
 #define YY_READ_BUF_SIZE 8192
+#endif /* __ia64__ */
 #endif

 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
@@ -1110,7 +1200,7 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 #define ECHO do { if (fwrite( yytext, yyleng, 1, yyout )) {} } while (0)
 %endif
 %if-c++-only C++ definition
-#define ECHO LexerOutput( yytext, yyleng )
+#define ECHO LexerOutput( reinterpret_cast<char*>(yytext), yyleng )
 %endif
 #endif
 ]])
@@ -1122,13 +1212,14 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 */
 #ifndef YY_INPUT
 #define YY_INPUT(buf,result,max_size) \
+do {\
 %% [5.0] fread()/read() definition of YY_INPUT goes here unless we're doing C++ \
 \
 %if-c++-only C++ definition \
 	if ( (result = LexerInput( (char *) buf, max_size )) < 0 ) \
-		YY_FATAL_ERROR( "input in flex scanner failed" );
-%endif
-
+		YY_FATAL_ERROR( "input in flex scanner failed" ); \
+%endif \
+} while(0)
 #endif
 ]])

@@ -1254,7 +1345,7 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
 /* Code executed at the end of each rule. */
 #ifndef YY_BREAK
-#define YY_BREAK break;
+#define YY_BREAK /*LINTED*/break;
 #endif
 ]])

@@ -1268,9 +1359,9 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 */
 YY_DECL
 {
-	register yy_state_type yy_current_state;
-	register char *yy_cp, *yy_bp;
-	register int yy_act;
+	yy_state_type yy_current_state;
+	YY_CHAR *yy_cp, *yy_bp;
+	int yy_act;
    M4_YY_DECL_GUTS_VAR();

 m4_ifdef( [[M4_YY_NOT_REENTRANT]],
@@ -1343,7 +1434,7 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 	{
 %% [7.0] user's declarations go here

-	while ( 1 )		/* loops until end-of-file is reached */
+	while ( /*CONSTCOND*/1 )		/* loops until end-of-file is reached */
 		{
 %% [8.0] yymore()-related code goes here
 		yy_cp = YY_G(yy_c_buf_p);
@@ -1405,7 +1496,7 @@ do_action:	/* This label is used only to access EOF actions. */
 		 * end-of-buffer state).  Contrast this with the test
 		 * in input().
 		 */
-		if ( YY_G(yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_G(yy_n_chars)] )
+		if ( ((YY_CHAR*)(YY_G(yy_c_buf_p))) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_G(yy_n_chars)] )
 			{ /* This was really a NUL. */
 			yy_state_type yy_next_state;

@@ -1483,8 +1574,9 @@ do_action:	/* This label is used only to access EOF actions. */
 				goto yy_match;

 			case EOB_ACT_LAST_MATCH:
-				YY_G(yy_c_buf_p) =
-				&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_G(yy_n_chars)];
+				YY_G(yy_c_buf_p) = ((YY_CHAR*)(
+				&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_G(yy_n_chars)]
+				));

 				yy_current_state = yy_get_previous_state( M4_YY_CALL_ONLY_ARG );

@@ -1603,6 +1695,57 @@ void yyFlexLexer::LexerOutput( const char* buf, int size )
 %ok-for-header
 %endif

+m4_ifdef( [[M4_YY_NOT_IN_HEADER]],[[
+m4_ifdef( [[M4_YY_CHARSET]],[[
+/* yycharset_convert - convert incoming data from arbitrary
+ *    charset into internal representation
+ */
+%if-c-only
+static size_t yycharset_convert YYFARGS5(
+        char*, source, size_t, source_bytes,
+        YY_CHAR*, target, size_t, target_length,
+        size_t*, converted_bytes) {
+%endif
+%if-c++-only
+size_t yyFlexLexer::yycharset_convert(
+        char* source, size_t source_bytes,
+        YY_CHAR* target, size_t target_length,
+        size_t* converted_bytes) {
+%endif
+    M4_YY_DECL_GUTS_VAR();
+    if(strcmp(yycharset, "M4_YY_CHARSET_SOURCE")==0) {
+        if(target_length < source_bytes)
+            YY_FATAL_ERROR("Too small buffer");
+        strncpy((char*)target, source, source_bytes);
+        *converted_bytes = source_bytes;
+        return source_bytes;
+    }
+%if-c-only
+    else if(yycharset_handler)
+%endif
+        return yycharset_handler(yycharset, source, source_bytes,
+            target, target_length, converted_bytes M4_YY_CALL_LAST_ARG);
+
+/* Code below just outputs an error message saying that selected encoding
+ * is not supported. In C scanner it is an end part of yycharset_convert,
+ * while in C++ scanner it is a default implementation of yycharset_handler */
+%if-c++-only
+}
+
+size_t yyFlexLexer::yycharset_handler(char *charset,
+        char *source, size_t source_bytes,
+        YY_CHAR *target, size_t target_length,
+        size_t *converted_bytes) {
+%endif
+    char msg[256];
+    snprintf(msg, sizeof(msg),
+        "Unsupported character encoding: %s", yycharset);
+    YY_FATAL_ERROR(msg);
+    return 0;
+}
+]])
+]])
+
 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
 /* yy_get_next_buffer - try to read in a new buffer
@@ -1620,12 +1763,12 @@ int yyFlexLexer::yy_get_next_buffer()
 %endif
 {
    M4_YY_DECL_GUTS_VAR();
-	register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf;
-	register char *source = YY_G(yytext_ptr);
-	register int number_to_move, i;
+	YY_CHAR *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf;
+	YY_CHAR *source = YY_G(yytext_ptr);
+	int number_to_move, i;
 	int ret_val;

-	if ( YY_G(yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_G(yy_n_chars) + 1] )
+	if ( YY_G(yy_c_buf_p) > ((YY_CHAR*)( &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[YY_G(yy_n_chars) + 1])) )
 		YY_FATAL_ERROR(
 		"fatal flex scanner internal error--end of buffer missed" );

@@ -1651,7 +1794,7 @@ int yyFlexLexer::yy_get_next_buffer()
 	/* Try to read more data. */

 	/* First move last chars to start of buffer. */
-	number_to_move = (int) (YY_G(yy_c_buf_p) - YY_G(yytext_ptr)) - 1;
+	number_to_move = (yy_size_t) (YY_G(yy_c_buf_p) - YY_G(yytext_ptr)) - 1;

 	for ( i = 0; i < number_to_move; ++i )
 		*(dest++) = *(source++);
@@ -1679,7 +1822,7 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 			YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE;

 			int yy_c_buf_p_offset =
-				(int) (YY_G(yy_c_buf_p) - b->yy_ch_buf);
+				(int) (YY_G(yy_c_buf_p) - ((YY_CHAR*)(b->yy_ch_buf)) );

 			if ( b->yy_is_our_buffer )
 				{
@@ -1690,10 +1833,10 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 				else
 					b->yy_buf_size *= 2;

-				b->yy_ch_buf = (char *)
+				b->yy_ch_buf = (YY_CHAR *)
 					/* Include room in for 2 EOB chars. */
 					yyrealloc( (void *) b->yy_ch_buf,
-							 b->yy_buf_size + 2 M4_YY_CALL_LAST_ARG );
+							 (b->yy_buf_size + 2)*sizeof(YY_CHAR) M4_YY_CALL_LAST_ARG );
 				}
 			else
 				/* Can't grow it, we don't own it. */
@@ -1714,6 +1857,43 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 			num_to_read = YY_READ_BUF_SIZE;

 		/* Read in more data. */
+m4_ifdef([[M4_YY_CHARSET]],[[
+		if(yycharset) {
+			const size_t max_size = YY_READ_BUF_SIZE * sizeof(YY_CHAR);
+			char buffer[max_size];
+			memcpy(buffer, YY_CURRENT_BUFFER_LVALUE->yy_input_buffer,
+					YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length);
+
+			size_t read_bytes, converted_characters;
+			YY_INPUT(
+					(&buffer[YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length]),
+					(read_bytes),
+					(max_size-YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length)
+			);
+			size_t converted_bytes = 0;
+			converted_characters = yycharset_convert(
+					buffer,
+					YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length+read_bytes,
+					&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move],
+					num_to_read,
+					&converted_bytes M4_YY_CALL_LAST_ARG);
+			if(converted_characters == 0 && read_bytes != 0)
+			    YY_FATAL_ERROR("Could not convert input characters");
+			YY_G(yy_n_chars) = converted_characters;
+
+			/* store left bytes in yy_input_buffer */
+			YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length +=
+					read_bytes-converted_bytes;
+
+			YY_CURRENT_BUFFER_LVALUE->yy_input_buffer = (char*)yyrealloc(
+					(void*) YY_CURRENT_BUFFER_LVALUE->yy_input_buffer,
+					YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length
+					M4_YY_CALL_LAST_ARG);
+			memcpy(YY_CURRENT_BUFFER_LVALUE->yy_input_buffer,
+					&buffer[converted_bytes],
+					YY_CURRENT_BUFFER_LVALUE->yy_input_buffer_length);
+		} else
+]])
 		YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]),
 			YY_G(yy_n_chars), num_to_read );

@@ -1742,8 +1922,8 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 	if ((yy_size_t) (YY_G(yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) {
 		/* Extend the array by 50%, plus the number we really need. */
 		yy_size_t new_size = YY_G(yy_n_chars) + number_to_move + (YY_G(yy_n_chars) >> 1);
-		YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc(
-			(void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, new_size M4_YY_CALL_LAST_ARG );
+		YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (YY_CHAR *) yyrealloc(
+			(void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, new_size*sizeof(YY_CHAR) M4_YY_CALL_LAST_ARG );
 		if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
 			YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" );
 	}
@@ -1768,8 +1948,8 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
    yy_state_type yyFlexLexer::yy_get_previous_state()
 %endif
 {
-	register yy_state_type yy_current_state;
-	register char *yy_cp;
+	yy_state_type yy_current_state;
+	YY_CHAR *yy_cp;
    M4_YY_DECL_GUTS_VAR();

 %% [15.0] code to get the start state into yy_current_state goes here
@@ -1795,7 +1975,7 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
    yy_state_type yyFlexLexer::yy_try_NUL_trans( yy_state_type yy_current_state )
 %endif
 {
-	register int yy_is_jam;
+	int yy_is_jam;
    M4_YY_DECL_GUTS_VAR(); /* This var may be unused depending upon options. */
 %% [17.0] code to find the next state, and perhaps do backing up, goes here

@@ -1804,16 +1984,17 @@ m4_ifdef( [[M4_YY_USES_REJECT]],
 }


+#ifndef YY_NO_UNPUT
 %if-c-only
 m4_ifdef( [[M4_YY_NO_UNPUT]],,
 [[
-    static void yyunput YYFARGS2( int,c, register char *,yy_bp)
+    static void yyunput YYFARGS2( int,c, YY_CHAR *,yy_bp)
 %endif
 %if-c++-only
-    void yyFlexLexer::yyunput( int c, register char* yy_bp)
+    void yyFlexLexer::yyunput( int c, YY_CHAR* yy_bp)
 %endif
 {
-	register char *yy_cp;
+	YY_CHAR *yy_cp;
    M4_YY_DECL_GUTS_VAR();

    yy_cp = YY_G(yy_c_buf_p);
@@ -1824,10 +2005,10 @@ m4_ifdef( [[M4_YY_NO_UNPUT]],,
 	if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 )
 		{ /* need to shift things up to make room */
 		/* +2 for EOB chars. */
-		register yy_size_t number_to_move = YY_G(yy_n_chars) + 2;
-		register char *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[
+		yy_size_t number_to_move = YY_G(yy_n_chars) + 2;
+		YY_CHAR *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[
 					YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2];
-		register char *source =
+		YY_CHAR *source =
 				&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move];

 		while ( source > YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
@@ -1859,6 +2040,7 @@ m4_ifdef( [[M4_YY_USE_LINENO]],
 %if-c-only
 ]])
 %endif
+#endif

 %if-c-only
 #ifndef YY_NO_INPUT
@@ -2044,12 +2226,12 @@ static void yy_load_buffer_state  YYFARGS0(void)
 	if ( ! b )
 		YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" );

-	b->yy_buf_size = size;
+	b->yy_buf_size = (yy_size_t)size;

 	/* yy_ch_buf has to be 2 characters longer than the size given because
 	 * we need to put in 2 end-of-buffer characters.
 	 */
-	b->yy_ch_buf = (char *) yyalloc( b->yy_buf_size + 2 M4_YY_CALL_LAST_ARG );
+	b->yy_ch_buf = (YY_CHAR *) yyalloc( (b->yy_buf_size + 2)*sizeof(YY_CHAR) M4_YY_CALL_LAST_ARG );
 	if ( ! b->yy_ch_buf )
 		YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" );

@@ -2134,6 +2316,11 @@ m4_ifdef( [[M4_YY_ALWAYS_INTERACTIVE]],
 	b->yy_is_interactive = 0;
 %endif
 	errno = oerrno;
+
+m4_ifdef([[M4_YY_CHARSET]],[[
+	b->yy_input_buffer = NULL;
+	b->yy_input_buffer_length = 0;
+]])
 }

 /** Discard all buffered characters. On the next scan, YY_INPUT will be called.
@@ -2277,7 +2464,7 @@ void yyFlexLexer::yyensure_buffer_stack(void)
 	if (YY_G(yy_buffer_stack_top) >= (YY_G(yy_buffer_stack_max)) - 1){

 		/* Increase the buffer to prepare for a possible push. */
-		int grow_size = 8 /* arbitrary grow size */;
+		yy_size_t grow_size = 8 /* arbitrary grow size */;

 		num_to_alloc = YY_G(yy_buffer_stack_max) + grow_size;
 		YY_G(yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc
@@ -2306,7 +2493,7 @@ m4_ifdef( [[M4_YY_NO_SCAN_BUFFER]],,
 * M4_YY_DOC_PARAM
 * @return the newly allocated buffer state object. 
 */
-YY_BUFFER_STATE yy_scan_buffer  YYFARGS2( char *,base, yy_size_t ,size)
+YY_BUFFER_STATE yy_scan_buffer  YYFARGS2( YY_CHAR *,base, yy_size_t ,size)
 {
 	YY_BUFFER_STATE b;
    m4_dnl M4_YY_DECL_GUTS_VAR();
@@ -2373,14 +2560,14 @@ m4_ifdef( [[M4_YY_NO_SCAN_BYTES]],,
 YY_BUFFER_STATE yy_scan_bytes  YYFARGS2( yyconst char *,yybytes, yy_size_t ,_yybytes_len)
 {
 	YY_BUFFER_STATE b;
-	char *buf;
+	YY_CHAR *buf;
 	yy_size_t n;
 	yy_size_t i;
    m4_dnl M4_YY_DECL_GUTS_VAR();

 	/* Get memory for full buffer, including space for trailing EOB's. */
 	n = _yybytes_len + 2;
-	buf = (char *) yyalloc( n M4_YY_CALL_LAST_ARG );
+	buf = (YY_CHAR *) yyalloc( n*sizeof(YY_CHAR) M4_YY_CALL_LAST_ARG );
 	if ( ! buf )
 		YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" );

@@ -2407,10 +2594,10 @@ YY_BUFFER_STATE yy_scan_bytes  YYFARGS2( yyconst char *,yybytes, yy_size_t ,_yyb
 m4_ifdef( [[M4_YY_NO_PUSH_STATE]],,
 [[
 %if-c-only
-    static void yy_push_state YYFARGS1( int ,new_state)
+    static void yy_push_state YYFARGS1( int ,_new_state)
 %endif
 %if-c++-only
-    void yyFlexLexer::yy_push_state( int new_state )
+    void yyFlexLexer::yy_push_state( int _new_state )
 %endif
 {
    M4_YY_DECL_GUTS_VAR();
@@ -2434,7 +2621,7 @@ m4_ifdef( [[M4_YY_NO_PUSH_STATE]],,

 	YY_G(yy_start_stack)[YY_G(yy_start_stack_ptr)++] = YY_START;

-	BEGIN(new_state);
+	BEGIN(_new_state);
 }
 ]])

@@ -2478,7 +2665,8 @@ m4_ifdef( [[M4_YY_NO_TOP_STATE]],,
 %if-c-only
 static void yy_fatal_error YYFARGS1(yyconst char*, msg)
 {
-    m4_dnl M4_YY_DECL_GUTS_VAR();
+	M4_YY_DECL_GUTS_VAR();
+	M4_YY_NOOP_GUTS_VAR();
 	(void) fprintf( stderr, "%s\n", msg );
 	exit( YY_EXIT_FAILURE );
 }
@@ -2566,7 +2754,55 @@ int yyget_column  YYFARGS0(void)
 }
 ]])
 ]])
+%endif

+m4_ifdef([[M4_YY_CHARSET]],[[
+    m4_ifdef([[M4_YY_REENTRANT]],[[
+
+        m4_ifdef([[M4_YY_NO_GET_CHARSET]],,[[
+            m4_define([[M4_YY_GET_CHARSET]],[[
+                char* yyget_charset YYFARGS0(void)
+            ]])
+        ]])
+
+        m4_ifdef([[M4_YY_NO_GET_CHARSET_HANDLER]],,[[
+            m4_define([[M4_YY_GET_CHARSET_HANDLER]],[[
+                yycharset_handler_t yyget_charset_handler YYFARGS0(void)
+            ]])
+        ]])
+    ]])
+
+    m4_ifdef([[M4_YY_CXX]], [[
+        m4_define( [[M4_YY_GET_CHARSET]], [[
+            char *FlexLexer::get_charset()
+        ]])
+    ]])
+]])
+
+m4_ifdef( [[M4_YY_GET_CHARSET]], [[
+/** Get the current charset name
+ * M4_YY_DOC_PARAM
+ * @return charset name
+ */
+M4_YY_GET_CHARSET
+{
+    M4_YY_DECL_GUTS_VAR();
+    return yycharset;
+}
+]])
+
+m4_ifdef( [[M4_YY_GET_CHARSET_HANDLER]],[[
+/** Get the currently set charset handler
+ * M4_YY_DOC_PARAM
+ */
+M4_YY_GET_CHARSET_HANDLER
+{
+    M4_YY_DECL_GUTS_VAR();
+    return yycharset_handler;
+}
+]])
+
+%if-c-only
 m4_ifdef( [[M4_YY_NO_GET_IN]],,
 [[
 /** Get the input stream.
@@ -2608,7 +2844,7 @@ yy_size_t yyget_leng  YYFARGS0(void)
 */
 m4_ifdef( [[M4_YY_NO_GET_TEXT]],,
 [[
-char *yyget_text  YYFARGS0(void)
+YY_CHAR *yyget_text  YYFARGS0(void)
 {
    M4_YY_DECL_GUTS_VAR();
    return yytext;
@@ -2633,10 +2869,10 @@ void yyset_extra YYFARGS1( YY_EXTRA_TYPE ,user_defined)
 m4_ifdef( [[M4_YY_NO_SET_LINENO]],,
 [[
 /** Set the current line number.
- * @param line_number
+ * @param _line_number line number
 * M4_YY_DOC_PARAM
 */
-void yyset_lineno YYFARGS1( int ,line_number)
+void yyset_lineno YYFARGS1( int ,_line_number)
 {
    M4_YY_DECL_GUTS_VAR();

@@ -2646,7 +2882,7 @@ void yyset_lineno YYFARGS1( int ,line_number)
        if (! YY_CURRENT_BUFFER )
           YY_FATAL_ERROR( "yyset_lineno called with no buffer" );
    ]])
-    yylineno = line_number;
+    yylineno = _line_number;
 }
 ]])

@@ -2655,10 +2891,10 @@ m4_ifdef( [[M4_YY_REENTRANT]],
 m4_ifdef( [[M4_YY_NO_SET_COLUMN]],,
 [[
 /** Set the current column.
- * @param line_number
+ * @param _column_no column number
 * M4_YY_DOC_PARAM
 */
-void yyset_column YYFARGS1( int , column_no)
+void yyset_column YYFARGS1( int , _column_no)
 {
    M4_YY_DECL_GUTS_VAR();

@@ -2668,33 +2904,81 @@ void yyset_column YYFARGS1( int , column_no)
        if (! YY_CURRENT_BUFFER )
           YY_FATAL_ERROR( "yyset_column called with no buffer" );
    ]])
-    yycolumn = column_no;
+    yycolumn = _column_no;
 }
 ]])
 ]])
+%endif

+m4_ifdef([[M4_YY_CHARSET]], [[
+    m4_ifdef([[M4_YY_REENTRANT]],[[

+        m4_ifdef([[M4_YY_NO_SET_CHARSET]],,[[
+            m4_define([[M4_YY_SET_CHARSET]], [[
+                void yyset_charset YYFARGS1(char*, charset)
+            ]])
+        ]])
+
+        m4_ifdef([[M4_YY_NO_SET_CHARSET_HANDLER]],,[[
+            m4_define([[M4_YY_SET_CHARSET_HANDLER]],[[
+                void yyset_charset_handler YYFARGS1(yycharset_handler_t, charset_handler)
+            ]])
+        ]])
+    ]])
+
+    m4_ifdef( [[M4_YY_CXX]],[[
+        m4_define( [[M4_YY_SET_CHARSET]], [[
+            void FlexLexer::set_charset(char *charset)
+        ]])
+    ]])
+]])
+
+m4_ifdef( [[M4_YY_SET_CHARSET]],[[
+/** Set the current charset name
+ * @param charset charset name
+ * M4_YY_DOC_PARAM
+ */
+M4_YY_SET_CHARSET
+{
+    M4_YY_DECL_GUTS_VAR();
+    yycharset = strdup(charset);
+}
+]])
+
+m4_ifdef( [[M4_YY_SET_CHARSET_HANDLER]],[[
+/** Set the current charset handler
+ * @param charset_handler handler function
+ * M4_YY_DOC_PARAM
+ */
+void yyset_charset_handler YYFARGS1( yycharset_handler_t, charset_handler)
+{
+    M4_YY_DECL_GUTS_VAR();
+    yycharset_handler = charset_handler;
+}
+]])
+
+%if-c-only
 m4_ifdef( [[M4_YY_NO_SET_IN]],,
 [[
 /** Set the input stream. This does not discard the current
 * input buffer.
- * @param in_str A readable stream.
+ * @param _in_str A readable stream.
 * M4_YY_DOC_PARAM
 * @see yy_switch_to_buffer
 */
-void yyset_in YYFARGS1( FILE * ,in_str)
+void yyset_in YYFARGS1( FILE * ,_in_str)
 {
    M4_YY_DECL_GUTS_VAR();
-    yyin = in_str ;
+    yyin = _in_str ;
 }
 ]])

 m4_ifdef( [[M4_YY_NO_SET_OUT]],,
 [[
-void yyset_out YYFARGS1( FILE * ,out_str)
+void yyset_out YYFARGS1( FILE * ,_out_str)
 {
    M4_YY_DECL_GUTS_VAR();
-    yyout = out_str ;
+    yyout = _out_str ;
 }
 ]])

@@ -2710,10 +2994,10 @@ int yyget_debug  YYFARGS0(void)

 m4_ifdef( [[M4_YY_NO_SET_DEBUG]],,
 [[
-void yyset_debug YYFARGS1( int ,bdebug)
+void yyset_debug YYFARGS1( int ,_bdebug)
 {
    M4_YY_DECL_GUTS_VAR();
-    yy_flex_debug = bdebug ;
+    yy_flex_debug = _bdebug ;
 }
 ]])
 %endif
@@ -2862,7 +3146,7 @@ m4_ifdef( [[M4_YY_USE_LINENO]],
    YY_G(yy_buffer_stack) = 0;
    YY_G(yy_buffer_stack_top) = 0;
    YY_G(yy_buffer_stack_max) = 0;
-    YY_G(yy_c_buf_p) = (char *) 0;
+    YY_G(yy_c_buf_p) = (YY_CHAR *) 0;
    YY_G(yy_init) = 0;
    YY_G(yy_start) = 0;

@@ -2888,6 +3172,11 @@ m4_ifdef( [[M4_YY_TEXT_IS_ARRAY]],
    YY_G(yy_prev_more_offset) = 0;
 ]])

+m4_ifdef( [[M4_YY_CHARSET]],[[
+    yycharset = NULL;
+    yycharset_handler = NULL;
+]])
+
 /* Defined in main.c */
 #ifdef YY_STDINIT
    yyin = stdin;
@@ -2959,9 +3248,12 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
 #ifndef yytext_ptr
-static void yy_flex_strncpy YYFARGS3( char*,s1, yyconst char *,s2, int,n)
+static void yy_flex_strncpy YYFARGS3( YY_CHAR*,s1, yyconst YY_CHAR *,s2, int,n)
 {
-	register int i;
+	M4_YY_DECL_GUTS_VAR();
+	M4_YY_NOOP_GUTS_VAR();
+
+	int i;
 	for ( i = 0; i < n; ++i )
 		s1[i] = s2[i];
 }
@@ -2973,7 +3265,7 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 #ifdef YY_NEED_STRLEN
 static int yy_flex_strlen YYFARGS1( yyconst char *,s)
 {
-	register int n;
+	int n;
 	for ( n = 0; s[n]; ++n )
 		;

@@ -2986,6 +3278,8 @@ m4_ifdef( [[M4_YY_NO_FLEX_ALLOC]],,
 [[
 void *yyalloc YYFARGS1( yy_size_t ,size)
 {
+	M4_YY_DECL_GUTS_VAR();
+	M4_YY_NOOP_GUTS_VAR();
 	return (void *) malloc( size );
 }
 ]])
@@ -2994,6 +3288,9 @@ m4_ifdef( [[M4_YY_NO_FLEX_REALLOC]],,
 [[
 void *yyrealloc  YYFARGS2( void *,ptr, yy_size_t ,size)
 {
+	M4_YY_DECL_GUTS_VAR();
+	M4_YY_NOOP_GUTS_VAR();
+
 	/* The cast to (char *) in the following accommodates both
 	 * implementations that use char* generic pointers, and those
 	 * that use void* generic pointers.  It works with the latter
@@ -3009,6 +3306,8 @@ m4_ifdef( [[M4_YY_NO_FLEX_FREE]],,
 [[
 void yyfree YYFARGS1( void *,ptr)
 {
+	M4_YY_DECL_GUTS_VAR();
+	M4_YY_NOOP_GUTS_VAR();
 	free( (char *) ptr );	/* see yyrealloc() for (char *) cast */
 }
 ]])
@@ -3054,7 +3353,7 @@ static int yytbl_read32 (void *v, struct yytbl_reader * rd)
 /** Read the header */
 static int yytbl_hdr_read YYFARGS2(struct yytbl_hdr *, th, struct yytbl_reader *, rd)
 {
-    int     bytes;
+    size_t  bytes;
    memset (th, 0, sizeof (struct yytbl_hdr));

    if (yytbl_read32 (&(th->th_magic), rd) != 0)
@@ -3103,8 +3402,11 @@ static int yytbl_hdr_read YYFARGS2(struct yytbl_hdr *, th, struct yytbl_reader *
 static struct yytbl_dmap *yytbl_dmap_lookup YYFARGS2(struct yytbl_dmap *, dmap,
                                                      int, id)
 {
+	M4_YY_DECL_GUTS_VAR();
+	M4_YY_NOOP_GUTS_VAR();
+
    while (dmap->dm_id)
-        if (dmap->dm_id == id)
+        if ((int)(dmap->dm_id) == id)
            return dmap;
        else
            dmap++;
@@ -40,7 +40,7 @@
 #endif

 /* AIX requires this to be the first thing in the file.  */
-#ifndef __GNUC__
+#if !defined(__GNUC__) && !defined(__lint__)
 # if HAVE_ALLOCA_H
 #  include <alloca.h>
 # else
@@ -395,6 +395,8 @@ char *alloca ();
 * yymore_really_used - whether to treat yymore() as really used, regardless
 *   of what we think based on references to it in the user's actions.
 * reject_really_used - same for REJECT
+ * trace_hex - use hexadecimal numbers in trace/debug outputs instead of octals
+ * charset_enabled - true if charset interface has been enabled
 */

 extern int printstats, syntaxerror, eofseen, ddebug, trace, nowarn,
@@ -409,7 +411,8 @@ extern int csize;
 extern int yymore_used, reject, real_reject, continued_action, in_rule;

 extern int yymore_really_used, reject_really_used;
-
+extern int trace_hex;
+extern bool charset_enabled;

 /* Variables used in the flex input routines:
 * datapos - characters on current output line
@@ -424,6 +427,7 @@ extern int yymore_really_used, reject_really_used;
 * infilename - name of input file
 * outfilename - name of output file
 * headerfilename - name of the .h file to generate
+ * headercharfilename - name of the .h file to fill with character type defintion
 * did_outfilename - whether outfilename was explicitly set
 * prefix - the prefix used for externally visible names ("yy" by default)
 * yyclass - yyFlexLexer subclass to use for YY_DECL
@@ -433,6 +437,8 @@ extern int yymore_really_used, reject_really_used;
 * num_input_files - size of input_files array
 * program_name - name with which program was invoked
 *
+ * charset_source - character set that has been declared as used in source file
+ *
 * action_array - array to hold the rule actions
 * action_size - size of action_array
 * defs1_offset - index where the user's section 1 definitions start
@@ -447,7 +453,7 @@ extern int datapos, dataline, linenum;
 extern FILE *skelfile, *yyin, *backing_up_file;
 extern const char *skel[];
 extern int skel_ind;
-extern char *infilename, *outfilename, *headerfilename;
+extern char *infilename, *outfilename, *headerfilename, *headercharfilename;
 extern int did_outfilename;
 extern char *prefix, *yyclass, *extra_type;
 extern int do_stdinit, use_stdout;
@@ -455,6 +461,8 @@ extern char **input_files;
 extern int num_input_files;
 extern char *program_name;

+extern char *charset_source;
+
 extern char *action_array;
 extern int action_size;
 extern int defs1_offset, prolog_offset, action_offset, action_index;
@@ -843,10 +851,10 @@ extern void action_define PROTO ((const char *defname, int value));
 extern void add_action PROTO ((const char *new_text));

 /* True if a string is all lower case. */
-extern int all_lower PROTO ((register char *));
+extern int all_lower PROTO ((char *));

 /* True if a string is all upper case. */
-extern int all_upper PROTO ((register char *));
+extern int all_upper PROTO ((char *));

 /* Compare two integers for use by qsort. */
 extern int intcmp PROTO ((const void *, const void *));
@@ -858,10 +866,10 @@ extern void check_char PROTO ((int c));
 extern Char clower PROTO ((int));

 /* Returns a dynamically allocated copy of a string. */
-extern char *copy_string PROTO ((register const char *));
+extern char *copy_string PROTO ((const char *));

 /* Returns a dynamically allocated copy of a (potentially) unsigned string. */
-extern Char *copy_unsigned_string PROTO ((register Char *));
+extern Char *copy_unsigned_string PROTO ((Char *));

 /* Compare two characters for use by qsort with '\0' sorting last. */
 extern int cclcmp PROTO ((const void *, const void *));
@@ -902,14 +910,19 @@ extern void flexfatal PROTO ((const char *));
 /* Convert a hexadecimal digit string to an integer value. */
 extern int htoi PROTO ((Char[]));

-/* Report an error message formatted with one integer argument. */
-extern void lerrif PROTO ((const char *, int));
+/* Report an error message formatted  */
+extern void lerr PROTO ((const char *, ...))
+#if defined(__GNUC__) && __GNUC__ >= 3
+    __attribute__((__format__(__printf__, 1, 2)))
+#endif
+;

-/* Report an error message formatted with one string argument. */
-extern void lerrsf PROTO ((const char *, const char *));
-
-/* Like lerrsf, but also exit after displaying message. */
-extern void lerrsf_fatal PROTO ((const char *, const char *));
+/* Like lerr, but also exit after displaying message. */
+extern void lerr_fatal PROTO ((const char *, ...))
+#if defined(__GNUC__) && __GNUC__ >= 3
+    __attribute__((__format__(__printf__, 1, 2)))
+#endif
+;

 /* Spit out a "#line" statement. */
 extern void line_directive_out PROTO ((FILE *, int));
@@ -987,7 +1000,7 @@ extern int link_machines PROTO ((int, int));
 /* Mark each "beginning" state in a machine as being a "normal" (i.e.,
 * not trailing context associated) state.
 */
-extern void mark_beginning_as_normal PROTO ((register int));
+extern void mark_beginning_as_normal PROTO ((int));

 /* Make a machine that branches to two machines. */
 extern int mkbranch PROTO ((int, int));
@@ -77,11 +77,32 @@ static const char *get_state_decl (void)
 		: "static yyconst yy_state_type * %s = 0;\n";
 }

+static const char *get_uint16_decl (void)
+{
+	return (gentables)
+		? "static yyconst flex_uint16_t %s[%d] =\n    {   0,\n"
+		: "static yyconst flex_uint16_t * %s = 0;\n";
+}
+
+static const char *get_uint32_decl (void)
+{
+	return (gentables)
+		? "static yyconst flex_uint32_t %s[%d] =\n    {   0,\n"
+		: "static yyconst flex_uint32_t * %s = 0;\n";
+}
+
+static const char *get_yy_char_decl (void)
+{
+	return (gentables)
+		? "static yyconst YY_CHAR %s[%d] =\n    {   0,\n"
+		: "static yyconst YY_CHAR * %s = 0;\n";
+}
+
 /* Indent to the current level. */

-void do_indent ()
+void do_indent (void)
 {
-	register int i = indent_level * 8;
+	int i = indent_level * 8;

 	while (i >= 8) {
 		outc ('\t');
@@ -121,7 +142,7 @@ static struct yytbl_data *mkeoltbl (void)
 }

 /* Generate the table for possible eol matches. */
-static void geneoltbl ()
+static void geneoltbl (void)
 {
 	int     i;

@@ -145,7 +166,7 @@ static void geneoltbl ()

 /* Generate the code to keep backing-up information. */

-void gen_backing_up ()
+void gen_backing_up (void)
 {
 	if (reject || num_backing_up == 0)
 		return;
@@ -166,7 +187,7 @@ void gen_backing_up ()

 /* Generate the code to perform the backing up. */

-void gen_bu_action ()
+void gen_bu_action (void)
 {
 	if (reject || num_backing_up == 0)
 		return;
@@ -201,7 +222,7 @@ void gen_bu_action ()

 static struct yytbl_data *mkctbl (void)
 {
-	register int i;
+	int i;
 	struct yytbl_data *tbl = 0;
 	flex_int32_t *tdata = 0, curr = 0;
 	int     end_of_buffer_action = num_rules + 1;
@@ -330,9 +351,9 @@ static struct yytbl_data *mkssltbl (void)

 /* genctbl - generates full speed compressed transition table */

-void genctbl ()
+void genctbl (void)
 {
-	register int i;
+	int i;
 	int     end_of_buffer_action = num_rules + 1;

 	/* Table of verify for transition and offset to next state. */
@@ -433,7 +454,7 @@ void genctbl ()

 struct yytbl_data *mkecstbl (void)
 {
-	register int i;
+	int i;
 	struct yytbl_data *tbl = 0;
 	flex_int32_t *tdata = 0;

@@ -453,19 +474,19 @@ struct yytbl_data *mkecstbl (void)

 	buf_prints (&yydmap_buf,
 		    "\t{YYTD_ID_EC, (void**)&yy_ec, sizeof(%s)},\n",
-		    "flex_int32_t");
+		    "YY_CHAR");

 	return tbl;
 }

 /* Generate equivalence-class tables. */

-void genecs ()
+void genecs (void)
 {
-	register int i, j;
+	int i, j;
 	int     numrows;

-	out_str_dec (get_int32_decl (), "yy_ec", csize);
+	out_str_dec (get_yy_char_decl (), "yy_ec", csize);

 	for (i = 1; i < csize; ++i) {
 		ecgroup[i] = ABS (ecgroup[i]);
@@ -495,7 +516,7 @@ void genecs ()

 /* Generate the code to find the action number. */

-void gen_find_action ()
+void gen_find_action (void)
 {
 	if (fullspd)
 		indent_puts ("yy_act = yy_current_state[-1].yy_nxt;");
@@ -507,7 +528,8 @@ void gen_find_action ()
 		indent_puts ("yy_current_state = *--YY_G(yy_state_ptr);");
 		indent_puts ("YY_G(yy_lp) = yy_accept[yy_current_state];");

-		outn ("find_rule: /* we branch to this label when backing up */");
+		if(reject_really_used)
+			outn ("find_rule: /* we branch to this label when backing up */");

 		indent_puts
 			("for ( ; ; ) /* until we find what rule we matched */");
@@ -637,7 +659,7 @@ void gen_find_action ()

 struct yytbl_data *mkftbl (void)
 {
-	register int i;
+	int i;
 	int     end_of_buffer_action = num_rules + 1;
 	struct yytbl_data *tbl;
 	flex_int32_t *tdata = 0;
@@ -654,7 +676,7 @@ struct yytbl_data *mkftbl (void)
 	dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action;

 	for (i = 1; i <= lastdfa; ++i) {
-		register int anum = dfaacc[i].dfaacc_state;
+		int anum = dfaacc[i].dfaacc_state;

 		tdata[i] = anum;

@@ -672,9 +694,9 @@ struct yytbl_data *mkftbl (void)

 /* genftbl - generate full transition table */

-void genftbl ()
+void genftbl (void)
 {
-	register int i;
+	int i;
 	int     end_of_buffer_action = num_rules + 1;

 	out_str_dec (long_align ? get_int32_decl () : get_int16_decl (),
@@ -683,7 +705,7 @@ void genftbl ()
 	dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action;

 	for (i = 1; i <= lastdfa; ++i) {
-		register int anum = dfaacc[i].dfaacc_state;
+		int anum = dfaacc[i].dfaacc_state;

 		mkdata (anum);

@@ -708,7 +730,7 @@ void genftbl ()
 void gen_next_compressed_state (char_map)
     char   *char_map;
 {
-	indent_put2s ("register YY_CHAR yy_c = %s;", char_map);
+	indent_put2s ("YY_CHAR yy_c = %s;", char_map);

 	/* Save the backing-up info \before/ computing the next state
 	 * because we always compute one more state than needed - we
@@ -750,7 +772,7 @@ void gen_next_compressed_state (char_map)

 /* Generate the code to find the next match. */

-void gen_next_match ()
+void gen_next_match (void)
 {
 	/* NOTE - changes in here should be reflected in gen_next_state() and
 	 * gen_NUL_trans().
@@ -794,8 +816,8 @@ void gen_next_match ()
 	else if (fullspd) {
 		indent_puts ("{");
 		indent_puts
-			("register yyconst struct yy_trans_info *yy_trans_info;\n");
-		indent_puts ("register YY_CHAR yy_c;\n");
+			("yyconst struct yy_trans_info *yy_trans_info;\n");
+		indent_puts ("YY_CHAR yy_c;\n");
 		indent_put2s ("for ( yy_c = %s;", char_map);
 		indent_puts
 			("      (yy_trans_info = &yy_current_state[(unsigned int) yy_c])->");
@@ -927,7 +949,7 @@ void gen_next_state (worry_about_NULs)

 /* Generate the code to make a NUL transition. */

-void gen_NUL_trans ()
+void gen_NUL_trans (void)
 {				/* NOTE - changes in here should be reflected in gen_next_match() */
 	/* Only generate a definition for "yy_cp" if we'll generate code
 	 * that uses it.  Otherwise lint and the like complain.
@@ -938,7 +960,7 @@ void gen_NUL_trans ()
 		/* We're going to need yy_cp lying around for the call
 		 * below to gen_backing_up().
 		 */
-		indent_puts ("register char *yy_cp = YY_G(yy_c_buf_p);");
+		indent_puts ("YY_CHAR *yy_cp = YY_G(yy_c_buf_p);");

 	outc ('\n');

@@ -959,10 +981,10 @@ void gen_NUL_trans ()

 	else if (fullspd) {
 		do_indent ();
-		out_dec ("register int yy_c = %d;\n", NUL_ec);
+		out_dec ("int yy_c = %d;\n", NUL_ec);

 		indent_puts
-			("register yyconst struct yy_trans_info *yy_trans_info;\n");
+			("yyconst struct yy_trans_info *yy_trans_info;\n");
 		indent_puts
 			("yy_trans_info = &yy_current_state[(unsigned int) yy_c];");
 		indent_puts ("yy_current_state += yy_trans_info->yy_nxt;");
@@ -1012,7 +1034,7 @@ void gen_NUL_trans ()

 /* Generate the code to find the start state. */

-void gen_start_state ()
+void gen_start_state (void)
 {
 	if (fullspd) {
 		if (bol_needed) {
@@ -1045,7 +1067,7 @@ void gen_start_state ()

 /* gentabs - generate data statements for the transition tables */

-void gentabs ()
+void gentabs (void)
 {
 	int     i, j, k, *accset, nacc, *acc_array, total_states;
 	int     end_of_buffer_action = num_rules + 1;
@@ -1271,10 +1293,10 @@ void gentabs ()
 			fputs (_("\n\nMeta-Equivalence Classes:\n"),
 			       stderr);

-		out_str_dec (get_int32_decl (), "yy_meta", numecs + 1);
+		out_str_dec (get_yy_char_decl (), "yy_meta", numecs + 1);
 		buf_prints (&yydmap_buf,
 			    "\t{YYTD_ID_META, (void**)&yy_meta, sizeof(%s)},\n",
-			    "flex_int32_t");
+			    "YY_CHAR");

 		for (i = 1; i <= numecs; ++i) {
 			if (trace)
@@ -1301,13 +1323,13 @@ void gentabs ()

 	/* Begin generating yy_base */
 	out_str_dec ((tblend >= INT16_MAX || long_align) ?
-		     get_int32_decl () : get_int16_decl (),
+		     get_uint32_decl () : get_uint16_decl (),
 		     "yy_base", total_states + 1);

 	buf_prints (&yydmap_buf,
 		    "\t{YYTD_ID_BASE, (void**)&yy_base, sizeof(%s)},\n",
 		    (tblend >= INT16_MAX
-		     || long_align) ? "flex_int32_t" : "flex_int16_t");
+		     || long_align) ? "flex_uint32_t" : "flex_uint16_t");
 	yybase_tbl =
 		(struct yytbl_data *) calloc (1,
 					      sizeof (struct yytbl_data));
@@ -1319,7 +1341,7 @@ void gentabs ()
 	yybase_curr = 1;

 	for (i = 1; i <= lastdfa; ++i) {
-		register int d = def[i];
+		int d = def[i];

 		if (base[i] == JAMSTATE)
 			base[i] = jambase;
@@ -1394,13 +1416,13 @@ void gentabs ()

 	/* Begin generating yy_nxt */
 	out_str_dec ((total_states >= INT16_MAX || long_align) ?
-		     get_int32_decl () : get_int16_decl (), "yy_nxt",
+		     get_uint32_decl () : get_uint16_decl (), "yy_nxt",
 		     tblend + 1);

 	buf_prints (&yydmap_buf,
 		    "\t{YYTD_ID_NXT, (void**)&yy_nxt, sizeof(%s)},\n",
 		    (total_states >= INT16_MAX
-		     || long_align) ? "flex_int32_t" : "flex_int16_t");
+		     || long_align) ? "flex_uint32_t" : "flex_uint16_t");

 	yynxt_tbl =
 		(struct yytbl_data *) calloc (1,
@@ -1499,10 +1521,10 @@ void indent_puts (str)
 /* make_tables - generate transition tables and finishes generating output file
 */

-void make_tables ()
+void make_tables (void)
 {
-	register int i;
-	int     did_eof_rule = false;
+	int i;
+	int did_eof_rule = false;
 	struct yytbl_data *yynultrans_tbl;


@@ -1687,7 +1709,7 @@ void make_tables ()
 			indent_puts
 				("static yy_state_type yy_last_accepting_state;");
 			indent_puts
-				("static char *yy_last_accepting_cpos;\n");
+				("static YY_CHAR *yy_last_accepting_cpos;\n");
 		}
 	}

@@ -1759,7 +1781,7 @@ void make_tables ()
 		/* Declare state buffer variables. */
 		if (!C_plus_plus && !reentrant) {
 			outn ("static yy_state_type *yy_state_buf=0, *yy_state_ptr=0;");
-			outn ("static char *yy_full_match;");
+			outn ("static YY_CHAR *yy_full_match;");
 			outn ("static int yy_lp;");
 		}

@@ -1854,14 +1876,14 @@ void make_tables ()
 			outn ("#define YYLMAX 8192");
 			outn ("#endif\n");
 			if (!reentrant){
-                outn ("char yytext[YYLMAX];");
-                outn ("char *yytext_ptr;");
+                outn ("YY_CHAR yytext[YYLMAX];");
+                outn ("YY_CHAR *yytext_ptr;");
            }
 		}

 		else {
 			if(! reentrant)
-                outn ("char *yytext;");
+                outn ("YY_CHAR *yytext;");
 		}
 	}

@@ -21,12 +21,13 @@
 /*  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR */
 /*  PURPOSE. */

-extern int yylex ();
+extern int yylex (void);

-int     main (argc, argv)
-     int     argc;
-     char   *argv[];
+int     main (int argc, char *argv[])
 {
+	(void)argc;
+	(void)argv;
+
 	while (yylex () != 0) ;

 	return 0;
@@ -21,6 +21,8 @@
 /*  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR */
 /*  PURPOSE. */

+int     yywrap (void);
+
 int     yywrap (void)
 {
 	return 1;
@@ -57,13 +57,14 @@ int     C_plus_plus, long_align, use_read, yytext_is_array, do_yywrap,
 int     reentrant, bison_bridge_lval, bison_bridge_lloc;
 int     yymore_used, reject, real_reject, continued_action, in_rule;
 int     yymore_really_used, reject_really_used;
+int     trace_hex = 0;
 int     datapos, dataline, linenum;
 FILE   *skelfile = NULL;
 int     skel_ind = 0;
 char   *action_array;
 int     action_size, defs1_offset, prolog_offset, action_offset,
 	action_index;
-char   *infilename = NULL, *outfilename = NULL, *headerfilename = NULL;
+char   *infilename = NULL, *outfilename = NULL, *headerfilename = NULL, *headercharfilename = NULL;
 int     did_outfilename;
 char   *prefix, *yyclass, *extra_type = NULL;
 int     do_stdinit, use_stdout;
@@ -99,13 +100,15 @@ int     sectnum, nummt, hshcol, dfaeql, numeps, eps2, num_reallocs;
 int     tmpuses, totnst, peakpairs, numuniq, numdup, hshsave;
 int     num_backing_up, bol_needed;
 FILE   *backing_up_file;
+FILE   *char_header_file = NULL;
 int     end_of_buffer_state;
 char  **input_files;
 int     num_input_files;
 jmp_buf flex_main_jmp_buf;
 bool   *rule_has_nl, *ccl_has_nl;
 int     nlch = '\n';
-bool    ansi_func_defs, ansi_func_protos;
+bool    ansi_func_defs, ansi_func_protos, charset_enabled = false;
+char   *charset_source = NULL;

 bool    tablesext, tablesverify, gentables;
 char   *tablesfilename=0,*tablesname=0;
@@ -117,13 +120,13 @@ struct yytbl_writer tableswr;
 char   *program_name = "flex";

 #ifndef SHORT_FILE_NAMES
-static char *outfile_template = "lex.%s.%s";
-static char *backing_name = "lex.backup";
-static char *tablesfile_template = "lex.%s.tables";
+static const char outfile_template[] = "lex.%s.%s";
+static const char backing_name[] = "lex.backup";
+static const char tablesfile_template[] = "lex.%s.tables";
 #else
-static char *outfile_template = "lex%s.%s";
-static char *backing_name = "lex.bck";
-static char *tablesfile_template = "lex%s.tbl";
+static const char outfile_template[] = "lex%s.%s";
+static const char backing_name[] = "lex.bck";
+static const char tablesfile_template[] = "lex%s.tbl";
 #endif

 #ifdef MS_DOS
@@ -301,6 +304,8 @@ void check_options ()
 	if (C_plus_plus && bison_bridge_lval)
 		flexerror (_("bison bridge not supported for the C++ scanner."));

+	if(C_plus_plus)
+		buf_m4_define( &m4defs_buf, "M4_YY_CXX", NULL);

 	if (useecs) {		/* Set up doubly-linked equivalence classes. */

@@ -354,7 +359,7 @@ void check_options ()
 		prev_stdout = freopen (outfilename, "w+", stdout);

 		if (prev_stdout == NULL)
-			lerrsf (_("could not create %s"), outfilename);
+			lerr (_("could not create %s"), outfilename);

 		outfile_created = 1;
 	}
@@ -367,6 +372,42 @@ void check_options ()
    filter_create_ext(output_chain, m4, "-P", 0);
    filter_create_int(output_chain, filter_fix_linedirs, NULL);

+    if(headerfilename && !headercharfilename) {
+        char *suffix = ".h";
+        char *basesuffix = "_char";
+        size_t suffix_len = strlen(suffix);
+        size_t basesuffix_len = strlen(basesuffix);
+
+        size_t headerfilename_len = strlen(headerfilename);
+
+        char *p = strstr(headerfilename, suffix);
+        if(p && headerfilename+headerfilename_len-suffix_len == p)
+            headerfilename_len -= suffix_len; /* cut out suffix */
+
+        headercharfilename = malloc(headerfilename_len+basesuffix_len+suffix_len+1);
+        p = headercharfilename;
+
+        strncpy(p, headerfilename, headerfilename_len);
+        p += headerfilename_len;
+
+        strncpy(p, basesuffix, basesuffix_len);
+        p += basesuffix_len;
+
+        strncpy(p, suffix, suffix_len);
+    }
+
+    /* Setup char header file */
+    if(headercharfilename) {
+        char_header_file = fopen(headercharfilename, "w");
+        fprintf(char_header_file, "#pragma once\n");
+        fprintf(char_header_file, "\n");
+        fflush(char_header_file);
+
+        buf_strappend(&userdef_buf, "#include \"");
+        buf_strappend(&userdef_buf, headercharfilename);
+        buf_strappend(&userdef_buf, "\"\n");
+    }
+
    /* For debugging, only run the requested number of filters. */
    if (preproc_level > 0) {
        filter_truncate(output_chain, preproc_level);
@@ -400,7 +441,7 @@ void check_options ()
 		}

 		if ((tablesout = fopen (tablesfilename, "w")) == NULL)
-			lerrsf (_("could not create %s"), tablesfilename);
+			lerr (_("could not create %s"), tablesfilename);
 		if (pname)
 			free (pname);
 		tablesfilename = 0;
@@ -417,7 +458,7 @@ void check_options ()
 	}

 	if (skelname && (skelfile = fopen (skelname, "r")) == NULL)
-		lerrsf (_("can't open skeleton file %s"), skelname);
+		lerr (_("can't open skeleton file %s"), skelname);

 	if (reentrant) {
        buf_m4_define (&m4defs_buf, "M4_YY_REENTRANT", NULL);
@@ -439,6 +480,12 @@ void check_options ()
 	if (do_yylineno)
 		buf_m4_define (&m4defs_buf, "M4_YY_USE_LINENO", NULL);

+	if(charset_enabled)
+		buf_m4_define(&m4defs_buf, "M4_YY_CHARSET", NULL);
+
+	if(charset_source)
+		buf_m4_define(&m4defs_buf, "M4_YY_CHARSET_SOURCE", charset_source);
+
 	/* Create the alignment type. */
 	buf_strdefine (&userdef_buf, "YY_INT_ALIGNED",
 		       long_align ? "long int" : "short int");
@@ -499,13 +546,17 @@ void flexend (exit_status)
 	if (++called_before)
 		FLEX_EXIT (exit_status);

+	if(char_header_file && fclose (char_header_file))
+			lerrsf (_("error closing char header file %s"),
+				headercharfilename);
+
 	if (skelfile != NULL) {
 		if (ferror (skelfile))
-			lerrsf (_("input error reading skeleton file %s"),
+			lerr (_("input error reading skeleton file %s"),
 				skelname);

 		else if (fclose (skelfile))
-			lerrsf (_("error closing skeleton file %s"),
+			lerr (_("error closing skeleton file %s"),
 				skelname);
 	}

@@ -690,7 +741,7 @@ void flexend (exit_status)
 		fprintf (header_out, "#endif /* %sHEADER_H */\n", prefix);

 		if (ferror (header_out))
-			lerrsf (_("error creating header file %s"),
+			lerr (_("error creating header file %s"),
 				headerfilename);
 		fflush (header_out);
 		fclose (header_out);
@@ -698,15 +749,15 @@ void flexend (exit_status)

 	if (exit_status != 0 && outfile_created) {
 		if (ferror (stdout))
-			lerrsf (_("error writing output file %s"),
+			lerr (_("error writing output file %s"),
 				outfilename);

 		else if ((_stdout_closed = 1) && fclose (stdout))
-			lerrsf (_("error closing output file %s"),
+			lerr (_("error closing output file %s"),
 				outfilename);

 		else if (unlink (outfilename))
-			lerrsf (_("error deleting output file %s"),
+			lerr (_("error deleting output file %s"),
 				outfilename);
 	}

@@ -724,11 +775,11 @@ void flexend (exit_status)
 				 _("Compressed tables always back up.\n"));

 		if (ferror (backing_up_file))
-			lerrsf (_("error writing backup file %s"),
+			lerr (_("error writing backup file %s"),
 				backing_name);

 		else if (fclose (backing_up_file))
-			lerrsf (_("error closing backup file %s"),
+			lerr (_("error closing backup file %s"),
 				backing_name);
 	}

@@ -1058,9 +1109,9 @@ void flexinit (argc, argv)
 					break;

 				default:
-					lerrif (_
+					lerr (_
 						("unknown -C option '%c'"),
-						(int) arg[i]);
+						arg[i]);
 					break;
 				}
 			break;
@@ -1421,7 +1472,8 @@ void flexinit (argc, argv)
 			//buf_strdefine (&userdef_buf, "YY_NO_SET_LLOC", "1");
            buf_m4_define( &m4defs_buf, "M4_YY_NO_SET_LLOC",0);
 			break;
-
+		case OPT_HEX:
+			trace_hex = 1;
 		}		/* switch */
 	}			/* while scanopt() */

@@ -1459,6 +1511,13 @@ void readin ()
 	static char yy_stdinit[] = "FILE *yyin = stdin, *yyout = stdout;";
 	static char yy_nostdinit[] =
 		"FILE *yyin = (FILE *) 0, *yyout = (FILE *) 0;";
+	static char character_type_uchar[] = "typedef unsigned char YY_CHAR;";
+	static char character_type_char[] = "typedef char YY_CHAR;";
+	static char character_defined[] = "#define YY_CHAR_DEFINED";
+
+	static char charset_handler_t[] = "typedef size_t(*yycharset_handler_t)(char*,char*,size_t,YY_CHAR*,size_t,size_t*);\n";
+	static char charset_handler_t_reentrant[] = "typedef size_t(*yycharset_handler_t)(char*,char*,size_t,YY_CHAR*,size_t,size_t*,yyscan_t);\n";
+

 	line_directive_out ((FILE *) 0, 1);

@@ -1494,7 +1553,7 @@ void readin ()
 	if (backing_up_report) {
 		backing_up_file = fopen (backing_name, "w");
 		if (backing_up_file == NULL)
-			lerrsf (_
+			lerr (_
 				("could not create backing-up info file %s"),
 				backing_name);
 	}
@@ -1575,11 +1634,12 @@ void readin ()
    }

 	if (!do_yywrap) {
-		if (!C_plus_plus)
+		if (!C_plus_plus) {
 			 if (reentrant)
-				outn ("\n#define yywrap(yyscanner) 1");
+				outn ("\n#define yywrap(yyscanner) (/*CONSTCOND*/1)");
 			 else
-				outn ("\n#define yywrap() 1");
+				outn ("\n#define yywrap() (/*CONSTCOND*/1)");
+		}
 		outn ("#define YY_SKIP_YYWRAP");
 	}

@@ -1587,12 +1647,27 @@ void readin ()
 		outn ("\n#define FLEX_DEBUG");

 	OUT_BEGIN_CODE ();
-	if (csize == 256)
-		outn ("typedef unsigned char YY_CHAR;");
-	else
-		outn ("typedef char YY_CHAR;");
+	outn("#ifndef YY_CHAR_DEFINED");
+	if (csize == 256) {
+		outn (character_type_uchar);
+		if(char_header_file)
+			fprintf(char_header_file, "%s\n", character_type_uchar);
+	} else {
+		outn (character_type_char);
+		if(char_header_file)
+			fprintf(char_header_file, "%s\n", character_type_char);
+	}
+	outn(character_defined);
+	outn("#endif");
+	if(char_header_file) {
+		fprintf(char_header_file, "%s\n", character_defined);
+		fflush(char_header_file);
+	}
 	OUT_END_CODE ();

+	if(charset_enabled)
+		outn ("#define YY_CHARSET");
+
 	if (C_plus_plus) {
 		outn ("#define yytext_ptr yytext");

@@ -1675,14 +1750,20 @@ void readin ()
 		 */
 		if (yytext_is_array) {
 			if (!reentrant)
-				outn ("extern char yytext[];\n");
+				outn ("extern YY_CHAR yytext[];\n");
 		}
 		else {
+			/* This prevents warning of "already defined macro" in multiple
+			 * non-reentrant scanners */
+			outn("#ifdef yytext_ptr");
+			outn("#undef yytext_ptr");
+			outn("#endif");
+
 			if (reentrant) {
 				outn ("#define yytext_ptr yytext_r");
 			}
 			else {
-				outn ("extern char *yytext;");
+				outn ("extern YY_CHAR *yytext;");
 				outn ("#define yytext_ptr yytext");
 			}
 		}
@@ -1692,6 +1773,22 @@ void readin ()
 				   ("%option yyclass only meaningful for C++ scanners"));
 	}

+	outn("");
+
+	if(charset_enabled) {
+		if(!reentrant)
+			outn(charset_handler_t);
+		else
+			outn(charset_handler_t_reentrant);
+
+		OUT_BEGIN_CODE ();
+		if(!C_plus_plus && !reentrant) {
+			outn("char *yycharset = NULL;");
+			outn("yycharset_handler_t yycharset_handler = NULL;");
+		}
+		OUT_END_CODE ();
+	}
+
 	if (useecs)
 		numecs = cre8ecs (nextecm, ecgroup, csize);
 	else
@@ -1818,6 +1915,7 @@ void usage ()
 		  "  -T, --trace             %s should run in trace mode\n"
 		  "  -w, --nowarn            do not generate warnings\n"
 		  "  -v, --verbose           write summary of scanner statistics to stdout\n"
+		  "      --hex               use hexadecimal numbers instead of octal in debug outputs\n"
 		  "\n" "Files:\n"
 		  "  -o, --outfile=FILE      specify output filename\n"
 		  "  -S, --skel=FILE         specify skeleton file\n"
@@ -113,11 +113,12 @@ void action_define (defname, value)
 }


+#ifdef notdef
 /** Append "m4_define([[defname]],[[value]])m4_dnl\n" to the running buffer.
 *  @param defname The macro name.
 *  @param value The macro value, can be NULL, which is the same as the empty string.
 */
-void action_m4_define (const char *defname, const char * value)
+static void action_m4_define (const char *defname, const char * value)
 {
 	char    buf[MAXLINE];

@@ -133,6 +134,7 @@ void action_m4_define (const char *defname, const char * value)
 	snprintf (buf, sizeof(buf), "m4_define([[%s]],[[%s]])m4_dnl\n", defname, value?value:"");
 	add_action (buf);
 }
+#endif

 /* Append "new_text" to the running buffer. */
 void add_action (new_text)
@@ -168,7 +170,7 @@ void   *allocate_array (size, element_size)
     int size;
     size_t element_size;
 {
-	register void *mem;
+	void *mem;
 	size_t  num_bytes = element_size * size;

 	mem = flex_alloc (num_bytes);
@@ -183,7 +185,7 @@ void   *allocate_array (size, element_size)
 /* all_lower - true if a string is all lower-case */

 int all_lower (str)
-     register char *str;
+     char *str;
 {
 	while (*str) {
 		if (!isascii ((Char) * str) || !islower ((Char) * str))
@@ -198,7 +200,7 @@ int all_lower (str)
 /* all_upper - true if a string is all upper-case */

 int all_upper (str)
-     register char *str;
+     char *str;
 {
 	while (*str) {
 		if (!isascii ((Char) * str) || !isupper ((Char) * str))
@@ -227,11 +229,11 @@ void check_char (c)
     int c;
 {
 	if (c >= CSIZE)
-		lerrsf (_("bad character '%s' detected in check_char()"),
+		lerr (_("bad character '%s' detected in check_char()"),
 			readable_form (c));

 	if (c >= csize)
-		lerrsf (_
+		lerr (_
 			("scanner requires -8 flag to use the character %s"),
 			readable_form (c));
 }
@@ -241,7 +243,7 @@ void check_char (c)
 /* clower - replace upper-case letter to lower-case */

 Char clower (c)
-     register int c;
+     int c;
 {
 	return (Char) ((isascii (c) && isupper (c)) ? tolower (c) : c);
 }
@@ -250,10 +252,10 @@ Char clower (c)
 /* copy_string - returns a dynamically allocated copy of a string */

 char   *copy_string (str)
-     register const char *str;
+     const char *str;
 {
-	register const char *c1;
-	register char *c2;
+	const char *c1;
+	char *c2;
 	char   *copy;
 	unsigned int size;

@@ -278,9 +280,9 @@ char   *copy_string (str)
 */

 Char   *copy_unsigned_string (str)
-     register Char *str;
+     Char *str;
 {
-	register Char *c;
+	Char *c;
 	Char   *copy;

 	/* find length */
@@ -383,41 +385,29 @@ int htoi (str)
 }


-/* lerrif - report an error message formatted with one integer argument */
+/* lerr - report an error message */

-void lerrif (msg, arg)
-     const char *msg;
-     int arg;
-{
+void lerr (const char *msg, ...) {
 	char    errmsg[MAXLINE];
+	va_list args;

-	snprintf (errmsg, sizeof(errmsg), msg, arg);
+	va_start(args, msg);
+	vsnprintf (errmsg, sizeof(errmsg), msg, args);
+	va_end(args);
 	flexerror (errmsg);
 }


-/* lerrsf - report an error message formatted with one string argument */
+/* lerr_fatal - as lerr, but call flexfatal */

-void lerrsf (msg, arg)
-	const char *msg, arg[];
+void lerr_fatal (const char *msg, ...)
 {
 	char    errmsg[MAXLINE];
+	va_list args;
+	va_start(args, msg);

-	snprintf (errmsg, sizeof(errmsg)-1, msg, arg);
-	errmsg[sizeof(errmsg)-1] = 0; /* ensure NULL termination */
-	flexerror (errmsg);
-}
-
-
-/* lerrsf_fatal - as lerrsf, but call flexfatal */
-
-void lerrsf_fatal (msg, arg)
-	const char *msg, arg[];
-{
-	char    errmsg[MAXLINE];
-
-	snprintf (errmsg, sizeof(errmsg)-1, msg, arg);
-	errmsg[sizeof(errmsg)-1] = 0; /* ensure NULL termination */
+	vsnprintf (errmsg, sizeof(errmsg), msg, args);
+	va_end(args);
 	flexfatal (errmsg);
 }

@@ -731,7 +721,6 @@ void outn (str)
 /** Print "m4_define( [[def]], [[val]])m4_dnl\n".
 * @param def The m4 symbol to define.
 * @param val The definition; may be NULL.
- * @return buf
 */
 void out_m4_define (const char* def, const char* val)
 {
@@ -746,9 +735,9 @@ void out_m4_define (const char* def, const char* val)
 */

 char   *readable_form (c)
-     register int c;
+     int c;
 {
-	static char rform[10];
+	static char rform[20];

 	if ((c >= 0 && c < 32) || c >= 127) {
 		switch (c) {
@@ -771,7 +760,10 @@ char   *readable_form (c)
 #endif

 		default:
-			snprintf (rform, sizeof(rform), "\\%.3o", (unsigned int) c);
+			if(trace_hex)
+				snprintf (rform, sizeof(rform), "\\x%.2x", (unsigned int) c);
+			else
+				snprintf (rform, sizeof(rform), "\\%.3o", (unsigned int) c);
 			return rform;
 		}
 	}
@@ -795,7 +787,7 @@ void   *reallocate_array (array, size, element_size)
     int size;
     size_t element_size;
 {
-	register void *new_array;
+	void *new_array;
 	size_t  num_bytes = element_size * size;

 	new_array = flex_realloc (array, num_bytes);
@@ -991,7 +983,7 @@ void zero_out (region_ptr, size_in_bytes)
     char   *region_ptr;
     size_t size_in_bytes;
 {
-	register char *rp, *rp_end;
+	char *rp, *rp_end;

 	rp = region_ptr;
 	rp_end = region_ptr + size_in_bytes;
@@ -340,7 +340,7 @@ int     link_machines (first, last)
 */

 void    mark_beginning_as_normal (mach)
-     register int mach;
+     int mach;
 {
 	switch (state_type[mach]) {
 	case STATE_NORMAL:
@@ -605,7 +605,7 @@ int     mkstate (sym)
 {
 	if (++lastnfa >= current_mns) {
 		if ((current_mns += MNS_INCREMENT) >= maximum_mns)
-			lerrif (_
+			lerr(_
 				("input rules are too complicated (>= %d NFA states)"),
 current_mns);

@@ -711,7 +711,7 @@ void    new_rule ()
 	}

 	if (num_rules > MAX_RULE)
-		lerrif (_("too many rules (> %d)!"), MAX_RULE);
+		lerr (_("too many rules (> %d)!"), MAX_RULE);

 	rule_linenum[num_rules] = linenum;
 	rule_useful[num_rules] = false;
@@ -117,6 +117,8 @@ optspec_t flexopts[] = {
 	,
 	{"--help", OPT_HELP, 0}
 	,			/* Produce this help message. */
+	{"--hex", OPT_HEX, 0}
+	,			/* Use hexadecimals in debug/trace outputs */
 	{"-I", OPT_INTERACTIVE, 0}
 	,
 	{"--interactive", OPT_INTERACTIVE, 0}
@@ -60,6 +60,7 @@ enum flexopt_flag_t {
 	OPT_FULL,
 	OPT_HEADER_FILE,
 	OPT_HELP,
+	OPT_HEX,
 	OPT_INTERACTIVE,
 	OPT_LEX_COMPAT,
 	OPT_POSIX_COMPAT,
@@ -1,8 +1,8 @@
 /* parse.y - parser for flex input */

 %token CHAR NUMBER SECTEND SCDECL XSCDECL NAME PREVCCL EOF_OP
-%token OPTION_OP OPT_OUTFILE OPT_PREFIX OPT_YYCLASS OPT_HEADER OPT_EXTRA_TYPE
-%token OPT_TABLES
+%token OPTION_OP OPT_OUTFILE OPT_PREFIX OPT_YYCLASS OPT_HEADER OPT_HEADER_CHAR
+%token OPT_EXTRA_TYPE OPT_TABLES OPT_CHARSET_SOURCE

 %token CCE_ALNUM CCE_ALPHA CCE_BLANK CCE_CNTRL CCE_DIGIT CCE_GRAPH
 %token CCE_LOWER CCE_PRINT CCE_PUNCT CCE_SPACE CCE_UPPER CCE_XDIGIT
@@ -204,8 +204,12 @@ option		:  OPT_OUTFILE '=' NAME
 			{ yyclass = copy_string( nmstr ); }
 		|  OPT_HEADER '=' NAME
 			{ headerfilename = copy_string( nmstr ); }
+		|  OPT_HEADER_CHAR '=' NAME
+			{ headercharfilename = copy_string( nmstr ); }
 	    |  OPT_TABLES '=' NAME
            { tablesext = true; tablesfilename = copy_string( nmstr ); }
+        |  OPT_CHARSET_SOURCE '=' NAME
+            { charset_source = copy_string(nmstr); }
 		;

 sect2		:  sect2 scon initforrule flexrule '\n'
@@ -953,7 +957,7 @@ string		:  string CHAR

 void build_eof_action()
 	{
-	register int i;
+	int i;
 	char action_text[MAXLINE];

 	for ( i = 1; i <= scon_stk_ptr; ++i )
@@ -1086,4 +1090,5 @@ int line;
 void yyerror( msg )
 const char *msg;
 	{
+		(void)msg;
 	}
@@ -113,7 +113,6 @@ char   *regmatch_cpy (regmatch_t * m, char *dest, const char *src)

 /** Get the length in characters of the match.
 * @param m A match as returned by regexec().
- * @param src The source string that was passed to regexec().
 * @return The length of the match.
 */
 int regmatch_len (regmatch_t * m)
@@ -95,7 +95,7 @@ extern const char *escaped_qstart, *escaped_qend;

 %}

-%option caseless nodefault stack noyy_top_state
+%option caseless nodefault noreject stack noyy_top_state
 %option nostdinit

 %x SECT2 SECT2PROLOG SECT3 CODEBLOCK PICKUPDEF SC CARETISBOL NUM QUOTE
@@ -349,6 +349,7 @@ M4QEND      "]]"
 	"c++"		C_plus_plus = option_sense;
 	caseful|case-sensitive		sf_set_case_ins(!option_sense);
 	caseless|case-insensitive	sf_set_case_ins(option_sense);
+	charset		charset_enabled = true;
 	debug		ddebug = option_sense;
 	default		spprdflt = ! option_sense;
 	ecs		useecs = option_sense;
@@ -425,7 +426,9 @@ M4QEND      "]]"
 	prefix		return OPT_PREFIX;
 	yyclass		return OPT_YYCLASS;
 	header(-file)?      return OPT_HEADER;
+	header-char(-file)? return OPT_HEADER_CHAR;
 	tables-file         return OPT_TABLES;
+	charset-source      return OPT_CHARSET_SOURCE;
 	tables-verify   {
                    tablesverify = option_sense;
                    if(!tablesext && option_sense)
@@ -661,7 +664,7 @@ M4QEND      "]]"
     * context.
     */
 	"{"{NAME}"}"[[:space:]]?	 {
-			register Char *nmdefptr;
+			Char *nmdefptr;
            int end_is_ws, end_ch;

            end_ch = yytext[yyleng-1];
@@ -993,7 +996,7 @@ char *file;
 		yyin = fopen( infilename, "r" );

 		if ( yyin == NULL )
-			lerrsf( _( "can't open %s" ), file );
+			lerr( _( "can't open %s" ), file );
 		}

 	else
@@ -61,8 +61,7 @@ sf_init (void)
    assert(_sf_stk == NULL);
    _sf_stk = (scanflags_t*) flex_alloc ( sizeof(scanflags_t) * (_sf_max = 32));
    if (!_sf_stk)
-        lerrsf_fatal(_("Unable to allocate %ld of stack"),
-            (long)sizeof(scanflags_t));
+        lerr_fatal(_("Unable to allocate %zu of stack"), sizeof(scanflags_t));
    _sf_stk[_sf_top_ix] = 0;
 }

@@ -46,7 +46,7 @@ static int STRCASECMP (a, b)
     const char *a;
     const char *b;
 {
-	while (tolower (*a++) == tolower (*b++)) ;
+	while (tolower ((unsigned char)*a++) == tolower ((unsigned char)*b++)) ;
 	return b - a;
 }
 #endif
@@ -82,7 +82,7 @@ static int PRINTLEN PROTO ((struct _scanopt_t *, int));
 static int RVAL PROTO ((struct _scanopt_t *, int));
 static int FLAGS PROTO ((struct _scanopt_t *, int));
 static const char *DESC PROTO ((struct _scanopt_t *, int));
-static int scanopt_err PROTO ((struct _scanopt_t *, int, int, int));
+static int scanopt_err PROTO ((struct _scanopt_t *, int, int));
 static int matchlongopt PROTO ((char *, char **, int *, char **, int *));
 static int find_opt
 PROTO ((struct _scanopt_t *, int, char *, int, int *, int *opt_offset));
@@ -212,7 +212,7 @@ scanopt_t *scanopt_init (options, argc, argv, flags)
 		aux->namelen = 0;
 		for (p = pname + 1; *p; p++) {
 			/* detect required arg */
-			if (*p == '=' || isspace (*p)
+			if (*p == '=' || isspace ((unsigned char)*p)
 			    || !(aux->flags & IS_LONG)) {
 				if (aux->namelen == 0)
 					aux->namelen = p - pname;
@@ -481,7 +481,7 @@ int     scanopt_usage (scanner, fp, usage)

 					while (*p && n < maxlen[1]
 					       && *p != '\n') {
-						if (isspace ((Char)(*p))
+						if (isspace ((unsigned char)(*p))
 						    || *p == '-') lastws =
 								p;
 						n++;
@@ -529,18 +529,13 @@ int     scanopt_usage (scanner, fp, usage)
 #endif /* no scanopt_usage */


-static int scanopt_err (s, opt_offset, is_short, err)
+static int scanopt_err (s, is_short, err)
     struct _scanopt_t *s;
-     int     opt_offset;
     int     is_short;
     int     err;
 {
 	const char *optname = "";
 	char    optchar[2];
-	const optspec_t *opt = NULL;
-
-	if (opt_offset >= 0)
-		opt = s->options + opt_offset;

 	if (!s->no_err_msg) {

@@ -749,7 +744,7 @@ int     scanopt (svoid, arg, optindex)
 			if (!find_opt
 			    (s, 1, optname, namelen, &errcode,
 			     &opt_offset)) {
-				scanopt_err (s, opt_offset, 0, errcode);
+				scanopt_err (s, 0, errcode);
 				return errcode;
 			}
 			/* We handle this below. */
@@ -784,7 +779,7 @@ int     scanopt (svoid, arg, optindex)

 		if (!find_opt
 		    (s, 0, pstart, namelen, &errcode, &opt_offset)) {
-			return scanopt_err (s, opt_offset, 1, errcode);
+			return scanopt_err (s, 1, errcode);
 		}

 		optarg = pstart + 1;
@@ -812,8 +807,7 @@ int     scanopt (svoid, arg, optindex)
 	/* case: no args allowed */
 	if (auxp->flags & ARG_NONE) {
 		if (optarg && !is_short) {
-			scanopt_err (s, opt_offset, is_short, errcode =
-				     SCANOPT_ERR_ARG_NOT_ALLOWED);
+			scanopt_err (s, is_short, errcode = SCANOPT_ERR_ARG_NOT_ALLOWED);
 			INC_INDEX (s, 1);
 			return errcode;
 		}
@@ -827,8 +821,7 @@ int     scanopt (svoid, arg, optindex)
 	/* case: required */
 	if (auxp->flags & ARG_REQ) {
 		if (!optarg && !has_next)
-			return scanopt_err (s, opt_offset, is_short,
-					    SCANOPT_ERR_ARG_NOT_FOUND);
+			return scanopt_err (s, is_short, SCANOPT_ERR_ARG_NOT_FOUND);

 		if (!optarg) {
 			/* Let the next argv element become the argument. */
@@ -59,12 +59,12 @@ static struct hash_entry *ccltab[CCL_HASH_SIZE];

 /* declare functions that have forward references */

-static int addsym PROTO ((register char[], char *, int, hash_table, int));
-static struct hash_entry *findsym PROTO ((register const char *sym,
+static int addsym PROTO ((char[], char *, int, hash_table, int));
+static struct hash_entry *findsym PROTO ((const char *sym,
 					  hash_table table,

 					  int table_size));
-static int hashfunct PROTO ((register const char *, int));
+static int hashfunct PROTO ((const char *, int));


 /* addsym - add symbol and definitions to symbol table
@@ -73,16 +73,16 @@ static int hashfunct PROTO ((register const char *, int));
 */

 static int addsym (sym, str_def, int_def, table, table_size)
-     register char sym[];
+     char sym[];
     char   *str_def;
     int     int_def;
     hash_table table;
     int     table_size;
 {
-	int     hash_val = hashfunct (sym, table_size);
-	register struct hash_entry *sym_entry = table[hash_val];
-	register struct hash_entry *new_entry;
-	register struct hash_entry *successor;
+	int    hash_val = hashfunct (sym, table_size);
+	struct hash_entry *sym_entry = table[hash_val];
+	struct hash_entry *new_entry;
+	struct hash_entry *successor;

 	while (sym_entry) {
 		if (!strcmp (sym, sym_entry->name)) {	/* entry already exists */
@@ -147,7 +147,7 @@ int     ccllookup (ccltxt)
 /* findsym - find symbol in symbol table */

 static struct hash_entry *findsym (sym, table, table_size)
-     register const char *sym;
+     const char *sym;
     hash_table table;
     int     table_size;
 {
@@ -155,7 +155,7 @@ static struct hash_entry *findsym (sym, table, table_size)
 		(struct hash_entry *) 0, (struct hash_entry *) 0,
 		(char *) 0, (char *) 0, 0,
 	};
-	register struct hash_entry *sym_entry =
+	struct hash_entry *sym_entry =

 		table[hashfunct (sym, table_size)];

@@ -171,11 +171,11 @@ static struct hash_entry *findsym (sym, table, table_size)
 /* hashfunct - compute the hash value for "str" and hash size "hash_size" */

 static int hashfunct (str, hash_size)
-     register const char *str;
+     const char *str;
     int     hash_size;
 {
-	register int hashval;
-	register int locstr;
+	int hashval;
+	int locstr;

 	hashval = 0;
 	locstr = 0;
@@ -65,7 +65,7 @@ static flex_int32_t yytbl_data_getijk (const struct yytbl_data *tbl, int i,

 /** Initialize the table writer.
 *  @param wr an uninitialized writer
- *  @param the output file
+ *  @param out the output file
 *  @return 0 on success
 */
 int yytbl_writer_init (struct yytbl_writer *wr, FILE * out)
@@ -96,7 +96,7 @@ int yytbl_hdr_init (struct yytbl_hdr *th, const char *version_str,
 }

 /** Allocate and initialize a table data structure.
- *  @param tbl a pointer to an uninitialized table
+ *  @param td a pointer to an uninitialized table
 *  @param id  the table identifier
 *  @return 0 on success
 */
@@ -137,7 +137,7 @@ static int yytbl_write_pad64 (struct yytbl_writer *wr)
 }

 /** write the header.
- *  @param out the output stream
+ *  @param wr the output stream
 *  @param th table header to be written
 *  @return -1 on error, or bytes written on success.
 */
@@ -183,7 +183,7 @@ int yytbl_hdr_fwrite (struct yytbl_writer *wr, const struct yytbl_hdr *th)


 /** Write this table.
- *  @param out the file writer
+ *  @param wr the file writer
 *  @param td table data to be written
 *  @return -1 on error, or bytes written on success.
 */
@@ -52,7 +52,7 @@ dnl

 /** Get the number of integers in this table. This is NOT the
 *  same thing as the number of elements.
- *  @param td the table 
+ *  @param tbl the table
 *  @return the number of integers in the table
 */
 yyskel_static flex_int32_t yytbl_calc_total_len (const struct yytbl_data *tbl)
@@ -36,7 +36,7 @@

 /* declarations for functions that have forward references */

-void mkentry PROTO ((register int *, int, int, int, int));
+void mkentry PROTO ((int *, int, int, int, int));
 void mkprot PROTO ((int[], int, int));
 void mktemplate PROTO ((int[], int, int));
 void mv2front PROTO ((int));
@@ -223,9 +223,9 @@ void    bldtbl (state, statenum, totaltrans, comstate, comfreq)

 void    cmptmps ()
 {
-	int     tmpstorage[CSIZE + 1];
-	register int *tmp = tmpstorage, i, j;
-	int     totaltrans, trans;
+	int tmpstorage[CSIZE + 1];
+	int *tmp = tmpstorage, i, j;
+	int totaltrans, trans;

 	peakpairs = numtemps * numecs + tblend;

@@ -291,7 +291,7 @@ void    cmptmps ()

 void    expand_nxt_chk ()
 {
-	register int old_max = current_max_xpairs;
+	int old_max = current_max_xpairs;

 	current_max_xpairs += MAX_XPAIRS_INCREMENT;

@@ -330,9 +330,9 @@ int     find_table_space (state, numtrans)
 	/* Firstfree is the position of the first possible occurrence of two
 	 * consecutive unused records in the chk and nxt arrays.
 	 */
-	register int i;
-	register int *state_ptr, *chk_ptr;
-	register int *ptr_to_last_entry_in_state;
+	int i;
+	int *state_ptr, *chk_ptr;
+	int *ptr_to_last_entry_in_state;

 	/* If there are too many out-transitions, put the state at the end of
 	 * nxt and chk.
@@ -421,7 +421,7 @@ int     find_table_space (state, numtrans)
 */
 void    inittbl ()
 {
-	register int i;
+	int i;

 	zero_out ((char *) chk,

@@ -501,11 +501,11 @@ void    mkdeftbl ()
 */

 void    mkentry (state, numchars, statenum, deflink, totaltrans)
-     register int *state;
-     int     numchars, statenum, deflink, totaltrans;
+     int *state;
+     int numchars, statenum, deflink, totaltrans;
 {
-	register int minec, maxec, i, baseaddr;
-	int     tblbase, tbllast;
+	int minec, maxec, i, baseaddr;
+	int tblbase, tbllast;

 	if (totaltrans == 0) {	/* there are no out-transitions */
 		if (deflink == JAMSTATE)
@@ -762,9 +762,9 @@ void    mv2front (qelm)
 void    place_state (state, statenum, transnum)
     int    *state, statenum, transnum;
 {
-	register int i;
-	register int *state_ptr;
-	int     position = find_table_space (state, transnum);
+	int i;
+	int *state_ptr;
+	int position = find_table_space (state, transnum);

 	/* "base" is the table of start positions. */
 	base[statenum] = position;
@@ -835,8 +835,8 @@ void    stack1 (statenum, sym, nextstate, deflink)
 int     tbldiff (state, pr, ext)
     int     state[], pr, ext[];
 {
-	register int i, *sp = state, *ep = ext, *protp;
-	register int numdiff = 0;
+	int i, *sp = state, *ep = ext, *protp;
+	int numdiff = 0;

 	protp = &protsave[numecs * (pr - 1)];

@@ -38,11 +38,11 @@

 /* yylex - scan for a regular expression token */

-int     yylex ()
+extern char *yytext;
+int     yylex (void)
 {
 	int     toktype;
 	static int beglin = false;
-	extern char *yytext;

 	if (eofseen)
 		toktype = EOF;
@@ -150,11 +150,12 @@ int     yylex ()
 				break;

 			default:
-				if (!isascii (yylval) || !isprint (yylval))
-					fprintf (stderr,
-						 "\\%.3o",
-						 (unsigned int) yylval);
-				else
+				if (!isascii (yylval) || !isprint (yylval)) {
+					if(trace_hex)
+						fprintf (stderr, "\\x%02x", (unsigned int) yylval);
+					else
+						fprintf (stderr, "\\%.3o", (unsigned int) yylval);
+				} else
 					(void) putc (yylval, stderr);
 				break;
 			}
@@ -1,11 +1,105 @@
-autom4te.cache
-config.cache
-configure
-config.in
-config.status
-config.h
-config.log
-Makefile
-aclocal.m4
-stamp-*
-Makefile.in
+*.log
+*.trs
+*.o
+*.tables
+alloc_extra
+alloc_extra.c
+array_nr
+array_nr.c
+array_r
+array_r.c
+basic_nr
+basic_nr.c
+basic_r
+basic_r.c
+bison_nr
+bison_nr_scanner.[ch]
+bison_nr_parser.[ch]
+bison_yylloc
+bison_yylloc_parser.[ch]
+bison_yylloc_scanner.[ch]
+bison_yylval
+bison_yylval_parser.[ch]
+bison_yylval_scanner.[ch]
+c_cxx_nr
+c_cxx_nr.cc
+c_cxx_r
+c_cxx_r.cc
+ccl
+ccl.c
+cxx_basic
+cxx_basic.cc
+cxx_multiple_scanners
+cxx_multiple_scanners_[12].cc
+cxx_yywrap.i3
+cxx_yywrap.cc
+debug_nr
+debug_nr.c
+debug_r
+debug_r.c
+extended
+extended.c
+header_nr
+header_nr_scanner.[ch]
+header_r
+header_r_scanner.[ch]
+include_by_buffer.direct
+include_by_buffer.direct.c
+include_by_push.direct
+include_by_push.direct.c
+include_by_reentrant.direct
+include_by_reentrant.direct.c
+lineno_nr.one
+lineno_nr.c
+lineno_r.one
+lineno_r.c
+lineno_trailing.one
+lineno_trailing.c
+mem_nr
+mem_nr.c
+mem_r
+mem_r.c
+multiple_scanners_nr
+multiple_scanners_nr_[12].[ch]
+multiple_scanners_r
+multiple_scanners_r_[12].[ch]
+noansi_nr
+noansi_nr.c
+noansi_r
+noansi_r.c
+posix
+posix.c
+posixly_correct
+posixly_correct.c
+prefix_nr
+prefix_nr.c
+prefix_r
+prefix_r.c
+pthread.pthread
+pthread.c
+reject_nr.reject
+reject_nr.reject.c
+reject_r.reject
+reject_r.reject.c
+reject_ver.table
+reject_ver.table.c
+reject_ser.table
+reject_ser.table.c
+rescan_nr.direct
+rescan_nr.direct.c
+rescan_r.direct
+rescan_r.direct.c
+quotes
+quotes.c
+string_nr
+string_nr.c
+string_r
+string_r.c
+top
+top.[ch]
+yyextra
+yyextra.c
+tableopts_*.c
+*.opt
+*.ser
+*.ver
@@ -19,129 +19,400 @@
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 # PURPOSE.

+TESTS = $(check_PROGRAMS) options.cn
+
+# The script testwrapper.sh will run most tests as is. A couple tests
+# in the suite end in .reject, .table and the like so that we can pass
+# different arguments to the test runner. We list those extensions so
+# automake knows how to distinguish between the various kinds of tests
+# we have.
+
+TEST_EXTENSIONS = .reject .table .direct .cn .i3 .pthread .one .opt .ser .ver
+
+LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_LOG_FLAGS = -d $(srcdir) -r
+
+REJECT_LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_REJECT_LOG_FLAGS = -d $(srcdir) -i reject.txt -r
+
+TABLE_LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_TABLE_LOG_FLAGS = -d $(srcdir) -i reject.txt -t
+
+DIRECT_LOG_COMPILER=$(srcdir)/testwrapper.sh
+AM_DIRECT_LOG_FLAGS = -d $(srcdir)
+
+CN_LOG_COMPILER=$(srcdir)/options.cn
+AM_CN_LOG_FLAGS=$(FLEX)
+
+I3_LOG_COMPILER=$(srcdir)/testwrapper.sh
+AM_I3_LOG_FLAGS=-i $(srcdir)/cxx_yywrap.txt -i $(srcdir)/cxx_yywrap.txt -i $(srcdir)/cxx_yywrap.txt
+
+PTHREAD_LOG_COMPILER=$(srcdir)/testwrapper.sh
+AM_PTHREAD_LOG_FLAGS=-i $(srcdir)/pthread_1.txt -i $(srcdir)/pthread_2.txt -i $(srcdir)/pthread_3.txt -i $(srcdir)/pthread_4.txt -i $(srcdir)/pthread_5.txt
+
+ONE_LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_ONE_LOG_FLAGS = -1 -d $(srcdir)
+
+AM_YFLAGS = -d -p test
+
+check_PROGRAMS = $(simple_tests) $(reject_tests) $(TABLE_TESTS) $(DIRECT_TESTS) $(I3_TESTS) $(PTHREAD_TESTS) $(ONE_TESTS) $(TABLEOPTS_TESTS)
+
+simple_tests = \
+	alloc_extra \
+	array_nr \
+	array_r \
+	basic_nr \
+	basic_r \
+	bison_nr \
+	bison_yylloc \
+	bison_yylval \
+	c_cxx_nr \
+	c_cxx_r \
+	ccl \
+	cxx_basic \
+	cxx_multiple_scanners \
+	debug_nr \
+	debug_r \
+	extended \
+	header_nr \
+	header_r \
+	mem_nr \
+	mem_r \
+	multiple_scanners_nr \
+	multiple_scanners_r \
+	noansi_nr \
+	noansi_r \
+	posix \
+	posixly_correct \
+	prefix_nr \
+	prefix_r \
+	quotes \
+	string_nr \
+string_r \
+	top \
+	yyextra
+
+reject_tests = \
+	reject_nr.reject \
+	reject_r.reject
+
+TABLE_TESTS = \
+	reject_ver.table \
+	reject_ser.table
+
+DIRECT_TESTS = \
+	include_by_buffer.direct \
+	include_by_push.direct \
+	include_by_reentrant.direct \
+	rescan_nr.direct \
+	rescan_r.direct
+
+ I3_TESTS = \
+	cxx_yywrap.i3
+
+if want_pthread
+PTHREAD_TESTS = \
+	pthread.pthread
+endif
+
+ONE_TESTS = \
+	lineno_nr.one \
+	lineno_r.one \
+	lineno_trailing.one
+
+alloc_extra_SOURCES = alloc_extra.l
+array_nr_SOURCES = array_nr.l
+array_r_SOURCES = array_r.l
+basic_nr_SOURCES = basic_nr.l
+basic_r_SOURCES = basic_r.l
+bison_nr_SOURCES = bison_nr_scanner.l bison_nr_parser.y bison_nr_main.c
+bison_yylloc_SOURCES = bison_yylloc_scanner.l bison_yylloc_parser.y bison_yylloc_main.c
+bison_yylval_SOURCES = bison_yylval_scanner.l bison_yylval_parser.y bison_yylval_main.c
+c_cxx_nr_SOURCES = c_cxx_nr.lll
+c_cxx_r_SOURCES = c_cxx_r.lll
+ccl_SOURCES = ccl.l
+cxx_basic_SOURCES = cxx_basic.ll
+cxx_multiple_scanners_SOURCES = cxx_multiple_scanners_main.cc cxx_multiple_scanners_1.ll cxx_multiple_scanners_2.ll
+cxx_yywrap_i3_SOURCES = cxx_yywrap.ll
+debug_nr_SOURCES = debug_nr.l
+debug_r_SOURCES = debug_r.l
+extended_SOURCES = extended.l
+header_nr_SOURCES = header_nr_scanner.l header_nr_main.c
+header_r_SOURCES = header_r_scanner.l header_r_main.c
+include_by_buffer_direct_SOURCES = include_by_buffer.direct.l
+include_by_push_direct_SOURCES = include_by_push.direct.l
+include_by_reentrant_direct_SOURCES = include_by_reentrant.direct.l
+lineno_nr_one_SOURCES = lineno_nr.l
+lineno_r_one_SOURCES = lineno_r.l
+lineno_trailing_one_SOURCES = lineno_trailing.l
+mem_nr_SOURCES = mem_nr.l
+mem_r_SOURCES = mem_r.l
+multiple_scanners_nr_SOURCES = multiple_scanners_nr_main.c multiple_scanners_nr_1.l multiple_scanners_nr_2.l
+multiple_scanners_r_SOURCES = multiple_scanners_r_main.c multiple_scanners_r_1.l multiple_scanners_r_2.l
+noansi_nr_SOURCES = noansi_nr.l
+noansi_r_SOURCES = noansi_r.l
+posix_SOURCES = posix.l
+posixly_correct_SOURCES = posixly_correct.l
+prefix_nr_SOURCES = prefix_nr.l
+prefix_r_SOURCES = prefix_r.l
+pthread_pthread_SOURCES = pthread.l
+quotes_SOURCES = quotes.l
+reject_nr_reject_SOURCES = reject.l4
+reject_r_reject_SOURCES = reject.l4
+reject_ver_table_SOURCES = reject.l4
+reject_ser_table_SOURCES = reject.l4
+rescan_nr_direct_SOURCES = rescan_nr.direct.l
+rescan_r_direct_SOURCES = rescan_r.direct.l
+string_nr_SOURCES = string_nr.l
+string_r_SOURCES = string_r.l
+top_SOURCES = top.l top_main.c
+yyextra_SOURCES = yyextra.l
+
+BUILT_SOURCES = \
+	bison_nr_scanner.h \
+	bison_nr_parser.h \
+	bison_yylloc_parser.h \
+	bison_yylloc_scanner.h \
+	bison_yylval_parser.h \
+	bison_yylval_scanner.h \
+	c_cxx_nr.cc \
+	c_cxx_r.cc \
+	header_nr_scanner.h \
+	header_r_scanner.h \
+	multiple_scanners_nr_1.h \
+	multiple_scanners_nr_2.h \
+	multiple_scanners_r_1.h \
+	multiple_scanners_r_2.h \
+	reject_nr.reject.c \
+	reject_r.reject.c \
+	reject_ver.table.c \
+	reject_ser.table.c \
+	top.h \
+	$(tableopts_c)
+
 EXTRA_DIST = \
-	README
+	README \
+	testwrapper.sh \
+	alloc_extra.txt \
+	array_nr.txt \
+	array_r.txt \
+basic_nr.txt \
+	basic_r.txt \
+	bison_nr_scanner.h \
+	bison_nr.txt \
+	bison_yylloc_scanner.h \
+	bison_yylloc.txt \
+	bison_yylval_scanner.h \
+	bison_yylval.txt \
+	c_cxx_nr.cc \
+	c_cxx_nr.txt \
+	c_cxx_r.cc \
+	c_cxx_r.txt \
+	ccl.txt \
+	cxx_basic.txt \
+	cxx_multiple_scanners.txt \
+	cxx_yywrap.txt \
+	debug_nr.txt \
+	debug_r.txt \
+	extended.txt \
+	header_nr_scanner.h \
+	header_nr.txt \
+	header_r_scanner.h \
+	header_r.txt \
+	include_by_buffer.direct.txt \
+	include_by_buffer.direct_2.txt \
+	include_by_buffer.direct_3.txt \
+	include_by_push.direct.txt \
+	include_by_push.direct_2.txt \
+	include_by_push.direct_3.txt \
+	include_by_reentrant.direct.txt \
+	include_by_reentrant.direct_2.txt \
+	include_by_reentrant.direct_3.txt \
+	lineno_nr.one.txt \
+	lineno_r.one.txt \
+	lineno_trailing.one.txt \
+	mem_nr.txt \
+	mem_r.txt \
+	noansi_nr.txt \
+	multiple_scanners_nr_1.h \
+	multiple_scanners_nr_2.h \
+	multiple_scanners_r_1.h \
+	multiple_scanners_r_2.h \
+	noansi_r.txt \
+	prefix_nr.txt \
+	prefix_r.txt \
+	pthread_1.txt \
+	pthread_2.txt \
+	pthread_3.txt \
+	pthread_4.txt \
+	pthread_5.txt \
+	reject.txt \
+	reject_nr.reject.c \
+	reject_r.reject.c \
+	reject_ver.table.c \
+	reject_ver.table.tables \
+	reject_ser.table.c \
+	reject_ser.table.tables \
+	rescan_nr.direct.txt \
+	rescan_r.direct.txt \
+	quotes.txt \
+	top.h \
+	top.txt \
+	yyextra.txt \
+	$(tableopts_c) \
+	$(tableopts_tables) \
+	tableopts.txt \
+	options.cn

-dist_noinst_SCRIPTS = \
-	create-test
+pthread_pthread_LDADD = -lpthread

-DIST_SUBDIRS = \
-	test-concatenated-options \
-	test-c++-yywrap \
-	test-extended \
-	test-ccl \
-	test-quotes \
-	test-rescan-r \
-	test-rescan-nr \
-	test-basic-nr \
-	test-basic-r \
-	test-bison-yylloc \
-	test-bison-yylval \
-	test-bison-nr \
-	test-multiple-scanners-nr \
-	test-multiple-scanners-r \
-	test-header-nr \
-	test-header-r \
-	test-reject \
-	test-c++-multiple-scanners \
-	test-c++-basic \
-	test-posixly-correct \
-	test-posix \
-	test-mem-r \
-	test-mem-nr \
-	test-debug-nr \
-	test-debug-r \
-	test-lineno-r \
-	test-lineno-nr \
-	test-lineno-trailing \
-	test-linedir-r \
-	TEMPLATE \
-	test-top \
-	test-array-nr \
-	test-array-r \
-	test-c-cpp-nr \
-	test-c-cpp-r \
-	test-include-by-buffer \
-	test-include-by-push \
-	test-include-by-reentrant \
-	test-prefix-nr \
-	test-prefix-r \
-	test-pthread \
-	test-string-nr \
-	test-string-r \
-	test-yyextra \
-	test-alloc-extra \
-	test-noansi-nr \
-	test-noansi-r \
-	test-table-opts
+# specify how to process .l files in order to test the flex built by make all

-SUBDIRS = \
-	test-concatenated-options \
-	test-c++-yywrap \
-	test-extended \
-	test-ccl \
-	test-quotes \
-	test-rescan-r \
-	test-rescan-nr \
-	test-basic-nr \
-	test-basic-r \
-	test-bison-yylloc \
-	test-bison-yylval \
-	test-bison-nr \
-	test-multiple-scanners-nr \
-	test-multiple-scanners-r \
-	test-header-nr \
-	test-header-r \
-	test-reject \
-	test-c++-multiple-scanners \
-	test-c++-basic \
-	test-posixly-correct \
-	test-posix \
-	test-mem-r \
-	test-mem-nr \
-	test-debug-nr \
-	test-debug-r \
-	test-lineno-r \
-	test-lineno-nr \
-	test-lineno-trailing \
-	test-linedir-r \
-	test-array-nr \
-	test-array-r \
-	test-c-cpp-nr \
-	test-c-cpp-r \
-	test-include-by-buffer \
-	test-include-by-push \
-	test-include-by-reentrant \
-	test-prefix-nr \
-	test-prefix-r \
-	test-pthread \
-	test-string-nr \
-	test-string-r \
-	test-yyextra \
-	test-alloc-extra \
-	test-noansi-nr \
-	test-noansi-r \
-	test-top \
-	test-table-opts
+FLEX = $(top_builddir)/src/flex

-# clean up before running the test suite so we dont test old builds of test code
+.l.c:
+	$(FLEX) -o $@ $<

-check-local: clean
-	NOK=;\
-	NFAIL=;\
-	for dir in $(SUBDIRS) ; do \
-		echo Executing test "$$dir" ; \
-		( cd "$$dir" && $(MAKE) test  > OUTPUT 2>&1 ) ; \
-		case $$? in \
-			0 )  echo Test "$$dir" succeeded.; \
-				NOK=0$$NOK;\
-				;; \
-			* )  echo Test "$$dir" FAILED. See "$$dir"/OUTPUT for details. ; \
-				NFAIL=0$$NFAIL; \
-				;; \
-		esac; \
-	done ; \
-	echo Results: ; \
-	echo Tests succeeded: `echo @ECHO_N@ "$$NOK@ECHO_C@"|wc -c`; \
-	echo Tests FAILED: `echo @ECHO_N@ "$$NFAIL@ECHO_C@"|wc -c` ; \
-	test "$$NFAIL" = ""
+.ll.cc:
+	$(FLEX) -+ -o $@ $<
+
+bison_nr_main.($OBJEXT): bison_nr_parser.h bison_nr_scanner.h
+bison_nr_scanner.h: bison_nr_scanner.c
+
+bison_yylloc_main.$(OBJEXT): bison_yylloc_parser.h bison_yylloc_scanner.h
+bison_yylloc_scanner.h: bison_yylloc_scanner.c
+
+bison_yylval_main.$(OBJEXT): bison_yylval_parser.h bison_yylval_scanner.h
+bison_yylval_scanner.h: bison_yylval_scanner.c
+
+# automake does not support compiling flex scanners output in C as C++
+# so we explicitly sayhow, using the .lll suffix for the lex input file
+
+.lll.cc:
+	$(FLEX) -o $@ $<
+
+header_nr_main.$(OBJEXT): header_nr_scanner.h
+header_nr_scanner.h: header_nr_scanner.c
+
+header_r_main.$(OBJEXT): header_r_scanner.h
+header_r_scanner.h: header_r_scanner.c
+
+include_by_buffer.direct$(EXE): include_by_buffer.direct.txt include_by_buffer.direct_2.txt include_by_buffer.direct_3.txt
+
+include_by_push.direct$(EXE): include_by_push.direct.txt include_by_push.direct_2.txt include_by_push.direct_3.txt
+
+include_by_reentrant.direct$(EXE): include_by_reentrant.direct.txt include_by_reentrant.direct_2.txt include_by_reentrant.direct_3.txt
+
+multiple_scanners_nr_main.$(OBJEXT): multiple_scanners_nr_1.h multiple_scanners_nr_2.h
+multiple_scanners_nr_1.h: multiple_scanners_nr_1.c
+multiple_scanners_nr_2.h: multiple_scanners_nr_2.c
+
+multiple_scanners_r_main.$(OBJEXT): multiple_scanners_r_1.h multiple_scanners_r_2.h
+multiple_scanners_r_1.h: multiple_scanners_r_1.c
+multiple_scanners_r_2.h: multiple_scanners_r_2.c
+
+posixly_correct.c: posixly_correct.l
+	POSIXLY_CORRECT=1 $(FLEX) -o $@ $<
+
+reject_nr.reject.c: reject.l4
+	$(FLEX) -o $@ $<
+
+reject_nr.reject$(EXEEXT): reject_nr.reject.$(OBJEXT)
+	$(LINK) $^
+
+reject_r.reject.c: reject.l4
+	$(FLEX) --reentrant -o $@ $<
+
+reject_r.reject.$(OBJEXT): reject_r.reject.c
+	$(COMPILE) -DTEST_IS_REENTRANT -c -o $@ $<
+
+reject_r.reject$(EXEEXT): reject_r.reject.$(OBJEXT)
+	$(LINK) $^
+
+reject_ver.table.c: reject.l4
+	$(FLEX) -o $@ --tables-verify --tables-file=$(basename $@).tables $<
+
+reject_ver.table.$(OBJEXT): reject_ver.table.c
+	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
+
+reject_ver.table$(EXEEXT): reject_ver.table.$(OBJEXT)
+	$(LINK) $^
+
+reject_ser.table.c: reject.l4
+	$(FLEX) -o $@ --tables-file=$(basename $@).tables $<
+
+reject_ser.table.$(OBJEXT): reject_ser.table.c
+	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
+
+reject_ser.table$(EXEEXT): reject_ser.table.$(OBJEXT)
+	$(LINK) $^
+
+top_main.$(OBJEXT): top.h
+top.h: top.c
+
+# We separate out the tableopts _SOURCES variables and the linking
+# rules for those programs because automake has no way to specify such
+# things with a loop in a variable (even though make can do such
+# things) and the resultant list is both long an unenlightening. And
+# it can be / is generated by a shell script, tableopts.sh.
+
+tableopts.am: tableopts.sh
+	$(srcdir)/tableopts.sh > $(srcdir)/tableopts.am
+
+include tableopts.am
+
+tableopts   := -Ca -Ce -Cf -CF -Cm -Cem -Cae -Caef -CaeF -Cam -Caem
+tableopts_opt_tests :=  $(foreach opt,$(tableopts), tableopts_opt_nr$(opt) tableopts_opt_r$(opt))
+tableopts_sertests :=  $(foreach opt,$(tableopts), tableopts_ser_nr$(opt) tableopts_ser_r$(opt))
+tableopts_vertests :=  $(foreach opt,$(tableopts), tableopts_ver_nr$(opt) tableopts_ver_r$(opt))
+tableopts_tests  := $(tableopts_opttests) $(tableopts_vertests) $(tableopts_sertests)
+
+tableopts_c := $(addsuffix .c,$(tableopts_tests))
+
+OPT_LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_OPT_LOG_FLAGS = -d $(srcdir) -i tableopts.txt -r
+
+tableopts_opt_nr%.c: tableopts.l4
+	$(FLEX) -P $(subst -,_,$(basename $(*F))) $* -o $@ $<
+
+tableopts_opt_nr%.$(OBJEXT): tableopts_opt_nr%.c 
+	$(COMPILE) -c -o $@ $<
+
+tableopts_opt_r%.c: tableopts.l4
+	$(FLEX) -P $(subst -,_,$(basename $(*F))) --reentrant $*  -o $@ $<
+
+tableopts_opt_r%.$(OBJEXT):  tableopts_opt_r%.c 
+	$(COMPILE) -DTEST_IS_REENTRANT -c -o $@ $<
+
+SER_LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_SER_LOG_FLAGS = -d $(srcdir) -i tableopts.txt -r -t
+
+tableopts_ser_nr%.c: tableopts.l4
+	$(FLEX) -P $(subst -,_,$(basename $(*F))) --tables-file="tableopts_ser_nr$*.ser.tables"  $* -o $@ $<
+
+tableopts_ser_nr%.$(OBJEXT): tableopts_ser_nr%.c 
+	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
+
+tableopts_ser_r%.c: tableopts.l4
+	$(FLEX) -P $(subst -,_,$(basename $(*F))) -R --tables-file="tableopts_ser_r$*.ser.tables" $*  -o $@ $<
+
+tableopts_ser_r%.$(OBJEXT):  tableopts_ser_r%.c
+	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -DTEST_IS_REENTRANT -c -o $@ $<
+
+VER_LOG_COMPILER = $(srcdir)/testwrapper.sh
+AM_VER_LOG_FLAGS = -d $(srcdir) -i tableopts.txt -r -t
+
+tableopts_ver_nr%.c: tableopts.l4
+	$(FLEX) -P $(subst -,_,$(basename $(*F))) --tables-file="tableopts_ver_nr$*.ver.tables" --tables-verify $* -o $@ $<
+
+tableopts_ver_nr%.$(OBJEXT): tableopts_ver_nr%.c 
+	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
+
+tableopts_ver_nr%.ver$(EXEEXT): tableopts_ver_nr%.$(OBJEXT)
+	$(LINK) -o $@ $^
+
+tableopts_ver_r%.c: tableopts.l4
+	$(FLEX) -P $(subst -,_,$(basename $(*F))) -R --tables-file="tableopts_ver_r$*.ver.tables" --tables-verify $*  -o $@ $<
+
+tableopts_ver_r%.$(OBJEXT):  tableopts_ver_r%.c  
+	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -DTEST_IS_REENTRANT -c -o $@ $<
@@ -8,14 +8,14 @@ this directory and its contents.

 * STRUCTURE OF THE TEST SUITE

-The test suite consists of several directories, each containing a
-scanner known to work with the most recent version of flex. In
+The testsuite consists of several tests. Each test is centered around
+a scanner known to work with the most recent version of flex. In
 general, after you modify your copy of the flex distribution, you
-should re-run the test suite. Some of the tests may require certain tools
-to be available (e.g., bison, diff). If any test returns an error or
-generates an error message, then your modifications *may* have broken
-a feature of flex. At a minimum, you'll want to investigate the
-failure and determine if it's truly significant.
+should re-run the test suite. Some of the tests may require certain
+tools to be available (e.g., bison, diff). If any test returns an
+error or generates an error message, then your modifications *may*
+have broken a feature of flex. At a minimum, you'll want to
+investigate the failure and determine if it's truly significant.

 * HOW TO RUN THE TEST SUITE

@@ -26,52 +26,32 @@ To build and execute all tests from the top level of the flex source tree:
 To build and execute a single test:

  $ cd tests/ # from the top level of the flex tree.
-  $ cd test-pthread-nr # for example
-  $ make test
+  $ make testname.log
+
+  where "testname" is the name of the test. This is an automake-ism
+  that will create (or re-create, if need be), a log of the particular
+  test run that you're working on.

 * HOW TO ADD A NEW TEST TO THE TEST SUITE

-**- RUN the script `create-test` found in this directory with a single
-argument of the name of the test you want to create.  If it fails with
-a message about a non-existent file `config.status', then run the
-configure script in the top-level directory and everything will be
-fine.  If it fails to work for you other than this, report it as a
-bug.
-
-** Modify the files in the newly created directory so that they test
-whatever feature of flex you are interested in.
+** List your test in the TESTS variable in Makefile.am in this
+   directory. Note that due to the large number of tests, we use
+   variables to group similar tests together. This also helps with
+   handling the automake test suite requirements. Hopefully your test
+   can be listed in SIMPLE_TESTS. You'll need to add the appropriate
+   automake _SOURCES variable as well. If you're unsure, then consult
+   the automake manual, paying attention to the parallel test harness
+   section.

 ** On success, your test should return zero.

 ** On error, your test should return 1 (one) and print a message to
-stderr, which will have been redirected to the file named, "OUTPUT",
-in your test's directory.
+stderr, which will have been redirected to the log file created by the
+automake test suite harness.

 ** If your test is skipped (e.g., because bison was not found), then
-   the test should return 2 (two). See "test-bison-nr/Makefile.am" for
-   an example.
+   the test should return 2 (two).

-** You must modify the last few lines of the top-level configure.in by
-   adding the Makefile for your test directory.  (This step is
-   done automatically by `create-test`.)
-
-** You must add the name of your test to the SUBDIRS variable in
-   tests/Makefile.am. (This is also done automatically for you by `create-test'.)
-
-** Add a description of your new test to the end of the file
-    `descriptions'. Remember to keep the description as brief as
-    possible, preferably to one line.
-
-** You will have to run the autogen.sh script in the top-level
-   directory as well as run the configure script in that
-   directory. (Note that running config.status may prove easier or
-   quicker.)
-
-** The easiest way for you to submit your new test to the flex
-   maintainers is by generating a patch. The flex maintainers only
-   need to have the Makefile.am, the flex input file, the test input
-   file (if there is one) and any other files necessary to compile the
-   test. You do not need to submit files generated by autoconf,
-   automake, configure etc. It would be helpful to include the file
-   .cvsignore which you will find in your test directory if you used
-   the `create-test' script.
+** Once your work is done, submit a patch via the flex development
+   mailing list, the github pull request mechanism or some other
+   suitable means.
@@ -1,8 +0,0 @@
-Makefile
-Makefile.in
-parser.c
-parser.h
-scanner.c
-TEMPLATE
-OUTPUT
-.deps
@@ -1,50 +0,0 @@
-# This file is part of flex.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-
-# Neither the name of the University nor the names of its contributors
-# may be used to endorse or promote products derived from this software
-# without specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE.
-
-FLEX = $(top_builddir)/flex
-
-builddir = @builddir@
-
-EXTRA_DIST = scanner.l parser.y test.input
-CLEANFILES = scanner.c scanner.h parser.c parser.h $(testname)$(EXEEXT) OUTPUT $(OBJS)
-OBJS = scanner.o # parser.o
-
-AM_CPPFLAGS = -I$(srcdir) -I$(builddir) -I$(top_srcdir) -I$(top_builddir)
-#LDFLAGS = $(top_srcdir)/libfl.a 
-#LFLAGS = --header="scanner.h"
-#YFLAGS = --defines --output=parser.c
-
-testname = TEMPLATE
-
-scanner.c: $(srcdir)/scanner.l
-	$(FLEX) $(LFLAGS) $<
-
-parser.c: $(srcdir)/parser.y
-	$(BISON) $(YFLAGS) $<
-
-$(testname)$(EXEEXT): $(OBJS)
-	$(CC) $(CFLAGS) -o $@ $(LDFLAGS) $(OBJS) $(LOADLIBES)
-
-test: $(testname)$(EXEEXT)
-	./$(testname)$(EXEEXT) < $(srcdir)/test.input
-
-.c.o:
-	$(CC) -c -o $@ $(AM_CPPFLAGS) $(CPPFLAGS) $(CFLAGS) $<
@@ -1,4 +0,0 @@
-Makefile
-parser.c
-parser.h
-scanner.c
@@ -1,61 +0,0 @@
-/*
- * This file is part of flex.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 
- * Neither the name of the University nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE.
- */
-
-/* A dummy template parser to build "parser.c" and "parser.h".
-   How to compile:
-   bison --defines --output-file="parser.c" --name-prefix="test" parser.y
- */
-%{
-#include <stdio.h>
-#include <stdlib.h>
-#include "config.h"
-
-#define YYERROR_VERBOSE 1       /* For debugging.   */
-/* #define YYPARSE_PARAM scanner  */ /* For pure bison parser. */
-/* #define YYLEX_PARAM   scanner  */ /* For reentrant flex. */
-
-
-int yyerror(char* msg);
-extern int testlex();
-
-%}
-
-%%
-
-rule:  ;
-
-%%
-
-int yyerror(char* msg) {
-    fprintf(stderr,"%s\n",msg);
-    return 0;
-}
-
-/*
-int
-main ( int argc, char** argv )
-{
-    yyparse ();
-    return 0;
-}
-*/
@@ -1,54 +0,0 @@
-/*
- * This file is part of flex.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 
- * Neither the name of the University nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE.
- */
-
-%{
-/* A template scanner file to build "scanner.c". */
-#include <stdio.h>
-#include <stdlib.h>
-#include "config.h"
-/*#include "parser.h" */
-
-%}
-
-%option 8bit outfile="scanner.c" prefix="test"
-%option nounput nomain noyywrap 
-%option warn
-
-
-%%
-
-.              { }
-
-%%
-
-int main(void);
-
-int
-main ()
-{
-    yyin = stdin;
-    yyout = stdout;
-    yylex();
-    printf("TEST RETURNING OK.\n");
-    return 0;
-}
@@ -48,8 +48,8 @@ void *yyalloc ( size_t size, yyscan_t  scanner );

 %}

-%option 8bit outfile="scanner.c" prefix="test"
-%option nounput nomain noyywrap nodefault
+%option 8bit  prefix="test"
+%option nounput nomain noyywrap nodefault noinput
 %option warn
 %option extra-type="struct Check *"
 %option reentrant
@@ -30,8 +30,8 @@

 %}

-%option 8bit outfile="scanner.c" prefix="test"
-%option nounput nomain noyywrap 
+%option 8bit prefix="test"
+%option nounput nomain noyywrap noinput
 %option warn array


@@ -30,8 +30,8 @@

 %}

-%option 8bit outfile="scanner.c" prefix="test"
-%option nounput nomain noyywrap 
+%option 8bit prefix="test"
+%option nounput nomain noyywrap noinput
 %option warn array reentrant


@@ -23,7 +23,6 @@

 /* TEST scanner.
   Basic non-reentrant scanner.
-   Compile with:  flex scanner.l

   Sample Input:
       # this is a comment
@@ -35,8 +34,8 @@
 #include "config.h"
 %}

-%option prefix="test" outfile="scanner.c" 
-%option nounput noyywrap noyylineno warn nodefault
+%option prefix="test"
+%option nounput noyywrap noyylineno warn nodefault noinput

 IDENT [[:alnum:]_-]
 WS    [[:blank:]]
@@ -63,4 +62,3 @@ int main ()
    printf("TEST RETURNING OK.\n");
    return 0;
 }
-
@@ -33,8 +33,8 @@
 #include "config.h"
 %}

-%option prefix="test" outfile="scanner.c" 
-%option nounput noyywrap noyylineno warn nodefault
+%option prefix="test"
+%option nounput noyywrap noyylineno warn nodefault noinput
 %option reentrant

 IDENT [[:alnum:]_-]
@@ -21,13 +21,17 @@
 * PURPOSE.
 */

-#include "parser.h"
-#include "scanner.h"
+#include "bison_nr_parser.h"
+#include "bison_nr_scanner.h"
+#include "bison_nr_scanner_char.h"

 extern int testparse(void);

 int main ( int argc, char** argv )
 {
+	(void)argc;
+	(void)argv;
+
    /*yydebug =1;*/
    testin = stdin;
    testparse ( );
@@ -30,17 +30,20 @@
 #include <stdlib.h>
 #include <string.h>
 #include "config.h"
+#include "bison_nr_parser.h"
+#include "bison_nr_scanner.h"
+#include "bison_nr_scanner_char.h"

 #define YYERROR_VERBOSE 1
 /* #define YYPARSE_PARAM scanner */
 /* #define YYLEX_PARAM   scanner */

-int yyerror(char* msg);
+int yyerror(const char* msg);
 extern int testget_lineno(void);


 /* A dummy function. A check against seg-faults in yylval->str. */
-int process_text(char* s) {
+int process_text(YY_CHAR* s) {
    int total =0;
    while(*s) {
        total += (int) *s;
@@ -56,7 +59,7 @@ int process_text(char* s) {

 %union  {
    int  lineno;
-    char * str;
+    YY_CHAR * str;
 }
 %token <str> IDENT
 %token <lineno> LINENO
@@ -89,7 +92,7 @@ line:

 %%

-int yyerror(char* msg) {
+int yyerror(const char* msg) {
    fprintf(stderr,"%s\n",msg);
    return 0;
 }
@@ -25,38 +25,33 @@
 /* The scanner expects to link to bison yylval . */
 #include <stdio.h>
 #include <stdlib.h>
-#include "parser.h"
+#include "bison_nr_parser.h"
+#include "strutils.h"
 #include "config.h"
-static char* STRDUP(char* s1);
 #define YY_EXTRA_TYPE int
 %}

-%option 8bit outfile="scanner.c" prefix="test"
+%option 8bit prefix="test"
 %option bison-locations yylineno
-%option nomain nounput noyy_top_state noyywrap nodefault warn
-%option prefix="test" header="scanner.h" yylineno
+%option prefix="test" header="bison_nr_scanner.h" yylineno
+%option nomain nounput noyy_top_state noyywrap nodefault noinput warn
+%option header-char="bison_nr_scanner_char.h"


 %%

 ^[[:digit:]]+  { 
        yylval->lineno = yylineno;
-        yylloc->first_line = (int)strtol(yytext,NULL,10);
+        yylloc->first_line = (int)yy_utils_strtol(yytext,NULL,10);
        return LINENO;
    }
 ":"  { return COLON; }
 " "  { return SPACE; }
 "="  { return EQUAL; }
-[[:alnum:]_]+ {  yylval->str = STRDUP(yytext); return IDENT;}
+[[:alnum:]_]+ {  yylval->str = yy_utils_strdup(yytext); return IDENT;}

 \r|\n { }
 .     { yyterminate();}
 %%


-static char* STRDUP(char* s1)
-{
-    char* s2 = (char*)malloc(strlen(s1)+1);
-    sprintf(s2,"%s",s1);
-    return s2;
-}
@@ -21,12 +21,17 @@
 * PURPOSE.
 */

-#include "parser.h"
-#include "scanner.h"
+#include "bison_yylloc_parser.h"
+#include "bison_yylloc_scanner.h"
+#include "bison_yylloc_scanner_char.h"

 int main ( int argc, char** argv )
 {
    yyscan_t scanner;
+
+    (void)argc;
+    (void)argv;
+
    /*yydebug =1;*/
    testlex_init ( &scanner );
    testset_in(stdin,scanner);
@@ -22,25 +22,30 @@
 */

 %parse-param { void* scanner }
+%lex-param { void* scanner }

 /* 
   How to compile:
-   bison --defines --output-file="parser.c" --name-prefix="test" parser.y
+   bison --defines --output-file="bison_yylloc_parser.c" --name-prefix="test" parser.y
 */
 %{
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "config.h"
+#include "bison_yylloc_parser.h"
+#include "bison_yylloc_scanner.h"
+#include "bison_yylloc_scanner_char.h"
+
+int yyerror(YYLTYPE *location, void* scanner, const char* msg);

 #define YYERROR_VERBOSE 1
-#define YYLEX_PARAM   scanner

 extern int testget_lineno(void*);


 /* A dummy function. A check against seg-faults in yylval->str. */
-int process_text(char* s) {
+int process_text(YY_CHAR* s) {
    int total =0;
    while(*s) {
        total += (int) *s;
@@ -52,11 +57,11 @@ int process_text(char* s) {

 %}

-%pure_parser
+%pure-parser

 %union  {
    int  lineno;
-    char * str;
+    YY_CHAR * str;
 }
 %token <str> IDENT
 %token <lineno> LINENO
@@ -78,7 +83,7 @@ line:
        /* Check lineno. */
        if( $1 != @1.first_line || $1 != testget_lineno(scanner))
        {
-            yyerror("Parse failed: Line numbers do not match.");
+            yyerror(0, 0, "Parse failed: Line numbers do not match.");
            YYABORT;
        }

@@ -89,7 +94,9 @@ line:

 %%

-int yyerror(void* scanner, char* msg) {
+int yyerror(YYLTYPE *location, void* scanner, const char* msg) {
+    (void)location;
+    (void)scanner;
    fprintf(stderr,"%s\n",msg);
    return 0;
 }
@@ -25,16 +25,18 @@
 /* The scanner expects to link to bison yylval . */
 #include <stdio.h>
 #include <stdlib.h>
-#include "parser.h"
+#include "bison_yylloc_parser.h"
 #include "config.h"
-static char* STRDUP(char* s1);
+#include "strutils.h"
 #define YY_EXTRA_TYPE int
 %}

-%option 8bit outfile="scanner.c" prefix="test"
+%option 8bit prefix="test"
 %option reentrant bison-bridge bison-locations yylineno
+%option header="bison_yylloc_scanner.h"
+%option nomain nounput noyy_top_state noyywrap nodefault noinput warn
 %option nomain nounput noyy_top_state noyywrap nodefault warn
-%option prefix="test" header="scanner.h"
+%option header-char="bison_yylloc_scanner_char.h"


 %%
@@ -44,22 +46,14 @@ static char* STRDUP(char* s1);

 ^[[:digit:]]+  { 
        yylval->lineno = yyextra++;
-        yylloc->first_line = (int)strtol(yytext,NULL,10);
+        yylloc->first_line = (int)yy_utils_strtol(yytext,NULL,10);
        return LINENO;
    }
 ":"  { return COLON; }
 " "  { return SPACE; }
 "="  { return EQUAL; }
-[[:alnum:]_]+ {  yylval->str = STRDUP(yytext); return IDENT;}
+[[:alnum:]_]+ {  yylval->str = yy_utils_strdup(yytext); return IDENT;}

 \r|\n { }
 .     { yyterminate();}
 %%
-
-
-static char* STRDUP(char* s1)
-{
-    char* s2 = (char*)malloc(strlen(s1)+1);
-    sprintf(s2,"%s",s1);
-    return s2;
-}
@@ -21,12 +21,17 @@
 * PURPOSE.
 */

-#include "parser.h"
-#include "scanner.h"
+<<<#include "bison_yylval_parser.h"
+#include "bison_yylval_scanner.h"
+#include "bison_yylval_scanner_char.h"

 int main ( int argc, char** argv )
 {
    yyscan_t scanner;
+
+    (void)argc;
+    (void)argv;
+
    /*yydebug =1;*/
    testlex_init ( &scanner );
    testset_in(stdin,scanner);
@@ -26,18 +26,22 @@
   bison --defines --output-file="parser.c" --name-prefix="test" parser.y
 */
 %parse-param { void* scanner }
+%lex-param { void* scanner }
 %{
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "config.h"
+#include "bison_yylval_parser.h"
+#include "bison_yylval_scanner.h"
+#include "bison_yylval_scanner_char.h"

 #define YYERROR_VERBOSE 1
-#define YYLEX_PARAM   scanner

+int yyerror(void* scanner, const char* msg);

 /* A dummy function. A check against seg-faults in yylval->str. */
-int process_text(char* s) {
+int process_text(YY_CHAR* s) {
    int total =0;
    while(*s) {
        total += (int) *s;
@@ -49,11 +53,11 @@ int process_text(char* s) {

 %}

-%pure_parser
+%pure-parser

 %union  {
    long unused;
-    char * str;
+    YY_CHAR * str;
 }

 %token <str> TAGNAME TEXT
@@ -74,7 +78,8 @@ starttag:  LT      TAGNAME GT { process_text($2); free($2);} ;
 endtag:    LTSLASH TAGNAME GT { process_text($2);free($2);} ;
 %%

-int yyerror(void* scanner, char* msg) {
+int yyerror(void* scanner, const char* msg) {
+    (void)scanner;
    fprintf(stderr,"%s\n",msg);
    return 0;
 }
@@ -25,18 +25,20 @@
 /* The scanner expects to link to bison yylval . */
 #include <stdio.h>
 #include <stdlib.h>
-#include "parser.h"
+#include "bison_yylval_parser.h"
 #include "config.h"
-static char* STRDUP(char* s1);
+#include "strutils.h"

 enum yesno_t { no=0, yes=1 };
 #define YY_EXTRA_TYPE  enum yesno_t
 %}

-%option 8bit outfile="scanner.c" prefix="test"
+%option 8bit prefix="test"
 %option reentrant bison-bridge
+%option header="bison_yylval_scanner.h"
+%option noyywrap nomain nounput noyy_top_state noyywrap nodefault noinput warn
 %option noyywrap nomain nounput noyy_top_state noyywrap nodefault warn
-%option prefix="test" header="scanner.h"
+%option header-char="bison_yylval_scanner_char.h"
 %option stack


@@ -52,14 +54,14 @@ enum yesno_t { no=0, yes=1 };
 "</"       { NEED_TAG_NAME= yes; yy_push_state( IN_TAG, yyscanner); return LTSLASH;}
 "<"[^[:alpha:]]  { yy_push_state(DISCARD_THRU_GT,yyscanner); }
 "<"        { NEED_TAG_NAME= yes; yy_push_state( IN_TAG, yyscanner); return  LT; }
-[^<]{1,512} { yyget_lval(yyscanner)->str = STRDUP(yytext); return TEXT;}
+[^<]{1,512} { yyget_lval(yyscanner)->str = yy_utils_strdup(yytext); return TEXT;}
 }
 <IN_TAG>{
 ">"  { yy_pop_state( yyscanner ); return GT; }
 [[:alpha:]][[:alnum:]]* {
        if( NEED_TAG_NAME == yes){
            NEED_TAG_NAME=no;
-            yylval->str = STRDUP(yytext);
+            yylval->str = yy_utils_strdup(yytext);
            return TAGNAME;
        }
    }
@@ -71,11 +73,3 @@ enum yesno_t { no=0, yes=1 };
 ">"        { yy_pop_state(yyscanner);}
 }
 %%
-
-
-static char* STRDUP(char* s1)
-{
-    char* s2 = (char*)malloc(strlen(s1)+1);
-    sprintf(s2,"%s",s1);
-    return s2;
-}
@@ -32,7 +32,7 @@

 %}

-%option 8bit outfile="scanner.cpp" prefix="test"
+%option 8bit prefix="test"
 %option nounput nomain noyywrap 
 %option warn

@@ -32,7 +32,7 @@

 %}

-%option 8bit outfile="scanner.cpp" prefix="test"
+%option 8bit prefix="test"
 %option nounput nomain noyywrap 
 %option warn reentrant

@@ -32,8 +32,8 @@
 #define a_ok()      do{printf("OK: flex line %d. input line %d.\n", __LINE__, yylineno); return 1;}while(0)
 %}

-%option 8bit outfile="scanner.c" prefix="test"
-%option nounput nomain noyywrap
+%option 8bit prefix="test"
+%option nounput nomain noyywrap noinput
 %option warn


@@ -1,33 +0,0 @@
-#!/bin/sh
-
-TESTFILES="Makefile.am scanner.l parser.y .cvsignore test.input"
-
-if [ ! $# -eq 1 ] ; then
-   echo 1>&2 Usage: $0 test-name
-   exit 1
-fi
-
-if test -e "$1" ; then
-    echo 1>&2 "$1 exists already"
-    exit 1
-fi
-
-mkdir $1 
-if test "$?" -ne 0 ; then
-    echo 1>&2 "mkdir $1 failed"
-    exit 1
-fi
-
-for i in $TESTFILES ; do
-    cp TEMPLATE/$i $1/$i
-done
-
-echo "$1" >> "$1"/.cvsignore
-
-sed -i '/--new-test-here--/i\
-tests/'"$1"'/Makefile' ../configure.in
-
-sed -i '/^\(DIST_\)\?SUBDIRS/a\
-	'"$1"' \\' Makefile.am
-
-sed -i "s:TEMPLATE:$1:g" "$1"/Makefile.am
@@ -27,10 +27,9 @@

 %}

-%option 8bit outfile="scanner.cpp" prefix="test"
-%option nounput nomain noyywrap 
+%option 8bit prefix="test"
 %option warn c++
-
+%option nounput nomain noinput noyywrap 

 %%

@@ -24,7 +24,7 @@

 %}

-%option 8bit outfile="scanner-1.cpp" prefix="S1_"
+%option 8bit prefix="S1_"
 %option nounput nomain noyywrap
 %option warn stack noyy_top_state

@@ -24,7 +24,7 @@

 %}

-%option 8bit outfile="scanner-2.cpp" prefix="S2_"
+%option 8bit prefix="S2_"
 %option nounput nomain
 %option warn stack noyy_top_state

@@ -38,13 +38,17 @@ main ( int argc, char** argv )
    S1_FlexLexer* S1 = new S1_FlexLexer;
    S2_FlexLexer* S2 = new S2_FlexLexer;

-    // scan simultaneously. 
-    while(S1_ok || S2_ok)
-    {
-        if (S1_ok)
-            S1_ok = S1->yylex();
-        if (S2_ok)
-            S2_ok = S2->yylex();
+    while(testlex(scanner)) {
+        YY_CHAR * text;
+        int line;
+        line = testget_lineno(scanner);
+        text = testget_text(scanner);
+        
+        if( (char*)testget_extra(scanner) != extra)
+            break;
+        
+        if ( !text || line < 0)
+            continue;
    }
    printf("TEST RETURNING OK.\n");
    delete S1;
@@ -28,7 +28,7 @@

 %}

-%option 8bit outfile="scanner.cpp" prefix="test"
+%option 8bit prefix="test"
 %option nounput nomain
 %option warn c++

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Mariusz Pluciński	227e731b76	Make charset support working with C++ scanners too	2014-11-30 19:22:43 -05:00
Mariusz Pluciński	4d6089fc83	Extend test-charset to check C++ scanner too	2014-11-30 19:22:43 -05:00
Mariusz Pluciński	114cc57e7f	Add implementation for charset support	2014-11-30 19:22:43 -05:00
Mariusz Pluciński	957471b78e	Add test cases for charset support	2014-11-30 19:21:31 -05:00
Mariusz Pluciński	f17e4fd073	Use YY_CHAR macro instead of `char` type in places where processed text is stored	2014-11-30 19:18:21 -05:00
Mariusz Pluciński	f2aa9e112f	Fix `label unused` warning	2014-11-26 15:31:53 -05:00
Mariusz Pluciński	a13574440b	Fix two "signed/unsigned" warnings	2014-11-26 13:26:49 -05:00
Mariusz Pluciński	874dd74ceb	Fix `unused function` warnings in tests	2014-11-26 13:26:49 -05:00
Mariusz Pluciński	17c292bf03	Fix `implicit function declaration` warnings in tests	2014-11-25 08:29:52 -05:00
Mariusz Pluciński	fef17765ca	Remove a few `unused variable` warnings	2014-11-25 08:28:24 -05:00
Mariusz Pluciński	9ba5477128	Remove unused variable from src/filter.c:filter_fix_linedirs	2014-11-25 08:23:06 -05:00
Mariusz Pluciński	6d1a6a8d59	Remove unused argument and variable from src/scanopt.c:scanopt_err	2014-11-23 20:29:49 -05:00
Mariusz Pluciński	71e207c1ac	Fix a few "unused parameter" warnings	2014-11-23 20:29:49 -05:00
Mariusz Pluciński	c0303615a6	Fix warning about redefined macro when multiple scanners are used.	2014-11-23 19:53:41 -05:00
Mariusz Pluciński	6920252234	Avoid passing `const char` argument as `char` in test-bison-nr	2014-11-23 19:53:41 -05:00
Alexis La Goutte	9c8daf867e	Fix -Wdocumentation warnings	2014-11-21 13:58:41 -05:00
Will Estes	d86e0bb1d4	document new suite layout	2014-11-16 16:33:15 -05:00
Will Estes	6b2c215cd7	remove unused files after test suite refactor	2014-11-16 16:32:59 -05:00
Will Estes	39413323dd	Ignore directories build-aux/, m4/ Since build-aux/ now contains a number of files previously at the top level, we ignore build-aux/. Therefore, it's not necessary to list any files that are now kept in it. Also, explicitly mark m4/ as a directory to ignore.	2014-11-14 08:41:37 -05:00
Will Estes	d6dfc4caa4	added back call to AC_CONFIG_AUX_DIR. In an effort to reduce top level directory clutter, reintroduced the call to AC_CONFIG_AUX_DIR. Moved the call to LT_INIT to after that call so configure will be able to find its files.	2014-11-14 08:29:44 -05:00
Will Estes	f4ec87b6d9	use unofficial bash strict mode and cleanups in supporting bash scripts	2014-11-12 05:14:28 -05:00
Will Estes	976cc15cf7	split apart table options tests into tests per threading, table option, serialization and verification	2014-11-12 05:14:28 -05:00
Will Estes	5336a9063d	add tableopts.sh script	2014-11-12 05:14:28 -05:00
Will Estes	2c64f2fa80	use automake conditional around pthread test	2014-11-12 05:14:28 -05:00
Will Estes	9146975ee4	refactor lineno_trailing test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	dd7eda5be0	refactor lineno_r test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	7442349dab	refactor lineno_nr test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	0c28dcc469	Remove linedir_r test. The linedir_r test tested the implementation of line number tracking, not its results.	2014-11-12 05:14:28 -05:00
Will Estes	5fb6428901	refactor pthread test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	01c6ea670b	build reject_[vs]er tests explicitly to pass proper compiler flags	2014-11-12 05:14:28 -05:00
Will Estes	9b9780e3c7	refactor cxx_yywrap test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	aaf002227e	refactor concatenated options test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	e98b2df022	refactor include_by_* tests for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	7563365413	refactor rescan_r test for new test suite layout	2014-11-12 05:14:28 -05:00
Will Estes	dd021b412c	Refactor rescan_nr test for new test suite layout. Also add -r option to testwrapper.sh to support passing input file as a command line argument to the test scanner without using shell redirection.	2014-11-12 05:14:27 -05:00
Will Estes	43e17a87db	correct use of objext to OBJEXT	2014-11-12 05:14:27 -05:00
Will Estes	db8e0d36da	refactor quotes test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	99e4880ccd	Refactor reject test for new test suite layout. Split out reject test into its constituant tests. Add .reject tests and .table tests for automake test log generation. Rewrite testwrapper.sh to handle running with a tables file and specifying optional input using command line options rather than positional parameters.	2014-11-12 05:14:27 -05:00
Will Estes	98d9a5af2b	Refactor multiple_scanners_r test for new test suite layout. Also, remove the use of table files from this test as that tests two features at once and we want to be as close to testing one feature at a time as we can be.	2014-11-12 05:14:27 -05:00
Will Estes	323c11e8ae	refactor multiple_scanners_nr test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	cc5649f1bd	refactor cxx_multiple_scanners test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	18abc5a0e6	refactor c_cxx_nr, c_cxx_r tests for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	278d524678	refactor cxx_basic test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	2d9cde08c7	refactor posixly_correct test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	435b336b2f	refactor yyextra test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	a0243585d2	refactor top test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	5c48969b76	refactor string_r test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	62d4a4a363	refactor string_nr test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	413d30f541	refactor prefix_r test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	47a32a9747	refactor prefix_nr for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	778b4742ed	Check if test input file exists. Not all tests have input files, so check if one exists and run the test program accordingly.	2014-11-12 05:14:27 -05:00
Will Estes	9334bcbe4a	refactor posix test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	df3b400a3e	refactor noansi_r test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	286625e8ea	refactor noansi_nr for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	a3a27a3cd9	refactor mem_r test for new test suite layout	2014-11-12 05:14:27 -05:00
Will Estes	6f00ec3623	refactor mem_nr test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	3086fc063c	refactor header_r test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	1c39506c5a	refactor header_nr test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	d342834a6b	refactor extended test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	cd1838ef1f	refactor debug_r test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	ccaa278506	refactor debug_nr test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	69d5f1065d	refactor ccl test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	ea001b9938	use underscores in test file names to silence automake warnings	2014-11-12 05:14:26 -05:00
Will Estes	120ce9c07d	do not track m4 subdirectory any more	2014-11-12 05:14:26 -05:00
Will Estes	53d5bc8876	upgrade automake version to 1.14.1 Unfortunately, automake 1.14.1 does not play nicely with AC_CONFIG_AUX_DIR, so upgrading the used automake version required removing the build-aux directory and letting autoconf's installed helper files live more over the tree.	2014-11-12 05:14:26 -05:00
Will Estes	6e57bbebe7	ignore more files generated by texinfo	2014-11-12 05:14:26 -05:00
Will Estes	c32c23bfc3	update gettext version to 0.19	2014-11-12 05:14:26 -05:00
Will Estes	1e5a73b1ce	call LT_INIT earlier in configure.ac	2014-11-12 05:14:26 -05:00
Will Estes	5b29e2ebb4	refactor bison-yylval test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	82f6a7cc2c	refactor bison-yylloc test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	bb00ab6892	refactor bison-nr test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	ce15230f47	refactor array-r test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	771464fb0a	list tests/README with other EXTRA_DIST files in tests/	2014-11-12 05:14:26 -05:00
Will Estes	3e580c57f1	refactor array-nr test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	fddc982e9f	make tests/testwrapper.sh more verbose; find input in srcdir Since output is redirected by the automake parallel test suite driver, turn on both -v and -x in bash for the testwrapper.sh shell script. This helps a ton in debugging problems with the test harness itself. In general, the input files are in automake's srcdir and the name of the test includes the relative path to it (even though that's supposed to be ./). Therefore, pass srcdir in AM_LOG_FLAGS and prepend that to the test name as part of constructing the input file's name.	2014-11-12 05:14:26 -05:00
Will Estes	4cf8cc7800	distribute tests/testwrapper.sh	2014-11-12 05:14:26 -05:00
Will Estes	5871608966	refacter alloc-extra for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	93b83d2cae	refactor basic-r test for new test suite layout	2014-11-12 05:14:26 -05:00
Will Estes	382be5cdc5	refactor basic-nr test for new test suite layout	2014-11-12 05:14:25 -05:00
Will Estes	52969184df	remove old tests/ subdirectories from build system	2014-11-12 05:14:25 -05:00
Will Estes	b6a001a167	add parallel test suite option to build system	2014-11-12 05:14:25 -05:00
Will Estes	ce21624b8b	ignore files for new test suite layout	2014-11-12 05:14:25 -05:00
Will Estes	57b1d32713	use automake parallel test suite option to build test suite	2014-11-12 05:14:25 -05:00
Manoj Srivastava	c376be574e	Do not use obsolete bison constructs in tests. In Bison 3.0, support for YYLEX_PARAM and YYPARSE_PARAM has been removed (deprecated in Bison 1.875): use %lex-param, %parse-param, or %param. This commit fixes the tests so they still work. Signed-off-by: Manoj Srivastava <srivasta@golden-gryphon.com>	2014-11-12 05:14:25 -05:00
Christos Zoulas	03a7f82cb4	NetBSD downstream patches. const fixes. -Wconversion fixes for the skeleton files. param namespace protection (add _ to inline function parameters). unused variable/code removal. rename warn to lwarn to avoid conflict with <err.h>. ctype.h function argument correction. merged the error functions lerrif and lerrsf -> lerr.	2014-11-11 21:09:36 -05:00
Mariusz Pluciński	2d257614dd	Allow error reporting routines to accept varying number of arguments in modern style	2014-07-25 09:48:16 -04:00
Mariusz Pluciński	681db87f8f	Fix warning on assigning from `const char` to `char`	2014-07-25 09:48:16 -04:00
Mariusz Pluciński	f46a8c9ae4	Add disambiguation braces in main.c	2014-07-25 08:16:30 -04:00
Yuri	8091dc9076	Removed deprecated 'register' storage class specifier. clang-3.5.0 now complains about them: warning: 'register' storage class specifier is deprecated [-Wdeprecated-register]	2014-07-25 08:13:41 -04:00
Mariusz Pluciński	3e0d96af4c	Change output formats from octal to hexadecimal	2014-06-26 16:39:39 -04:00
Will Estes	c979ed27bb	update version number to 2.6.0-pre	2014-06-12 16:43:01 -04:00
Will Estes	71aeaac9e8	check limits before using array index cclp; resolves sf-166	2014-05-03 09:23:49 -04:00
Sean McBride	4e470b426e	Suppress clang warning about empty @param paragraph; resolves sf#158 Signed-off-by: Will Estes <westes575@gmail.com>	2014-04-02 20:49:43 -04:00
Will Estes	1bcf050ca5	fix punction when talking about colon-bracket delimiters; resolves sf#167	2014-04-02 20:32:33 -04:00
Will Estes	5a5c70c3e9	remove proofreading comment	2014-04-02 20:23:13 -04:00
Will Estes	a87ac71ed8	Put angle brackets around start condition name; resolves bug #168 .	2014-04-02 20:18:08 -04:00
Michael Haubenwallner	6c429958e7	Fix malloc/realloc replacement, bug#151. Signed-off-by: Will Estes <westes575@gmail.com>	2014-04-02 16:51:22 -04:00
Will Estes	a7fa4719ba	change crlf line ending to lf	2014-04-02 15:59:14 -04:00
Will Estes	dc21228233	mention flex 2.5.39 release in NEWS file	2014-03-26 16:17:05 -04:00
Will Estes	fcac15c52f	initial default control file for shipper	2014-03-26 08:46:37 -04:00
Cyril Brulebois	14a1724dde	Adjust buffer sizes on ia64. From the debian change entry: > Finish fixing the ia64 buffer issue. Previous commits increased YY_READ_BUF_SIZE (where __ia64__ is defined) but left YY_BUF_SIZE unchanged, so that didn't fix the problem in the end. In the general case, the latter is twice the former. Therefore set it to the same ratio in the ia64 case. In general, this sort of architecture specific fix is not the path we want to take, but the cleanup should be done in a more organized way in the future and getting it working would be preferrable now.	2014-03-05 12:17:25 -05:00
Will Estes	5dbce367dc	move m4 make variable to src/Makefile.am. This prevents an error when building skel.c caused by the $(m4) make variable not being defined. Particularly nasty since skel.c would still be created, thus causing make to think skel.c was up to date.	2014-02-18 14:34:20 -05:00
Will Estes	e43ca6dd7e	remove unused devel/ subdirectory from codebase	2014-02-18 13:07:29 -05:00
Will Estes	4c3209e737	Add make rule to rsync latest .po files from translation project. The rule assumes that rsync is on the path and that there is exactly one domain listed in the DOMAIN make variable. The intent is that the rule will work with vpath builds.	2014-02-16 09:19:51 -05:00
Will Estes	cb88397c36	git ignore directories of the form flex-*	2014-02-16 09:19:51 -05:00
Will Estes	99613af984	list source files for translation as now being in src/	2014-02-16 09:19:51 -05:00
Will Estes	45fe3acd7e	Add tools/ directory. Since tools/git2cl is a dependency of ChangeLog, not distributing tools/git2cl with flex causes the "make dist" target to fail in the distributed tar ball.	2014-02-16 09:19:51 -05:00
Will Estes	b451ff4576	removes extraneous files from EXTRA_DIST as automake picks them up better without mentioning them	2014-02-16 09:19:51 -05:00
Will Estes	0177dfdc23	mention version 2.6.0 in release news	2014-02-16 09:19:51 -05:00
Will Estes	269620b900	move flex program sources into src/ directory The *.[chly] sources are now in the src directory. This implies a bunch of changes in Makefile.am and friends to account for the new location. The .gitignore files are now more local to places where various object files and generated source files occur.	2014-02-16 09:19:51 -05:00
Will Estes	638d8a1448	increment flex version to 2.6.0	2014-02-16 09:19:51 -05:00
				`@@ -1 +0,0 @@`
				`EXTRA_DIST = codeset.m4 gettext.m4 glibc21.m4 iconv.m4 intdiv0.m4 inttypes-pri.m4 inttypes.m4 inttypes_h.m4 isc-posix.m4 lcmessage.m4 lib-ld.m4 lib-link.m4 lib-prefix.m4 progtest.m4 stdint_h.m4 uintmax_t.m4 ulonglong.m4 nls.m4 po.m4`