shithub: trueawk

--- a/.gitignore

+++ b/.gitignore

@@ -2,3 +2,8 @@

 maketab

 proctab.c

*.o

+ytab*

+testdir/foo*

+testdir/temp*

+*.pdf

+*.mail

--- a/FIXES

+++ b/FIXES

@@ -22,1408 +22,28 @@

 THIS SOFTWARE.

 ****************************************************************/

-This file lists all bug fixes, changes, etc., made since the AWK book

-was sent to the printers in August 1987.

+This file lists all bug fixes, changes, etc., made since the

+second edition of the AWK book was published in September 2023.

-Sep 06, 2023:

-	Fix edge case where FS is changed on commandline. Thanks to

-	Gordon Shephard and Miguel Pineiro Jr.

+[This entry is a summary, not a precise list of changes.]

-	Fix regular expression clobbering in the lexer, where lexer does

-	not make a copy of regexp literals. also makedfa memory leaks have

-	been plugged. Thanks to Miguel Pineiro Jr.

-Dec 15, 2022:

-	Force hex escapes in strings to be no more than two characters,

-	as they already are in regular expressions. This brings internal

-	consistency, as well as consistency with gawk. Thanks to

-	Arnold Robbins.

+	Added --csv option to enable processing of comma-separated

+	values inputs.  When --csv is enabled, fields are separated

+	by commas, fields may be quoted with " double quotes, fields

+	may contain embedded newlines.

-Sep 12, 2022:

-	adjbuf minlen error (cannot be 0) in cat, resulting in NULL pbuf.

-	discovered by todd miller. also use-after-free issue with

-	tempfree in cat, thanks to Miguel Pineiro Jr and valgrind.

+	If no explicit separator argument is provided, split() uses

+	the setting of --csv to determine how fields are split.

-Aug 30, 2022:

-	Various leaks and use-after-free issues plugged/fixed.

-	Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

+	Strings may now contain UTF-8 code points (not necessarily

+	characters).  Functions that operate on characters, like

+	length, substr, index, match, etc., use UTF-8, so the length

+	of a string of 3 emojis is 3, not 12 as it would be if bytes

+	were counted.

-May 23, 2022:

-	Memory leak when assigning a string to some of the built-in

-	variables. allocated string erroneously marked DONTFREE.

-	Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

+	Regular expressions are processes as UTF-8.

-Mar 14, 2022:

-	Historic bug: command-line "name=value" assignment had been

-	truncating its entry in ARGV. (circa 1989) Thanks to

-	Miguel Pineiro Jr. <mpj@pineiro.cc>.

-Mar 3, 2022:

-	Fixed file management memory leak that appears to have been

-	there since the files array was first initialized with stdin,

-	stdout, and stderr (circa 1992). Thanks to Miguel Pineiro Jr.

-	<mpj@pineiro.cc>.

-December 8, 2021:

-	The error handling in closefile and closeall was mangled. Long

-	standing warnings had been made fatal and some fatal errors went

-	undetected. Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

-Nov 03, 2021:

-        getline accesses uninitialized data after getrec()

-	returns 0 on EOF and leaves the contents of buf unchanged.

-	Thanks to Volodymyr Gubarkov, and Todd C Miller.

-Oct 12, 2021:

-	The fix for #83 changed the code to insert 2 chars, but the

-	call to adjbuf just above it only allows for 1 char. This can

-	cause a heap buffer overflow.

-July 27, 2021:

-	As per IEEE Std 1003.1-2008, -F "str" is now consistent with

-	-v FS="str" when str is null. Thanks to Warner Losh.

-July 24, 2021:

-	Fix readrec's definition of a record. This fixes an issue

-	with NetBSD's RS regular expression support that can cause

-	an infinite read loop. Thanks to Miguel Pineiro Jr.

-	Fix regular expression RS ^-anchoring. RS ^-anchoring needs to

-	know if it is reading the first record of a file. This change

-	restores a missing line that was overlooked when porting NetBSD's

-	RS regex functionality. Thanks to Miguel Pineiro Jr.

-	Fix size computation in replace_repeat() for special case

-	REPEAT_WITH_Q. Thanks to Todd C. Miller.

-February 15, 2021:

-	Small fix so that awk will compile again with g++. Thanks to

-	Arnold Robbins.

-January 06, 2021:

-	Fix a decision bug with trailing stuff in lib.c:is_valid_number

-	after recent changes. Thanks to Ozan Yigit.

-December 18, 2020:

-	Fix problems converting inf and NaN values in lib.c:is_valid_number.

-	Enhance number to string conversion to do the right thing for

-	NaN and inf values.  Things are now pretty much the same as in

-	gawk.  (Found a gawk bug while we're at it.) Added a torture

-	test for these values.  Thanks to Arnold Robbins.  Allows closing

-	of PR #101.

-December 15, 2020:

-	Merge PR #99, which gets the right header for strcasecmp.

-	Thanks to GitHub user michaelforney.

-December 8, 2020:

-	Merge PR #98: Disallow hex data. Allow only +nan, -nan,

-	+inf, -inf (case independent) to give NaN and infinity values.

-	Improve things so that string to double conversion is only

-	done once, yielding something of a speedup.  This obviate

-	PR #95. Thanks to Arnold Robbins.

-December 3, 2020:

-	Fix to argument parsing to avoid printing spurious newlines.

-	Thanks to Todd Miller. Merges PR #97.

-October 13, 2020:

-	Add casts before all the calls to malloc/calloc/realloc in order

-	to get it to compile with g++. Thanks to Arnold Robbins.

-August 16, 2020:

-	Additional fixes for DJGPP. Thanks to Eli Zaretskii for

-	the testing.

-August 7, 2020:

-	Merge PR #93, which adds casts to (void*) for debug prints

-	using the %p format specifier. Thanks to GitHub user YongHaoWu

-	("Chris") for the fixes.

-August 4, 2020:

-	In run.c, use non-restartable multibyte routines to attain

-	portability to DJGPP. Should fix Issue 92. Thanks to Albert Wik

-	for the report and to Todd Miller for the suggested fix.

-July 30, 2020:

-	Merge PRs 88-91 which fix small bugs. Thanks to Todd Miller and

-	Tim van der Molen for the fixes.

-	In order to make life easier, we move exclusively to bison

-	as the parser generator.

-July 2, 2020:

-	Merge PRs 85 and 86 which fix regressions. Thanks to

-	Tim van der Molen for the fixes.

-June 25, 2020:

-	Merge PRs 82 and 84. The latter fixes issue #83. Thanks to

-	Todd Miller and awkfan77.

-June 12, 2020:

-	Clear errno before calling errcheck to avoid any spurious errors

-	left over from previous calls that may have set it. Thanks to

-	Todd Miller for the fix, from PR #80.

-	Fix Issue #78 by allowing \r to follow floating point numbers in

-	lib.c:is_number. Thanks to GitHub user ajcarr for the report

-	and to Arnold Robbins for the fix.

-June 5, 2020:

-	In fldbld(), make sure that inputFS is set before trying to

-	use it. Thanks to  Steffen Nurpmeso <steffen@sdaoden.eu>

-	for the report.

-May 5, 2020:

-	Fix checks for compilers that can handle noreturn. Thanks to

-	GitHub user enh-google for pointing it out. Closes Issue #79.

-April 16, 2020:

-	Handle old compilers that don't support C11 (for noreturn).

-	Thanks to Arnold Robbins.

-April 5, 2020:

-	Use <stdnoreturn.h> and noreturn instead of GCC attributes.

-	Thanks to GitHub user awkfan77. Closes PR #77.

-February 28, 2020:

-	More cleanups from Christos Zoulas: notably backslash continuation

-	inside strings removes the newline and a fix for RS = "^a".

-	Fix for address sanitizer-found problem. Thanks to GitHub user

-	enh-google.

-February 19, 2020:

-	More small cleanups from Christos Zoulas.

-February 18, 2020:

-	Additional cleanups from Christos Zoulas. It's no longer necessary

-	to use the -y flag to bison.

-February 6, 2020:

-	Additional small cleanups from Christos Zoulas. awk is now

-	a little more robust about reporting I/O errors upon exit.

-January 31, 2020:

-	Merge PR #70, which avoids use of variable length arrays. Thanks

-	to GitHub user michaelforney.  Fix issue #60 ({0} in interval

-	expressions doesn't work).  Also get all tests working again.

-	Thanks to Arnold Robbins.

-January 24, 2020:

-	A number of small cleanups from Christos Zoulas.  Add the close

-	on exec flag to files/pipes opened for redirection; courtesy of

-	Arnold Robbins.

-January 19, 2020:

-	If POSIXLY_CORRECT is set in the environment, then sub and gsub

-	use POSIX rules for multiple backslashes.  This fixes Issue #66,

-	while maintaining backwards compatibility.

-January 9, 2020:

-	Input/output errors on closing files are now fatal instead of

-	mere warnings. Thanks to Martijn Dekker <martijn@inlv.org>.

-January 5, 2020:

-	Fix a bug in the concatentation of two string constants into

-	one done in the grammar.  Fixes GitHub issue #61.  Thanks

-	to GitHub user awkfan77 for pointing out the direction for

-	the fix.  New test T.concat added to the test suite.

-	Fix a few memory leaks reported by valgrind, as well.

-December 27, 2019:

-	Fix a bug whereby a{0,3} could match four a's.  Thanks to

-	"Anonymous AWK fan" for the report.

-December 11, 2019:

-	Further printf-related fixes for 32 bit systems.

-	Thanks again to Christos Zoulas.

-December 8, 2019:

-	Fix the return value of sprintf("%d") on 32 bit systems.

-	Thanks to Jim Lowe for the report and to Christos Zoulas

-	for the fix.

-November 10, 2019:

-	Convert a number of Boolean integer variables into

-	actual bools. Convert compile_time variable into an

-	enum and simplify some of the related code.  Thanks

-	to Arnold Robbins.

-November 8, 2019:

-	Fix from Ori Bernstein to get UTF-8 characters instead of

-	bytes when FS = "".  This is currently the only bit of

-	the One True Awk that understands multibyte characters.

-	From Arnold Robbins, apply some cleanups in the test suite.

-October 25, 2019:

-	More fixes and cleanups from NetBSD, courtesy of Christos

-	Zoulas. Merges PRs 54 and 55.

-October 24, 2019:

-	Import second round of code cleanups from NetBSD. Much thanks

-	to Christos Zoulas (GitHub user zoulasc). Merges PR 53.

-	Add an optimization for string concatenation, also from

-	Christos.

-October 17, 2019:

-	Import code cleanups from NetBSD. Much thanks to Christos

-	Zoulas (GitHub user zoulasc). Merges PR 51.

-October 6, 2019:

-	Import code from NetBSD awk that implements RS as a regular

-	expression.

-September 10, 2019:

-	Fixes for various array / memory overruns found via gcc's

-	-fsanitize=unknown. Thanks to Alexander Richardson (GitHub

-	user arichardson). Merges PRs 47 and 48.

-July 28, 2019:

-	Import grammar optimization from NetBSD: Two string constants

-	concatenated together get turned into a single string.

-July 26, 2019:

-	Support POSIX-specified C-style escape sequences "\a" (alarm)

-	and "\v" (vertical tab) in command line arguments and regular

-	expressions, further to the support for them in strings added on

-	Apr 9, 1989. These now no longer match as literal "a" and "v"

-	characters (as they don't on other awk implementations).

-	Thanks to Martijn Dekker.

-July 17, 2019:

-	Pull in a number of code cleanups and minor fixes from

-	Warner Losh's bsd-ota branch.  The only user visible change

-	is the use of random(3) as the random number generator.

-	Thanks to Warner Losh for collecting all these fixes in

-	one easy place to get them from.

-July 16, 2019:

-	Fix field splitting to use FS value as of the time a record

-	was read or assigned to.  Thanks to GitHub user Cody Mello (melloc)

-	for the fix. (Merged from his branch, via PR #42.) Updated

-	testdir/T.split per said PR as well.

-June 24, 2019:

-	Extract awktest.tar into testdir directory. Add some very

-	simple mechanics to the makefile for running the tests and

-	for cleaning up. No changes to awk itself.

-June 17, 2019:

-	Disallow deleting SYMTAB and its elements, which creates

-	use-after-free bugs. Thanks to GitHub user Cody Mello (melloc)

-	for the fix. (Merged from PR #43.)

-June 5, 2019:

-	Allow unmatched right parenthesis in a regular expression to

-	be treated literally. Fixes Issue #40. Thanks to GitHub user

-	Warner Losh (bsdimp) for the report. Thanks to Arnold Robbins

-	for the fix.

-May 29,2019:

-	Fix check for command line arguments to no longer require that

-	first character after '=' not be another '='. Reverts change of

-	August 11, 1989. Thanks to GitHub user Jamie Landeg Jones for

-	pointing out the issue; from Issue #38.

-Apr 7, 2019:

-	Update awktest.tar(p.50) to use modern options to sort. Needed

-	for Android development. Thanks to GitHub user mohd-akram (Mohamed

-	Akram).  From Issue #33.

-Mar 12, 2019:

-	Added very simplistic support for cross-compiling in the

-	makefile.  We are NOT going to go in the direction of the

-	autotools, though.  Thanks to GitHub user nee-san for

-	the basic change. (Merged from PR #34.)

-Mar 5, 2019:

-	Added support for POSIX-standard interval expressions (a.k.a.

-	bounds, a.k.a. repetition expressions) in regular expressions,

-	backported (via NetBSD) from Apple awk-24 (20070501).

-	Thanks to Martijn Dekker <martijn@inlv.org> for the port.

-	(Merged from PR #30.)

-Mar 3, 2019:

-	Merge PRs as follows:

-	#12: Avoid undefined behaviour when using ctype(3) functions in

-	     relex(). Thanks to GitHub user iamleot.

-	#31: Make getline handle numeric strings, and update FIXES. Thanks

-	     to GitHub user Arnold Robbins (arnoldrobbins)

-	#32: maketab: support build systems with read-only source. Thanks

-	     to GitHub user enh.

-Jan 25, 2019:

-	Make getline handle numeric strings properly in all cases.

-	(Thanks, Arnold.)

-Jan 21, 2019:

-	Merged a number of small fixes from GitHub pull requests.

-	Thanks to GitHub users Arnold Robbins (arnoldrobbins),

-	Cody Mello (melloc) and Christoph Junghans (junghans).

-	PR numbers: 13-21, 23, 24, 27.

-Oct 25, 2018:

-	Added test in maketab.c to prevent generating a proctab entry

-	for YYSTYPE_IS_DEFINED.  It was harmless but some gcc settings

-	generated a warning message.  Thanks to Nan Xiao for report.

-Aug 27, 2018:

-	Disallow '$' in printf formats; arguments evaluated in order

-	and printed in order.

-	Added some casts to silence warnings on debugging printfs.

-	(Thanks, Arnold.)

-Aug 23, 2018:

-        A long list of fixes courtesy of Arnold Robbins,

-        to whom profound thanks.

-        1. ofs-rebuild: OFS value used to rebuild the record was incorrect.

-        Fixed August 19, 2014. Revised fix August 2018.

-        2. system-status: Instead of a floating-point division by 256, use

-        the wait(2) macros to create a reasonable exit status.

-        Fixed March 12, 2016.

-        3. space: Use provided xisblank() function instead of ispace() for

-        matching [[:blank:]].

-        4. a-format: Add POSIX standard %a and %A to supported formats. Check

-        at runtime that this format is available.

-        5. decr-NF: Decrementing NF did not change $0. This is a decades-old

-        bug. There are interactions with the old and new value of OFS as well.

-        Most of the fix came from the NetBSD awk.

-        6. string-conv: String conversions of scalars were sticky.  Once a

-        conversion to string happened, even with OFMT, that value was used until

-        a new numeric value was assigned, even if OFMT differed from CONVFMT,

-        and also if CONVFMT changed.

-        7. unary-plus: Unary plus on a string constant returned the string.

-        Instead, it should convert the value to numeric and give that value.

-	Also added Arnold's tests for these to awktest.tar as T.arnold.

-Aug 15, 2018:

-	fixed mangled awktest.tar (thanks, Arnold), posted all

-	current (very minor) fixes to github / onetrueawk

-Jun 7, 2018:

-	(yes, a long layoff)

-	Updated some broken tests (beebe.tar, T.lilly)

-	[thanks to Arnold Robbins]

-Mar 26, 2015:

-	buffer overflow in error reporting; thanks to tobias ulmer

-	and john-mark gurney for spotting it and the fix.

-Feb 4, 2013:

-	cleaned up a handful of tests that didn't seem to actually

-	test for correct behavior: T.latin1, T.gawk.

-Jan 5, 2013:

-	added ,NULL initializer to static Cells in run.c; not really

-	needed but cleaner.  Thanks to Michael Bombardieri.

-Dec 20, 2012:

-	fiddled makefile to get correct yacc and bison flags.  pick yacc

-	(linux) or bison (mac) as necessary.

-	added  __attribute__((__noreturn__)) to a couple of lines in

-	proto.h, to silence someone's enthusiastic checker.

-	fixed obscure call by value bug in split(a[1],a) reported on

-	9fans.  the management of temporary values is just a mess; i

-	took a shortcut by making an extra string copy.  thanks

-	to paul patience and arnold robbins for passing it on and for

-	proposed patches.

-	tiny fiddle in setfval to eliminate -0 results in T.expr, which

-	has irritated me for 20+ years.

-Aug 10, 2011:

-	another fix to avoid core dump with delete(ARGV); again, many thanks

-	to ruslan ermilov.

-Aug 7, 2011:

-	split(s, a, //) now behaves the same as split(s, a, "")

-Jun 12, 2011:

-	/pat/, \n /pat/ {...} is now legal, though bad style to use.

-	added checks to new -v code that permits -vnospace; thanks to

-	ruslan ermilov for spotting this and providing the patch.

-	removed fixed limit on number of open files; thanks to aleksey

-	cheusov and christos zoulos.

-	fixed day 1 bug that resurrected deleted elements of ARGV when

-	used as filenames (in lib.c).

-	minor type fiddles to make gcc -Wall -pedantic happier (but not

-	totally so); turned on -fno-strict-aliasing in makefile.

-May 6, 2011:

-	added #ifdef for isblank.

-	now allows -ffoo as well as -f foo arguments.

-	(thanks, ruslan)

-May 1, 2011:

-	after advice from todd miller, kevin lo, ruslan ermilov,

-	and arnold robbins, changed srand() to return the previous

-	seed (which is 1 on the first call of srand).  the seed is

-	an Awkfloat internally though converted to unsigned int to

-	pass to the library srand().  thanks, everyone.

-	fixed a subtle (and i hope low-probability) overflow error

-	in fldbld, by adding space for one extra \0.  thanks to

-	robert bassett for spotting this one and providing a fix.

-	removed the files related to compilation on windows.  i no

-	longer have anything like a current windows environment, so

-	i can't test any of it.

-May 23, 2010:

-	fixed long-standing overflow bug in run.c; many thanks to

-	nelson beebe for spotting it and providing the fix.

-	fixed bug that didn't parse -vd=1 properly; thanks to santiago

-	vila for spotting it.

-Feb 8, 2010:

-	i give up.  replaced isblank with isspace in b.c; there are

-	no consistent header files.

-Nov 26, 2009:

-	fixed a long-standing issue with when FS takes effect.  a

-	change to FS is now noticed immediately for subsequent splits.

-	changed the name getline() to awkgetline() to avoid yet another

-	name conflict somewhere.

-Feb 11, 2009:

-	temporarily for now defined HAS_ISBLANK, since that seems to

-	be the best way through the thicket.  isblank arrived in C99,

-	but seems to be arriving at different systems at different

-	times.

-Oct 8, 2008:

-	fixed typo in b.c that set tmpvec wrongly.  no one had ever

-	run into the problem, apparently.  thanks to alistair crooks.

-Oct 23, 2007:

-	minor fix in lib.c: increase inputFS to 100, change malloc

-	for fields to n+1.

-	fixed memory fault caused by out of order test in setsval.

-	thanks to david o'brien, freebsd, for both fixes.

-May 1, 2007:

-	fiddle in makefile to fix for BSD make; thanks to igor sobrado.

-Mar 31, 2007:

-	fixed some null pointer refs calling adjbuf.

-Feb 21, 2007:

-	fixed a bug in matching the null RE in sub and gsub.  thanks to al aho

-	who actually did the fix (in b.c), and to wolfgang seeberg for finding

-	it and providing a very compact test case.

-	fixed quotation in b.c; thanks to Hal Pratt and the Princeton Dante

-	Project.

-	removed some no-effect asserts in run.c.

-	fiddled maketab.c to not complain about bison-generated values.

-	removed the obsolete -V argument; fixed --version to print the

-	version and exit.

-	fixed wording and an outright error in the usage message; thanks to igor

-	sobrado and jason mcintyre.

-	fixed a bug in -d that caused core dump if no program followed.

-Jan 1, 2007:

-	dropped mac.code from makefile; there are few non-MacOSX

-	mac's these days.

-Jan 17, 2006:

-	system() not flagged as unsafe in the unadvertised -safe option.

-	found it while enhancing tests before shipping the ;login: article.

-	practice what you preach.

-	removed the 9-years-obsolete -mr and -mf flags.

-	added -version and --version options.

-	core dump on linux with BEGIN {nextfile}, now fixed.

-	removed some #ifdef's in run.c and lex.c that appear to no

-	longer be necessary.

-Apr 24, 2005:

-	modified lib.c so that values of $0 et al are preserved in the END

-	block, apparently as required by posix.  thanks to havard eidnes

-	for the report and code.

-Jan 14, 2005:

-	fixed infinite loop in parsing, originally found by brian tsang.

-	thanks to arnold robbins for a suggestion that started me

-	rethinking it.

-Dec 31, 2004:

-	prevent overflow of -f array in main, head off potential error in

-	call of SYNTAX(), test malloc return in lib.c, all with thanks to

-	todd miller.

-Dec 22, 2004:

-	cranked up size of NCHARS; coverity thinks it can be overrun with

-	smaller size, and i think that's right.  added some assertions to b.c

-	to catch places where it might overrun.  the RE code is still fragile.

-Dec 5, 2004:

-	fixed a couple of overflow problems with ridiculous field numbers:

-	e.g., print $(2^32-1).  thanks to ruslan ermilov, giorgos keramidas

-	and david o'brien at freebsd.org for patches.  this really should

-	be re-done from scratch.

-Nov 21, 2004:

-	fixed another 25-year-old RE bug, in split.  it's another failure

-	to (re-)initialize.  thanks to steve fisher for spotting this and

-	providing a good test case.

-Nov 22, 2003:

-	fixed a bug in regular expressions that dates (so help me) from 1977;

-	it's been there from the beginning.  an anchored longest match that

-	was longer than the number of states triggered a failure to initialize

-	the machine properly.  many thanks to moinak ghosh for not only finding

-	this one but for providing a fix, in some of the most mysterious

-	code known to man.

-	fixed a storage leak in call() that appears to have been there since

-	1983 or so -- a function without an explicit return that assigns a

-	string to a parameter leaked a Cell.  thanks to moinak ghosh for

-	spotting this very subtle one.

-Jul 31, 2003:

-	fixed, thanks to andrey chernov and ruslan ermilov, a bug in lex.c

-	that mis-handled the character 255 in input.  (it was being compared

-	to EOF with a signed comparison.)

-Jul 29, 2003:

-	fixed (i think) the long-standing botch that included the beginning of

-	line state ^ for RE's in the set of valid characters; this led to a

-	variety of odd problems, including failure to properly match certain

-	regular expressions in non-US locales.  thanks to ruslan for keeping

-	at this one.

-Jul 28, 2003:

-	n-th try at getting internationalization right, with thanks to volker

-	kiefel, arnold robbins and ruslan ermilov for advice, though they

-	should not be blamed for the outcome.  according to posix, "."  is the

-	radix character in programs and command line arguments regardless of

-	the locale; otherwise, the locale should prevail for input and output

-	of numbers.  so it's intended to work that way.

-	i have rescinded the attempt to use strcoll in expanding shorthands in

-	regular expressions (cclenter).  its properties are much too

-	surprising; for example [a-c] matches aAbBc in locale en_US but abBcC

-	in locale fr_CA.  i can see how this might arise by implementation

-	but i cannot explain it to a human user.  (this behavior can be seen

-	in gawk as well; we're leaning on the same library.)

-	the issue appears to be that strcoll is meant for sorting, where

-	merging upper and lower case may make sense (though note that unix

-	sort does not do this by default either).  it is not appropriate

-	for regular expressions, where the goal is to match specific

-	patterns of characters.  in any case, the notations [:lower:], etc.,

-	are available in awk, and they are more likely to work correctly in

-	most locales.

-	a moratorium is hereby declared on internationalization changes.

-	i apologize to friends and colleagues in other parts of the world.

-	i would truly like to get this "right", but i don't know what

-	that is, and i do not want to keep making changes until it's clear.

-Jul 4, 2003:

-	fixed bug that permitted non-terminated RE, as in "awk /x".

-Jun 1, 2003:

-	subtle change to split: if source is empty, number of elems

-	is always 0 and the array is not set.

-Mar 21, 2003:

-	added some parens to isblank, in another attempt to make things

-	internationally portable.

-Mar 14, 2003:

-	the internationalization changes, somewhat modified, are now

-	reinstated.  in theory awk will now do character comparisons

-	and case conversions in national language, but "." will always

-	be the decimal point separator on input and output regardless

-	of national language.  isblank(){} has an #ifndef.

-	this no longer compiles on windows: LC_MESSAGES isn't defined

-	in vc6++.

-	fixed subtle behavior in field and record splitting: if FS is

-	a single character and RS is not empty, \n is NOT a separator.

-	this tortuous reading is found in the awk book; behavior now

-	matches gawk and mawk.

-Dec 13, 2002:

-	for the moment, the internationalization changes of nov 29 are

-	rolled back -- programs like x = 1.2 don't work in some locales,

-	because the parser is expecting x = 1,2.  until i understand this

-	better, this will have to wait.

-Nov 29, 2002:

-	modified b.c (with tiny changes in main and run) to support

-	locales, using strcoll and iswhatever tests for posix character

-	classes.  thanks to ruslan ermilov (ru@freebsd.org) for code.

-	the function isblank doesn't seem to have propagated to any

-	header file near me, so it's there explicitly.  not properly

-	tested on non-ascii character sets by me.

-Jun 28, 2002:

-	modified run/format() and tran/getsval() to do a slightly better

-	job on using OFMT for output from print and CONVFMT for other

-	number->string conversions, as promised by posix and done by

-	gawk and mawk.  there are still places where it doesn't work

-	right if CONVFMT is changed; by then the STR attribute of the

-	variable has been irrevocably set.  thanks to arnold robbins for

-	code and examples.

-	fixed subtle bug in format that could get core dump.  thanks to

-	Jaromir Dolecek <jdolecek@NetBSD.org> for finding and fixing.

-	minor cleanup in run.c / format() at the same time.

-	added some tests for null pointers to debugging printf's, which

-	were never intended for external consumption.  thanks to dave

-	kerns (dkerns@lucent.com) for pointing this out.

-	GNU compatibility: an empty regexp matches anything (thanks to

-	dag-erling smorgrav, des@ofug.org).  subject to reversion if

-	this does more harm than good.

-	pervasive small changes to make things more const-correct, as

-	reported by gcc's -Wwrite-strings.  as it says in the gcc manual,

-	this may be more nuisance than useful.  provoked by a suggestion

-	and code from arnaud desitter, arnaud@nimbus.geog.ox.ac.uk

-	minor documentation changes to note that this now compiles out

-	of the box on Mac OS X.

-Feb 10, 2002:

-	changed types in posix chars structure to quiet solaris cc.

-Jan 1, 2002:

-	fflush() or fflush("") flushes all files and pipes.

-	length(arrayname) returns number of elements; thanks to

-	arnold robbins for suggestion.

-	added a makefile.win to make it easier to build on windows.

-	based on dan allen's buildwin.bat.

-Nov 16, 2001:

-	added support for posix character class names like [:digit:],

-	which are not exactly shorter than [0-9] and perhaps no more

-	portable.  thanks to dag-erling smorgrav for code.

-Feb 16, 2001:

-	removed -m option; no longer needed, and it was actually

-	broken (noted thanks to volker kiefel).

-Feb 10, 2001:

-	fixed an appalling bug in gettok: any sequence of digits, +,-, E, e,

-	and period was accepted as a valid number if it started with a period.

-	this would never have happened with the lex version.

-	other 1-character botches, now fixed, include a bare $ and a

-	bare " at the end of the input.

-Feb 7, 2001:

-	more (const char *) casts in b.c and tran.c to silence warnings.

-Nov 15, 2000:

-	fixed a bug introduced in august 1997 that caused expressions

-	like $f[1] to be syntax errors.  thanks to arnold robbins for

-	noticing this and providing a fix.

-Oct 30, 2000:

-	fixed some nextfile bugs: not handling all cases.  thanks to

-	arnold robbins for pointing this out.  new regressions added.

-	close() is now a function.  it returns whatever the library

-	fclose returns, and -1 for closing a file or pipe that wasn't

-	opened.

-Sep 24, 2000:

-	permit \n explicitly in character classes; won't work right

-	if comes in as "[\n]" but ok as /[\n]/, because of multiple

-	processing of \'s.  thanks to arnold robbins.

-July 5, 2000:

-	minor fiddles in tran.c to keep compilers happy about uschar.

-	thanks to norman wilson.

-May 25, 2000:

-	yet another attempt at making 8-bit input work, with another

-	band-aid in b.c (member()), and some (uschar) casts to head

-	off potential errors in subscripts (like isdigit).  also

-	changed HAT to NCHARS-2.  thanks again to santiago vila.

-	changed maketab.c to ignore apparently out of range definitions

-	instead of halting; new freeBSD generates one.  thanks to

-	jon snader <jsnader@ix.netcom.com> for pointing out the problem.

-May 2, 2000:

-	fixed an 8-bit problem in b.c by making several char*'s into

-	unsigned char*'s.  not clear i have them all yet.  thanks to

-	Santiago Vila <sanvila@unex.es> for the bug report.

-Apr 21, 2000:

-	finally found and fixed a memory leak in function call; it's

-	been there since functions were added ~1983.  thanks to

-	jon bentley for the test case that found it.

-	added test in envinit to catch environment "variables" with

-	names beginning with '='; thanks to Berend Hasselman.

-Jul 28, 1999:

-	added test in defn() to catch function foo(foo), which

-	otherwise recurses until core dump.  thanks to arnold

-	robbins for noticing this.

-Jun 20, 1999:

-	added *bp in gettok in lex.c; appears possible to exit function

-	without terminating the string.  thanks to russ cox.

-Jun 2, 1999:

-	added function stdinit() to run to initialize files[] array,

-	in case stdin, etc., are not constants; some compilers care.

-May 10, 1999:

-	replaced the ERROR ... FATAL, etc., macros with functions

-	based on vprintf, to avoid problems caused by overrunning

-	fixed-size errbuf array.  thanks to ralph corderoy for the

-	impetus, and for pointing out a string termination bug in

-	qstring as well.

-Apr 21, 1999:

-	fixed bug that caused occasional core dumps with commandline

-	variable with value ending in \.  (thanks to nelson beebe for

-	the test case.)

-Apr 16, 1999:

-	with code kindly provided by Bruce Lilly, awk now parses

-	/=/ and similar constructs more sensibly in more places.

-	Bruce also provided some helpful test cases.

-Apr 5, 1999:

-	changed true/false to True/False in run.c to make it

-	easier to compile with C++.  Added some casts on malloc

-	and realloc to be honest about casts; ditto.  changed

-	ltype int to long in struct rrow to reduce some 64-bit

-	complaints; other changes scattered throughout for the

-	same purpose.  thanks to Nelson Beebe for these portability

-	improvements.

-	removed some horrible pointer-int casting in b.c and elsewhere

-	by adding ptoi and itonp to localize the casts, which are

-	all benign.  fixed one incipient bug that showed up on sgi

-	in 64-bit mode.

-	reset lineno for new source file; include filename in error

-	message.  also fixed line number error in continuation lines.

-	(thanks to Nelson Beebe for both of these.)

-Mar 24, 1999:

-	Nelson Beebe notes that irix 5.3 yacc dies with a bogus

-	error; use a newer version or switch to bison, since sgi

-	is unlikely to fix it.

-Mar 5, 1999:

-	changed isnumber to is_number to avoid the problem caused by

-	versions of ctype.h that include the name isnumber.

-	distribution now includes a script for building on a Mac,

-	thanks to Dan Allen.

-Feb 20, 1999:

-	fixed memory leaks in run.c (call) and tran.c (setfval).

-	thanks to Stephen Nutt for finding these and providing the fixes.

-Jan 13, 1999:

-	replaced srand argument by (unsigned int) in run.c;

-	avoids problem on Mac and potentially on Unix & Windows.

-	thanks to Dan Allen.

-	added a few (int) casts to silence useless compiler warnings.

-	e.g., errorflag= in run.c jump().

-	added proctab.c to the bundle outout; one less thing

-	to have to compile out of the box.

-	added calls to _popen and _pclose to the win95 stub for

-	pipes (thanks to Steve Adams for this helpful suggestion).

-	seems to work, though properties are not well understood

-	by me, and it appears that under some circumstances the

-	pipe output is truncated.  Be careful.

-Oct 19, 1998:

-	fixed a couple of bugs in getrec: could fail to update $0

-	after a getline var; because inputFS wasn't initialized,

-	could split $0 on every character, a misleading diversion.

-	fixed caching bug in makedfa: LRU was actually removing

-	least often used.

-	thanks to ross ridge for finding these, and for providing

-	great bug reports.

-May 12, 1998:

-	fixed potential bug in readrec: might fail to update record

-	pointer after growing.  thanks to dan levy for spotting this

-	and suggesting the fix.

-Mar 12, 1998:

-	added -V to print version number and die.

-[notify dave kerns, dkerns@dacsoup.ih.lucent.com]

-Feb 11, 1998:

-	subtle silent bug in lex.c: if the program ended with a number

-	longer than 1 digit, part of the input would be pushed back and

-	parsed again because token buffer wasn't terminated right.

-	example:  awk 'length($0) > 10'.  blush.  at least i found it

-	myself.

-Aug 31, 1997:

-	s/adelete/awkdelete/: SGI uses this in malloc.h.

-	thanks to nelson beebe for pointing this one out.

-Aug 21, 1997:

-	fixed some bugs in sub and gsub when replacement includes \\.

-	this is a dark, horrible corner, but at least now i believe that

-	the behavior is the same as gawk and the intended posix standard.

-	thanks to arnold robbins for advice here.

-Aug 9, 1997:

-	somewhat regretfully, replaced the ancient lex-based lexical

-	analyzer with one written in C.  it's longer, generates less code,

-	and more portable; the old one depended too much on mysterious

-	properties of lex that were not preserved in other environments.

-	in theory these recognize the same language.

-	now using strtod to test whether a string is a number, instead of

-	the convoluted original function.  should be more portable and

-	reliable if strtod is implemented right.

-	removed now-pointless optimization in makefile that tries to avoid

-	recompilation when awkgram.y is changed but symbols are not.

-	removed most fixed-size arrays, though a handful remain, some

-	of which are unchecked.  you have been warned.

-Aug 4, 1997:

-	with some trepidation, replaced the ancient code that managed

-	fields and $0 in fixed-size arrays with arrays that grow on

-	demand.  there is still some tension between trying to make this

-	run fast and making it clean; not sure it's right yet.

-	the ill-conceived -mr and -mf arguments are now useful only

-	for debugging.  previous dynamic string code removed.

-	numerous other minor cleanups along the way.

-Jul 30, 1997:

-	using code provided by dan levy (to whom profuse thanks), replaced

-	fixed-size arrays and awkward kludges by a fairly uniform mechanism

-	to grow arrays as needed for printf, sub, gsub, etc.

-Jul 23, 1997:

-	falling off the end of a function returns "" and 0, not 0.

-	thanks to arnold robbins.

-Jun 17, 1997:

-	replaced several fixed-size arrays by dynamically-created ones

-	in run.c; added overflow tests to some previously unchecked cases.

-	getline, toupper, tolower.

-	getline code is still broken in that recursive calls may wind

-	up using the same space.  [fixed later]

-	increased RECSIZE to 8192 to push problems further over the horizon.

-	added \r to \n as input line separator for programs, not data.

-	damn CRLFs.

-	modified format() to permit explicit printf("%c", 0) to include

-	a null byte in output.  thanks to ken stailey for the fix.

-	added a "-safe" argument that disables file output (print >,

-	print >>), process creation (cmd|getline, print |, system), and

-	access to the environment (ENVIRON).  this is a first approximation

-	to a "safe" version of awk, but don't rely on it too much.  thanks

-	to joan feigenbaum and matt blaze for the inspiration long ago.

-Jul 8, 1996:

-	fixed long-standing bug in sub, gsub(/a/, "\\\\&"); thanks to

-	ralph corderoy.

-Jun 29, 1996:

-	fixed awful bug in new field splitting; didn't get all the places

-	where input was done.

-Jun 28, 1996:

-	changed field-splitting to conform to posix definition: fields are

-	split using the value of FS at the time of input; it used to be

-	the value when the field or NF was first referred to, a much less

-	predictable definition.  thanks to arnold robbins for encouragement

-	to do the right thing.

-May 28, 1996:

-	fixed appalling but apparently unimportant bug in parsing octal

-	numbers in reg exprs.

-	explicit hex in reg exprs now limited to 2 chars: \xa, \xaa.

-May 27, 1996:

-	cleaned up some declarations so gcc -Wall is now almost silent.

-	makefile now includes backup copies of ytab.c and lexyy.c in case

-	one makes before looking; it also avoids recreating lexyy.c unless

-	really needed.

-	s/aprintf/awkprint, s/asprintf/awksprintf/ to avoid some name clashes

-	with unwisely-written header files.

-	thanks to jeffrey friedl for several of these.

-May 26, 1996:

-	an attempt to rationalize the (unsigned) char issue.  almost all

-	instances of unsigned char have been removed; the handful of places

-	in b.c where chars are used as table indices have been hand-crafted.

-	added some latin-1 tests to the regression, but i'm not confident;

-	none of my compilers seem to care much.  thanks to nelson beebe for

-	pointing out some others that do care.

-May 2, 1996:

-	removed all register declarations.

-	enhanced split(), as in gawk, etc:  split(s, a, "") splits s into

-	a[1]...a[length(s)] with each character a single element.

-	made the same changes for field-splitting if FS is "".

-	added nextfile, as in gawk: causes immediate advance to next

-	input file. (thanks to arnold robbins for inspiration and code).

-	small fixes to regexpr code:  can now handle []], [[], and

-	variants;  [] is now a syntax error, rather than matching

-	everything;  [z-a] is now empty, not z.  far from complete

-	or correct, however.  (thanks to jeffrey friedl for pointing out

-	some awful behaviors.)

-Apr 29, 1996:

-	replaced uchar by uschar everywhere; apparently some compilers

-	usurp this name and this causes conflicts.

-	fixed call to time in run.c (bltin); arg is time_t *.

-	replaced horrible pointer/long punning in b.c by a legitimate

-	union.  should be safer on 64-bit machines and cleaner everywhere.

-	(thanks to nelson beebe for pointing out some of these problems.)

-	replaced nested comments by #if 0...#endif in run.c, lib.c.

-	removed getsval, setsval, execute macros from run.c and lib.c.

-	machines are 100x faster than they were when these macros were

-	first used.

-	revised filenames: awk.g.y => awkgram.y, awk.lx.l => awklex.l,

-	y.tab.[ch] => ytab.[ch], lex.yy.c => lexyy.c, all in the aid of

-	portability to nameless systems.

-	"make bundle" now includes yacc and lex output files for recipients

-	who don't have yacc or lex.

-Aug 15, 1995:

-	initialized Cells in setsymtab more carefully; some fields

-	were not set.  (thanks to purify, all of whose complaints i

-	think i now understand.)

-	fixed at least one error in gsub that looked at -1-th element

-	of an array when substituting for a null match (e.g., $).

-	delete arrayname is now legal; it clears the elements but leaves

-	the array, which may not be the right behavior.

-	modified makefile: my current make can't cope with the test used

-	to avoid unnecessary yacc invocations.

-Jul 17, 1995:

-	added dynamically growing strings to awk.lx.l and b.c

-	to permit regular expressions to be much bigger.

-	the state arrays can still overflow.

-Aug 24, 1994:

-	detect duplicate arguments in function definitions (mdm).

-May 11, 1994:

-	trivial fix to printf to limit string size in sub().

-Apr 22, 1994:

-	fixed yet another subtle self-assignment problem:

-	$1 = $2; $1 = $1 clobbered $1.

-	Regression tests now use private echo, to avoid quoting problems.

-Feb 2, 1994:

-	changed error() to print line number as %d, not %g.

-Jul 23, 1993:

-	cosmetic changes: increased sizes of some arrays,

-	reworded some error messages.

-	added CONVFMT as in posix (just replaced OFMT in getsval)

-	FILENAME is now "" until the first thing that causes a file

-	to be opened.

-Nov 28, 1992:

-	deleted yyunput and yyoutput from proto.h;

-	different versions of lex give these different declarations.

-May 31, 1992:

-	added -mr N and -mf N options: more record and fields.

-	these really ought to adjust automatically.

-	cleaned up some error messages; "out of space" now means

-	malloc returned NULL in all cases.

-	changed rehash so that if it runs out, it just returns;

-	things will continue to run slow, but maybe a bit longer.

-Apr 24, 1992:

-	remove redundant close of stdin when using -f -.

-	got rid of core dump with -d; awk -d just prints date.

-Apr 12, 1992:

-	added explicit check for /dev/std(in,out,err) in redirection.

-	unlike gawk, no /dev/fd/n yet.

-	added (file/pipe) builtin.  hard to test satisfactorily.

-	not posix.

-Feb 20, 1992:

-	recompile after abortive changes;  should be unchanged.

-Dec 2, 1991:

-	die-casting time:  converted to ansi C, installed that.

-Nov 30, 1991:

-	fixed storage leak in freefa, failing to recover [N]CCL.

-	thanks to Bill Jones (jones@cs.usask.ca)

-Nov 19, 1991:

-	use RAND_MAX instead of literal in builtin().

-Nov 12, 1991:

-	cranked up some fixed-size arrays in b.c, and added a test for

-	overflow in penter.  thanks to mark larsen.

-Sep 24, 1991:

-	increased buffer in gsub.  a very crude fix to a general problem.

-	and again on Sep 26.

-Aug 18, 1991:

-	enforce variable name syntax for commandline variables: has to

-	start with letter or _.

-Jul 27, 1991:

-	allow newline after ; in for statements.

-Jul 21, 1991:

-	fixed so that in self-assignment like $1=$1, side effects

-	like recomputing $0 take place.  (this is getting subtle.)

-Jun 30, 1991:

-	better test for detecting too-long output record.

-Jun 2, 1991:

-	better defense against very long printf strings.

-	made break and continue illegal outside of loops.

-May 13, 1991:

-	removed extra arg on gettemp, tempfree.  minor error message rewording.

-May 6, 1991:

-	fixed silly bug in hex parsing in hexstr().

-	removed an apparently unnecessary test in isnumber().

-	warn about weird printf conversions.

-	fixed unchecked array overwrite in relex().

-	changed for (i in array) to access elements in sorted order.

-	then unchanged it -- it really does run slower in too many cases.

-	left the code in place, commented out.

-Feb 10, 1991:

-	check error status on all writes, to avoid banging on full disks.

-Jan 28, 1991:

-	awk -f - reads the program from stdin.

-Jan 11, 1991:

-	failed to set numeric state on $0 in cmd|getline context in run.c.

-Nov 2, 1990:

-	fixed sleazy test for integrality in getsval;  use modf.

-Oct 29, 1990:

-	fixed sleazy buggy code in lib.c that looked (incorrectly) for

-	too long input lines.

-Oct 14, 1990:

-	fixed the bug on p. 198 in which it couldn't deduce that an

-	argument was an array in some contexts.  replaced the error

-	message in intest() by code that damn well makes it an array.

-Oct 8, 1990:

-	fixed horrible bug:  types and values were not preserved in

-	some kinds of self-assignment. (in assign().)

-Aug 24, 1990:

-	changed NCHARS to 256 to handle 8-bit characters in strings

-	presented to match(), etc.

-Jun 26, 1990:

-	changed struct rrow (awk.h) to use long instead of int for lval,

-	since cfoll() stores a pointer in it.  now works better when int's

-	are smaller than pointers!

-May 6, 1990:

-	AVA fixed the grammar so that ! is uniformly of the same precedence as

-	unary + and -.  This renders illegal some constructs like !x=y, which

-	now has to be parenthesized as !(x=y), and makes others work properly:

-	!x+y is (!x)+y, and x!y is x !y, not two pattern-action statements.

-	(These problems were pointed out by Bob Lenk of Posix.)

-	Added \x to regular expressions (already in strings).

-	Limited octal to octal digits; \8 and \9 are not octal.

-	Centralized the code for parsing escapes in regular expressions.

-	Added a bunch of tests to T.re and T.sub to verify some of this.

-Feb 9, 1990:

-	fixed null pointer dereference bug in main.c:  -F[nothing].  sigh.

-	restored srand behavior:  it returns the current seed.

-Jan 18, 1990:

-	srand now returns previous seed value (0 to start).

-Jan 5, 1990:

-	fix potential problem in tran.c -- something was freed,

-	then used in freesymtab.

-Oct 18, 1989:

-	another try to get the max number of open files set with

-	relatively machine-independent code.

-	small fix to input() in case of multiple reads after EOF.

-Oct 11, 1989:

-	FILENAME is now defined in the BEGIN block -- too many old

-	programs broke.

-	"-" means stdin in getline as well as on the commandline.

-	added a bunch of casts to the code to tell the truth about

-	char * vs. unsigned char *, a right royal pain.  added a

-	setlocale call to the front of main, though probably no one

-	has it usefully implemented yet.

-Aug 24, 1989:

-	removed redundant relational tests against nullnode if parse

-	tree already had a relational at that point.

-Aug 11, 1989:

-	fixed bug:  commandline variable assignment has to look like

-	var=something.  (consider the man page for =, in file =.1)

-	changed number of arguments to functions to static arrays

-	to avoid repeated malloc calls.

-Aug 2, 1989:

-	restored -F (space) separator

-Jul 30, 1989:

-	added -v x=1 y=2 ... for immediate commandline variable assignment;

-	done before the BEGIN block for sure.  they have to precede the

-	program if the program is on the commandline.

-	Modified Aug 2 to require a separate -v for each assignment.

-Jul 10, 1989:

-	fixed ref-thru-zero bug in environment code in tran.c

-Jun 23, 1989:

-	add newline to usage message.

-Jun 14, 1989:

-	added some missing ansi printf conversion letters: %i %X %E %G.

-	no sensible meaning for h or L, so they may not do what one expects.

-	made %* conversions work.

-	changed x^y so that if n is a positive integer, it's done

-	by explicit multiplication, thus achieving maximum accuracy.

-	(this should be done by pow() but it seems not to be locally.)

-	done to x ^= y as well.

-Jun 4, 1989:

-	ENVIRON array contains environment: if shell variable V=thing,

-		ENVIRON["V"] is "thing"

-	multiple -f arguments permitted.  error reporting is naive.

-	(they were permitted before, but only the last was used.)

-	fixed a really stupid botch in the debugging macro dprintf

-	fixed order of evaluation of commandline assignments to match

-	what the book claims:  an argument of the form x=e is evaluated

-	at the time it would have been opened if it were a filename (p 63).

-	this invalidates the suggested answer to ex 4-1 (p 195).

-	removed some code that permitted -F (space) fieldseparator,

-	since it didn't quite work right anyway.  (restored aug 2)

-Apr 27, 1989:

-	Line number now accumulated correctly for comment lines.

-Apr 26, 1989:

-	Debugging output now includes a version date,

-	if one compiles it into the source each time.

-Apr 9, 1989:

-	Changed grammar to prohibit constants as 3rd arg of sub and gsub;

-	prevents class of overwriting-a-constant errors.  (Last one?)

-	This invalidates the "banana" example on page 43 of the book.

-	Added \a ("alert"), \v (vertical tab), \xhhh (hexadecimal),

-	as in ANSI, for strings.  Rescinded the sloppiness that permitted

-	non-octal digits in \ooo.  Warning:  not all compilers and libraries

-	will be able to deal with \x correctly.

-Jan 9, 1989:

-	Fixed bug that caused tempcell list to contain a duplicate.

-	The fix is kludgy.

-Dec 17, 1988:

-	Catches some more commandline errors in main.

-	Removed redundant decl of modf in run.c (confuses some compilers).

-	Warning:  there's no single declaration of malloc, etc., in awk.h

-	that seems to satisfy all compilers.

-Dec 7, 1988:

-	Added a bit of code to error printing to avoid printing nulls.

-	(Not clear that it actually would.)

-Nov 27, 1988:

-	With fear and trembling, modified the grammar to permit

-	multiple pattern-action statements on one line without

-	an explicit separator.  By definition, this capitulation

-	to the ghost of ancient implementations remains undefined

-	and thus subject to change without notice or apology.

-	DO NOT COUNT ON IT.

-Oct 30, 1988:

-	Fixed bug in call() that failed to recover storage.

-	A warning is now generated if there are more arguments

-	in the call than in the definition (in lieu of fixing

-	another storage leak).

-Oct 20, 1988:

-	Fixed %c:  if expr is numeric, use numeric value;

-	otherwise print 1st char of string value.  still

-	doesn't work if the value is 0 -- won't print \0.

-	Added a few more checks for running out of malloc.

-Oct 12, 1988:

-	Fixed bug in call() that freed local arrays twice.

-	Fixed to handle deletion of non-existent array right;

-	complains about attempt to delete non-array element.

-Sep 30, 1988:

-	Now guarantees to evaluate all arguments of built-in

-	functions, as in C;  the appearance is that arguments

-	are evaluated before the function is called.  Places

-	affected are sub (gsub was ok), substr, printf, and

-	all the built-in arithmetic functions in bltin().

-	A warning is generated if a bltin() is called with

-	the wrong number of arguments.

-	This requires changing makeprof on p167 of the book.

-Aug 23, 1988:

-	setting FILENAME in BEGIN caused core dump, apparently

-	because it was freeing space not allocated by malloc.

-July 24, 1988:

-	fixed egregious error in toupper/tolower functions.

-	still subject to rescinding, however.

-July 2, 1988:

-	flush stdout before opening file or pipe

-July 2, 1988:

-	performance bug in b.c/cgoto(): not freeing some sets of states.

-	partial fix only right now, and the number of states increased

-	to make it less obvious.

-June 1, 1988:

-	check error status on close

-May 28, 1988:

-	srand returns seed value it's using.

-	see 1/18/90

-May 22, 1988:

-	Removed limit on depth of function calls.

-May 10, 1988:

-	Fixed lib.c to permit _ in commandline variable names.

-Mar 25, 1988:

-	main.c fixed to recognize -- as terminator of command-

-	line options.  Illegal options flagged.

-	Error reporting slightly cleaned up.

-Dec 2, 1987:

-	Newer C compilers apply a strict scope rule to extern

-	declarations within functions.  Two extern declarations in

-	lib.c and tran.c have been moved to obviate this problem.

-Oct xx, 1987:

-	Reluctantly added toupper and tolower functions.

-	Subject to rescinding without notice.

-Sep 17, 1987:

-	Error-message printer had printf(s) instead of

-	printf("%s",s);  got core dumps when the message

-	included a %.

-Sep 12, 1987:

-	Very long printf strings caused core dump;

-	fixed aprintf, asprintf, format to catch them.

-	Can still get a core dump in printf itself.

+	Unicode literals can be written as \u followed by one

+	to eight hexadecimal digits.  These may appear in strings and

+	regular expressions.

--- /dev/null

+++ b/FIXES.1e

@@ -1,0 +1,1421 @@

+/****************************************************************

+Copyright (C) Lucent Technologies 1997

+All Rights Reserved

+Permission to use, copy, modify, and distribute this software and

+its documentation for any purpose and without fee is hereby

+granted, provided that the above copyright notice appear in all

+copies and that both that the copyright notice and this

+permission notice and warranty disclaimer appear in supporting

+documentation, and that the name Lucent Technologies or any of

+its entities not be used in advertising or publicity pertaining

+to distribution of the software without specific, written prior

+permission.

+LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,

+INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.

+IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY

+SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER

+IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,

+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF

+THIS SOFTWARE.

+****************************************************************/

+This file lists all bug fixes, changes, etc., made since the AWK book

+was sent to the printers in August 1987.

+Dec 15, 2022:

+	Force hex escapes in strings to be no more than two characters,

+	as they already are in regular expressions. This brings internal

+	consistency, as well as consistency with gawk. Thanks to

+	Arnold Robbins.

+Sep 12, 2022:

+	adjbuf minlen error (cannot be 0) in cat, resulting in NULL pbuf.

+	discovered by todd miller. also use-after-free issue with

+	tempfree in cat, thanks to Miguel Pineiro Jr and valgrind.

+Aug 30, 2022:

+	Various leaks and use-after-free issues plugged/fixed.

+	Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

+May 23, 2022:

+	Memory leak when assigning a string to some of the built-in

+	variables. allocated string erroneously marked DONTFREE.

+	Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

+Mar 14, 2022:

+	Historic bug: command-line "name=value" assignment had been

+	truncating its entry in ARGV. (circa 1989) Thanks to

+	Miguel Pineiro Jr. <mpj@pineiro.cc>.

+Mar 3, 2022:

+	Fixed file management memory leak that appears to have been

+	there since the files array was first initialized with stdin,

+	stdout, and stderr (circa 1992). Thanks to Miguel Pineiro Jr.

+	<mpj@pineiro.cc>.

+December 8, 2021:

+	The error handling in closefile and closeall was mangled. Long

+	standing warnings had been made fatal and some fatal errors went

+	undetected. Thanks to Miguel Pineiro Jr. <mpj@pineiro.cc>.

+Nov 03, 2021:

+        getline accesses uninitialized data after getrec()

+	returns 0 on EOF and leaves the contents of buf unchanged.

+	Thanks to Volodymyr Gubarkov, and Todd C Miller.

+Oct 12, 2021:

+	The fix for #83 changed the code to insert 2 chars, but the

+	call to adjbuf just above it only allows for 1 char. This can

+	cause a heap buffer overflow.

+July 27, 2021:

+	As per IEEE Std 1003.1-2008, -F "str" is now consistent with

+	-v FS="str" when str is null. Thanks to Warner Losh.

+July 24, 2021:

+	Fix readrec's definition of a record. This fixes an issue

+	with NetBSD's RS regular expression support that can cause

+	an infinite read loop. Thanks to Miguel Pineiro Jr.

+	Fix regular expression RS ^-anchoring. RS ^-anchoring needs to

+	know if it is reading the first record of a file. This change

+	restores a missing line that was overlooked when porting NetBSD's

+	RS regex functionality. Thanks to Miguel Pineiro Jr.

+	Fix size computation in replace_repeat() for special case

+	REPEAT_WITH_Q. Thanks to Todd C. Miller.

+February 15, 2021:

+	Small fix so that awk will compile again with g++. Thanks to

+	Arnold Robbins.

+January 06, 2021:

+	Fix a decision bug with trailing stuff in lib.c:is_valid_number

+	after recent changes. Thanks to Ozan Yigit.

+December 18, 2020:

+	Fix problems converting inf and NaN values in lib.c:is_valid_number.

+	Enhance number to string conversion to do the right thing for

+	NaN and inf values.  Things are now pretty much the same as in

+	gawk.  (Found a gawk bug while we're at it.) Added a torture

+	test for these values.  Thanks to Arnold Robbins.  Allows closing

+	of PR #101.

+December 15, 2020:

+	Merge PR #99, which gets the right header for strcasecmp.

+	Thanks to GitHub user michaelforney.

+December 8, 2020:

+	Merge PR #98: Disallow hex data. Allow only +nan, -nan,

+	+inf, -inf (case independent) to give NaN and infinity values.

+	Improve things so that string to double conversion is only

+	done once, yielding something of a speedup.  This obviate

+	PR #95. Thanks to Arnold Robbins.

+December 3, 2020:

+	Fix to argument parsing to avoid printing spurious newlines.

+	Thanks to Todd Miller. Merges PR #97.

+October 13, 2020:

+	Add casts before all the calls to malloc/calloc/realloc in order

+	to get it to compile with g++. Thanks to Arnold Robbins.

+August 16, 2020:

+	Additional fixes for DJGPP. Thanks to Eli Zaretskii for

+	the testing.

+August 7, 2020:

+	Merge PR #93, which adds casts to (void*) for debug prints

+	using the %p format specifier. Thanks to GitHub user YongHaoWu

+	("Chris") for the fixes.

+August 4, 2020:

+	In run.c, use non-restartable multibyte routines to attain

+	portability to DJGPP. Should fix Issue 92. Thanks to Albert Wik

+	for the report and to Todd Miller for the suggested fix.

+July 30, 2020:

+	Merge PRs 88-91 which fix small bugs. Thanks to Todd Miller and

+	Tim van der Molen for the fixes.

+	In order to make life easier, we move exclusively to bison

+	as the parser generator.

+July 2, 2020:

+	Merge PRs 85 and 86 which fix regressions. Thanks to

+	Tim van der Molen for the fixes.

+June 25, 2020:

+	Merge PRs 82 and 84. The latter fixes issue #83. Thanks to

+	Todd Miller and awkfan77.

+June 12, 2020:

+	Clear errno before calling errcheck to avoid any spurious errors

+	left over from previous calls that may have set it. Thanks to

+	Todd Miller for the fix, from PR #80.

+	Fix Issue #78 by allowing \r to follow floating point numbers in

+	lib.c:is_number. Thanks to GitHub user ajcarr for the report

+	and to Arnold Robbins for the fix.

+June 5, 2020:

+	In fldbld(), make sure that inputFS is set before trying to

+	use it. Thanks to  Steffen Nurpmeso <steffen@sdaoden.eu>

+	for the report.

+May 5, 2020:

+	Fix checks for compilers that can handle noreturn. Thanks to

+	GitHub user enh-google for pointing it out. Closes Issue #79.

+April 16, 2020:

+	Handle old compilers that don't support C11 (for noreturn).

+	Thanks to Arnold Robbins.

+April 5, 2020:

+	Use <stdnoreturn.h> and noreturn instead of GCC attributes.

+	Thanks to GitHub user awkfan77. Closes PR #77.

+February 28, 2020:

+	More cleanups from Christos Zoulas: notably backslash continuation

+	inside strings removes the newline and a fix for RS = "^a".

+	Fix for address sanitizer-found problem. Thanks to GitHub user

+	enh-google.

+February 19, 2020:

+	More small cleanups from Christos Zoulas.

+February 18, 2020:

+	Additional cleanups from Christos Zoulas. It's no longer necessary

+	to use the -y flag to bison.

+February 6, 2020:

+	Additional small cleanups from Christos Zoulas. awk is now

+	a little more robust about reporting I/O errors upon exit.

+January 31, 2020:

+	Merge PR #70, which avoids use of variable length arrays. Thanks

+	to GitHub user michaelforney.  Fix issue #60 ({0} in interval

+	expressions doesn't work).  Also get all tests working again.

+	Thanks to Arnold Robbins.

+January 24, 2020:

+	A number of small cleanups from Christos Zoulas.  Add the close

+	on exec flag to files/pipes opened for redirection; courtesy of

+	Arnold Robbins.

+January 19, 2020:

+	If POSIXLY_CORRECT is set in the environment, then sub and gsub

+	use POSIX rules for multiple backslashes.  This fixes Issue #66,

+	while maintaining backwards compatibility.

+January 9, 2020:

+	Input/output errors on closing files are now fatal instead of

+	mere warnings. Thanks to Martijn Dekker <martijn@inlv.org>.

+January 5, 2020:

+	Fix a bug in the concatentation of two string constants into

+	one done in the grammar.  Fixes GitHub issue #61.  Thanks

+	to GitHub user awkfan77 for pointing out the direction for

+	the fix.  New test T.concat added to the test suite.

+	Fix a few memory leaks reported by valgrind, as well.

+December 27, 2019:

+	Fix a bug whereby a{0,3} could match four a's.  Thanks to

+	"Anonymous AWK fan" for the report.

+December 11, 2019:

+	Further printf-related fixes for 32 bit systems.

+	Thanks again to Christos Zoulas.

+December 8, 2019:

+	Fix the return value of sprintf("%d") on 32 bit systems.

+	Thanks to Jim Lowe for the report and to Christos Zoulas

+	for the fix.

+November 10, 2019:

+	Convert a number of Boolean integer variables into

+	actual bools. Convert compile_time variable into an

+	enum and simplify some of the related code.  Thanks

+	to Arnold Robbins.

+November 8, 2019:

+	Fix from Ori Bernstein to get UTF-8 characters instead of

+	bytes when FS = "".  This is currently the only bit of

+	the One True Awk that understands multibyte characters.

+	From Arnold Robbins, apply some cleanups in the test suite.

+October 25, 2019:

+	More fixes and cleanups from NetBSD, courtesy of Christos

+	Zoulas. Merges PRs 54 and 55.

+October 24, 2019:

+	Import second round of code cleanups from NetBSD. Much thanks

+	to Christos Zoulas (GitHub user zoulasc). Merges PR 53.

+	Add an optimization for string concatenation, also from

+	Christos.

+October 17, 2019:

+	Import code cleanups from NetBSD. Much thanks to Christos

+	Zoulas (GitHub user zoulasc). Merges PR 51.

+October 6, 2019:

+	Import code from NetBSD awk that implements RS as a regular

+	expression.

+September 10, 2019:

+	Fixes for various array / memory overruns found via gcc's

+	-fsanitize=unknown. Thanks to Alexander Richardson (GitHub

+	user arichardson). Merges PRs 47 and 48.

+July 28, 2019:

+	Import grammar optimization from NetBSD: Two string constants

+	concatenated together get turned into a single string.

+July 26, 2019:

+	Support POSIX-specified C-style escape sequences "\a" (alarm)

+	and "\v" (vertical tab) in command line arguments and regular

+	expressions, further to the support for them in strings added on

+	Apr 9, 1989. These now no longer match as literal "a" and "v"

+	characters (as they don't on other awk implementations).

+	Thanks to Martijn Dekker.

+July 17, 2019:

+	Pull in a number of code cleanups and minor fixes from

+	Warner Losh's bsd-ota branch.  The only user visible change

+	is the use of random(3) as the random number generator.

+	Thanks to Warner Losh for collecting all these fixes in

+	one easy place to get them from.

+July 16, 2019:

+	Fix field splitting to use FS value as of the time a record

+	was read or assigned to.  Thanks to GitHub user Cody Mello (melloc)

+	for the fix. (Merged from his branch, via PR #42.) Updated

+	testdir/T.split per said PR as well.

+June 24, 2019:

+	Extract awktest.tar into testdir directory. Add some very

+	simple mechanics to the makefile for running the tests and

+	for cleaning up. No changes to awk itself.

+June 17, 2019:

+	Disallow deleting SYMTAB and its elements, which creates

+	use-after-free bugs. Thanks to GitHub user Cody Mello (melloc)

+	for the fix. (Merged from PR #43.)

+June 5, 2019:

+	Allow unmatched right parenthesis in a regular expression to

+	be treated literally. Fixes Issue #40. Thanks to GitHub user

+	Warner Losh (bsdimp) for the report. Thanks to Arnold Robbins

+	for the fix.

+May 29,2019:

+	Fix check for command line arguments to no longer require that

+	first character after '=' not be another '='. Reverts change of

+	August 11, 1989. Thanks to GitHub user Jamie Landeg Jones for

+	pointing out the issue; from Issue #38.

+Apr 7, 2019:

+	Update awktest.tar(p.50) to use modern options to sort. Needed

+	for Android development. Thanks to GitHub user mohd-akram (Mohamed

+	Akram).  From Issue #33.

+Mar 12, 2019:

+	Added very simplistic support for cross-compiling in the

+	makefile.  We are NOT going to go in the direction of the

+	autotools, though.  Thanks to GitHub user nee-san for

+	the basic change. (Merged from PR #34.)

+Mar 5, 2019:

+	Added support for POSIX-standard interval expressions (a.k.a.

+	bounds, a.k.a. repetition expressions) in regular expressions,

+	backported (via NetBSD) from Apple awk-24 (20070501).

+	Thanks to Martijn Dekker <martijn@inlv.org> for the port.

+	(Merged from PR #30.)

+Mar 3, 2019:

+	Merge PRs as follows:

+	#12: Avoid undefined behaviour when using ctype(3) functions in

+	     relex(). Thanks to GitHub user iamleot.

+	#31: Make getline handle numeric strings, and update FIXES. Thanks

+	     to GitHub user Arnold Robbins (arnoldrobbins)

+	#32: maketab: support build systems with read-only source. Thanks

+	     to GitHub user enh.

+Jan 25, 2019:

+	Make getline handle numeric strings properly in all cases.

+	(Thanks, Arnold.)

+Jan 21, 2019:

+	Merged a number of small fixes from GitHub pull requests.

+	Thanks to GitHub users Arnold Robbins (arnoldrobbins),

+	Cody Mello (melloc) and Christoph Junghans (junghans).

+	PR numbers: 13-21, 23, 24, 27.

+Oct 25, 2018:

+	Added test in maketab.c to prevent generating a proctab entry

+	for YYSTYPE_IS_DEFINED.  It was harmless but some gcc settings

+	generated a warning message.  Thanks to Nan Xiao for report.

+Aug 27, 2018:

+	Disallow '$' in printf formats; arguments evaluated in order

+	and printed in order.

+	Added some casts to silence warnings on debugging printfs.

+	(Thanks, Arnold.)

+Aug 23, 2018:

+        A long list of fixes courtesy of Arnold Robbins,

+        to whom profound thanks.

+        1. ofs-rebuild: OFS value used to rebuild the record was incorrect.

+        Fixed August 19, 2014. Revised fix August 2018.

+        2. system-status: Instead of a floating-point division by 256, use

+        the wait(2) macros to create a reasonable exit status.

+        Fixed March 12, 2016.

+        3. space: Use provided xisblank() function instead of ispace() for

+        matching [[:blank:]].

+        4. a-format: Add POSIX standard %a and %A to supported formats. Check

+        at runtime that this format is available.

+        5. decr-NF: Decrementing NF did not change $0. This is a decades-old

+        bug. There are interactions with the old and new value of OFS as well.

+        Most of the fix came from the NetBSD awk.

+        6. string-conv: String conversions of scalars were sticky.  Once a

+        conversion to string happened, even with OFMT, that value was used until

+        a new numeric value was assigned, even if OFMT differed from CONVFMT,

+        and also if CONVFMT changed.

+        7. unary-plus: Unary plus on a string constant returned the string.

+        Instead, it should convert the value to numeric and give that value.

+	Also added Arnold's tests for these to awktest.tar as T.arnold.

+Aug 15, 2018:

+	fixed mangled awktest.tar (thanks, Arnold), posted all

+	current (very minor) fixes to github / onetrueawk

+Jun 7, 2018:

+	(yes, a long layoff)

+	Updated some broken tests (beebe.tar, T.lilly)

+	[thanks to Arnold Robbins]

+Mar 26, 2015:

+	buffer overflow in error reporting; thanks to tobias ulmer

+	and john-mark gurney for spotting it and the fix.

+Feb 4, 2013:

+	cleaned up a handful of tests that didn't seem to actually

+	test for correct behavior: T.latin1, T.gawk.

+Jan 5, 2013:

+	added ,NULL initializer to static Cells in run.c; not really

+	needed but cleaner.  Thanks to Michael Bombardieri.

+Dec 20, 2012:

+	fiddled makefile to get correct yacc and bison flags.  pick yacc

+	(linux) or bison (mac) as necessary.

+	added  __attribute__((__noreturn__)) to a couple of lines in

+	proto.h, to silence someone's enthusiastic checker.

+	fixed obscure call by value bug in split(a[1],a) reported on

+	9fans.  the management of temporary values is just a mess; i

+	took a shortcut by making an extra string copy.  thanks

+	to paul patience and arnold robbins for passing it on and for

+	proposed patches.

+	tiny fiddle in setfval to eliminate -0 results in T.expr, which

+	has irritated me for 20+ years.

+Aug 10, 2011:

+	another fix to avoid core dump with delete(ARGV); again, many thanks

+	to ruslan ermilov.

+Aug 7, 2011:

+	split(s, a, //) now behaves the same as split(s, a, "")

+Jun 12, 2011:

+	/pat/, \n /pat/ {...} is now legal, though bad style to use.

+	added checks to new -v code that permits -vnospace; thanks to

+	ruslan ermilov for spotting this and providing the patch.

+	removed fixed limit on number of open files; thanks to aleksey

+	cheusov and christos zoulos.

+	fixed day 1 bug that resurrected deleted elements of ARGV when

+	used as filenames (in lib.c).

+	minor type fiddles to make gcc -Wall -pedantic happier (but not

+	totally so); turned on -fno-strict-aliasing in makefile.

+May 6, 2011:

+	added #ifdef for isblank.

+	now allows -ffoo as well as -f foo arguments.

+	(thanks, ruslan)

+May 1, 2011:

+	after advice from todd miller, kevin lo, ruslan ermilov,

+	and arnold robbins, changed srand() to return the previous

+	seed (which is 1 on the first call of srand).  the seed is

+	an Awkfloat internally though converted to unsigned int to

+	pass to the library srand().  thanks, everyone.

+	fixed a subtle (and i hope low-probability) overflow error

+	in fldbld, by adding space for one extra \0.  thanks to

+	robert bassett for spotting this one and providing a fix.

+	removed the files related to compilation on windows.  i no

+	longer have anything like a current windows environment, so

+	i can't test any of it.

+May 23, 2010:

+	fixed long-standing overflow bug in run.c; many thanks to

+	nelson beebe for spotting it and providing the fix.

+	fixed bug that didn't parse -vd=1 properly; thanks to santiago

+	vila for spotting it.

+Feb 8, 2010:

+	i give up.  replaced isblank with isspace in b.c; there are

+	no consistent header files.

+Nov 26, 2009:

+	fixed a long-standing issue with when FS takes effect.  a

+	change to FS is now noticed immediately for subsequent splits.

+	changed the name getline() to awkgetline() to avoid yet another

+	name conflict somewhere.

+Feb 11, 2009:

+	temporarily for now defined HAS_ISBLANK, since that seems to

+	be the best way through the thicket.  isblank arrived in C99,

+	but seems to be arriving at different systems at different

+	times.

+Oct 8, 2008:

+	fixed typo in b.c that set tmpvec wrongly.  no one had ever

+	run into the problem, apparently.  thanks to alistair crooks.

+Oct 23, 2007:

+	minor fix in lib.c: increase inputFS to 100, change malloc

+	for fields to n+1.

+	fixed memory fault caused by out of order test in setsval.

+	thanks to david o'brien, freebsd, for both fixes.

+May 1, 2007:

+	fiddle in makefile to fix for BSD make; thanks to igor sobrado.

+Mar 31, 2007:

+	fixed some null pointer refs calling adjbuf.

+Feb 21, 2007:

+	fixed a bug in matching the null RE in sub and gsub.  thanks to al aho

+	who actually did the fix (in b.c), and to wolfgang seeberg for finding

+	it and providing a very compact test case.

+	fixed quotation in b.c; thanks to Hal Pratt and the Princeton Dante

+	Project.

+	removed some no-effect asserts in run.c.

+	fiddled maketab.c to not complain about bison-generated values.

+	removed the obsolete -V argument; fixed --version to print the

+	version and exit.

+	fixed wording and an outright error in the usage message; thanks to igor

+	sobrado and jason mcintyre.

+	fixed a bug in -d that caused core dump if no program followed.

+Jan 1, 2007:

+	dropped mac.code from makefile; there are few non-MacOSX

+	mac's these days.

+Jan 17, 2006:

+	system() not flagged as unsafe in the unadvertised -safe option.

+	found it while enhancing tests before shipping the ;login: article.

+	practice what you preach.

+	removed the 9-years-obsolete -mr and -mf flags.

+	added -version and --version options.

+	core dump on linux with BEGIN {nextfile}, now fixed.

+	removed some #ifdef's in run.c and lex.c that appear to no

+	longer be necessary.

+Apr 24, 2005:

+	modified lib.c so that values of $0 et al are preserved in the END

+	block, apparently as required by posix.  thanks to havard eidnes

+	for the report and code.

+Jan 14, 2005:

+	fixed infinite loop in parsing, originally found by brian tsang.

+	thanks to arnold robbins for a suggestion that started me

+	rethinking it.

+Dec 31, 2004:

+	prevent overflow of -f array in main, head off potential error in

+	call of SYNTAX(), test malloc return in lib.c, all with thanks to

+	todd miller.

+Dec 22, 2004:

+	cranked up size of NCHARS; coverity thinks it can be overrun with

+	smaller size, and i think that's right.  added some assertions to b.c

+	to catch places where it might overrun.  the RE code is still fragile.

+Dec 5, 2004:

+	fixed a couple of overflow problems with ridiculous field numbers:

+	e.g., print $(2^32-1).  thanks to ruslan ermilov, giorgos keramidas

+	and david o'brien at freebsd.org for patches.  this really should

+	be re-done from scratch.

+Nov 21, 2004:

+	fixed another 25-year-old RE bug, in split.  it's another failure

+	to (re-)initialize.  thanks to steve fisher for spotting this and

+	providing a good test case.

+Nov 22, 2003:

+	fixed a bug in regular expressions that dates (so help me) from 1977;

+	it's been there from the beginning.  an anchored longest match that

+	was longer than the number of states triggered a failure to initialize

+	the machine properly.  many thanks to moinak ghosh for not only finding

+	this one but for providing a fix, in some of the most mysterious

+	code known to man.

+	fixed a storage leak in call() that appears to have been there since

+	1983 or so -- a function without an explicit return that assigns a

+	string to a parameter leaked a Cell.  thanks to moinak ghosh for

+	spotting this very subtle one.

+Jul 31, 2003:

+	fixed, thanks to andrey chernov and ruslan ermilov, a bug in lex.c

+	that mis-handled the character 255 in input.  (it was being compared

+	to EOF with a signed comparison.)

+Jul 29, 2003:

+	fixed (i think) the long-standing botch that included the beginning of

+	line state ^ for RE's in the set of valid characters; this led to a

+	variety of odd problems, including failure to properly match certain

+	regular expressions in non-US locales.  thanks to ruslan for keeping

+	at this one.

+Jul 28, 2003:

+	n-th try at getting internationalization right, with thanks to volker

+	kiefel, arnold robbins and ruslan ermilov for advice, though they

+	should not be blamed for the outcome.  according to posix, "."  is the

+	radix character in programs and command line arguments regardless of

+	the locale; otherwise, the locale should prevail for input and output

+	of numbers.  so it's intended to work that way.

+	i have rescinded the attempt to use strcoll in expanding shorthands in

+	regular expressions (cclenter).  its properties are much too

+	surprising; for example [a-c] matches aAbBc in locale en_US but abBcC

+	in locale fr_CA.  i can see how this might arise by implementation

+	but i cannot explain it to a human user.  (this behavior can be seen

+	in gawk as well; we're leaning on the same library.)

+	the issue appears to be that strcoll is meant for sorting, where

+	merging upper and lower case may make sense (though note that unix

+	sort does not do this by default either).  it is not appropriate

+	for regular expressions, where the goal is to match specific

+	patterns of characters.  in any case, the notations [:lower:], etc.,

+	are available in awk, and they are more likely to work correctly in

+	most locales.

+	a moratorium is hereby declared on internationalization changes.

+	i apologize to friends and colleagues in other parts of the world.

+	i would truly like to get this "right", but i don't know what

+	that is, and i do not want to keep making changes until it's clear.

+Jul 4, 2003:

+	fixed bug that permitted non-terminated RE, as in "awk /x".

+Jun 1, 2003:

+	subtle change to split: if source is empty, number of elems

+	is always 0 and the array is not set.

+Mar 21, 2003:

+	added some parens to isblank, in another attempt to make things

+	internationally portable.

+Mar 14, 2003:

+	the internationalization changes, somewhat modified, are now

+	reinstated.  in theory awk will now do character comparisons

+	and case conversions in national language, but "." will always

+	be the decimal point separator on input and output regardless

+	of national language.  isblank(){} has an #ifndef.

+	this no longer compiles on windows: LC_MESSAGES isn't defined

+	in vc6++.

+	fixed subtle behavior in field and record splitting: if FS is

+	a single character and RS is not empty, \n is NOT a separator.

+	this tortuous reading is found in the awk book; behavior now

+	matches gawk and mawk.

+Dec 13, 2002:

+	for the moment, the internationalization changes of nov 29 are

+	rolled back -- programs like x = 1.2 don't work in some locales,

+	because the parser is expecting x = 1,2.  until i understand this

+	better, this will have to wait.

+Nov 29, 2002:

+	modified b.c (with tiny changes in main and run) to support

+	locales, using strcoll and iswhatever tests for posix character

+	classes.  thanks to ruslan ermilov (ru@freebsd.org) for code.

+	the function isblank doesn't seem to have propagated to any

+	header file near me, so it's there explicitly.  not properly

+	tested on non-ascii character sets by me.

+Jun 28, 2002:

+	modified run/format() and tran/getsval() to do a slightly better

+	job on using OFMT for output from print and CONVFMT for other

+	number->string conversions, as promised by posix and done by

+	gawk and mawk.  there are still places where it doesn't work

+	right if CONVFMT is changed; by then the STR attribute of the

+	variable has been irrevocably set.  thanks to arnold robbins for

+	code and examples.

+	fixed subtle bug in format that could get core dump.  thanks to

+	Jaromir Dolecek <jdolecek@NetBSD.org> for finding and fixing.

+	minor cleanup in run.c / format() at the same time.

+	added some tests for null pointers to debugging printf's, which

+	were never intended for external consumption.  thanks to dave

+	kerns (dkerns@lucent.com) for pointing this out.

+	GNU compatibility: an empty regexp matches anything (thanks to

+	dag-erling smorgrav, des@ofug.org).  subject to reversion if

+	this does more harm than good.

+	pervasive small changes to make things more const-correct, as

+	reported by gcc's -Wwrite-strings.  as it says in the gcc manual,

+	this may be more nuisance than useful.  provoked by a suggestion

+	and code from arnaud desitter, arnaud@nimbus.geog.ox.ac.uk

+	minor documentation changes to note that this now compiles out

+	of the box on Mac OS X.

+Feb 10, 2002:

+	changed types in posix chars structure to quiet solaris cc.

+Jan 1, 2002:

+	fflush() or fflush("") flushes all files and pipes.

+	length(arrayname) returns number of elements; thanks to

+	arnold robbins for suggestion.

+	added a makefile.win to make it easier to build on windows.

+	based on dan allen's buildwin.bat.

+Nov 16, 2001:

+	added support for posix character class names like [:digit:],

+	which are not exactly shorter than [0-9] and perhaps no more

+	portable.  thanks to dag-erling smorgrav for code.

+Feb 16, 2001:

+	removed -m option; no longer needed, and it was actually

+	broken (noted thanks to volker kiefel).

+Feb 10, 2001:

+	fixed an appalling bug in gettok: any sequence of digits, +,-, E, e,

+	and period was accepted as a valid number if it started with a period.

+	this would never have happened with the lex version.

+	other 1-character botches, now fixed, include a bare $ and a

+	bare " at the end of the input.

+Feb 7, 2001:

+	more (const char *) casts in b.c and tran.c to silence warnings.

+Nov 15, 2000:

+	fixed a bug introduced in august 1997 that caused expressions

+	like $f[1] to be syntax errors.  thanks to arnold robbins for

+	noticing this and providing a fix.

+Oct 30, 2000:

+	fixed some nextfile bugs: not handling all cases.  thanks to

+	arnold robbins for pointing this out.  new regressions added.

+	close() is now a function.  it returns whatever the library

+	fclose returns, and -1 for closing a file or pipe that wasn't

+	opened.

+Sep 24, 2000:

+	permit \n explicitly in character classes; won't work right

+	if comes in as "[\n]" but ok as /[\n]/, because of multiple

+	processing of \'s.  thanks to arnold robbins.

+July 5, 2000:

+	minor fiddles in tran.c to keep compilers happy about uschar.

+	thanks to norman wilson.

+May 25, 2000:

+	yet another attempt at making 8-bit input work, with another

+	band-aid in b.c (member()), and some (uschar) casts to head

+	off potential errors in subscripts (like isdigit).  also

+	changed HAT to NCHARS-2.  thanks again to santiago vila.

+	changed maketab.c to ignore apparently out of range definitions

+	instead of halting; new freeBSD generates one.  thanks to

+	jon snader <jsnader@ix.netcom.com> for pointing out the problem.

+May 2, 2000:

+	fixed an 8-bit problem in b.c by making several char*'s into

+	unsigned char*'s.  not clear i have them all yet.  thanks to

+	Santiago Vila <sanvila@unex.es> for the bug report.

+Apr 21, 2000:

+	finally found and fixed a memory leak in function call; it's

+	been there since functions were added ~1983.  thanks to

+	jon bentley for the test case that found it.

+	added test in envinit to catch environment "variables" with

+	names beginning with '='; thanks to Berend Hasselman.

+Jul 28, 1999:

+	added test in defn() to catch function foo(foo), which

+	otherwise recurses until core dump.  thanks to arnold

+	robbins for noticing this.

+Jun 20, 1999:

+	added *bp in gettok in lex.c; appears possible to exit function

+	without terminating the string.  thanks to russ cox.

+Jun 2, 1999:

+	added function stdinit() to run to initialize files[] array,

+	in case stdin, etc., are not constants; some compilers care.

+May 10, 1999:

+	replaced the ERROR ... FATAL, etc., macros with functions

+	based on vprintf, to avoid problems caused by overrunning

+	fixed-size errbuf array.  thanks to ralph corderoy for the

+	impetus, and for pointing out a string termination bug in

+	qstring as well.

+Apr 21, 1999:

+	fixed bug that caused occasional core dumps with commandline

+	variable with value ending in \.  (thanks to nelson beebe for

+	the test case.)

+Apr 16, 1999:

+	with code kindly provided by Bruce Lilly, awk now parses

+	/=/ and similar constructs more sensibly in more places.

+	Bruce also provided some helpful test cases.

+Apr 5, 1999:

+	changed true/false to True/False in run.c to make it

+	easier to compile with C++.  Added some casts on malloc

+	and realloc to be honest about casts; ditto.  changed

+	ltype int to long in struct rrow to reduce some 64-bit

+	complaints; other changes scattered throughout for the

+	same purpose.  thanks to Nelson Beebe for these portability

+	improvements.

+	removed some horrible pointer-int casting in b.c and elsewhere

+	by adding ptoi and itonp to localize the casts, which are

+	all benign.  fixed one incipient bug that showed up on sgi

+	in 64-bit mode.

+	reset lineno for new source file; include filename in error

+	message.  also fixed line number error in continuation lines.

+	(thanks to Nelson Beebe for both of these.)

+Mar 24, 1999:

+	Nelson Beebe notes that irix 5.3 yacc dies with a bogus

+	error; use a newer version or switch to bison, since sgi

+	is unlikely to fix it.

+Mar 5, 1999:

+	changed isnumber to is_number to avoid the problem caused by

+	versions of ctype.h that include the name isnumber.

+	distribution now includes a script for building on a Mac,

+	thanks to Dan Allen.

+Feb 20, 1999:

+	fixed memory leaks in run.c (call) and tran.c (setfval).

+	thanks to Stephen Nutt for finding these and providing the fixes.

+Jan 13, 1999:

+	replaced srand argument by (unsigned int) in run.c;

+	avoids problem on Mac and potentially on Unix & Windows.

+	thanks to Dan Allen.

+	added a few (int) casts to silence useless compiler warnings.

+	e.g., errorflag= in run.c jump().

+	added proctab.c to the bundle outout; one less thing

+	to have to compile out of the box.

+	added calls to _popen and _pclose to the win95 stub for

+	pipes (thanks to Steve Adams for this helpful suggestion).

+	seems to work, though properties are not well understood

+	by me, and it appears that under some circumstances the

+	pipe output is truncated.  Be careful.

+Oct 19, 1998:

+	fixed a couple of bugs in getrec: could fail to update $0

+	after a getline var; because inputFS wasn't initialized,

+	could split $0 on every character, a misleading diversion.

+	fixed caching bug in makedfa: LRU was actually removing

+	least often used.

+	thanks to ross ridge for finding these, and for providing

+	great bug reports.

+May 12, 1998:

+	fixed potential bug in readrec: might fail to update record

+	pointer after growing.  thanks to dan levy for spotting this

+	and suggesting the fix.

+Mar 12, 1998:

+	added -V to print version number and die.

+[notify dave kerns, dkerns@dacsoup.ih.lucent.com]

+Feb 11, 1998:

+	subtle silent bug in lex.c: if the program ended with a number

+	longer than 1 digit, part of the input would be pushed back and

+	parsed again because token buffer wasn't terminated right.

+	example:  awk 'length($0) > 10'.  blush.  at least i found it

+	myself.

+Aug 31, 1997:

+	s/adelete/awkdelete/: SGI uses this in malloc.h.

+	thanks to nelson beebe for pointing this one out.

+Aug 21, 1997:

+	fixed some bugs in sub and gsub when replacement includes \\.

+	this is a dark, horrible corner, but at least now i believe that

+	the behavior is the same as gawk and the intended posix standard.

+	thanks to arnold robbins for advice here.

+Aug 9, 1997:

+	somewhat regretfully, replaced the ancient lex-based lexical

+	analyzer with one written in C.  it's longer, generates less code,

+	and more portable; the old one depended too much on mysterious

+	properties of lex that were not preserved in other environments.

+	in theory these recognize the same language.

+	now using strtod to test whether a string is a number, instead of

+	the convoluted original function.  should be more portable and

+	reliable if strtod is implemented right.

+	removed now-pointless optimization in makefile that tries to avoid

+	recompilation when awkgram.y is changed but symbols are not.

+	removed most fixed-size arrays, though a handful remain, some

+	of which are unchecked.  you have been warned.

+Aug 4, 1997:

+	with some trepidation, replaced the ancient code that managed

+	fields and $0 in fixed-size arrays with arrays that grow on

+	demand.  there is still some tension between trying to make this

+	run fast and making it clean; not sure it's right yet.

+	the ill-conceived -mr and -mf arguments are now useful only

+	for debugging.  previous dynamic string code removed.

+	numerous other minor cleanups along the way.

+Jul 30, 1997:

+	using code provided by dan levy (to whom profuse thanks), replaced

+	fixed-size arrays and awkward kludges by a fairly uniform mechanism

+	to grow arrays as needed for printf, sub, gsub, etc.

+Jul 23, 1997:

+	falling off the end of a function returns "" and 0, not 0.

+	thanks to arnold robbins.

+Jun 17, 1997:

+	replaced several fixed-size arrays by dynamically-created ones

+	in run.c; added overflow tests to some previously unchecked cases.

+	getline, toupper, tolower.

+	getline code is still broken in that recursive calls may wind

+	up using the same space.  [fixed later]

+	increased RECSIZE to 8192 to push problems further over the horizon.

+	added \r to \n as input line separator for programs, not data.

+	damn CRLFs.

+	modified format() to permit explicit printf("%c", 0) to include

+	a null byte in output.  thanks to ken stailey for the fix.

+	added a "-safe" argument that disables file output (print >,

+	print >>), process creation (cmd|getline, print |, system), and

+	access to the environment (ENVIRON).  this is a first approximation

+	to a "safe" version of awk, but don't rely on it too much.  thanks

+	to joan feigenbaum and matt blaze for the inspiration long ago.

+Jul 8, 1996:

+	fixed long-standing bug in sub, gsub(/a/, "\\\\&"); thanks to

+	ralph corderoy.

+Jun 29, 1996:

+	fixed awful bug in new field splitting; didn't get all the places

+	where input was done.

+Jun 28, 1996:

+	changed field-splitting to conform to posix definition: fields are

+	split using the value of FS at the time of input; it used to be

+	the value when the field or NF was first referred to, a much less

+	predictable definition.  thanks to arnold robbins for encouragement

+	to do the right thing.

+May 28, 1996:

+	fixed appalling but apparently unimportant bug in parsing octal

+	numbers in reg exprs.

+	explicit hex in reg exprs now limited to 2 chars: \xa, \xaa.

+May 27, 1996:

+	cleaned up some declarations so gcc -Wall is now almost silent.

+	makefile now includes backup copies of ytab.c and lexyy.c in case

+	one makes before looking; it also avoids recreating lexyy.c unless

+	really needed.

+	s/aprintf/awkprint, s/asprintf/awksprintf/ to avoid some name clashes

+	with unwisely-written header files.

+	thanks to jeffrey friedl for several of these.

+May 26, 1996:

+	an attempt to rationalize the (unsigned) char issue.  almost all

+	instances of unsigned char have been removed; the handful of places

+	in b.c where chars are used as table indices have been hand-crafted.

+	added some latin-1 tests to the regression, but i'm not confident;

+	none of my compilers seem to care much.  thanks to nelson beebe for

+	pointing out some others that do care.

+May 2, 1996:

+	removed all register declarations.

+	enhanced split(), as in gawk, etc:  split(s, a, "") splits s into

+	a[1]...a[length(s)] with each character a single element.

+	made the same changes for field-splitting if FS is "".

+	added nextfile, as in gawk: causes immediate advance to next

+	input file. (thanks to arnold robbins for inspiration and code).

+	small fixes to regexpr code:  can now handle []], [[], and

+	variants;  [] is now a syntax error, rather than matching

+	everything;  [z-a] is now empty, not z.  far from complete

+	or correct, however.  (thanks to jeffrey friedl for pointing out

+	some awful behaviors.)

+Apr 29, 1996:

+	replaced uchar by uschar everywhere; apparently some compilers

+	usurp this name and this causes conflicts.

+	fixed call to time in run.c (bltin); arg is time_t *.

+	replaced horrible pointer/long punning in b.c by a legitimate

+	union.  should be safer on 64-bit machines and cleaner everywhere.

+	(thanks to nelson beebe for pointing out some of these problems.)

+	replaced nested comments by #if 0...#endif in run.c, lib.c.

+	removed getsval, setsval, execute macros from run.c and lib.c.

+	machines are 100x faster than they were when these macros were

+	first used.

+	revised filenames: awk.g.y => awkgram.y, awk.lx.l => awklex.l,

+	y.tab.[ch] => ytab.[ch], lex.yy.c => lexyy.c, all in the aid of

+	portability to nameless systems.

+	"make bundle" now includes yacc and lex output files for recipients

+	who don't have yacc or lex.

+Aug 15, 1995:

+	initialized Cells in setsymtab more carefully; some fields

+	were not set.  (thanks to purify, all of whose complaints i

+	think i now understand.)

+	fixed at least one error in gsub that looked at -1-th element

+	of an array when substituting for a null match (e.g., $).

+	delete arrayname is now legal; it clears the elements but leaves

+	the array, which may not be the right behavior.

+	modified makefile: my current make can't cope with the test used

+	to avoid unnecessary yacc invocations.

+Jul 17, 1995:

+	added dynamically growing strings to awk.lx.l and b.c

+	to permit regular expressions to be much bigger.

+	the state arrays can still overflow.

+Aug 24, 1994:

+	detect duplicate arguments in function definitions (mdm).

+May 11, 1994:

+	trivial fix to printf to limit string size in sub().

+Apr 22, 1994:

+	fixed yet another subtle self-assignment problem:

+	$1 = $2; $1 = $1 clobbered $1.

+	Regression tests now use private echo, to avoid quoting problems.

+Feb 2, 1994:

+	changed error() to print line number as %d, not %g.

+Jul 23, 1993:

+	cosmetic changes: increased sizes of some arrays,

+	reworded some error messages.

+	added CONVFMT as in posix (just replaced OFMT in getsval)

+	FILENAME is now "" until the first thing that causes a file

+	to be opened.

+Nov 28, 1992:

+	deleted yyunput and yyoutput from proto.h;

+	different versions of lex give these different declarations.

+May 31, 1992:

+	added -mr N and -mf N options: more record and fields.

+	these really ought to adjust automatically.

+	cleaned up some error messages; "out of space" now means

+	malloc returned NULL in all cases.

+	changed rehash so that if it runs out, it just returns;

+	things will continue to run slow, but maybe a bit longer.

+Apr 24, 1992:

+	remove redundant close of stdin when using -f -.

+	got rid of core dump with -d; awk -d just prints date.

+Apr 12, 1992:

+	added explicit check for /dev/std(in,out,err) in redirection.

+	unlike gawk, no /dev/fd/n yet.

+	added (file/pipe) builtin.  hard to test satisfactorily.

+	not posix.

+Feb 20, 1992:

+	recompile after abortive changes;  should be unchanged.

+Dec 2, 1991:

+	die-casting time:  converted to ansi C, installed that.

+Nov 30, 1991:

+	fixed storage leak in freefa, failing to recover [N]CCL.

+	thanks to Bill Jones (jones@cs.usask.ca)

+Nov 19, 1991:

+	use RAND_MAX instead of literal in builtin().

+Nov 12, 1991:

+	cranked up some fixed-size arrays in b.c, and added a test for

+	overflow in penter.  thanks to mark larsen.

+Sep 24, 1991:

+	increased buffer in gsub.  a very crude fix to a general problem.

+	and again on Sep 26.

+Aug 18, 1991:

+	enforce variable name syntax for commandline variables: has to

+	start with letter or _.

+Jul 27, 1991:

+	allow newline after ; in for statements.

+Jul 21, 1991:

+	fixed so that in self-assignment like $1=$1, side effects

+	like recomputing $0 take place.  (this is getting subtle.)

+Jun 30, 1991:

+	better test for detecting too-long output record.

+Jun 2, 1991:

+	better defense against very long printf strings.

+	made break and continue illegal outside of loops.

+May 13, 1991:

+	removed extra arg on gettemp, tempfree.  minor error message rewording.

+May 6, 1991:

+	fixed silly bug in hex parsing in hexstr().

+	removed an apparently unnecessary test in isnumber().

+	warn about weird printf conversions.

+	fixed unchecked array overwrite in relex().

+	changed for (i in array) to access elements in sorted order.

+	then unchanged it -- it really does run slower in too many cases.

+	left the code in place, commented out.

+Feb 10, 1991:

+	check error status on all writes, to avoid banging on full disks.

+Jan 28, 1991:

+	awk -f - reads the program from stdin.

+Jan 11, 1991:

+	failed to set numeric state on $0 in cmd|getline context in run.c.

+Nov 2, 1990:

+	fixed sleazy test for integrality in getsval;  use modf.

+Oct 29, 1990:

+	fixed sleazy buggy code in lib.c that looked (incorrectly) for

+	too long input lines.

+Oct 14, 1990:

+	fixed the bug on p. 198 in which it couldn't deduce that an

+	argument was an array in some contexts.  replaced the error

+	message in intest() by code that damn well makes it an array.

+Oct 8, 1990:

+	fixed horrible bug:  types and values were not preserved in

+	some kinds of self-assignment. (in assign().)

+Aug 24, 1990:

+	changed NCHARS to 256 to handle 8-bit characters in strings

+	presented to match(), etc.

+Jun 26, 1990:

+	changed struct rrow (awk.h) to use long instead of int for lval,

+	since cfoll() stores a pointer in it.  now works better when int's

+	are smaller than pointers!

+May 6, 1990:

+	AVA fixed the grammar so that ! is uniformly of the same precedence as

+	unary + and -.  This renders illegal some constructs like !x=y, which

+	now has to be parenthesized as !(x=y), and makes others work properly:

+	!x+y is (!x)+y, and x!y is x !y, not two pattern-action statements.

+	(These problems were pointed out by Bob Lenk of Posix.)

+	Added \x to regular expressions (already in strings).

+	Limited octal to octal digits; \8 and \9 are not octal.

+	Centralized the code for parsing escapes in regular expressions.

+	Added a bunch of tests to T.re and T.sub to verify some of this.

+Feb 9, 1990:

+	fixed null pointer dereference bug in main.c:  -F[nothing].  sigh.

+	restored srand behavior:  it returns the current seed.

+Jan 18, 1990:

+	srand now returns previous seed value (0 to start).

+Jan 5, 1990:

+	fix potential problem in tran.c -- something was freed,

+	then used in freesymtab.

+Oct 18, 1989:

+	another try to get the max number of open files set with

+	relatively machine-independent code.

+	small fix to input() in case of multiple reads after EOF.

+Oct 11, 1989:

+	FILENAME is now defined in the BEGIN block -- too many old

+	programs broke.

+	"-" means stdin in getline as well as on the commandline.

+	added a bunch of casts to the code to tell the truth about

+	char * vs. unsigned char *, a right royal pain.  added a

+	setlocale call to the front of main, though probably no one

+	has it usefully implemented yet.

+Aug 24, 1989:

+	removed redundant relational tests against nullnode if parse

+	tree already had a relational at that point.

+Aug 11, 1989:

+	fixed bug:  commandline variable assignment has to look like

+	var=something.  (consider the man page for =, in file =.1)

+	changed number of arguments to functions to static arrays

+	to avoid repeated malloc calls.

+Aug 2, 1989:

+	restored -F (space) separator

+Jul 30, 1989:

+	added -v x=1 y=2 ... for immediate commandline variable assignment;

+	done before the BEGIN block for sure.  they have to precede the

+	program if the program is on the commandline.

+	Modified Aug 2 to require a separate -v for each assignment.

+Jul 10, 1989:

+	fixed ref-thru-zero bug in environment code in tran.c

+Jun 23, 1989:

+	add newline to usage message.

+Jun 14, 1989:

+	added some missing ansi printf conversion letters: %i %X %E %G.

+	no sensible meaning for h or L, so they may not do what one expects.

+	made %* conversions work.

+	changed x^y so that if n is a positive integer, it's done

+	by explicit multiplication, thus achieving maximum accuracy.

+	(this should be done by pow() but it seems not to be locally.)

+	done to x ^= y as well.

+Jun 4, 1989:

+	ENVIRON array contains environment: if shell variable V=thing,

+		ENVIRON["V"] is "thing"

+	multiple -f arguments permitted.  error reporting is naive.

+	(they were permitted before, but only the last was used.)

+	fixed a really stupid botch in the debugging macro dprintf

+	fixed order of evaluation of commandline assignments to match

+	what the book claims:  an argument of the form x=e is evaluated

+	at the time it would have been opened if it were a filename (p 63).

+	this invalidates the suggested answer to ex 4-1 (p 195).

+	removed some code that permitted -F (space) fieldseparator,

+	since it didn't quite work right anyway.  (restored aug 2)

+Apr 27, 1989:

+	Line number now accumulated correctly for comment lines.

+Apr 26, 1989:

+	Debugging output now includes a version date,

+	if one compiles it into the source each time.

+Apr 9, 1989:

+	Changed grammar to prohibit constants as 3rd arg of sub and gsub;

+	prevents class of overwriting-a-constant errors.  (Last one?)

+	This invalidates the "banana" example on page 43 of the book.

+	Added \a ("alert"), \v (vertical tab), \xhhh (hexadecimal),

+	as in ANSI, for strings.  Rescinded the sloppiness that permitted

+	non-octal digits in \ooo.  Warning:  not all compilers and libraries

+	will be able to deal with \x correctly.

+Jan 9, 1989:

+	Fixed bug that caused tempcell list to contain a duplicate.

+	The fix is kludgy.

+Dec 17, 1988:

+	Catches some more commandline errors in main.

+	Removed redundant decl of modf in run.c (confuses some compilers).

+	Warning:  there's no single declaration of malloc, etc., in awk.h

+	that seems to satisfy all compilers.

+Dec 7, 1988:

+	Added a bit of code to error printing to avoid printing nulls.

+	(Not clear that it actually would.)

+Nov 27, 1988:

+	With fear and trembling, modified the grammar to permit

+	multiple pattern-action statements on one line without

+	an explicit separator.  By definition, this capitulation

+	to the ghost of ancient implementations remains undefined

+	and thus subject to change without notice or apology.

+	DO NOT COUNT ON IT.

+Oct 30, 1988:

+	Fixed bug in call() that failed to recover storage.

+	A warning is now generated if there are more arguments

+	in the call than in the definition (in lieu of fixing

+	another storage leak).

+Oct 20, 1988:

+	Fixed %c:  if expr is numeric, use numeric value;

+	otherwise print 1st char of string value.  still

+	doesn't work if the value is 0 -- won't print \0.

+	Added a few more checks for running out of malloc.

+Oct 12, 1988:

+	Fixed bug in call() that freed local arrays twice.

+	Fixed to handle deletion of non-existent array right;

+	complains about attempt to delete non-array element.

+Sep 30, 1988:

+	Now guarantees to evaluate all arguments of built-in

+	functions, as in C;  the appearance is that arguments

+	are evaluated before the function is called.  Places

+	affected are sub (gsub was ok), substr, printf, and

+	all the built-in arithmetic functions in bltin().

+	A warning is generated if a bltin() is called with

+	the wrong number of arguments.

+	This requires changing makeprof on p167 of the book.

+Aug 23, 1988:

+	setting FILENAME in BEGIN caused core dump, apparently

+	because it was freeing space not allocated by malloc.

+July 24, 1988:

+	fixed egregious error in toupper/tolower functions.

+	still subject to rescinding, however.

+July 2, 1988:

+	flush stdout before opening file or pipe

+July 2, 1988:

+	performance bug in b.c/cgoto(): not freeing some sets of states.

+	partial fix only right now, and the number of states increased

+	to make it less obvious.

+June 1, 1988:

+	check error status on close

+May 28, 1988:

+	srand returns seed value it's using.

+	see 1/18/90

+May 22, 1988:

+	Removed limit on depth of function calls.

+May 10, 1988:

+	Fixed lib.c to permit _ in commandline variable names.

+Mar 25, 1988:

+	main.c fixed to recognize -- as terminator of command-

+	line options.  Illegal options flagged.

+	Error reporting slightly cleaned up.

+Dec 2, 1987:

+	Newer C compilers apply a strict scope rule to extern

+	declarations within functions.  Two extern declarations in

+	lib.c and tran.c have been moved to obviate this problem.

+Oct xx, 1987:

+	Reluctantly added toupper and tolower functions.

+	Subject to rescinding without notice.

+Sep 17, 1987:

+	Error-message printer had printf(s) instead of

+	printf("%s",s);  got core dumps when the message

+	included a %.

+Sep 12, 1987:

+	Very long printf strings caused core dump;

+	fixed aprintf, asprintf, format to catch them.

+	Can still get a core dump in printf itself.

--- a/README.md

+++ b/README.md

@@ -1,9 +1,39 @@

 # The One True Awk

 This is the version of `awk` described in _The AWK Programming Language_,

-by Al Aho, Brian Kernighan, and Peter Weinberger

-(Addison-Wesley, 1988, ISBN 0-201-07981-X).

+Second Edition, by Al Aho, Brian Kernighan, and Peter Weinberger

+(Addison-Wesley, 2024, ISBN-13 978-0138269722, ISBN-10 0138269726).

+## What's New? ##

+This version of Awk handles UTF-8 and comma-separated values (CSV) input.

+### Strings ###

+Functions that process strings now count Unicode code points, not bytes;

+this affects `length`, `substr`, `index`, `match`, `split`,

+`sub`, `gsub`, and others.  Note that code

+points are not necessarily characters.

+UTF-8 sequences may appear in literal strings and regular expressions.

+Aribtrary characters may be included with `\u` followed by 1 to 8 hexadecimal digits.

+### Regular expressions ###

+Regular expressions may include UTF-8 code points, including `\u`.

+Character classes are likely to be limited to about 256 characters

+when expanded.

+### CSV ###

+The option `--csv` turns on CSV processing of input:

+fields are separated by commas, fields may be quoted with

+double-quote (`"`) characters, fields may contain embedded newlines.

+In CSV mode, `FS` is ignored.

+If no explicit separator argument is provided,

+field-splitting in `split` is determined by CSV mode.

 ## Copyright

 Copyright (C) Lucent Technologies 1997<br/>

@@ -67,22 +97,22 @@

 which should produce a sequence of messages roughly like this:

-	yacc -d awkgram.y

-	conflicts: 43 shift/reduce, 85 reduce/reduce

-	mv y.tab.c ytab.c

-	mv y.tab.h ytab.h

-	cc -c ytab.c

-	cc -c b.c

-	cc -c main.c

-	cc -c parse.c

-	cc maketab.c -o maketab

-	./maketab >proctab.c

-	cc -c proctab.c

-	cc -c tran.c

-	cc -c lib.c

-	cc -c run.c

-	cc -c lex.c

-	cc ytab.o b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o -lm

+	bison -d  awkgram.y

+	awkgram.y: warning: 44 shift/reduce conflicts [-Wconflicts-sr]

+	awkgram.y: warning: 85 reduce/reduce conflicts [-Wconflicts-rr]

+	awkgram.y: note: rerun with option '-Wcounterexamples' to generate conflict counterexamples

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o awkgram.tab.o awkgram.tab.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o b.o b.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o main.o main.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o parse.o parse.c

+	gcc -g -Wall -pedantic -Wcast-qual -O2 maketab.c -o maketab

+	./maketab awkgram.tab.h >proctab.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o proctab.o proctab.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o tran.o tran.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o lib.o lib.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o run.o run.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2   -c -o lex.o lex.c

+	gcc -g -Wall -pedantic -Wcast-qual   -O2 awkgram.tab.o b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o   -lm

 This produces an executable `a.out`; you will eventually want to

 move this to some place like `/usr/bin/awk`.

@@ -102,14 +132,9 @@

 You can also use `make CC=g++` to build with the GNU C++ compiler,

 should you choose to do so.

-The version of `malloc` that comes with some systems is sometimes

-astonishly slow.  If `awk` seems slow, you might try fixing that.

-More generally, turning on optimization can significantly improve

-`awk`'s speed, perhaps by 1/3 for highest levels.

 ## A Note About Releases

-We don't usually do releases.

+We don't usually do releases.

 ## A Note About Maintenance

@@ -120,5 +145,4 @@

 #### Last Updated

-Sun 23 Jan 2022 03:48:01 PM EST

+Sun Sep  3 09:26:43 EDT 2023

--- a/awk.1

+++ b/awk.1

@@ -20,6 +20,8 @@

 .BI \-F

 .I fs

+|

+.B \-\^\-csv

 .BI \-v

@@ -76,6 +78,12 @@

 .I fs

 option defines the input field separator to be the regular expression

 .IR fs .

+The

+.B \-\^\-csv

+option causes

+.I awk

+to process records using (more or less) standard comma-separated values

+(CSV) format.

.PP

 An input line is normally made up of fields separated by white space,

 or by the regular expression

@@ -202,9 +210,9 @@

 .B sqrt

 are built in.

 Other built-in functions:

-.TF length

+.TF "\fBlength(\fR[\fIv\^\fR]\fB)\fR"

.TP

-.B length

+\fBlength(\fR[\fIv\^\fR]\fB)\fR

 the length of its argument

 taken as a string,

 number of elements in an array for an array argument,

@@ -212,15 +220,15 @@

 .B $0

 if no argument.

.TP

-.B rand

+.B rand()

 random number on [0,1).

.TP

-.B srand

+\fBsrand(\fR[\fIs\^\fR]\fB)\fR

 sets seed for

 .B rand

 and returns the previous seed.

.TP

-.B int

+.BI int( x\^ )

 truncates to an integer value.

.TP

 \fBsubstr(\fIs\fB, \fIm\fR [\fB, \fIn\^\fR]\fB)\fR

@@ -389,7 +397,7 @@

 A pattern may consist of two patterns separated by a comma;

 in this case, the action is performed for all lines

 from an occurrence of the first pattern

-though an occurrence of the second.

+through an occurrence of the second, inclusive.

.PP

 A relational expression is one of the following:

.IP

@@ -399,7 +407,7 @@

.br

 .IB expression " in " array-name

.br

-.BI ( expr , expr,... ") in " array-name

+.BI ( expr ,\| expr ,\| ... ") in " array-name

.PP

 where a

 .I relop

@@ -499,7 +507,7 @@

 Functions may be defined (at the position of a pattern-action statement) thus:

.IP

.B

-function foo(a, b, c) { ...; return x }

+function foo(a, b, c) { ... }

.PP

 Parameters are passed by value if scalar and by reference if array name;

 functions may be called recursively.

@@ -565,8 +573,8 @@

 .IR sed (1)

.br

 A. V. Aho, B. W. Kernighan, P. J. Weinberger,

-.IR "The AWK Programming Language" ,

-Addison-Wesley, 1988.  ISBN 0-201-07981-X.

+.IR "The AWK Programming Language, Second Edition" ,

+Addison-Wesley, 2024.  ISBN 978-0-13-826972-2, 0-13-826972-6.

 .SH BUGS

 There are no explicit conversions between numbers and strings.

 To force an expression to be treated as a number add 0 to it;

@@ -576,7 +584,8 @@

 The scope rules for variables in functions are a botch;

 the syntax is worse.

.PP

-Only eight-bit characters sets are handled correctly.

+Input is expected to be UTF-8 encoded. Other multibyte

+character sets are not handled.

 .SH UNUSUAL FLOATING-POINT VALUES

 .I Awk

 was designed before IEEE 754 arithmetic defined Not-A-Number (NaN)

--- a/awk.h

+++ b/awk.h

@@ -78,6 +78,8 @@

 extern Awkfloat *RSTART;

 extern Awkfloat *RLENGTH;

+extern bool	CSV;		/* true for csv input */

 extern char	*record;	/* points to $0 */

 extern int	lineno;		/* line number in awk program */

 extern int	errorflag;	/* 1 if error has occurred */

@@ -225,7 +227,8 @@

 /* structures used by regular expression matching machinery, mostly b.c: */

-#define NCHARS	(256+3)		/* 256 handles 8-bit chars; 128 does 7-bit */

+#define NCHARS	(1256+3)		/* 256 handles 8-bit chars; 128 does 7-bit */

+				/* BUG: some overflows (caught) if we use 256 */

 				/* watch out in match(), etc. */

 #define	HAT	(NCHARS+2)	/* matches ^ in regular expr */

 #define NSTATES	32

@@ -236,12 +239,19 @@

 		int i;

 		Node *np;

 		uschar *up;

+		int *rp; /* rune representation of char class */

 	} lval;		/* because Al stores a pointer in it! */

 	int	*lfollow;

 } rrow;

+typedef struct gtt { /* gototab entry */

+	unsigned int ch;

+	unsigned int state;

+} gtt;

 typedef struct fa {

-	unsigned int	**gototab;

+	gtt	**gototab;

+	int	gototab_len;

 	uschar	*out;

 	uschar	*restr;

 	int	**posns;

--- a/b.c

+++ b/b.c

@@ -80,6 +80,41 @@

 fa	*fatab[NFA];

 int	nfatab	= 0;	/* entries in fatab */

+/* utf-8 mechanism:

+   For most of Awk, utf-8 strings just "work", since they look like

+   null-terminated sequences of 8-bit bytes.

+   Functions like length(), index(), and substr() have to operate

+   in units of utf-8 characters.  The u8_* functions in run.c

+   handle this.

+   Regular expressions are more complicated, since the basic

+   mechanism of the goto table used 8-bit byte indices into the

+   gototab entries to compute the next state.  Unicode is a lot

+   bigger, so the gototab entries are now structs with a character

+   and a next state, and there is a linear search of the characters

+   to find the state.  (Yes, this is slower, by a significant

+   amount.  Tough.)

+   Throughout the RE mechanism in b.c, utf-8 characters are

+   converted to their utf-32 value.  This mostly shows up in

+   cclenter, which expands character class ranges like a-z and now

+   alpha-omega.  The size of a gototab array is still about 256.

+   This should be dynamic, but for now things work ok for a single

+   code page of Unicode, which is the most likely case.

+   The code changes are localized in run.c and b.c.  I have added a

+   handful of functions to somewhat better hide the implementation,

+   but a lot more could be done.

+ */

+static int get_gototab(fa*, int, int);

+static int set_gototab(fa*, int, int, int);

+extern int u8_rune(int *, const uschar *);

 static int *

 intalloc(size_t n, const char *f)

@@ -105,7 +140,7 @@

 static void

 resize_state(fa *f, int state)

-	unsigned int **p;

+	gtt **p;

 	uschar *p2;

 	int **p3;

 	int i, new_count;

@@ -115,7 +150,7 @@

 	new_count = state + 10; /* needs to be tuned */

-	p = (unsigned int **) realloc(f->gototab, new_count * sizeof(f->gototab[0]));

+	p = (gtt **) realloc(f->gototab, new_count * sizeof(f->gototab[0]));

 	if (p == NULL)

 		goto out;

 	f->gototab = p;

@@ -131,12 +166,13 @@

 	f->posns = p3;

 	for (i = f->state_count; i < new_count; ++i) {

-		f->gototab[i] = (unsigned int *) calloc(NCHARS, sizeof(**f->gototab));

+		f->gototab[i] = (gtt *) calloc(NCHARS, sizeof(**f->gototab));

 		if (f->gototab[i] == NULL)

 			goto out;

 		f->out[i]  = 0;

 		f->posns[i] = NULL;

+	f->gototab_len = NCHARS; /* should be variable, growable */

 	f->state_count = new_count;

 	return;

 out:

@@ -231,7 +267,7 @@

 	if ((f->posns[2])[1] == f->accept)

 		f->out[2] = 1;

 	for (i = 0; i < NCHARS; i++)

-		f->gototab[2][i] = 0;

+		set_gototab(f, 2, 0, 0); /* f->gototab[2][i] = 0; */

 	f->curstat = cgoto(f, 2, HAT);

 	if (anchor) {

 		*f->posns[2] = k-1;	/* leave out position 0 */

@@ -300,13 +336,13 @@

 /* in the parsing of regular expressions, metacharacters like . have */

 /* to be seen literally;  \056 is not a metacharacter. */

-int hexstr(const uschar **pp)	/* find and eval hex string at pp, return new p */

+int hexstr(const uschar **pp, int max)	/* find and eval hex string at pp, return new p */

 {			/* only pick up one 8-bit byte (2 chars) */

 	const uschar *p;

 	int n = 0;

 	int i;

-	for (i = 0, p = *pp; i < 2 && isxdigit(*p); i++, p++) {

+	for (i = 0, p = *pp; i < max && isxdigit(*p); i++, p++) {

 		if (isdigit(*p))

 			n = 16 * n + *p - '0';

 		else if (*p >= 'a' && *p <= 'f')

@@ -318,6 +354,8 @@

 	return n;

 #define isoctdigit(c) ((c) >= '0' && (c) <= '7')	/* multiple use of arg */

 int quoted(const uschar **pp)	/* pick up next thing after a \\ */

@@ -326,24 +364,28 @@

 	const uschar *p = *pp;

 	int c;

-	if ((c = *p++) == 't')

+/* BUG: should advance by utf-8 char even if makes no sense */

+	if ((c = *p++) == 't') {

 		c = '\t';

-	else if (c == 'n')

+	} else if (c == 'n') {

 		c = '\n';

-	else if (c == 'f')

+	} else if (c == 'f') {

 		c = '\f';

-	else if (c == 'r')

+	} else if (c == 'r') {

 		c = '\r';

-	else if (c == 'b')

+	} else if (c == 'b') {

 		c = '\b';

-	else if (c == 'v')

+	} else if (c == 'v') {

 		c = '\v';

-	else if (c == 'a')

+	} else if (c == 'a') {

 		c = '\a';

-	else if (c == '\\')

+	} else if (c == '\\') {

 		c = '\\';

-	else if (c == 'x') {	/* hexadecimal goo follows */

-		c = hexstr(&p);	/* this adds a null if number is invalid */

+	} else if (c == 'x') {	/* 2 hex digits follow */

+		c = hexstr(&p, 2);	/* this adds a null if number is invalid */

+	} else if (c == 'u') {	/* unicode char number up to 8 hex digits */

+		c = hexstr(&p, 8);

 	} else if (isoctdigit(c)) {	/* \d \dd \ddd */

 		int n = c - '0';

 		if (isoctdigit(*p)) {

@@ -358,27 +400,31 @@

 	return c;

-char *cclenter(const char *argp)	/* add a character class */

+int *cclenter(const char *argp)	/* add a character class */

 	int i, c, c2;

-	const uschar *op, *p = (const uschar *) argp;

-	uschar *bp;

-	static uschar *buf = NULL;

+	int n;

+	const uschar *p = (const uschar *) argp;

+	int *bp, *retp;

+	static int *buf = NULL;

 	static int bufsz = 100;

-	op = p;

-	if (buf == NULL && (buf = (uschar *) malloc(bufsz)) == NULL)

+	if (buf == NULL && (buf = (int *) calloc(bufsz, sizeof(int))) == NULL)

 		FATAL("out of space for character class [%.10s...] 1", p);

 	bp = buf;

-	for (i = 0; (c = *p++) != 0; ) {

+	for (i = 0; *p != 0; ) {

+		n = u8_rune(&c, p);

+		p += n;

 		if (c == '\\') {

 			c = quoted(&p);

 		} else if (c == '-' && i > 0 && bp[-1] != 0) {

 			if (*p != 0) {

 				c = bp[-1];

-				c2 = *p++;

+				/* c2 = *p++; */

+				n = u8_rune(&c2, p);

+				p += n;

 				if (c2 == '\\')

-					c2 = quoted(&p);

+					c2 = quoted(&p); /* BUG: sets p, has to be u8 size */

 				if (c > c2) {	/* empty; ignore */

 					bp--;

 					i--;

@@ -385,8 +431,13 @@

 					continue;

 				while (c < c2) {

-					if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "cclenter1"))

-						FATAL("out of space for character class [%.10s...] 2", p);

+					if (i >= bufsz) {

+						bufsz *= 2;

+						buf = (int *) realloc(buf, bufsz * sizeof(int));

+						if (buf == NULL)

+							FATAL("out of space for character class [%.10s...] 2", p);

+						bp = buf + i;

+					}

 					*bp++ = ++c;

 					i++;

@@ -393,15 +444,23 @@

 				continue;

-		if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "cclenter2"))

-			FATAL("out of space for character class [%.10s...] 3", p);

+		if (i >= bufsz) {

+			bufsz *= 2;

+			buf = (int *) realloc(buf, bufsz * sizeof(int));

+			if (buf == NULL)

+				FATAL("out of space for character class [%.10s...] 2", p);

+			bp = buf + i;

+		}

 		*bp++ = c;

 		i++;

 	*bp = 0;

-	DPRINTF("cclenter: in = |%s|, out = |%s|\n", op, buf);

-	xfree(op);

-	return (char *) tostring((char *) buf);

+	/* DPRINTF("cclenter: in = |%s|, out = |%s|\n", op, buf); BUG: can't print array of int */

+	/* xfree(op);  BUG: what are we freeing here? */

+	retp = (int *) calloc(bp-buf+1, sizeof(int));

+	for (i = 0; i < bp-buf+1; i++)

+		retp[i] = buf[i];

+	return retp;

 void overflo(const char *s)

@@ -524,9 +583,9 @@

-int member(int c, const char *sarg)	/* is c in s? */

+int member(int c, int *sarg)	/* is c in s? */

-	const uschar *s = (const uschar *) sarg;

+	int *s = (int *) sarg;

 	while (*s)

 		if (c == *s++)

@@ -534,11 +593,41 @@

 	return(0);

+static int get_gototab(fa *f, int state, int ch) /* hide gototab inplementation */

+{

+	int i;

+	for (i = 0; i < f->gototab_len; i++) {

+		if (f->gototab[state][i].ch == 0)

+			break;

+		if (f->gototab[state][i].ch == ch)

+			return f->gototab[state][i].state;

+	}

+	return 0;

+}

+static int set_gototab(fa *f, int state, int ch, int val) /* hide gototab inplementation */

+{

+	int i;

+	for (i = 0; i < f->gototab_len; i++) {

+		if (f->gototab[state][i].ch == 0 || f->gototab[state][i].ch == ch) {

+			f->gototab[state][i].ch = ch;

+			f->gototab[state][i].state = val;

+			return val;

+		}

+	}

+	overflo(__func__);

+	return val; /* not used anywhere at the moment */

+}

 int match(fa *f, const char *p0)	/* shortest match ? */

 	int s, ns;

+	int n;

+	int rune;

 	const uschar *p = (const uschar *) p0;

+	/* return pmatch(f, p0); does it matter whether longest or shortest? */

 	s = f->initstat;

 	assert (s < f->state_count);

@@ -546,13 +635,17 @@

 		return(1);

 	do {

 		/* assert(*p < NCHARS); */

-		if ((ns = f->gototab[s][*p]) != 0)

+		n = u8_rune(&rune, p);

+		if ((ns = get_gototab(f, s, rune)) != 0)

 			s = ns;

 		else

-			s = cgoto(f, s, *p);

+			s = cgoto(f, s, rune);

 		if (f->out[s])

 			return(1);

-	} while (*p++ != 0);

+		if (*p == 0)

+			break;

+		p += n;

+	} while (1);  /* was *p++ != 0 */

 	return(0);

@@ -559,6 +652,8 @@

 int pmatch(fa *f, const char *p0)	/* longest match, for sub */

 	int s, ns;

+	int n;

+	int rune;

 	const uschar *p = (const uschar *) p0;

 	const uschar *q;

@@ -573,10 +668,11 @@

 			if (f->out[s])		/* final state */

 				patlen = q-p;

 			/* assert(*q < NCHARS); */

-			if ((ns = f->gototab[s][*q]) != 0)

+			n = u8_rune(&rune, q);

+			if ((ns = get_gototab(f, s, rune)) != 0)

 				s = ns;

 			else

-				s = cgoto(f, s, *q);

+				s = cgoto(f, s, rune);

 			assert(s < f->state_count);

@@ -588,7 +684,11 @@

 				else

 					goto nextin;	/* no match */

-		} while (*q++ != 0);

+			if (*q == 0)

+				break;

+			q += n;

+		} while (1);

+		q++;  /* was *q++ */

 		if (f->out[s])

 			patlen = q-p-1;	/* don't count $ */

 		if (patlen >= 0) {

@@ -597,7 +697,11 @@

 	nextin:

 		s = 2;

-	} while (*p++);

+		if (*p == 0)

+			break;

+		n = u8_rune(&rune, p);

+		p += n;

+	} while (1); /* was *p++ */

 	return (0);

@@ -604,6 +708,8 @@

 int nematch(fa *f, const char *p0)	/* non-empty match, for sub */

 	int s, ns;

+        int n;

+        int rune;

 	const uschar *p = (const uschar *) p0;

 	const uschar *q;

@@ -618,10 +724,11 @@

 			if (f->out[s])		/* final state */

 				patlen = q-p;

 			/* assert(*q < NCHARS); */

-			if ((ns = f->gototab[s][*q]) != 0)

+			n = u8_rune(&rune, q);

+			if ((ns = get_gototab(f, s, rune)) != 0)

 				s = ns;

 			else

-				s = cgoto(f, s, *q);

+				s = cgoto(f, s, rune);

 			if (s == 1) {	/* no transition */

 				if (patlen > 0) {

 					patbeg = (const char *) p;

@@ -629,7 +736,11 @@

 				} else

 					goto nnextin;	/* no nonempty match */

-		} while (*q++ != 0);

+			if (*q == 0)

+				break;

+			q += n;

+		} while (1);

+		q++;

 		if (f->out[s])

 			patlen = q-p-1;	/* don't count $ */

 		if (patlen > 0 ) {

@@ -643,7 +754,36 @@

 	return (0);

+static int getrune(FILE *fp, char **pbuf, int *pbufsize, int quantum,

+		   int *curpos, int *lastpos)

+{

+	int c = 0;

+	char *buf = *pbuf;

+	static const int max_bytes = 4;	// max multiple bytes in UTF-8 is 4

+	int i, rune;

+	uschar private_buf[max_bytes + 1];

+	for (i = 0; i <= max_bytes; i++) {

+		if (++*curpos == *lastpos) {

+			if (*lastpos == *pbufsize)

+				if (!adjbuf((char **) pbuf, pbufsize, *pbufsize+1, quantum, 0, "getrune"))

+					FATAL("stream '%.30s...' too long", buf);

+			buf[(*lastpos)++] = (c = getc(fp)) != EOF ? c : 0;

+			private_buf[i] = c;

+		}

+		if (c == 0 || c < 128 ||  (c >> 6) == 4) { // 10xxxxxx starts a new character

+			ungetc(c, fp);

+			private_buf[i] = 0;

+			break;

+		}

+	}

+	u8_rune(& rune, private_buf);

+	return rune;

+}

/*

  * NAME

  *     fnematch

@@ -664,6 +804,7 @@

 	char *buf = *pbuf;

 	int bufsize = *pbufsize;

 	int c, i, j, k, ns, s;

+	int rune;

 	s = pfa->initstat;

 	patlen = 0;

@@ -687,12 +828,19 @@

 				buf[k++] = (c = getc(f)) != EOF ? c : 0;

 			c = (uschar)buf[j];

-			/* assert(c < NCHARS); */

+			if (c < 128)

+				rune = c;

+			else {

+				j--;

+				k--;

+				ungetc(c, f);

+				rune = getrune(f, &buf, &bufsize, quantum, &j, &k);

+			}

-			if ((ns = pfa->gototab[s][c]) != 0)

+			if ((ns = get_gototab(pfa, s, rune)) != 0)

 				s = ns;

 			else

-				s = cgoto(pfa, s, c);

+				s = cgoto(pfa, s, rune);

 			if (pfa->out[s]) {	/* final state */

 				patlen = j - i + 1;

@@ -1019,6 +1167,8 @@

 	return 0;

+extern int u8_rune(int *, const uschar *); /* run.c; should be in header file */

 int relex(void)		/* lexical analyzer for reparse */

 	int c, n;

@@ -1036,6 +1186,12 @@

 rescan:

 	starttok = prestr;

+	if ((n = u8_rune(&rlxval, prestr)) > 1) {

+		prestr += n;

+		starttok = prestr;

+		return CHAR;

+	}

 	switch (c = *prestr++) {

 	case '|': return OR;

 	case '*': return STAR;

@@ -1073,10 +1229,15 @@

 		else

 			cflag = 0;

-		n = 2 * strlen((const char *) prestr)+1;

+		n = 5 * strlen((const char *) prestr)+1; /* BUG: was 2.  what value? */

 		if (!adjbuf((char **) &buf, &bufsz, n, n, (char **) &bp, "relex1"))

 			FATAL("out of space for reg expr %.10s...", lastre);

 		for (; ; ) {

+			if ((n = u8_rune(&rlxval, prestr)) > 1) {

+				for (i = 0; i < n; i++)

+					*bp++ = *prestr++;

+				continue;

+			}

 			if ((c = *prestr++) == '\\') {

 				*bp++ = '\\';

 				if ((c = *prestr++) == '\0')

@@ -1243,7 +1404,7 @@

 	int *p, *q;

 	int i, j, k;

-	assert(c == HAT || c < NCHARS);

+	/* assert(c == HAT || c < NCHARS);  BUG: seg fault if disable test */

 	while (f->accept >= maxsetvec) {	/* guessing here! */

 		resizesetvec(__func__);

@@ -1259,8 +1420,8 @@

 			 || (k == DOT && c != 0 && c != HAT)

 			 || (k == ALL && c != 0)

 			 || (k == EMPTYRE && c != 0)

-			 || (k == CCL && member(c, (char *) f->re[p[i]].lval.up))

-			 || (k == NCCL && !member(c, (char *) f->re[p[i]].lval.up) && c != 0 && c != HAT)) {

+			 || (k == CCL && member(c, (int *) f->re[p[i]].lval.rp))

+			 || (k == NCCL && !member(c, (int *) f->re[p[i]].lval.rp) && c != 0 && c != HAT)) {

 				q = f->re[p[i]].lfollow;

 				for (j = 1; j <= *q; j++) {

 					if (q[j] >= maxsetvec) {

@@ -1292,7 +1453,7 @@

 				goto different;

 		/* setvec is state i */

 		if (c != HAT)

-			f->gototab[s][c] = i;

+			set_gototab(f, s, c, i);

 		return i;

 	  different:;

@@ -1301,13 +1462,13 @@

 	++(f->curstat);

 	resize_state(f, f->curstat);

 	for (i = 0; i < NCHARS; i++)

-		f->gototab[f->curstat][i] = 0;

+		set_gototab(f, f->curstat, 0, 0);

 	xfree(f->posns[f->curstat]);

 	p = intalloc(setcnt + 1, __func__);

 	f->posns[f->curstat] = p;

 	if (c != HAT)

-		f->gototab[s][c] = f->curstat;

+		set_gototab(f, s, c, f->curstat);

 	for (i = 0; i <= setcnt; i++)

 		p[i] = tmpset[i];

 	if (setvec[f->accept])

--- a/lex.c

+++ b/lex.c

@@ -368,6 +368,8 @@

+extern int runetochar(char *str, int c);

 int string(void)

 	int c, n;

@@ -415,7 +417,7 @@

 				*bp++ = n;

 				break;

-			case 'x':	/* hex  \x0-9a-fA-F + */

+			case 'x':	/* hex  \x0-9a-fA-F (exactly two) */

 				int i;

@@ -438,6 +440,27 @@

 					*bp++ = n;

 				else

 					unput(c);

+				break;

+			    }

+			case 'u':	/* utf  \u0-9a-fA-F (1..8) */

+			    {

+				int i;

+				n = 0;

+				for (i = 0; i < 8; i++) {

+					c = input();

+					if (!isxdigit(c) || c == 0)

+						break;

+					c = tolower(c);

+					n *= 16;

+					if (isdigit(c))

+						n += (c - '0');

+					else

+						n += 10 + (c - 'a');

+				}

+				unput(c);

+				bp += runetochar(bp, n);

 				break;

--- a/lib.c

+++ b/lib.c

@@ -34,6 +34,8 @@

 #include <math.h>

 #include "awk.h"

+extern int u8_nextlen(const char *s);

 char	EMPTY[] = { '\0' };

 FILE	*infile	= NULL;

 bool	innew;		/* true = infile has not been read by readrec */

@@ -219,14 +221,19 @@

 	argno++;

+extern int readcsvrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag);

 int readrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag)	/* read one record into buf */

-	int sep, c, isrec;

-	char *rr, *buf = *pbuf;

+	int sep, c, isrec; // POTENTIAL BUG? isrec is a macro in awk.h

+	char *rr = *pbuf, *buf = *pbuf;

 	int bufsize = *pbufsize;

 	char *rs = getsval(rsloc);

-	if (*rs && rs[1]) {

+	if (CSV) {

+		c = readcsvrec(pbuf, pbufsize, inf, newflag);

+		isrec = (c == EOF && rr == buf) ? false : true;

+	} else if (*rs && rs[1]) {

 		bool found;

 		fa *pfa = makedfa(rs, 1);

@@ -241,6 +248,7 @@

 		if (found)

 			setptr(patbeg, '\0');

 		isrec = (found == 0 && *buf == '\0') ? false : true;

 	} else {

 		if ((sep = *rs) == 0) {

 			sep = '\n';

@@ -278,6 +286,52 @@

 	return isrec;

+/*******************

+ * loose ends here:

+ *   \r\n should become \n

+ *   what about bare \r?  Excel uses that for embedded newlines

+ *   can't have "" in unquoted fields, according to RFC 4180

+*/

+int readcsvrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag) /* csv can have \n's */

+{			/* so read a complete record that might be multiple lines */

+	int sep, c;

+	char *rr = *pbuf, *buf = *pbuf;

+	int bufsize = *pbufsize;

+	bool in_quote = false;

+	sep = '\n'; /* the only separator; have to skip over \n embedded in "..." */

+	rr = buf;

+	while ((c = getc(inf)) != EOF) {

+		if (c == sep) {

+			if (! in_quote)

+				break;

+			if (rr > buf && rr[-1] == '\r')	// remove \r if was \r\n

+				rr--;

+		}

+		if (rr-buf+1 > bufsize)

+			if (!adjbuf(&buf, &bufsize, 1+rr-buf,

+			    recsize, &rr, "readcsvrec 1"))

+				FATAL("input record `%.30s...' too long", buf);

+		*rr++ = c;

+		if (c == '"')

+			in_quote = ! in_quote;

+ 	}

+	if (c == '\n' && rr > buf && rr[-1] == '\r') 	// remove \r if was \r\n

+		rr--;

+	if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readcsvrec 4"))

+		FATAL("input record `%.30s...' too long", buf);

+	*rr = 0;

+	*pbuf = buf;

+	*pbufsize = bufsize;

+	DPRINTF("readcsvrec saw <%s>, returns %d\n", buf, c);

+	return c;

+}

 char *getargv(int n)	/* get ARGV[n] */

 	Cell *x;

@@ -299,6 +353,9 @@

 	Cell *q;

 	double result;

+/* commit f3d9187d4e0f02294fb1b0e31152070506314e67 broke T.argv test */

+/* I don't understand why it was changed. */

 	for (p=s; *p != '='; p++)

 	e = p;

@@ -343,7 +400,7 @@

 		savefs();

 	if (strlen(inputFS) > 1) {	/* it's a regular expression */

 		i = refldbld(r, inputFS);

-	} else if ((sep = *inputFS) == ' ') {	/* default whitespace */

+	} else if (!CSV && (sep = *inputFS) == ' ') {	/* default whitespace */

 		for (i = 0; ; ) {

 			while (*r == ' ' || *r == '\t' || *r == '\n')

 				r++;

@@ -362,26 +419,58 @@

 			*fr++ = 0;

 		*fr = 0;

-	} else if ((sep = *inputFS) == 0) {		/* new: FS="" => 1 char/field */

-		for (i = 0; *r != '\0'; r += n) {

-			char buf[MB_LEN_MAX + 1];

+	} else if (CSV) {	/* CSV processing.  no error handling */

+		if (*r != 0) {

+			for (;;) {

+				i++;

+				if (i > nfields)

+					growfldtab(i);

+				if (freeable(fldtab[i]))

+					xfree(fldtab[i]->sval);

+				fldtab[i]->sval = fr;

+				fldtab[i]->tval = FLD | STR | DONTFREE;

+				if (*r == '"' ) { /* start of "..." */

+					for (r++ ; *r != '\0'; ) {

+						if (*r == '"' && r[1] != '\0' && r[1] == '"') {

+							r += 2; /* doubled quote */

+							*fr++ = '"';

+						} else if (*r == '"' && (r[1] == '\0' || r[1] == ',')) {

+							r++; /* skip over closing quote */

+							break;

+						} else {

+							*fr++ = *r++;

+						}

+					}

+					*fr++ = 0;

+				} else {	/* unquoted field */

+					while (*r != ',' && *r != '\0')

+						*fr++ = *r++;

+					*fr++ = 0;

+				}

+				if (*r++ == 0)

+					break;

+			}

+		}

+		*fr = 0;

+	} else if ((sep = *inputFS) == 0) {	/* new: FS="" => 1 char/field */

+		for (i = 0; *r != '\0'; ) {

+			char buf[10];

 			i++;

 			if (i > nfields)

 				growfldtab(i);

 			if (freeable(fldtab[i]))

 				xfree(fldtab[i]->sval);

-			n = mblen(r, MB_LEN_MAX);

-			if (n < 0)

-				n = 1;

-			memcpy(buf, r, n);

-			buf[n] = '\0';

+			n = u8_nextlen(r);

+			for (j = 0; j < n; j++)

+				buf[j] = *r++;

+			buf[j] = '\0';

 			fldtab[i]->sval = tostring(buf);

 			fldtab[i]->tval = FLD | STR;

 		*fr = 0;

 	} else if (*r != 0) {	/* if 0, it's a null field */

-		/* subtlecase : if length(FS) == 1 && length(RS > 0)

+		/* subtle case: if length(FS) == 1 && length(RS > 0)

 		 * \n is NOT a field separator (cf awk book 61,84).

 		 * this variable is tested in the inner while loop.

*/

@@ -796,11 +885,11 @@

 	while (isspace(*s))

 		s++;

-	// no hex floating point, sorry

+	/* no hex floating point, sorry */

 	if (s[0] == '0' && tolower(s[1]) == 'x')

 		return false;

-	// allow +nan, -nan, +inf, -inf, any other letter, no

+	/* allow +nan, -nan, +inf, -inf, any other letter, no */

 	if (s[0] == '+' || s[0] == '-') {

 		is_nan = (strncasecmp(s+1, "nan", 3) == 0);

 		is_inf = (strncasecmp(s+1, "inf", 3) == 0);

@@ -834,7 +923,7 @@

 	if (no_trailing != NULL)

 		*no_trailing = (*ep == '\0');

-        // return true if found the end, or trailing stuff is allowed

+        /* return true if found the end, or trailing stuff is allowed */

 	retval = *ep == '\0' || trailing_stuff_ok;

 	return retval;

--- a/main.c

+++ b/main.c

@@ -49,6 +49,8 @@

 static size_t	npfile;		/* number of filenames */

 static size_t	curpfile;	/* current filename */

+bool	CSV = false;	/* true for csv input */

 bool	safe = false;	/* true => "safe" mode */

 static noreturn void fpecatch(int n

@@ -149,6 +151,12 @@

 			argc--;

 			argv++;

 			break;

+		}

+		if (strcmp(argv[1], "--csv") == 0) {	/* turn on csv input processing */

+			CSV = true;

+			argc--;

+			argv++;

+			continue;

 		switch (argv[1][1]) {

 		case 's':

--- a/proto.h

+++ b/proto.h

@@ -43,14 +43,13 @@

 extern	int	makeinit(fa *, bool);

 extern	void	penter(Node *);

 extern	void	freetr(Node *);

-extern	int	hexstr(const uschar **);

 extern	int	quoted(const uschar **);

-extern	char	*cclenter(const char *);

+extern	int	*cclenter(const char *);

 extern	noreturn void	overflo(const char *);

 extern	void	cfoll(fa *, Node *);

 extern	int	first(Node *);

 extern	void	follow(Node *);

-extern	int	member(int, const char *);

+extern	int	member(int, int *);

 extern	int	match(fa *, const char *);

 extern	int	pmatch(fa *, const char *);

 extern	int	nematch(fa *, const char *);

--- a/run.c

+++ b/run.c

@@ -26,7 +26,6 @@

 #include <stdio.h>

 #include <ctype.h>

 #include <errno.h>

-#include <wchar.h>

 #include <wctype.h>

 #include <fcntl.h>

 #include <setjmp.h>

@@ -40,8 +39,10 @@

 #include "awk.h"

 #include "awkgram.tab.h"

 static void stdinit(void);

 static void flush_all(void);

+static char *wide_char_to_byte_str(int rune, size_t *outlen);

 #if 1

 #define tempfree(x)	do { if (istemp(x)) tfree(x); } while (/*CONSTCOND*/0)

@@ -579,11 +580,225 @@

+/* ======== utf-8 code ========== */

+/*

+ * Awk strings can contain ascii, random 8-bit items (eg Latin-1),

+ * or utf-8.  u8_isutf tests whether a string starts with a valid

+ * utf-8 sequence, and returns 0 if not (e.g., high bit set).

+ * u8_nextlen returns length of next valid sequence, which is

+ * 1 for ascii, 2..4 for utf-8, or 1 for high bit non-utf.

+ * u8_strlen returns length of string in valid utf-8 sequences

+ * and/or high-bit bytes.  Conversion functions go between byte

+ * number and character number.

+ *

+ * In theory, this behaves the same as before for non-utf8 bytes.

+ *

+ * Limited checking! This is a potential security hole.

+ */

+/* is s the beginning of a valid utf-8 string? */

+/* return length 1..4 if yes, 0 if no */

+int u8_isutf(const char *s)

+{

+	int n, ret;

+	unsigned char c;

+	c = s[0];

+	if (c < 128)

+		return 1; /* what if it's 0? */

+	n = strlen(s);

+	if (n >= 2 && ((c>>5) & 0x7) == 0x6 && (s[1] & 0xC0) == 0x80) {

+		ret = 2; /* 110xxxxx 10xxxxxx */

+	} else if (n >= 3 && ((c>>4) & 0xF) == 0xE && (s[1] & 0xC0) == 0x80

+			 && (s[2] & 0xC0) == 0x80) {

+		ret = 3; /* 1110xxxx 10xxxxxx 10xxxxxx */

+	} else if (n >= 4 && ((c>>3) & 0x1F) == 0x1E && (s[1] & 0xC0) == 0x80

+			 && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) {

+		ret = 4; /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */

+	} else {

+		ret = 0;

+	}

+	return ret;

+}

+/* Convert (prefix of) utf8 string to utf-32 rune. */

+/* Sets *rune to the value, returns the length. */

+/* No error checking: watch out. */

+int u8_rune(int *rune, const char *s)

+{

+	int n, ret;

+	unsigned char c;

+	c = s[0];

+	if (c < 128) {

+		*rune = c;

+		return 1;

+	}

+	n = strlen(s);

+	if (n >= 2 && ((c>>5) & 0x7) == 0x6 && (s[1] & 0xC0) == 0x80) {

+		*rune = ((c & 0x1F) << 6) | (s[1] & 0x3F); /* 110xxxxx 10xxxxxx */

+		ret = 2;

+	} else if (n >= 3 && ((c>>4) & 0xF) == 0xE && (s[1] & 0xC0) == 0x80

+			  && (s[2] & 0xC0) == 0x80) {

+		*rune = ((c & 0xF) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F);

+			/* 1110xxxx 10xxxxxx 10xxxxxx */

+		ret = 3;

+	} else if (n >= 4 && ((c>>3) & 0x1F) == 0x1E && (s[1] & 0xC0) == 0x80

+			  && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) {

+		*rune = ((c & 0x7) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F);

+			/* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */

+		ret = 4;

+	} else {

+		*rune = c;

+		ret = 1;

+	}

+	return ret; /* returns one byte if sequence doesn't look like utf */

+}

+/* return length of next sequence: 1 for ascii or random, 2..4 for valid utf8 */

+int u8_nextlen(const char *s)

+{

+	int len;

+	len = u8_isutf(s);

+	if (len == 0)

+		len = 1;

+	return len;

+}

+/* return number of utf characters or single non-utf bytes */

+int u8_strlen(const char *s)

+{

+	int i, len, n, totlen;

+	unsigned char c;

+	n = strlen(s);

+	totlen = 0;

+	for (i = 0; i < n; i += len) {

+		c = s[i];

+		if (c < 128) {

+			len = 1;

+		} else {

+			len = u8_nextlen(&s[i]);

+		}

+		totlen++;

+		if (i > n)

+			FATAL("bad utf count [%s] n=%d i=%d\n", s, n, i);

+	}

+	return totlen;

+}

+/* convert utf-8 char number in a string to its byte offset */

+int u8_char2byte(const char *s, int charnum)

+{

+	int n;

+	int bytenum = 0;

+	while (charnum > 0) {

+		n = u8_nextlen(s);

+		s += n;

+		bytenum += n;

+		charnum--;

+	}

+	return bytenum;

+}

+/* convert byte offset in s to utf-8 char number that starts there */

+int u8_byte2char(const char *s, int bytenum)

+{

+	int i, len, b;

+	int charnum = 0; /* BUG: what origin? */

+	/* should be 0 to match start==0 which means no match */

+	b = strlen(s);

+	if (bytenum >= b) {

+		return -1; /* ??? */

+	}

+	for (i = 0; i <= bytenum; i += len) {

+		len = u8_nextlen(s+i);

+		charnum++;

+	}

+	return charnum;

+}

+/* runetochar() adapted from rune.c in the Plan 9 distributione */

+enum

+{

+	Runeerror = 128, /* from somewhere else */

+	Runemax = 0x10FFFF,

+	Bit1    = 7,

+	Bitx    = 6,

+	Bit2    = 5,

+	Bit3    = 4,

+	Bit4    = 3,

+	Bit5    = 2,

+	T1      = ((1<<(Bit1+1))-1) ^ 0xFF,     /* 0000 0000 */

+	Tx      = ((1<<(Bitx+1))-1) ^ 0xFF,     /* 1000 0000 */

+	T2      = ((1<<(Bit2+1))-1) ^ 0xFF,     /* 1100 0000 */

+	T3      = ((1<<(Bit3+1))-1) ^ 0xFF,     /* 1110 0000 */

+	T4      = ((1<<(Bit4+1))-1) ^ 0xFF,     /* 1111 0000 */

+	T5      = ((1<<(Bit5+1))-1) ^ 0xFF,     /* 1111 1000 */

+	Rune1   = (1<<(Bit1+0*Bitx))-1,	 	/* 0000 0000 0000 0000 0111 1111 */

+	Rune2   = (1<<(Bit2+1*Bitx))-1,	 	/* 0000 0000 0000 0111 1111 1111 */

+	Rune3   = (1<<(Bit3+2*Bitx))-1,	 	/* 0000 0000 1111 1111 1111 1111 */

+	Rune4   = (1<<(Bit4+3*Bitx))-1,	 	/* 0011 1111 1111 1111 1111 1111 */

+	Maskx   = (1<<Bitx)-1,		  	/* 0011 1111 */

+	Testx   = Maskx ^ 0xFF,		 	/* 1100 0000 */

+};

+int runetochar(char *str, int c)

+{

+	/* one character sequence 00000-0007F => 00-7F */

+	if (c <= Rune1) {

+		str[0] = c;

+		return 1;

+	}

+	/* two character sequence 00080-007FF => T2 Tx */

+	if (c <= Rune2) {

+		str[0] = T2 | (c >> 1*Bitx);

+		str[1] = Tx | (c & Maskx);

+		return 2;

+	}

+	/* three character sequence 00800-0FFFF => T3 Tx Tx */

+	if (c > Runemax)

+		c = Runeerror;

+	if (c <= Rune3) {

+		str[0] = T3 |  (c >> 2*Bitx);

+		str[1] = Tx | ((c >> 1*Bitx) & Maskx);

+		str[2] = Tx |  (c & Maskx);

+		return 3;

+	}

+	/* four character sequence 010000-1FFFFF => T4 Tx Tx Tx */

+	str[0] = T4 |  (c >> 3*Bitx);

+	str[1] = Tx | ((c >> 2*Bitx) & Maskx);

+	str[2] = Tx | ((c >> 1*Bitx) & Maskx);

+	str[3] = Tx |  (c & Maskx);

+	return 4;

+}

+/* ========== end of utf8 code =========== */

 Cell *matchop(Node **a, int n)	/* ~ and match() */

 	Cell *x, *y;

 	char *s, *t;

 	int i;

+	int cstart, cpatlen, len;

 	fa *pfa;

 	int (*mf)(fa *, const char *) = match, mode = 0;

@@ -604,9 +819,21 @@

 	tempfree(x);

 	if (n == MATCHFCN) {

-		int start = patbeg - s + 1;

-		if (patlen < 0)

-			start = 0;

+		int start = patbeg - s + 1; /* origin 1 */

+		if (patlen < 0) {

+			start = 0; /* not found */

+		} else {

+			cstart = u8_byte2char(s, start-1);

+			cpatlen = 0;

+			for (i = 0; i < patlen; i += len) {

+				len = u8_nextlen(patbeg+i);

+				cpatlen++;

+			}

+			start = cstart;

+			patlen = cpatlen;

+		}

 		setfval(rstartloc, (Awkfloat) start);

 		setfval(rlengthloc, (Awkfloat) patlen);

 		x = gettemp();

@@ -657,10 +884,15 @@

 	int i;

 	Cell *x, *y;

 	Awkfloat j;

+	bool x_is_nan, y_is_nan;

 	x = execute(a[0]);

 	y = execute(a[1]);

+	x_is_nan = isnan(x->fval);

+	y_is_nan = isnan(y->fval);

 	if (x->tval&NUM && y->tval&NUM) {

+		if ((x_is_nan || y_is_nan) && n != NE)

+			return(False);

 		j = x->fval - y->fval;

 		i = j<0? -1: (j>0? 1: 0);

 	} else {

@@ -673,7 +905,8 @@

 			else return(False);

 	case LE:	if (i<=0) return(True);

 			else return(False);

-	case NE:	if (i!=0) return(True);

+	case NE:	if (x_is_nan && y_is_nan) return(True);

+			else if (i!=0) return(True);

 			else return(False);

 	case EQ:	if (i == 0) return(True);

 			else return(False);

@@ -742,6 +975,7 @@

 Cell *substr(Node **a, int nnn)		/* substr(a[0], a[1], a[2]) */

 	int k, m, n;

+	int mb, nb;

 	char *s;

 	int temp;

 	Cell *x, *y, *z = NULL;

@@ -777,12 +1011,16 @@

 		n = 0;

 	else if (n > k - m)

 		n = k - m;

+	/* m is start, n is length from there */

 	DPRINTF("substr: m=%d, n=%d, s=%s\n", m, n, s);

 	y = gettemp();

-	temp = s[n+m-1];	/* with thanks to John Linderman */

-	s[n+m-1] = '\0';

-	setsval(y, s + m - 1);

-	s[n+m-1] = temp;

+	mb = u8_char2byte(s, m-1); /* byte offset of start char in s */

+	nb = u8_char2byte(s, m-1+n);  /* byte offset of end+1 char in s */

+	temp = s[nb];	/* with thanks to John Linderman */

+	s[nb] = '\0';

+	setsval(y, s + mb);

+	s[nb] = temp;

 	tempfree(x);

 	return(y);

@@ -803,7 +1041,15 @@

 		for (q = p1, p2 = s2; *p2 != '\0' && *q == *p2; q++, p2++)

 			continue;

 		if (*p2 == '\0') {

-			v = (Awkfloat) (p1 - s1 + 1);	/* origin 1 */

+			/* v = (Awkfloat) (p1 - s1 + 1);	 origin 1 */

+		   /* should be a function: used in match() as well */

+			int i, len;

+			v = 0;

+			for (i = 0; i < p1-s1+1; i += len) {

+				len = u8_nextlen(s1+i);

+				v++;

+			}

 			break;

@@ -813,6 +1059,18 @@

 	return(z);

+int has_utf8(char *s)	/* return 1 if s contains any utf-8 (2 bytes or more) character */

+{

+	int n;

+	for (n = 0; *s != 0; s += n) {

+		n = u8_nextlen(s);

+		if (n > 1)

+			return 1;

+	}

+	return 0;

+}

 #define	MAXNUMSIZE	50

 int format(char **pbuf, int *pbufsize, const char *s, Node *a)	/* printf-like conversions */

@@ -855,7 +1113,6 @@

 			s += 2;

 			continue;

-		/* have to be real careful in case this is a huge number, eg, %100000d */

 		fmtwd = atoi(s+1);

 		if (fmtwd < 0)

 			fmtwd = -fmtwd;

@@ -928,7 +1185,8 @@

 			n = fmtwd;

 		adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format5");

 		switch (flag) {

-		case '?':	snprintf(p, BUFSZ(p), "%s", fmt);	/* unknown, so dump it too */

+		case '?':

+			snprintf(p, BUFSZ(p), "%s", fmt);	/* unknown, so dump it too */

 			t = getsval(x);

 			n = strlen(t);

 			if (fmtwd > n)

@@ -942,29 +1200,176 @@

 		case 'f':	snprintf(p, BUFSZ(p), fmt, getfval(x)); break;

 		case 'd':	snprintf(p, BUFSZ(p), fmt, (intmax_t) getfval(x)); break;

 		case 'u':	snprintf(p, BUFSZ(p), fmt, (uintmax_t) getfval(x)); break;

-		case 's':

+		case 's': {

 			t = getsval(x);

 			n = strlen(t);

-			if (fmtwd > n)

-				n = fmtwd;

-			if (!adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format7"))

-				FATAL("huge string/format (%d chars) in printf %.30s... ran format() out of memory", n, t);

-			snprintf(p, BUFSZ(p), fmt, t);

+			/* if simple format or no utf-8 in the string, sprintf works */

+			if (!has_utf8(t) || strcmp(fmt,"%s") == 0) {

+				if (fmtwd > n)

+					n = fmtwd;

+				if (!adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format7"))

+					FATAL("huge string/format (%d chars) in printf %.30s..." \

+						" ran format() out of memory", n, t);

+				snprintf(p, BUFSZ(p), fmt, t);

+				break;

+			}

+			/* get here if string has utf-8 chars and fmt is not plain %s */

+			/* "%-w.ps", where -, w and .p are all optional */

+			/* '0' before the w is a flag character */

+			/* fmt points at % */

+			int ljust = 0, wid = 0, prec = n, pad = 0;

+			char *f = fmt+1;

+			if (f[0] == '-') {

+				ljust = 1;

+				f++;

+			}

+			// flags '0' and '+' are recognized but skipped

+			if (f[0] == '0') {

+				f++;

+				if (f[0] == '+')

+					f++;

+			}

+			if (f[0] == '+') {

+				f++;

+				if (f[0] == '0')

+					f++;

+			}

+			if (isdigit(f[0])) { /* there is a wid */

+				wid = strtol(f, &f, 10);

+			}

+			if (f[0] == '.') { /* there is a .prec */

+				prec = strtol(++f, &f, 10);

+			}

+			if (prec > u8_strlen(t))

+				prec = u8_strlen(t);

+			pad = wid>prec ? wid - prec : 0;  // has to be >= 0

+			int i, k, n;

+			if (ljust) { // print prec chars from t, then pad blanks

+				n = u8_char2byte(t, prec);

+				for (k = 0; k < n; k++) {

+					//putchar(t[k]);

+					*p++ = t[k];

+				}

+				for (i = 0; i < pad; i++) {

+					//printf(" ");

+					*p++ = ' ';

+				}

+			} else { // print pad blanks, then prec chars from t

+				for (i = 0; i < pad; i++) {

+					//printf(" ");

+					*p++ = ' ';

+				}

+				n = u8_char2byte(t, prec);

+				for (k = 0; k < n; k++) {

+					//putchar(t[k]);

+					*p++ = t[k];

+				}

+			}

+			*p = 0;

 			break;

-		case 'c':

+		}

+               case 'c': {

+			/*

+			 * If a numeric value is given, awk should just turn

+			 * it into a character and print it:

+			 *      BEGIN { printf("%c\n", 65) }

+			 * prints "A".

+			 *

+			 * But what if the numeric value is > 128 and

+			 * represents a valid Unicode code point?!? We do

+			 * our best to convert it back into UTF-8. If we

+			 * can't, we output the encoding of the Unicode

+			 * "invalid character", 0xFFFD.

+			 */

 			if (isnum(x)) {

-				if ((int)getfval(x))

-					snprintf(p, BUFSZ(p), fmt, (int) getfval(x));

-				else {

+				int charval = (int) getfval(x);

+				if (charval != 0) {

+					if (charval < 128)

+						snprintf(p, BUFSZ(p), fmt, charval);

+					else {

+						// possible unicode character

+						size_t count;

+						char *bs = wide_char_to_byte_str(charval, &count);

+						if (bs == NULL)	{ // invalid character

+							// use unicode invalid character, 0xFFFD

+							bs = "\357\277\275";

+							count = 3;

+						}

+						t = bs;

+						n = count;

+						goto format_percent_c;

+					}

+				} else {

 					*p++ = '\0'; /* explicit null byte */

 					*p = '\0';   /* next output will start here */

-			} else

+				break;

+			}

+			t = getsval(x);

+			n = u8_nextlen(t);

+		format_percent_c:

+			if (n < 2) { /* not utf8 */

 				snprintf(p, BUFSZ(p), fmt, getsval(x)[0]);

+				break;

+			}

+			// utf8 character, almost same song and dance as for %s

+			int ljust = 0, wid = 0, prec = n, pad = 0;

+			char *f = fmt+1;

+			if (f[0] == '-') {

+				ljust = 1;

+				f++;

+			}

+			// flags '0' and '+' are recognized but skipped

+			if (f[0] == '0') {

+				f++;

+				if (f[0] == '+')

+					f++;

+			}

+			if (f[0] == '+') {

+				f++;

+				if (f[0] == '0')

+					f++;

+			}

+			if (isdigit(f[0])) { /* there is a wid */

+				wid = strtol(f, &f, 10);

+			}

+			if (f[0] == '.') { /* there is a .prec */

+				prec = strtol(++f, &f, 10);

+			}

+			if (prec > 1)           // %c --> only one character

+				prec = 1;

+			pad = wid>prec ? wid - prec : 0;  // has to be >= 0

+			int i;

+			if (ljust) { // print one char from t, then pad blanks

+				for (int i = 0; i < n; i++)

+					*p++ = t[i];

+				for (i = 0; i < pad; i++) {

+					//printf(" ");

+					*p++ = ' ';

+				}

+			} else { // print pad blanks, then prec chars from t

+				for (i = 0; i < pad; i++) {

+					//printf(" ");

+					*p++ = ' ';

+				}

+				for (int i = 0; i < n; i++)

+					*p++ = t[i];

+			}

+			*p = 0;

 			break;

+		}

 		default:

 			FATAL("can't happen: bad conversion %c in format()", flag);

 		tempfree(x);

 		p += strlen(p);

 		s++;

@@ -1265,6 +1670,7 @@

 	int sep;

 	char temp, num[50];

 	int n, tempstat, arg3type;

+	int j;

 	double result;

 	y = execute(a[0]);	/* source string */

@@ -1271,18 +1677,20 @@

 	origs = s = strdup(getsval(y));

 	tempfree(y);

 	arg3type = ptoi(a[3]);

-	if (a[2] == NULL)		/* fs string */

+	if (a[2] == NULL) {		/* BUG: CSV should override implicit fs but not explicit */

 		fs = getsval(fsloc);

-	else if (arg3type == STRING) {	/* split(str,arr,"string") */

+	} else if (arg3type == STRING) {	/* split(str,arr,"string") */

 		x = execute(a[2]);

 		fs = origfs = strdup(getsval(x));

 		tempfree(x);

-	} else if (arg3type == REGEXPR)

+	} else if (arg3type == REGEXPR) {

 		fs = "(regexpr)";	/* split(str,arr,/regexpr/) */

-	else

+	} else {

 		FATAL("illegal type of split");

+	}

 	sep = *fs;

 	ap = execute(a[1]);	/* array name */

+/* BUG 7/26/22: this appears not to reset array: see C1/asplit */

 	freesymtab(ap);

 	DPRINTF("split: s=|%s|, a=%s, sep=|%s|\n", s, NN(ap->nval), fs);

 	ap->tval &= ~STR;

@@ -1336,7 +1744,41 @@

 			setsymtab(num, s, 0.0, STR, (Array *) ap->sval);

   spdone:

 		pfa = NULL;

-	} else if (sep == ' ') {

+	} else if (a[2] == NULL && CSV) {	/* CSV only if no explicit separator */

+		char *newt = (char *) malloc(strlen(s)); /* for building new string; reuse for each field */

+		for (;;) {

+			char *fr = newt;

+			n++;

+			if (*s == '"' ) { /* start of "..." */

+				for (s++ ; *s != '\0'; ) {

+					if (*s == '"' && s[1] != '\0' && s[1] == '"') {

+						s += 2; /* doubled quote */

+						*fr++ = '"';

+					} else if (*s == '"' && (s[1] == '\0' || s[1] == ',')) {

+						s++; /* skip over closing quote */

+						break;

+					} else {

+						*fr++ = *s++;

+					}

+				}

+				*fr++ = 0;

+			} else {	/* unquoted field */

+				while (*s != ',' && *s != '\0')

+					*fr++ = *s++;

+				*fr++ = 0;

+			}

+			snprintf(num, sizeof(num), "%d", n);

+			if (is_number(newt, &result))

+				setsymtab(num, newt, result, STR|NUM, (Array *) ap->sval);

+			else

+				setsymtab(num, newt, 0.0, STR, (Array *) ap->sval);

+			if (*s++ == '\0')

+				break;

+		}

+		free(newt);

+	} else if (!CSV && sep == ' ') { /* usual case: split on white space */

 		for (n = 0; ; ) {

 #define ISWS(c)	((c) == ' ' || (c) == '\t' || (c) == '\n')

 			while (ISWS(*s))

@@ -1359,19 +1801,25 @@

 			if (*s != '\0')

 				s++;

 	} else if (sep == 0) {	/* new: split(s, a, "") => 1 char/elem */

-		for (n = 0; *s != '\0'; s++) {

-			char buf[2];

+		for (n = 0; *s != '\0'; s += u8_nextlen(s)) {

+			char buf[10];

 			n++;

 			snprintf(num, sizeof(num), "%d", n);

-			buf[0] = *s;

-			buf[1] = '\0';

+			for (j = 0; j < u8_nextlen(s); j++) {

+				buf[j] = s[j];

+			}

+			buf[j] = '\0';

 			if (isdigit((uschar)buf[0]))

 				setsymtab(num, buf, atof(buf), STR|NUM, (Array *) ap->sval);

 			else

 				setsymtab(num, buf, 0.0, STR, (Array *) ap->sval);

-	} else if (*s != '\0') {

+	} else if (*s != '\0') {  /* some random single character */

 		for (;;) {

 			n++;

 			t = s;

@@ -1530,6 +1978,7 @@

 	size_t n       = 0;

 	wchar_t wc;

 	size_t sz = MB_CUR_MAX;

+	int unused;

 	if (sz == 1) {

 		buf = tostring(s);

@@ -1549,7 +1998,7 @@

 		 * doesn't work.)

 		 * Increment said variable to avoid a different warning.

*/

-		int unused = wctomb(NULL, L'\0');

+		unused = wctomb(NULL, L'\0');

 		unused++;

 		ps   = s;

@@ -1603,6 +2052,8 @@

 	return nawk_convert(s, tolower, towlower);

 Cell *bltin(Node **a, int n)	/* builtin functions. a[0] is type, a[1] is arg list */

 	Cell *x, *y;

@@ -1622,7 +2073,7 @@

 		if (isarr(x))

 			u = ((Array *) x->sval)->nelem;	/* GROT.  should be function*/

 		else

-			u = strlen(getsval(x));

+			u = u8_strlen(getsval(x));

 		break;

 	case FLOG:

 		errno = 0;

@@ -2143,4 +2594,42 @@

 	*pb_ptr = pb;

 	*sptr_ptr = sptr;

+}

+static char *wide_char_to_byte_str(int rune, size_t *outlen)

+{

+	static char buf[5];

+	int len;

+	if (rune < 0 || rune > 0x10FFFF)

+		return NULL;

+	memset(buf, 0, sizeof(buf));

+	len = 0;

+	if (rune <= 0x0000007F) {

+		buf[len++] = rune;

+	} else if (rune <= 0x000007FF) {

+		// 110xxxxx 10xxxxxx

+		buf[len++] = 0xC0 | (rune >> 6);

+		buf[len++] = 0x80 | (rune & 0x3F);

+	} else if (rune <= 0x0000FFFF) {

+		// 1110xxxx 10xxxxxx 10xxxxxx

+		buf[len++] = 0xE0 | (rune >> 12);

+		buf[len++] = 0x80 | ((rune >> 6) & 0x3F);

+		buf[len++] = 0x80 | (rune & 0x3F);

+	} else {

+		// 0x00010000 - 0x10FFFF

+		// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

+		buf[len++] = 0xF0 | (rune >> 18);

+		buf[len++] = 0x80 | ((rune >> 12) & 0x3F);

+		buf[len++] = 0x80 | ((rune >> 6) & 0x3F);

+		buf[len++] = 0x80 | (rune & 0x3F);

+	}

+	*outlen = len;

+	buf[len++] = '\0';

+	return buf;

--- /dev/null

+++ b/testdir/T.csv

@@ -1,0 +1,81 @@

+#!/bin/sh

+echo T.csv: tests of csv field splitting, no embedded newlines

+awk=${awk-../a.out}

+$awk '

+BEGIN {

+	FS = "\t"

+	awk = "../a.out --csv"

+}

+NF == 0 || $1 ~ /^#/ {

+	next

+}

+$1 ~ /try/ {	# new test

+	nt++

+	sub(/try /, "")

+	prog = $0

+	printf("%3d  %s\n", nt, prog)

+	prog = sprintf("%s -F\"\\t\" '"'"'%s'"'"'", awk, prog)

+	# print "prog is", prog

+	nt2 = 0

+	while (getline > 0) {

+		if (NF == 0)	# blank line terminates a sequence

+			break

+		input = $1

+		for (i = 2; i < NF; i++)	# input data

+			input = input "\t" $i

+		test = sprintf("./echo '"'"'%s'"'"' | %s >foo1; ",

+			input, prog)

+		if ($NF == "\"\"")

+			output = ">foo2;"

+		else

+			output = sprintf("./echo '"'"'%s'"'"' >foo2; ", $NF)

+		gsub(/\\t/, "\t", output)

+		gsub(/\\n/, "\n", output)

+		run = sprintf("cmp foo1 foo2 || echo test %d.%d failed",

+			nt, ++nt2)

+		# print  "input is", input

+		# print  "test is", test

+		# print  "output is", output

+		# print  "run is", run

+		system(test output run)

+	}

+	tt += nt2

+}

+END { print tt, "tests" }

+' <<\!!!!

+# General format:

+# try program as rest of line

+# $1	$2	$3	output1  (\t for tab, \n for newline,

+# $1	$2	$3	output2  ("" for null)

+# ... terminated by blank line

+try  { for (i=1; i<=NF; i++) printf("[%s]", $i); printf("\n") }

+a	[a]

+  a	[  a]

+,a	[][a]

+ , a	[ ][ a]

+a,b	[a][b]

+a,b,c	[a][b][c]

+""	[]

+"abc"	[abc]

+"a""b"	[a"b]

+"a","b"	[a][b]

+a""b	[a""b]

+"a,b"	[a,b]

+""""	["]

+""""""	[""]

+"""x"""	["x"]

+""","""	[","]

+,,""	[][][]

+a""b	[a""b]

+a''b	[a''b]

+,,	[][][]

+a,	[a][]

+"",	[][]

+,	[][]

+a"b	[a"b]

+!!!!

--- /dev/null

+++ b/testdir/T.utf

@@ -1,0 +1,194 @@

+#!/bin/sh

+echo T.utf: tests of utf functions

+awk=${awk-../a.out}

+$awk '

+BEGIN {

+	FS = "\t"

+	awk = "../a.out"

+}

+NF == 0 || $1 ~ /^#/ {

+	next

+}

+$1 ~ /try/ {	# new test

+	nt++

+	sub(/try [a-zA-Z_0-9]+ /, "")

+	prog = $0

+	printf("try %3d %s\n", nt, prog)

+	prog = sprintf("%s -F\"\\t\" '"'"'%s'"'"'", awk, prog)

+	# print "prog is", prog

+	nt2 = 0

+	while (getline > 0) {

+		if (NF == 0)	# blank line terminates a sequence

+			break

+		input = $1

+		for (i = 2; i < NF; i++)	# input data

+			input = input "\t" $i

+		test = sprintf("./echo '"'"'%s'"'"' | %s >foo1; ",

+			input, prog)

+		if ($NF == "\"\"")

+			output = ">foo2;"

+		else

+			output = sprintf("./echo '"'"'%s'"'"' >foo2; ", $NF)

+		gsub(/\\t/, "\t", output)

+		gsub(/\\n/, "\n", output)

+		run = sprintf("diff foo1 foo2 || echo test %d.%d failed",

+			nt, ++nt2)

+		# print  "input is", input

+		# print  "test is", test

+		# print  "output is", output

+		# print  "run is", run

+		system(test output run)

+	}

+	tt += nt2

+}

+END { print tt, "tests" }

+' <<\!!!!

+# General format:

+# try program as rest of line

+# $1	$2	$3	output1  (\t for tab, \n for newline,

+# $1	$2	$3	output2  ("" for null)

+# ... terminated by blank line

+# try another program...

+try length { print length($1) }

+	0

+a	1

+の今がその時だ	7

+Сейчас	6

+现在是时候了	6

+给所有的好男	6

+来参加聚会。	6

+😀	1

+🖕 finger	8

+Τωρα	4

+για	3

+να	2

+עכשיו	5

+לכל	3

+לבוא	4

+の今がその時だ	7

+지금이	3

+모든	2

+파티에	3

+Сейчас	6

+для	3

+прийти	6

+try index { print index($1, $2) }

+abc	a	1

+abc	b	2

+abc	x	0

+现在是时候了	""	0

+现在是时候了	了	6

+现在是时候了	在是	2

+现在是时候了	x	0

+现x在是时候了	x	2

+🖕 fingerすべての善人のためにすべての善人のために	f	3

+🖕 finger🖕	r🖕	8

+try substr { print substr($0, 2, 3) }

+abcdef	bcd

+Τωρα ειναι η	ωρα

+Τω	ω

+지금 이절호의	금 이

+xпyрийти	пyр

+try rematch { print $1 ~ $2 }

+abc	a	1

+abc	x	0

+すべての善人のために	の	1

+すべての善人のために	の.*の	1

+すべての善人のために	の.*て	0

+Τωρα	ω+	1

+# replace first occurrence of $2 by $3 in $1

+try sub { n = sub($2, $3, $1); print n, $1 }

+abcdef	bc	XYZ	1 aXYZdef

+abcdef	xy	XYZ	0 abcdef

+の今がその時だ	の	NO	1 NO今がその時だ

+🖕 finger	🖕.*g	FING	1 FINGer

+Сейчас	.	x	1 xейчас

+# replace all occurrences of $2 by $3 in $1

+try gsub { n = gsub($2, $3, $1); print n, $1 }

+abcdef	bc	XYZ	1 aXYZdef

+abcdef	xy	XYZ	0 abcdef

+の今がその時だ	の	NO	2 NO今がそNO時だ

+🖕 finger	🖕.*g	FING	1 FINGer

+Сейчас	.	x	6 xxxxxx

+try match { print match($1, $2), RSTART, RLENGTH }

+abc	[^a]	2 2 1

+abc	[^ab]	3 3 1

+すべての善人のために	[^す]	2 2 1

+すべての善人のために	[^ぁ-ゖ]	5 5 1

+abc	a	1 1 1

+abc	x	0 0 -1

+すべての善人のために	の	4 4 1

+すべての善人のために	の.*の	4 4 4

+すべての善人のために	の.*て	0 0 -1

+Τωρα	ω+	2 2 1

+Τωρα	x+	0 0 -1

+Τωρα	ω.	2 2 2

+すべての善人のために	[の]	4 4 1

+すべての善人のために	[ぁ-え]	0 0 -1

+すべての善人のために	[^ぁ-え]	1 1 1

+Τωρα ειναι η	[α-ω]	2 2 1

+Τωρα ειναι η	[α-ω]+	2 2 3

+xxxΤωρα ειναι η	[Α-Ω]	4 4 1

+για όλους τους καλούς ά	α.*α	3 3 15

+να έρθει στο πά	[^ν]	2 2 1

+# FS="" should split into unicode chars

+try emptyFS BEGIN {FS=""} {print NF}

+すべての善人のために	10

+の今がその時だ	7

+Сейчас	6

+现在是时候了	6

+给所有的好男	6

+来参加聚会。	6

+😀	1

+🖕 finger	8

+# printf(%N.Ns) for utf8 strings

+try printfs1 {printf("[%5.2s][%-5.2s]\n"), $1, $1}

+abcd	[   ab][ab   ]

+现在abc	[   现在][现在   ]

+现ωabc	[   现ω][现ω   ]

+ωabc	[   ωa][ωa   ]

+Сейчас	[   Се][Се   ]

+Сейxyz	[   Се][Се   ]

+😀	[    😀][😀    ]

+# printf(%N.Ns) for utf8 strings

+try printfs2 {printf("[%5s][%-5s]\n"), $1, $1}

+abcd	[ abcd][abcd ]

+现在ab	[ 现在ab][现在ab ]

+a现在ab	[a现在ab][a现在ab]

+a现在abc	[a现在abc][a现在abc]

+现ωab	[ 现ωab][现ωab ]

+ωabc	[ ωabc][ωabc ]

+Сейчас	[Сейчас][Сейчас]

+😀	[    😀][😀    ]

+# printf(%N.Ns) for utf8 strings

+try printfs3 {printf("[%.2s][%-.2s]\n"), $1, $1}

+abcd	[ab][ab]

+现在abc	[现在][现在]

+现ωabc	[现ω][现ω]

+ω	[ω][ω]

+😀	[😀][😀]

+# printf(%c) for utf

+try printfc {printf("%c %c\n", $1, substr($1,2,1))}

+すべての善人のために	す べ

+の今がその時だ	の 今

+Сейчас	С е

+现在是时候了	现 在

+😀🖕	😀 🖕

+!!!!

--- /dev/null

+++ b/testdir/T.utfre

@@ -1,0 +1,234 @@

+echo T.utfre: tests of regular expression code for Unicode/utf-8

+# adapted from T.re

+awk '

+BEGIN {

+	FS = "\t"

+	awk = "../a.out"

+}

+NF == 0 {

+	next

+}

+$1 != "" {	# new test

+	re = $1

+}

+$2 != "" {	# either ~ or !~

+	op = $2

+	if (op == "~")

+		neg = "!"

+	else if (op == "!~")

+		neg = ""

+}

+$3 != "" {	# new test string

+	str = $3

+}

+$3 == "\"\"" {	# explicit empty line

+	$3 = ""

+}

+NF > 2 {	# generate a test

+	input = $3

+	test = sprintf("./echo '"'"'%s'"'"' | %s '"'"'%s/%s/ {print \"%d fails %s %s %s\"}'"'"'",

+		input, awk, neg, re, NR, re, op, input)

+	# printf(" %3d   %s %s %s:\n", NR, re, op, input)

+	# print "test is |" test "|"

+	system(test)

+	# system("bprint -c ../a.out")

+	nt++

+}

+END { print "	" nt, "tests" }

+' <<\!!!!

+	~	🖕

+		🖕🖕

+		🖕🖕🖕

+		""

+🖕	~	🖕

+		b🖕

+		b🖕b

+	!~	""

+		时

+		xxxxx

+.	~	时

+		x时x

+		🙂

+	!~	""

+.の	~	xの

+		xxの

+		xのx

+	!~	の

+		のx

+		""

+$	~	x

+		🙂

+		""

+.$	~	모

+		xx모

+		x모x

+	!~	""

+д$	~	д

+		bд

+		bbbд

+	!~	дb

+		x

+		""

+^	~	и

+		""

+		^

+^λ$	~	λ

+	!~	xλ

+		λx

+		xλx

+		""

+^λ.$	~	λx

+		λλ

+	!~	xλ

+		λλλ

+		λxy

+		""

+^$	~	""

+	!~	に

+		^

+^.해	~	め해

+		め해해

+	!~	해

+		""

+^.*해	~	해

+		め해

+		めめめめめめ해

+	!~	""

+^.+해	~	め해

+		めめめめめめ해

+	!~	""

+		해

+		해め

+해*	~	""

+		해

+		해해해해

+		め해

+		めめめめ

+해해*	~	해

+		해해해

+		め해

+	!~	めめめめ

+		""

+\$	~	🖕$

+		$

+		$🖕

+		🖕$🖕

+	!~	""

+		🖕

+\.	~	.

+	!~	🖕

+		""

+xθ+y	~	xθy

+		xθθy

+		xθθθθθθy

+	!~	θy

+		xy

+		xθ

+xθ?y	~	xy

+		xθy

+	!~	xθθy

+θ?b?の?	~	""

+		x

+^a?b?め	~	め

+		aめ

+		bめ

+		abめ

+		めa

+	!~	""

+		ab

+		aba

+[Α-Ω]	~	Α

+		aΔb

+		xyΩ

+	!~	abc

+		β

+		""

+[^Α-Ω]	~	δ

+		aΔb

+		xyΩ

+	!~	Α

+		Δ

+		""

+[Α-ΔΦ-Ω]	~	Α

+		Β

+		Δ

+		Φ

+		Ω

+	!~	Π

+		Σ

+Π[[:lower:]]+	~	Πa

+		Πab

+	!~	Π

+		ΠX

+		Π:

+		Π[

+		Π]

+の[0-9]+に	~	の0に

+		の23に

+		の12345に

+	!~	0に

+		のに

+の[0-9]?に	~	のに

+		の1に

+	!~	の23に

+の[[]に	~	の[に

+	!~	のに

+		の[[]に

+		の]に

+の[[-]に	~	の[に

+		の-に

+	!~	のに

+		の[[]に

+		の]に

+の[[-a]に	~	の[に

+		のaに

+		の]に

+	!~	のに

+		の[[]に

+		の-に

+の[]-a]に	~	の]に

+		のaに

+	!~	のに

+		の[に

+		の-に

+の[]]に	~	の]に

+	!~	のに

+		の[]]に

+		の[に

+の[^[]に	~	のaに

+	!~	の[に

+の[-]に	~	の-に

+	!~	のに

+		の+に

+の[^-]に	~	の+に

+	!~	の-に

+		のに

+の[][]に	~	の[に

+		の]に

+	!~	のに

+		の][に

+		の[]に

+の[z-a]に	~	のに

+	!~	の

+		に

+		のaに

+		のzに

+		の-に

+に|だ	~	だ

+		に

+		だに

+	!~	a

+		""

+^στο|τους$	~	στο

+		στοd

+		aτους

+		τους

+	!~	xστο

+		τουςa

+^(στο|τους)$	~	στο

+		τους

+	!~	στοτους

+		στοx

+		cτους

+!!!!

--- a/testdir/tt.15

+++ b/testdir/tt.15

@@ -15,7 +15,7 @@

 END  { printline() }

 function addword(w) {

-    ## print "adding [", w, "] ", length(w), length(line), maxlen

+    print "adding [", w, "] ", length(w), length(line), maxlen

     if (length(line) + length(w) > maxlen)

         printline()

     if (length(w) > 2 && ( w ~ /[\.!]["?)]?$/ || w ~ /[?!]"?$/) &&

--- a/testdir/xc

+++ /dev/null

@@ -1,17 +1,0 @@

-for i in $*

-do

-	echo $i >/dev/tty

-	echo $i '<<<'

-	cd ..

-	echo testdir/$i:

-	ind <testdir/$i

-	a.out -f testdir/$i >drek.c

-	cat drek.c

-	make drek || ( echo $i '	' bad compile; echo $i '	' bad compile >/dev/tty; continue )

-	cd testdir

-	time awk -f $i test.countries >foo1

-	time ../drek test.countries >foo2

-	cmp foo1 foo2 || ( echo $i '	' bad; echo $i '	' bad >/dev/tty; diff foo1 foo2 )

-	echo '>>>' $i

-	echo

-done

--- a/tran.c

+++ b/tran.c

@@ -308,7 +308,7 @@

 	} else if (&vp->fval == NF) {

 		donerec = false;	/* mark $0 invalid */

 		setlastfld(f);

-		DPRINTF("setting NF to %g\n", f);

+		DPRINTF("setfval: setting NF to %g\n", f);

 	} else if (isrec(vp)) {

 		donefld = false;	/* mark $1... invalid */

 		donerec = true;

@@ -348,6 +348,10 @@

 		(void*)vp, NN(vp->nval), s, vp->tval, donerec, donefld);

 	if ((vp->tval & (NUM | STR)) == 0)

 		funnyvar(vp, "assign to");

+	if (CSV && (vp == rsloc))

+		WARNING("danger: don't set RS when --csv is in effect");

+	if (CSV && (vp == fsloc))

+		WARNING("danger: don't set FS when --csv is in effect");

 	if (isfld(vp)) {

 		donerec = false;	/* mark $0 invalid */

 		fldno = atoi(vp->nval);

@@ -375,7 +379,7 @@

 		donerec = false;	/* mark $0 invalid */

 		f = getfval(vp);

 		setlastfld(f);

-		DPRINTF("setting NF to %g\n", f);

+		DPRINTF("setsval: setting NF to %g\n", f);

 	return(vp->sval);

--

⑨