/**
 *	\file "lexer/hackt-lex.ll"
 *	vi: ft=lex
 *	Will generate .cc (C++) file for the token-scanner.  
 *	$Id: hackt-lex.ll,v 1.29 2009/02/20 20:39:40 fang Exp $
 *	This file was originally:
 *	Id: art++-lex.ll,v 1.17 2005/06/21 21:26:35 fang Exp
 *	in prehistory.  
 */

/***************** FOREWORD ***************************************************
	THESE COMMENTS NEED TO BE UPDATED.

	Do not use [f]lex++, just write C++ and compile with C++ compiler.  

	quoted strings are restricted to single line, however, 
		the parser will concatenate sequences of strings
		into a single string.  

	yylval, passed to parser is a union of double, long, char*.  
		The char* member, string, is just pointer-copied, 
		with no memory management.  Thus, the parser should
		actually strcpy() the string into it's own structures.  
		strcpy() is probably performed by the constructors.  

	yylineno: seems buggy, so not using the option, keeping
		track manually with mylineno.  
		Thus no other pattern may allow newline characters, we must 
		force a stop at each newline, and treat it separately.  

	column_position: manually keeping track of column position of tokens
		get moderately complicated for multi-line tokens, 
		such as raw-source and extended comments.  
		How is it done?  Since comments and sources may span multiple
		lines and start anywhere on a line, and we only have access
		to the global yyleng upon each match, we need to remember
		how much we've matched since the last newline.  
		case 1) start of source (anywhere): while the source
		is on the 

	TODO:
		Modified to throw exception on lexical error, 
		but parser needs to catch it to properly unwind the stack!

		After static type are implemented in unions, 
		use the appropriate yylval union member.  

		Implement the #line directive to interpret the output
		of flattened, preprocessed files.  

		Use fast token allocation from util library.  

	KNOWN ISSUE: the headers in this source file do NOT appear
		first in the generated lexer source code, which 
		presents a problem in the renaming scheme that uses
		an include with preprocessor defines.  

******************************************************************************/
/****** DEFINITIONS **********************************************************/

%{
/* scanner-specific header */

#define	ENABLE_STACKTRACE		(0 && !defined(LIBBOGUS))

#include <iostream>
#include <iomanip>
#include <cstdlib>
#include <stack>

#ifdef	LIBBOGUS
// HACK: prevent the inclusion of "parser/hackt-prefix.h"
#define	__LEXYACC_HACKT__PREFIX_H__
#endif

#include "util/macros.h"
#include "util/using_ostream.h"
#include "parser/hackt-prefix.h"
#include "AST/AST.h"		/* everything needed for "y.tab.h" */
#include "lexer/input_manager.h"
#include "lexer/file_manager.h"
using namespace HAC::parser;

// DIRTY MAKE HACK ALERT
#if	defined(LIBBOGUS)
#include "parser/hackt-parse.h"		/* symbols generated by yacc */
#else
#include "parser/hackt-parse-real.h"	/* symbols generated by yacc */
#endif
// END DIRTY HACK ALERT

#include "lexer/hac_lex.h"
#include "lexer/hackt-lex-options.h"
#include "lexer/flex_lexer_state.h"
#include "parser/hackt-union.h"
#include "util/stacktrace.h"
#include "util/format_IO.h"
#include "util/sstream.h"
#include "util/libc.h"			/* for strsep */
using flex::lexer_state;

/**
	If lexyacc_test links against main parser, 
	we will want to avoid a symbol conflict.
 */
#if	defined(LIBBOGUS)
#define	hackt_parse_file_manager	yy_parse_file_manager
#define	hackt_embedded_file_stack	yy_embedded_file_stack
#endif
/**
	This is the file stack and include path manager for 
	the hackt parser.  
	This is globally visible and accessible (unfortunately).  
 */
HAC::lexer::file_manager
hackt_parse_file_manager;

/**
	Auxiliary stack for file-embedding construct, whereby the
	lexer pushes a fake file onto this stack, and the parser
	pops it off the stack when the file construct is reduced.  
 */
HAC::lexer::embedded_file_stack_type
hackt_embedded_file_stack;


/// generated in "parser/hackt-union.cc" for deleting tokens
extern
void
yy_union_lookup_delete(const YYSTYPE&, const int);

extern
std::ostream&
yy_union_lookup_dump(const YYSTYPE&, const int, std::ostream&);

namespace HAC {

// defined in "main/main_funcs.cc"
extern count_ptr<root_body>
parse_to_AST(FILE*);

/**
	Namespace for the lexer variables and functions.  
 */
namespace lexer {

/**
	Maximum string length.  Can be extended arbitrarily.  
 */
#define	STRING_MAX_LEN		1024

/** line and position tracking data for tokens */
#define	CURRENT		hackt_parse_file_manager.current_position()
static	token_position comment_pos(1, 0, 1);
static	token_position string_pos(1, 0, 1);
	/* even though strings may not be multi-line */
/***
	Observation: comments will never contain strings to lex, 
		nor will strings ever lex comments.  
	Also note that a file cannot be referenced while inside a 
		comment or string.  
***/

/* for string matching */
static	char string_buf[STRING_MAX_LEN];
static	char* string_buf_ptr = string_buf;

/**
	Thie macro is intended for use with ostream& operator << .
	\param c is a token_position.  
 */
#define	LINE_COL(c)	"on line " << c.line << ":" << c.col

int allow_nested_comments = 0;
static int comment_level = 0;		/* useful for nested comments */

/* debugging switches -- consider making these macro-defined */
static const int token_feedback = 0;
static const int string_feedback = 0;
static const int comment_feedback = 0;		/* reporting of comment state */
	/*	0 = off, 
		1 = nested levels only, 
		2 = null and endline comments, 
		3 = ignored text feedback details
	*/

/**
	Debugging tool.  
	This will generate excessive feedback for every detailed
	action of the lexer, even in the middle of tokenizing.  
 */
static inline void
DEFAULT_TOKEN_FEEDBACK(const lexer_state& foo) {
	if (token_feedback) {
		cerr << "token = " << yytext <<
			" " LINE_COL(CURRENT) << endl;
	}
}

/* macros for tracking single line tokens (no new line) */

static inline void
TOKEN_UPDATE(const lexer_state& foo) {
	DEFAULT_TOKEN_FEEDBACK(foo);
	CURRENT.col += yyleng;
}

static inline void
NEWLINE_UPDATE(void) {
	CURRENT.line++; CURRENT.col = 1;
#if 0
	cerr << "Line number advanced to " << CURRENT.line << endl;
#endif
}

static inline void
KEYWORD_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._keyword_position = new keyword_position(yytext, CURRENT);
#if ENABLE_STACKTRACE
	STACKTRACE_INDENT_PRINT("new keyword at " <<
		hackt_lval._keyword_position << endl);
#endif
	TOKEN_UPDATE(foo);
}

static inline void
LINKAGE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
#if 0
	hackt_lval._token_keyword = new token_keyword(yytext);
	TOKEN_UPDATE(foo);
#else
	KEYWORD_UPDATE(hackt_lval, foo);
#endif
}

static inline void
ELSE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_else = new token_else(yytext);
	TOKEN_UPDATE(foo);
}

static inline void
BOOL_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_bool = new token_bool(yytext);
	TOKEN_UPDATE(foo);
}

static inline void
INT_TYPE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_int_type = new token_int_type(yytext);
	TOKEN_UPDATE(foo);
}

static inline void
BOOL_TYPE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_bool_type = new token_bool_type(yytext);
	TOKEN_UPDATE(foo);
}

static inline void
PINT_TYPE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_pint_type = new token_pint_type(yytext);
	TOKEN_UPDATE(foo);
}

static inline void
PBOOL_TYPE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_pbool_type = new token_pbool_type(yytext);
	TOKEN_UPDATE(foo);
}

static inline void
PREAL_TYPE_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._token_preal_type = new token_preal_type(yytext);
	TOKEN_UPDATE(foo);
}

/***
static inline void
MULTICHAR_UPDATE(const lexer_state& foo) {
	hackt_lval._token_string = new token_string(yytext); TOKEN_UPDATE(foo);
}
***/

static inline void
NODE_POSITION_UPDATE(YYSTYPE& hackt_lval, const lexer_state& foo) {
	hackt_lval._node_position = new node_position(yytext, CURRENT);
	TOKEN_UPDATE(foo);
}

static inline void
STRING_UPDATE(const lexer_state& foo) {
	TOKEN_UPDATE(foo);
	assert(string_buf_ptr -string_buf < STRING_MAX_LEN);
}

static inline void
STRING_FINISH(YYSTYPE& hackt_lval, lexer_state& foo) {
	STRING_UPDATE(foo);
	hackt_lval._token_quoted_string = new token_quoted_string(string_buf);
}

/* macros for tracking long, multiline tokens */
/* pass into p either source_pos or comment_pos */
static inline void
MULTILINE_START(token_position& p, const lexer_state& foo) {
	p.col = CURRENT.col;
	p.line = CURRENT.line;
	p.leng = yyleng;
	TOKEN_UPDATE(foo);
}

static inline void
MULTILINE_MORE(const token_position& p, const lexer_state& foo) {
	if (p.line == CURRENT.line) {
		CURRENT.col = yyleng +p.col
#if USE_TOKEN_POSITION_OFFSET
			+p.off
#endif
		;
	} else {
		CURRENT.col = yyleng -p.leng;
	}
}

static inline void
MULTILINE_NEWLINE(token_position& p, const lexer_state& foo) {
	p.leng = yyleng -1; NEWLINE_UPDATE();
}

static
void
hackt_lex_expect(YYSTYPE&, lexer_state&, const int, const char*);

}	/* end namespace lexer */
}	/* end namespace HAC */

using namespace HAC::lexer;

%}

DIGIT		[0-9]
HEXDIGIT	[0-9A-Fa-f]
IDHEAD		[a-zA-Z_]
IDBODY		[a-zA-Z0-9_]
INT		{DIGIT}+
SIGN_INT	[+-]?{INT}
EXP		[eE]{SIGN_INT}
FRACTIONAL	"."{INT}
FLOAT		({INT}{FRACTIONAL}{EXP}?)|({INT}{FRACTIONAL}?{EXP})
HEX		0[xX]{HEXDIGIT}+
BIN		0[bB][01]+
QUAD		0[qQ][0-3]+
/* OCTAL? */

/* note: '-' signed ints are lexed as two tokens and combined in the parser
	as a unary expr, thus, no numerical tokens in the language 
	*start* with a sign.  
	A sign in the exponent of a floating-point number is acceptable.  
*/

ID		{IDHEAD}{IDBODY}*
BADID		({INT}{ID})|({FLOAT}{ID})
BADHEX		{HEX}{ID}
WHITESPACE	[ \t]+
NEWLINE		"\n"
WS		{WHITESPACE}

POSITIONTOKEN	[][(){}<>*%/=:;|!?~&^.,@#$+-]

PLUSPLUS	"++"
MINUSMINUS	"--"
LARROW		"<-"
RARROW		"->"
HASH_ARROW	"#>"
EQUAL		"=="
NOTEQUAL	"!="
LE		"<="
GE		">="
IMPLIES		"=>"
INSERT		">>"
EXTRACT		"<<"
FWDSLASH	"\\"
LOGICAL_AND	"&&"
LOGICAL_OR	"||"
ASSIGN		":="

/** syntactic sugar tokens, value is not important, return _node_position */
BEGINLOOP	"*["
BEGINPROB	"%["
ENDPROB		"]%"
THICKBAR	"[]"
SCOPE		"::"
DEFINEOP	"<:"
RANGE		".."

ENDLINECOMMENT	"//"(.*)$
OPENCOMMENT	"/*"
CLOSECOMMENT	[*]+"/"
CLOSEINCOMMENT	"*/"

OPENSTRING	"\""
MORESTRING	[^\\\"\n]+
CLOSESTRING	"\""
FILESTRING	{OPENSTRING}{MORESTRING}{CLOSESTRING}

OCTAL_ESCAPE	"\\"[0-7]{1,3}
BAD_ESCAPE	"\\"[0-9]+

/****** keywords ****/
IMPORT		"import"
NAMESPACE	"namespace"
OPEN		"open"
AS		"as"
TEMPLATE	"template"
DEFINE		"define"
DEFCHAN		"defchan"
DEFTYPE		"deftype"
DEFPROC		"defproc"
TYPEDEF		"typedef"
CHP		"chp"
HSE		"hse"
PRS		"prs"
SPEC		"spec"
TREE		"tree"
SUBCKT		"subckt"
SKIP		"skip"
ELSE		"else"
LOG		"log"
SEND		"send"
RECV		"recv"
SET		"set"
GET		"get"
ENUM		"enum"
INT_TYPE	"int"
BOOL_TYPE	"bool"
PINT_TYPE	"pint"
PBOOL_TYPE	"pbool"
PREAL_TYPE	"preal"
CHANNEL		"chan"
TRUE		"true"
FALSE		"false"
EXTERN		"extern"
STATIC		"static"
EXPORT		"export"
/* just like C preprocessor built-in definitions */
FILENAME	"__FILE__"
LINENO		"__LINE__"

/* whole line  for line directive */
LINE		^#{WS}{INT}{WS}{FILESTRING}.*$

EMBEDFILE	^#FILE

/* consider recording all tokens' (including punctuation) positions? */

/****** states ******/
%s incomment
%s instring
%s inescape

/*
	Explicitly stating options to guarantee proper definition of 
	macros in the generated source file, because I've turned on
	-Wundef for all translation units.  
 */
%option never-interactive
%option nomain
%option nostack
/** I wish! **/
/** 	%option reentrant	**/

/****** rules ****************************************************************/
%%

<INITIAL>{

{LE}		{ NODE_POSITION_UPDATE(*hackt_lval, foo); return LE; }
{GE}		{ NODE_POSITION_UPDATE(*hackt_lval, foo); return GE; }
{EQUAL}		{ NODE_POSITION_UPDATE(*hackt_lval, foo); return EQUAL; }
{NOTEQUAL}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return NOTEQUAL; }
{IMPLIES}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return IMPLIES; }
{RARROW}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return RARROW; }
{HASH_ARROW}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return HASH_ARROW; }
{PLUSPLUS}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return PLUSPLUS; }
{MINUSMINUS}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return MINUSMINUS; }
{LOGICAL_AND}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return LOGICAL_AND; }
{LOGICAL_OR}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return LOGICAL_OR; }
{INSERT}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return INSERT; }
{EXTRACT}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return EXTRACT; }
{ASSIGN}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return ASSIGN; }

{BEGINLOOP}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return BEGINLOOP; }
{BEGINPROB}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return BEGINPROB; }
{ENDPROB}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return ENDPROB; }
{THICKBAR}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return THICKBAR; }
{SCOPE}		{ NODE_POSITION_UPDATE(*hackt_lval, foo); return SCOPE; }
{RANGE}		{ NODE_POSITION_UPDATE(*hackt_lval, foo); return RANGE; }
{DEFINEOP}	{ NODE_POSITION_UPDATE(*hackt_lval, foo); return DEFINEOP; }


{LINE} {
/**
	line directive processing:
	# INT STRING
	NOTE: file name ("quoted") is *required* unlike ANSI C
	NOTE: whitespace between arguments is mandatory
	NOTE: tokens after the file name are ignored to end-of-line
**/
	STACKTRACE("lexing #line directive");
	static const char delim[] = " \t\n";
	char** stringp = &yytext;
	char* last = strsep(stringp, delim);	// eat the #
	NEVER_NULL(last);
	last = strsep(stringp, delim);		// expect line number
	NEVER_NULL(last);
	int v;
	std::istringstream iss(last);	/* slower, but safer */
	iss >> v;
	last = strsep(stringp, delim);		// expect quoted filename
	NEVER_NULL(last);
	// NOTE: strips quotes without honoring string escape sequences!
	++last;
	stringp = &last;
	last = strsep(stringp, "\"");
	NEVER_NULL(last);
	// TODO: error handling, completely absent
#if 0
	cout << "line = " << v << endl;
	cout << "file = " << last << endl;
#endif
	// coercively set current file and position
	hackt_parse_file_manager.coerce_line_directive(last, v);
	// read and ignore the rest of the line
	// don't return, continue lexing next token
}

{IMPORT} {
	STACKTRACE("lexing import");
#if ENABLE_STACKTRACE
#define	DUMP_FILE_NAME_STACK(ostr)					\
	hackt_parse_file_manager.dump_file_names(ostr) << endl
#else
#define	DUMP_FILE_NAME_STACK(ostr)
#endif

	DUMP_FILE_NAME_STACK(cerr);
/***
	That's right, manually opening the file in the lexer.  
	Can't necessarily count on the [LA]LR parser to do this properly.  
***/
	/* need some string-hacking to extract file name */
	KEYWORD_UPDATE(*hackt_lval, foo);
	excl_ptr<const keyword_position>
		kw_import(hackt_lval->_keyword_position);
	NEVER_NULL(kw_import);
	// meh, just re-use the passed in lval, rather than local temporary
	YYSTYPE& temp(*hackt_lval);
	// YYSTYPE temp;
	/* OK to reuse this lexer state in recursive lex */
	hackt_lex_expect(temp, foo, STRING, "Expecting \"file\" after import.");
	STRING_FINISH(temp, foo);
	/* excl_ptr will delete token if unused */
	excl_ptr<const token_quoted_string>
		fsp(temp._token_quoted_string);
	NEVER_NULL(fsp);
	
	const string& fstr(*fsp);
	/* claim the semicolon first before opening file */
	hackt_lex_expect(temp, foo, ';',
		"Expecting \';\' after import \"...\".");
	/* NODE_POSITION_UPDATE(*hackt_lval, foo); */
	/* excl_ptr will delete token if unused */
	const excl_ptr<const node_position>
		ssp(temp._node_position);
	// cerr << fstr << endl;
	// cerr << "current FILE* (before) = " << yyin << endl;
/***
	From: http://developer.apple.com/documentation/DeveloperTools/flex/flex_9.html

	If the scanner reaches an end-of-file, subsequent calls are undefined 
	unless either yyin is pointed at a new input file (in which case 
	scanning continues from that file), or `yyrestart()' is called. 
	`yyrestart()' takes one argument, a `FILE *' pointer (which can be nil,
	if you've set up YY_INPUT to scan from a source other than yyin), and 
	initializes yyin for scanning from that file. Essentially there is no 
	difference between just assigning yyin to a new input file or using 
	`yyrestart()' to do so; the latter is available for compatibility with 
	previous versions of flex, and because it can be used to switch input 
	files in the middle of scanning. It can also be used to throw away the 
	current input buffer, by calling it with an argument of yyin; but 
	better is to use YY_FLUSH_BUFFER (see above). Note that `yyrestart()' 
	does not reset the start condition to INITIAL (see Start Conditions, 
	below).
***/
	// TODO: better error reporting and handling, when I have time...
	// try to open the file, using search paths (true)
	const input_manager ym(hackt_parse_file_manager, fstr.c_str(), true);
	// get the full path to the file name
	const file_status::status stat = ym.get_status();
	switch(stat) {
	case file_status::NEW_FILE: {
		STACKTRACE("import new file");
		DUMP_FILE_NAME_STACK(cerr);
		const string& pstr(hackt_parse_file_manager.top_FILE_name());
		excl_ptr<root_body>
			_root(HAC::parse_to_AST(ym.get_file())
				.exclusive_release());	// take from count_ptr
		if (!_root) {
			// presumably already have error message from callee
			cerr << "From: \"" << pstr << "\":" <<
				kw_import->leftmost().line << ':' << endl;
			THROW_EXIT;
		}
		// false: this is first time seeing this file
		// NOTE: creating the imported root here will use
		// the wrong token_position because file stack already-adjusted
		hackt_lval->_imported_root =
			new imported_root(_root, kw_import, fsp, pstr, false);
		MUST_BE_NULL(kw_import);
		MUST_BE_NULL(fsp);
		MUST_BE_NULL(_root);	// transferred ownership
		return IMPORT;
	}
	case file_status::SEEN_FILE: {
		STACKTRACE("old file");
		DUMP_FILE_NAME_STACK(cerr);
		// true: already seen file, this will be a placeholder
		excl_ptr<root_body> null;
		hackt_lval->_imported_root =
			new imported_root(null, kw_import, fsp, string(), true);
		MUST_BE_NULL(kw_import);
		MUST_BE_NULL(fsp);
		return IMPORT;
	}
	case file_status::CYCLE: {
		STACKTRACE("cyclic file!");
		hackt_parse_file_manager.dump_file_stack(cerr);
		cerr << "Detected cyclic file dependency: " << fstr << endl;
		THROW_EXIT;
	}
	case file_status::NOT_FOUND: {
		STACKTRACE("file not found");
		hackt_parse_file_manager.dump_file_stack(cerr);
		cerr << "Unable to open file: " << fstr << endl;
		THROW_EXIT;
	}
	default:
		abort();
	}
#undef	DUMP_FILE_NAME_STACK
}

{NAMESPACE}	{ KEYWORD_UPDATE(*hackt_lval, foo); return NAMESPACE; }
{OPEN}		{ KEYWORD_UPDATE(*hackt_lval, foo); return OPEN; }
{AS}		{ // KEYWORD_UPDATE(*hackt_lval, foo); return AS;
	cerr << "WARNING: \'as\' is a deprecated keyword.  "
		"Assuming you want \'->\' instead..." << endl;
	cerr << "(Eventually, this keyword will be removed.)" << endl;
	NODE_POSITION_UPDATE(*hackt_lval, foo); return RARROW;
	}
{TEMPLATE}	{ KEYWORD_UPDATE(*hackt_lval, foo); return TEMPLATE; }
{DEFINE}	{ KEYWORD_UPDATE(*hackt_lval, foo); return DEFINE; }
{DEFCHAN}	{ KEYWORD_UPDATE(*hackt_lval, foo); return DEFCHAN; }
{DEFTYPE}	{ KEYWORD_UPDATE(*hackt_lval, foo); return DEFTYPE; }
{DEFPROC}	{ KEYWORD_UPDATE(*hackt_lval, foo); return DEFPROC; }
{TYPEDEF}	{ KEYWORD_UPDATE(*hackt_lval, foo); return TYPEDEF; }
{CHP}		{ KEYWORD_UPDATE(*hackt_lval, foo); return CHP_LANG; }
{HSE}		{ KEYWORD_UPDATE(*hackt_lval, foo); return HSE_LANG; }
{PRS}		{ KEYWORD_UPDATE(*hackt_lval, foo); return PRS_LANG; }
{TREE}		{ KEYWORD_UPDATE(*hackt_lval, foo); return TREE_LANG; }
{SUBCKT}	{ KEYWORD_UPDATE(*hackt_lval, foo); return SUBCKT_LANG; }
{SPEC}		{ KEYWORD_UPDATE(*hackt_lval, foo); return SPEC_LANG; }
{SKIP}		{ KEYWORD_UPDATE(*hackt_lval, foo); return SKIP; }
{ELSE}		{ ELSE_UPDATE(*hackt_lval, foo); return ELSE; }
{LOG}		{ KEYWORD_UPDATE(*hackt_lval, foo); return LOG; }
{SEND}		{ KEYWORD_UPDATE(*hackt_lval, foo); return SEND; }
{RECV}		{ KEYWORD_UPDATE(*hackt_lval, foo); return RECV; }
{SET}		{ KEYWORD_UPDATE(*hackt_lval, foo); return SET; }
{GET}		{ KEYWORD_UPDATE(*hackt_lval, foo); return GET; }
{ENUM}		{ KEYWORD_UPDATE(*hackt_lval, foo); return ENUM; }
{CHANNEL}	{ KEYWORD_UPDATE(*hackt_lval, foo); return CHANNEL; }
{INT_TYPE}	{ INT_TYPE_UPDATE(*hackt_lval, foo); return INT_TYPE; }
{BOOL_TYPE}	{ BOOL_TYPE_UPDATE(*hackt_lval, foo); return BOOL_TYPE; }
{PINT_TYPE}	{ PINT_TYPE_UPDATE(*hackt_lval, foo); return PINT_TYPE; }
{PBOOL_TYPE}	{ PBOOL_TYPE_UPDATE(*hackt_lval, foo); return PBOOL_TYPE; }
{PREAL_TYPE}	{ PREAL_TYPE_UPDATE(*hackt_lval, foo); return PREAL_TYPE; }
{TRUE}		{ BOOL_UPDATE(*hackt_lval, foo); return BOOL_TRUE; }
{FALSE}		{ BOOL_UPDATE(*hackt_lval, foo); return BOOL_FALSE; }
{EXTERN}	{ LINKAGE_UPDATE(*hackt_lval, foo); return EXTERN; }
{STATIC}	{ LINKAGE_UPDATE(*hackt_lval, foo); return STATIC; }
{EXPORT}	{ LINKAGE_UPDATE(*hackt_lval, foo); return EXPORT; }
{FILENAME}	{ 
	/* substitute __FILE__ with current file name */
	hackt_lval->_token_quoted_string =
		new token_quoted_string(hackt_parse_file_manager.top_FILE_name());
	TOKEN_UPDATE(foo);
	return STRING;
}
{LINENO}	{
	/* substitute __LINE__ with current line number */
	hackt_lval->_token_int =
		new token_int(hackt_parse_file_manager.current_position().line);
	TOKEN_UPDATE(foo);
	return INT;
}
{EMBEDFILE}	{
	KEYWORD_UPDATE(*hackt_lval, foo);
	excl_ptr<const keyword_position>
		kw_file(hackt_lval->_keyword_position);
	NEVER_NULL(kw_file);

	YYSTYPE temp;
	hackt_lex_expect(temp, foo, STRING, "Expecting \"file\" after #FILE.");
	STRING_FINISH(temp, foo);
	/* excl_ptr will delete token if unused */
	excl_ptr<const token_quoted_string>
		fsp(temp._token_quoted_string);
	NEVER_NULL(fsp);
	const string& fstr(*fsp);
	hackt_lval->_imported_root = new imported_root(kw_file, fsp);
	NEVER_NULL(hackt_lval->_imported_root);
	// push a fake file onto the file-manager stack
	// emulate the construction action of a input_manager
	hackt_embedded_file_stack.push(
		embedded_file_stack_type::value_type(
			new file_manager::embed_manager(
				hackt_parse_file_manager, fstr)));
	// pop should be done by the corresponding reduce action
	return EMBEDFILE;
}

{WHITESPACE}	TOKEN_UPDATE(foo);
{NEWLINE}	NEWLINE_UPDATE();

{ENDLINECOMMENT} { 
	if (comment_feedback > 1) {
		cerr << "end-of-line comment ignored " <<
			LINE_COL(CURRENT) << endl;
	}
	TOKEN_UPDATE(foo);
}

{FLOAT} {
	if (token_feedback) {
		cerr << "float = " << yytext << " " LINE_COL(CURRENT) << endl;
	}
	/* TODO: error handling of value-ranges */
	/* consider using stream conversions to avoid precision errors */
	HAC::entity::preal_value_type v;
	std::istringstream iss(yytext);	/* slower, but safer */
	iss >> v;
	/* could try to use faster, but unsafe istrstream (deprecated) */
	hackt_lval->_token_float = new token_float(v);
	/* hackt_lval->_token_float = new token_float(atof(yytext)); */
	TOKEN_UPDATE(foo);
	return FLOAT;
}

{INT}	{
	if (token_feedback) {
		cerr << "int = " << yytext << " " << LINE_COL(CURRENT) << endl;
	}
	/* TODO: error handling of value-ranges */
	/* consider using stream conversions to avoid precision errors */
	/* what if we need atol? */
	HAC::entity::pint_value_type v;
	std::istringstream iss(yytext);	/* slower, but safer */
	iss >> v;
	/* could try to use faster, but unsafe istrstream (deprecated) */
	hackt_lval->_token_int = new token_int(v);
	/* hackt_lval->_token_int = new token_int(atoi(yytext)); */
	TOKEN_UPDATE(foo);
	return INT;
}

{HEX}	{
	if (token_feedback) {
		cerr << "int = " << yytext << " " << LINE_COL(CURRENT) << endl;
	}
	/* TODO: error handling of value-ranges */
	/* consider using stream conversions to avoid precision errors */
	/* what if we need atol? */
	HAC::entity::pint_value_type v;
	std::istringstream iss(yytext);	/* slower, but safer */
	iss >> std::hex >> v;
	/* could try to use faster, but unsafe istrstream (deprecated) */
	hackt_lval->_token_int = new token_int(v);
	/* hackt_lval->_token_int = new token_int(atoi(yytext)); */
	TOKEN_UPDATE(foo);
	return INT;
}

{QUAD}	{
	if (token_feedback) {
		cerr << "int = " << yytext << " " << LINE_COL(CURRENT) << endl;
	}
	/* TODO: error handling of value-ranges */
	/* consider using stream conversions to avoid precision errors */
	/* what if we need atol? */
	size_t v = 0;
	// std::istringstream iss(yytext +2);	/* skip the '0q' prefix */
	// iss >> std::setbase(4) >> v;	/* setbase doesn't work for input */
	util::string_to_int_base(yytext +2, 4, v);
	hackt_lval->_token_int = new token_int(HAC::entity::pint_value_type(v));
	TOKEN_UPDATE(foo);
	return INT;
}

{BIN}	{
	if (token_feedback) {
		cerr << "int = " << yytext << " " << LINE_COL(CURRENT) << endl;
	}
	/* TODO: error handling of value-ranges */
	/* consider using stream conversions to avoid precision errors */
	/* what if we need atol? */
	size_t v = 0;
	// std::istringstream iss(yytext +2);	/* skip the '0b' prefix */
	// iss >> std::setbase(2) >> v;	/* setbase doesn't work for input */
	util::string_to_int_binary(yytext +2, v);
	hackt_lval->_token_int = new token_int(HAC::entity::pint_value_type(v));
	TOKEN_UPDATE(foo);
	return INT;
}

{ID}	{
	if (token_feedback) {
		cerr << "identifier = \"" << yytext << "\" " << 
			LINE_COL(CURRENT) << endl;
	}
	hackt_lval->_token_identifier = new token_identifier(yytext);
	TOKEN_UPDATE(foo);
	return ID;
}

{CLOSECOMMENT} {
	hackt_parse_file_manager.dump_file_stack(cerr);
	cerr << "*/ (close-comment) found outside of <comment> " <<
		LINE_COL(CURRENT) << endl;
	TOKEN_UPDATE(foo);
	THROW_EXIT;
}

{OPENCOMMENT} {
	/* crazy... allowing nested comments */
	comment_level++;
	if (comment_feedback) {
		cerr << "start of comment-level " << comment_level << " " <<
			LINE_COL(CURRENT) << endl;
	}
	yymore();
	MULTILINE_START(comment_pos, foo);
	BEGIN(incomment); 
}

{OPENSTRING}	{
	if (string_feedback) {
		cerr << "start of quoted-string " << LINE_COL(CURRENT) << endl;
	}
	/* no yymore(), skip open quote */
	string_buf_ptr = string_buf;
	MULTILINE_START(string_pos, foo);	/* just for column positioning */
	BEGIN(instring); 
}

{BADHEX}	{ 
	hackt_parse_file_manager.dump_file_stack(cerr);
	cerr << "bad hexadecimal integer: \"" << yytext << "\" " <<
		LINE_COL(CURRENT) << endl;
	TOKEN_UPDATE(foo);
	hackt_lval->_token_identifier = NULL;
	THROW_EXIT;
}

{BADID}	{ 
	hackt_parse_file_manager.dump_file_stack(cerr);
	cerr << "bad identifier: \"" << yytext << "\" " <<
		LINE_COL(CURRENT) << endl;
	TOKEN_UPDATE(foo);
	hackt_lval->_token_identifier = NULL;
	THROW_EXIT;
}

{POSITIONTOKEN} { NODE_POSITION_UPDATE(*hackt_lval, foo); return yytext[0]; }

.	{
	hackt_parse_file_manager.dump_file_stack(cerr);
	/* for everything else that doesn't match... */
	cerr << "unexpected character: \'" << yytext << "\' " <<
		LINE_COL(CURRENT) << endl;
	TOKEN_UPDATE(foo);
	THROW_EXIT;
}
}

<incomment>{

[^*/\n]*	|
[/][^*\n]*	{
	if (comment_feedback > 2) {
		cerr << "eaten up more comment " << LINE_COL(CURRENT) << endl;
	}
	MULTILINE_MORE(comment_pos, foo);
	yymore();
}

[^/\n]*{CLOSEINCOMMENT} {
	MULTILINE_MORE(comment_pos, foo);
	if (comment_feedback) {
		cerr << "end of comment-level " << comment_level << " " <<
			LINE_COL(CURRENT) << endl;
	}
	comment_level--;
	if (!comment_level) {
		BEGIN(INITIAL); 
	} else {
		yymore();		// doesn't seem to make a difference
	}
}

[*]+[^/\n]*	{
	if (comment_feedback > 2) {
		cerr << "eaten up more comment " << LINE_COL(CURRENT) << endl;
	}
	MULTILINE_MORE(comment_pos, foo);
	yymore();
}

{OPENCOMMENT} {
	if (allow_nested_comments) {
		comment_level++;
		if (comment_feedback) {
			cerr << "start of comment-level " << comment_level <<
				" " << LINE_COL(CURRENT) << endl;
		}
		MULTILINE_MORE(comment_pos, foo);
		yymore();
	} else {
		hackt_parse_file_manager.dump_file_stack(cerr);
		cerr << "nested comments forbidden, found /* " <<
			LINE_COL(CURRENT) << endl;
		THROW_EXIT;
	}
}

{NEWLINE}	{
	if (comment_feedback > 1) {
		cerr << "new-line in comment at end of line "
			<< CURRENT.line << endl;
	}
	MULTILINE_NEWLINE(comment_pos, foo);
	yymore();
}

<<EOF>>	{
	// cerr << "in-comment-EOF!" << endl;
	// in this case, yywrap() is already called too early, and we don't get
	// the full file-stack, thus we must push some error handling into
	// yywrap() and detect the state there.  
	MULTILINE_MORE(comment_pos, foo);
	hackt_parse_file_manager.dump_file_stack_top(cerr);
	cerr << "unterminated comment, starting on line "
		<< comment_pos.line << ", got <<EOF>>" << endl;
	THROW_EXIT;
}

}

<instring>{
{NEWLINE}	{
	hackt_parse_file_manager.dump_file_stack(cerr);
	cerr << "unterminated quoted-string on line " << CURRENT.line <<
		", got \\n" << endl;
	STRING_UPDATE(foo);
	THROW_EXIT;
}

{MORESTRING}	{
	// do escape seqences later... need a string buffer
	char *copy = yytext;
	while (*copy) {			/* until null-termination */
		*string_buf_ptr++ = *copy++;
	}
	/* don't really need yymore, just for tracking column position here */
	STRING_UPDATE(foo);
}

{CLOSESTRING}	{
	*string_buf_ptr = '\0';		/* null-terminate */
	if (string_feedback) {
		cerr << "end of quoted-string: \"" << string_buf << "\" " <<
			LINE_COL(CURRENT) << endl;
	}
	STRING_FINISH(*hackt_lval, foo);
	BEGIN(INITIAL);
	return STRING;
}

"\\0"	{ *string_buf_ptr++ = '\0';	STRING_UPDATE(foo);	}
"\\b"	{ *string_buf_ptr++ = '\b';	STRING_UPDATE(foo);	}
"\\f"	{ *string_buf_ptr++ = '\f';	STRING_UPDATE(foo);	}
"\\n"	{ *string_buf_ptr++ = '\n';	STRING_UPDATE(foo);	}
"\\r"	{ *string_buf_ptr++ = '\r';	STRING_UPDATE(foo);	}
"\\t"	{ *string_buf_ptr++ = '\t';	STRING_UPDATE(foo);	}
"\\\\"	{ *string_buf_ptr++ = '\\';	STRING_UPDATE(foo);	}
"\\\'"	{ *string_buf_ptr++ = '\'';	STRING_UPDATE(foo);	}
"\\\""	{ *string_buf_ptr++ = '\"';	STRING_UPDATE(foo);	}

{OCTAL_ESCAPE}	{
	unsigned int result;
	sscanf(yytext +1, "%o", &result);
	if ( result > 0xff ) {
		hackt_parse_file_manager.dump_file_stack(cerr);
		cerr << "bad octal escape sequence " << yytext << " " <<
			LINE_COL(CURRENT) << endl;
		THROW_EXIT;
	}
	*string_buf_ptr++ = result;
	STRING_UPDATE(foo);
}

{BAD_ESCAPE}	{
	hackt_parse_file_manager.dump_file_stack(cerr);
	cerr << "bad octal escape sequence " << yytext << " " <<
		LINE_COL(CURRENT) << endl;
	THROW_EXIT;
}
<<EOF>>	{
	// for same reason as stated above, in EOF in COMMENT
	// we push some error handling into yywrap().  
	hackt_parse_file_manager.dump_file_stack(cerr);
	cerr << "unterminated string, starting on line " << CURRENT.line << 
		", got <<EOF>>" << endl;
	THROW_EXIT;
}
}

%%
/****** user-code ************************************************************/


namespace HAC {
namespace lexer {

//- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
/**
	\param temp the return slot for the constructed token.
	\param foo the lexer state structure.
	\param tok the expected enumeration of upcoming token.
	\param errmsg error message: expecting blah...
	\throw std:::exception upon error.  
 */
// static
void
hackt_lex_expect(YYSTYPE& temp, lexer_state& foo, 
		const int tok, const char* errmsg) {
	const int expect = __hackt_lex(&temp, foo);
	if (expect != tok) {
		cerr << errmsg << endl;
		if (!foo.at_eof()) {
			yy_union_lookup_dump(temp, expect,
				cerr << "got: ") << endl;
			yy_union_lookup_delete(temp, expect);
		} else {
			// a.k.a. yytname[0]
			cerr << "got: $end" << endl;
		}
		THROW_EXIT;
	}
}

//=============================================================================
}	/* end namespace lexer */
}	/* end namespace HAC */

