package java_cup;
import java.util.Hashtable;
import java_cup.runtime.str_token;
import java_cup.runtime.token;
/** This class implements a small scanner (aka lexical analyzer or lexer) for
* the JavaCup specification. This scanner reads characters from standard
* input (System.in) and returns integers corresponding to the terminal
* number of the next token. Once end of input is reached the EOF token is
* returned on every subsequent call.<p>
* Tokens currently returned include: <pre>
* Symbol Constant Returned Symbol Constant Returned
* ------ ----------------- ------ -----------------
* "package" PACKAGE "import" IMPORT
* "code" CODE "action" ACTION
* "parser" PARSER "terminal" TERMINAL
* "non" NON "init" INIT
* "scan" SCAN "with" WITH
* "start" START ; SEMI
* , COMMA * STAR
* . DOT : COLON
* ::= COLON_COLON_EQUALS | BAR
* identifier ID {:...:} CODE_STRING
* "debug" DEBUG
* </pre>
* All symbol constants are defined in sym.java which is generated by
* JavaCup from parser.cup.<p>
*
* In addition to the scanner proper (called first via init() then with
* next_token() to get each token) this class provides simple error and
* warning routines and keeps a count of errors and warnings that is
* publicly accessible.<p>
*
* This class is "static" (i.e., it has only static members and methods).
*
* @version last updated: 11/25/95
* @author Scott Hudson
*/
public class lexer {
/*-----------------------------------------------------------*/
/*--- Constructor(s) ----------------------------------------*/
/*-----------------------------------------------------------*/
/** The only constructor is private, so no instances can be created. */
private lexer() { }
/*-----------------------------------------------------------*/
/*--- Static (Class) Variables ------------------------------*/
/*-----------------------------------------------------------*/
/** First character of lookahead. */
protected static int next_char;
/** Second character of lookahead. */
protected static int next_char2;
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** EOF constant. */
protected static final int EOF_CHAR = -1;
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Table of keywords. Keywords are initially treated as identifiers.
* Just before they are returned we look them up in this table to see if
* they match one of the keywords. The string of the name is the key here,
* which indexes Integer objects holding the symbol number.
*/
protected static Hashtable keywords = new Hashtable(23);
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Table of single character symbols. For ease of implementation, we
* store all unambiguous single character tokens in this table of Integer
* objects keyed by Integer objects with the numerical value of the
* appropriate char (currently Character objects have a bug which precludes
* their use in tables).
*/
protected static Hashtable char_symbols = new Hashtable(11);
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Current line number for use in error messages. */
protected static int current_line = 1;
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Character position in current line. */
protected static int current_position = 1;
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Count of total errors detected so far. */
public static int error_count = 0;
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Count of warnings issued so far */
public static int warning_count = 0;
/*-----------------------------------------------------------*/
/*--- Static Methods ----------------------------------------*/
/*-----------------------------------------------------------*/
/** Initialize the scanner. This sets up the keywords and char_symbols
* tables and reads the first two characters of lookahead.
*/
public static void init() throws java.io.IOException
{
/* set up the keyword table */
keywords.put("package", new Integer(sym.PACKAGE));
keywords.put("import", new Integer(sym.IMPORT));
keywords.put("code", new Integer(sym.CODE));
keywords.put("action", new Integer(sym.ACTION));
keywords.put("parser", new Integer(sym.PARSER));
keywords.put("terminal", new Integer(sym.TERMINAL));
keywords.put("non", new Integer(sym.NON));
keywords.put("init", new Integer(sym.INIT));
keywords.put("scan", new Integer(sym.SCAN));
keywords.put("with", new Integer(sym.WITH));
keywords.put("start", new Integer(sym.START));
keywords.put("debug", new Integer(sym.DEBUG));
/* set up the table of single character symbols */
char_symbols.put(new Integer(';'), new Integer(sym.SEMI));
char_symbols.put(new Integer(','), new Integer(sym.COMMA));
char_symbols.put(new Integer('*'), new Integer(sym.STAR));
char_symbols.put(new Integer('.'), new Integer(sym.DOT));
char_symbols.put(new Integer('|'), new Integer(sym.BAR));
/* read two characters of lookahead */
next_char = System.in.read();
if (next_char == EOF_CHAR)
next_char2 = EOF_CHAR;
else
next_char2 = System.in.read();
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Advance the scanner one character in the input stream. This moves
* next_char2 to next_char and then reads a new next_char2.
*/
protected static void advance() throws java.io.IOException
{
int old_char;
old_char = next_char;
next_char = next_char2;
if (next_char == EOF_CHAR)
next_char2 = EOF_CHAR;
else
next_char2 = System.in.read();
/* count this */
current_position++;
if (old_char == '\n')
{
current_line++;
current_position = 1;
}
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Emit an error message. The message will be marked with both the
* current line number and the position in the line. Error messages
* are printed on standard error (System.err).
* @param message the message to print.
*/
public static void emit_error(String message)
{
System.err.println("Error at " + current_line + "(" + current_position +
"): " + message);
error_count++;
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Emit a warning message. The message will be marked with both the
* current line number and the position in the line. Messages are
* printed on standard error (System.err).
* @param message the message to print.
*/
public static void emit_warn(String message)
{
System.err.println("Warning at " + current_line + "(" + current_position +
"): " + message);
warning_count++;
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Determine if a character is ok to start an id.
* @param ch the character in question.
*/
protected static boolean id_start_char(int ch)
{
return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
(ch == '_');
// later need to deal with non-8-bit chars here
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Determine if a character is ok for the middle of an id.
* @param ch the character in question.
*/
protected static boolean id_char(int ch)
{
return id_start_char(ch) || (ch >= '0' && ch <= '9');
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Try to look up a single character symbol, returns -1 for not found.
* @param ch the character in question.
*/
protected static int find_single_char(int ch)
{
Integer result;
result = (Integer)char_symbols.get(new Integer((char)ch));
if (result == null)
return -1;
else
return result.intValue();
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Handle swallowing up a comment. Both old style C and new style C++
* comments are handled.
*/
protected static void swallow_comment() throws java.io.IOException
{
/* next_char == '/' at this point */
/* is it a traditional comment */
if (next_char2 == '*')
{
/* swallow the opener */
advance(); advance();
/* swallow the comment until end of comment or EOF */
for (;;)
{
/* if its EOF we have an error */
if (next_char == EOF_CHAR)
{
emit_error("Specification file ends inside a comment");
return;
}
/* if we can see the closer we are done */
if (next_char == '*' && next_char2 == '/')
{
advance();
advance();
return;
}
/* otherwise swallow char and move on */
advance();
}
}
/* is its a new style comment */
if (next_char2 == '/')
{
/* swallow the opener */
advance(); advance();
/* swallow to '\n', '\f', or EOF */
while (next_char != '\n' && next_char != '\f' && next_char!=EOF_CHAR)
advance();
return;
}
/* shouldn't get here, but... if we get here we have an error */
emit_error("Malformed comment in specification -- ignored");
advance();
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Swallow up a code string. Code strings begin with "{:" and include
all characters up to the first occurrence of ":}" (there is no way to
include ":}" inside a code string). The routine returns an str_token
object suitable for return by the scanner.
*/
protected static token do_code_string() throws java.io.IOException
{
StringBuffer result = new StringBuffer();
/* at this point we have lookahead of "{:" -- swallow that */
advance(); advance();
/* save chars until we see ":}" */
while (!(next_char == ':' && next_char2 == '}'))
{
/* if we have run off the end issue a message and break out of loop */
if (next_char == EOF_CHAR)
{
emit_error("Specification file ends inside a code string");
break;
}
/* otherwise record the char and move on */
result.append(new Character((char)next_char));
advance();
}
/* advance past the closer and build a return token */
advance(); advance();
return new str_token(sym.CODE_STRING, result.toString());
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Process an identifier. Identifiers begin with a letter, underscore,
* or dollar sign, which is followed by zero or more letters, numbers,
* underscores or dollar signs. This routine returns an str_token suitable
* for return by the scanner.
*/
protected static token do_id() throws java.io.IOException
{
StringBuffer result = new StringBuffer();
String result_str;
Integer keyword_num;
char buffer[] = new char[1];
/* next_char holds first character of id */
buffer[0] = (char)next_char;
result.append(buffer,0,1);
advance();
/* collect up characters while they fit in id */
while(id_char(next_char))
{
buffer[0] = (char)next_char;
result.append(buffer,0,1);
advance();
}
/* extract a string and try to look it up as a keyword */
result_str = result.toString();
keyword_num = (Integer)keywords.get(result_str);
/* if we found something, return that keyword */
if (keyword_num != null)
return new token(keyword_num.intValue());
/* otherwise build and return an id token with an attached string */
return new str_token(sym.ID, result_str);
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Return one token. This is the main external interface to the scanner.
* It consumes sufficient characters to determine the next input token
* and returns it. To help with debugging, this routine actually calls
* real_next_token() which does the work. If you need to debug the
* parser, this can be changed to call debug_next_token() which prints
* a debugging message before returning the token.
*/
public static token next_token() throws java.io.IOException
{
return real_next_token();
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** Debugging version of next_token(). This routine calls the real scanning
* routine, prints a message on System.out indicating what the token is,
* then returns it.
*/
public static token debug_next_token() throws java.io.IOException
{
token result = real_next_token();
System.out.println("# next_token() => " + result.sym);
return result;
}
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
/** The actual routine to return one token. This is normally called from
* next_token(), but for debugging purposes can be called indirectly from
* debug_next_token().
*/
protected static token real_next_token() throws java.io.IOException
{
int sym_num;
for (;;)
{
/* look for white space */
if (next_char == ' ' || next_char == '\t' || next_char == '\n' ||
next_char == '\f' || next_char == '\r')
{
/* advance past it and try the next character */
advance();
continue;
}
/* look for a single character symbol */
sym_num = find_single_char(next_char);
if (sym_num != -1)
{
/* found one -- advance past it and return a token for it */
advance();
return new token(sym_num);
}
/* look for : or ::= */
if (next_char == ':')
{
/* if we don't have a second ':' return COLON */
if (next_char2 != ':')
{
advance();
return new token(sym.COLON);
}
/* move forward and look for the '=' */
advance();
if (next_char2 == '=')
{
advance(); advance();
return new token(sym.COLON_COLON_EQUALS);
}
else
{
/* return just the colon (already consumed) */
return new token(sym.COLON);
}
}
/* look for a comment */
if (next_char == '/' && (next_char2 == '*' || next_char2 == '/'))
{
/* swallow then continue the scan */
swallow_comment();
continue;
}
/* look for start of code string */
if (next_char == '{' && next_char2 == ':')
return do_code_string();
/* look for an id or keyword */
if (id_start_char(next_char)) return do_id();
/* look for EOF */
if (next_char == EOF_CHAR) return new token(sym.EOF);
/* if we get here, we have an unrecognized character */
emit_warn("Unrecognized character '" +
new Character((char)next_char) + "'(" + next_char +
") -- ignored");
/* advance past it */
advance();
}
}
/*-----------------------------------------------------------*/
};