From 68271c1b714ea927370b86e22489a32fffaedf10 Mon Sep 17 00:00:00 2001
From: David Walter Seikel
Date: Tue, 17 Jan 2012 13:56:22 +1000
Subject: Store the lexer provided length.

---
 LuaSL/src/LuaSL_lexer.l | 133 ++++++++++++++++++++++++------------------------
 1 file changed, 67 insertions(+), 66 deletions(-)

(limited to 'LuaSL/src/LuaSL_lexer.l')

diff --git a/LuaSL/src/LuaSL_lexer.l b/LuaSL/src/LuaSL_lexer.l
index bb452f3..5945599 100644
--- a/LuaSL/src/LuaSL_lexer.l
+++ b/LuaSL/src/LuaSL_lexer.l
@@ -3,7 +3,7 @@
 #define excludeLexer
 #include "LuaSL.h"
 
-int common(YYSTYPE *lval, char *text, LuaSL_compiler *compiler, boolean checkIgnorable, int type);
+int common(YYSTYPE *lval, char *text, int len, LuaSL_compiler *compiler, boolean checkIgnorable, int type);
 
 %}
 
@@ -31,95 +31,96 @@ STRING      \"(\\.|[^\\"\n])*\"
  /* The order here is important, in mysterious ways. The more specific the lower in case of ambiguities like "floats contain integers". I think, not tested that well yet. */
 
  /* Ignorables. */
-[[:space:]]+    %{ common(yylval, yytext, yyextra, FALSE, LSL_SPACE); %}
+[[:space:]]+    %{ common(yylval, yytext, yyleng, yyextra, FALSE, LSL_SPACE); %}
  /* Yes I know this will have problems with huge comments, just being simple to get it to work for now. */
-"/*"([^"*/"]*)"*/"   %{ common(yylval, yytext, yyextra, FALSE, LSL_COMMENT); %}
-"//"[^\n]*      %{ common(yylval, yytext, yyextra, FALSE, LSL_COMMENT_LINE); %}
+"/*"([^"*/"]*)"*/"   %{ common(yylval, yytext, yyleng, yyextra, FALSE, LSL_COMMENT); %}
+"//"[^\n]*      %{ common(yylval, yytext, yyleng, yyextra, FALSE, LSL_COMMENT_LINE); %}
 
  /* Operations. */
-"&&"            { return common(yylval, yytext, yyextra, TRUE, LSL_BOOL_AND); }
-"||"            { return common(yylval, yytext, yyextra, TRUE, LSL_BOOL_OR); }
-"|"             { return common(yylval, yytext, yyextra, TRUE, LSL_BIT_OR); }
-"^"             { return common(yylval, yytext, yyextra, TRUE, LSL_BIT_XOR); }
-"&"             { return common(yylval, yytext, yyextra, TRUE, LSL_BIT_AND); }
-"!="            { return common(yylval, yytext, yyextra, TRUE, LSL_NOT_EQUAL); }
-"=="            { return common(yylval, yytext, yyextra, TRUE, LSL_EQUAL); }
-">="            { return common(yylval, yytext, yyextra, TRUE, LSL_GREATER_EQUAL); }
-"<="            { return common(yylval, yytext, yyextra, TRUE, LSL_LESS_EQUAL); }
-">"             { return common(yylval, yytext, yyextra, TRUE, LSL_GREATER_THAN); }
-"<"             { return common(yylval, yytext, yyextra, TRUE, LSL_LESS_THAN); }
-">>"            { return common(yylval, yytext, yyextra, TRUE, LSL_RIGHT_SHIFT); }
-"<<"            { return common(yylval, yytext, yyextra, TRUE, LSL_LEFT_SHIFT); }
-"+"             { return common(yylval, yytext, yyextra, TRUE, LSL_ADD); }
-"-"             { return common(yylval, yytext, yyextra, TRUE, LSL_SUBTRACT); }
-"*"             { return common(yylval, yytext, yyextra, TRUE, LSL_MULTIPLY); }
-"%"             { return common(yylval, yytext, yyextra, TRUE, LSL_MODULO); }
-"/"             { return common(yylval, yytext, yyextra, TRUE, LSL_DIVIDE); }
-"!"             { return common(yylval, yytext, yyextra, TRUE, LSL_BOOL_NOT); }
-"~"             { return common(yylval, yytext, yyextra, TRUE, LSL_BIT_NOT); }
-"["             { return common(yylval, yytext, yyextra, TRUE, LSL_BRACKET_OPEN); }
-"]"             { return common(yylval, yytext, yyextra, TRUE, LSL_BRACKET_CLOSE); }
-"("             { return common(yylval, yytext, yyextra, TRUE, LSL_PARENTHESIS_OPEN); }
-")"             { return common(yylval, yytext, yyextra, TRUE, LSL_PARENTHESIS_CLOSE); }
-"+="            { return common(yylval, yytext, yyextra, TRUE, LSL_ASSIGNMENT_ADD); }
-"-="            { return common(yylval, yytext, yyextra, TRUE, LSL_ASSIGNMENT_SUBTRACT); }
-"*="            { return common(yylval, yytext, yyextra, TRUE, LSL_ASSIGNMENT_MULTIPLY); }
-"%="            { return common(yylval, yytext, yyextra, TRUE, LSL_ASSIGNMENT_MODULO); }
-"/="            { return common(yylval, yytext, yyextra, TRUE, LSL_ASSIGNMENT_DIVIDE); }
-"="             { return common(yylval, yytext, yyextra, TRUE, LSL_ASSIGNMENT_PLAIN); }
-"."             { return common(yylval, yytext, yyextra, TRUE, LSL_DOT); }
-"--"            { return common(yylval, yytext, yyextra, TRUE, LSL_DECREMENT_PRE); }
-"++"            { return common(yylval, yytext, yyextra, TRUE, LSL_INCREMENT_PRE); }
-","             { return common(yylval, yytext, yyextra, TRUE, LSL_COMMA); }
+"&&"            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BOOL_AND); }
+"||"            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BOOL_OR); }
+"|"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BIT_OR); }
+"^"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BIT_XOR); }
+"&"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BIT_AND); }
+"!="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_NOT_EQUAL); }
+"=="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_EQUAL); }
+">="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_GREATER_EQUAL); }
+"<="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_LESS_EQUAL); }
+">"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_GREATER_THAN); }
+"<"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_LESS_THAN); }
+">>"            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_RIGHT_SHIFT); }
+"<<"            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_LEFT_SHIFT); }
+"+"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ADD); }
+"-"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_SUBTRACT); }
+"*"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_MULTIPLY); }
+"%"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_MODULO); }
+"/"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_DIVIDE); }
+"!"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BOOL_NOT); }
+"~"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BIT_NOT); }
+"["             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BRACKET_OPEN); }
+"]"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BRACKET_CLOSE); }
+"("             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_PARENTHESIS_OPEN); }
+")"             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_PARENTHESIS_CLOSE); }
+"+="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ASSIGNMENT_ADD); }
+"-="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ASSIGNMENT_SUBTRACT); }
+"*="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ASSIGNMENT_MULTIPLY); }
+"%="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ASSIGNMENT_MODULO); }
+"/="            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ASSIGNMENT_DIVIDE); }
+"="             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ASSIGNMENT_PLAIN); }
+"."             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_DOT); }
+"--"            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_DECREMENT_PRE); }
+"++"            { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_INCREMENT_PRE); }
+","             { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_COMMA); }
 
  /* Other symbols. */
-"@"             %{ return common(yylval, yytext, yyextra, TRUE, LSL_LABEL); %}
-"{"             %{ beginBlock(yyextra, yylval);  return common(yylval, yytext, yyextra, TRUE, LSL_BLOCK_OPEN); %}
-"}"             %{ endBlock(yyextra, yylval);    return common(yylval, yytext, yyextra, TRUE, LSL_BLOCK_CLOSE); %}
-";"             %{ return common(yylval, yytext, yyextra, TRUE, LSL_STATEMENT); %}
+"@"             %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_LABEL); %}
+"{"             %{ beginBlock(yyextra, yylval);  return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BLOCK_OPEN); %}
+"}"             %{ endBlock(yyextra, yylval);    return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_BLOCK_CLOSE); %}
+";"             %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_STATEMENT); %}
 
  /* Type keywords. */
-"float"         %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_FLOAT); %}
-"integer"       %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_INTEGER); %}
-"key"           %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_KEY); %}
-"list"          %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_LIST); %}
-"quaternion"    %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_ROTATION); %}
-"rotation"      %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_ROTATION); %}
-"string"        %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_STRING); %}
-"vector"        %{ return common(yylval, yytext, yyextra, TRUE, LSL_TYPE_VECTOR); %}
+"float"         %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_FLOAT); %}
+"integer"       %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_INTEGER); %}
+"key"           %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_KEY); %}
+"list"          %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_LIST); %}
+"quaternion"    %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_ROTATION); %}
+"rotation"      %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_ROTATION); %}
+"string"        %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_STRING); %}
+"vector"        %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_TYPE_VECTOR); %}
 
  /* Statement keywords. */
-"do"            %{ return common(yylval, yytext, yyextra, TRUE, LSL_DO); %}
-"for"           %{ return common(yylval, yytext, yyextra, TRUE, LSL_FOR); %}
-"else"          %{ return common(yylval, yytext, yyextra, TRUE, LSL_ELSE); %}
-"if"            %{ return common(yylval, yytext, yyextra, TRUE, LSL_IF); %}
-"jump"          %{ return common(yylval, yytext, yyextra, TRUE, LSL_JUMP); %}
-"return"        %{ return common(yylval, yytext, yyextra, TRUE, LSL_RETURN); %}
-"state"         %{ return common(yylval, yytext, yyextra, TRUE, LSL_STATE_CHANGE); %}
-"while"         %{ return common(yylval, yytext, yyextra, TRUE, LSL_WHILE); %}
+"do"            %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_DO); %}
+"for"           %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_FOR); %}
+"else"          %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_ELSE); %}
+"if"            %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_IF); %}
+"jump"          %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_JUMP); %}
+"return"        %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_RETURN); %}
+"state"         %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_STATE_CHANGE); %}
+"while"         %{ return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_WHILE); %}
 
-{IDENTIFIER}    %{ yylval->value.stringValue = strdup(yytext); return common(yylval, yytext, yyextra, TRUE, LSL_IDENTIFIER); %}
+{IDENTIFIER}    %{ yylval->value.stringValue = strdup(yytext); return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_IDENTIFIER); %}
 
  /* Types. */
-{INTEGER}       %{ yylval->value.integerValue = atoi(yytext); return common(yylval, yytext, yyextra, TRUE, LSL_INTEGER); %}
-{FLOAT}         %{ yylval->value.floatValue = atof(yytext); return common(yylval, yytext, yyextra, TRUE, LSL_FLOAT); %}
-{KEY}           %{ yylval->value.stringValue = strdup(yytext); return common(yylval, yytext, yyextra, TRUE, LSL_KEY); %}
-{STRING}        %{ yylval->value.stringValue = strdup(yytext); return common(yylval, yytext, yyextra, TRUE, LSL_STRING); %}
+{INTEGER}       %{ yylval->value.integerValue = atoi(yytext);  return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_INTEGER); %}
+{FLOAT}         %{ yylval->value.floatValue = atof(yytext);    return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_FLOAT); %}
+{KEY}           %{ yylval->value.stringValue = strdup(yytext); return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_KEY); %}
+{STRING}        %{ yylval->value.stringValue = strdup(yytext); return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_STRING); %}
 
-<<EOF>>         { return common(yylval, yytext, yyextra, TRUE, LSL_SCRIPT); }
+<<EOF>>         { return common(yylval, yytext, yyleng, yyextra, TRUE, LSL_SCRIPT); }
 
  /* Everything else */
-.               %{ printf(" unexpected character.\n"); yylval->value.unknownValue = strdup(yytext); common(yylval, yytext, yyextra, TRUE, LSL_UNKNOWN); %}
+.               %{ printf(" unexpected character.\n"); yylval->value.unknownValue = strdup(yytext); common(yylval, yytext, yyleng, yyextra, TRUE, LSL_UNKNOWN); %}
 
 %%
 
-int common(YYSTYPE *lval, char *text, LuaSL_compiler *compiler, boolean checkIgnorable, int type)
+int common(YYSTYPE *lval, char *text, int len, LuaSL_compiler *compiler, boolean checkIgnorable, int type)
 {
     char *p;
 
     lval->token = tokens[type - lowestToken];
     lval->line = compiler->line;
     lval->column = compiler->column;
+    lval->len = len;
 
     for (p = text; *p; p++)
     {
-- 
cgit v1.1