Skip to content
Snippets Groups Projects
Commit 45c00246 authored by Tom Lane's avatar Tom Lane
Browse files

Remove fixed-size literal buffer from ecpg's lexer (same 

fix recently applied to backend's lexer).  I see that YY_USES_REJECT
still gets defined for this lexer, which means it's going to have trouble
parsing really long tokens.  Not sure if it's worth doing anything about
that or not; I don't have the interest right now to understand why
ecpg's additions to the syntax cause this problem...
parent abceb20a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/interfaces/ecpg/preproc/Makefile
+ 11
− 13
View file @ 45c00246
...@@ -15,11 +15,19 @@ OBJ=preproc.o pgc.o type.o ecpg.o ecpg_keywords.o \ ...@@ -15,11 +15,19 @@ OBJ=preproc.o pgc.o type.o ecpg.o ecpg_keywords.o \
all:: ecpg all:: ecpg
# Rule that really do something.
ecpg: $(OBJ)
$(CC) -o ecpg $(OBJ) $(LEXLIB) $(LDFLAGS)
preproc.c preproc.h: preproc.y preproc.c preproc.h: preproc.y
$(YACC) $(YFLAGS) $< $(YACC) $(YFLAGS) $<
mv y.tab.c preproc.c mv y.tab.c preproc.c
mv y.tab.h preproc.h mv y.tab.h preproc.h
pgc.c: pgc.l
$(LEX) $<
mv lex.yy.c pgc.c
clean: clean:
rm -f *.o core a.out ecpg$(X) *~ *.output rm -f *.o core a.out ecpg$(X) *~ *.output
# And the garbage that might have been left behind by partial build: # And the garbage that might have been left behind by partial build:
...@@ -33,19 +41,9 @@ install: all ...@@ -33,19 +41,9 @@ install: all
uninstall: uninstall:
rm -f $(BINDIR)/ecpg rm -f $(BINDIR)/ecpg
# Rule that really do something. preproc.o: preproc.h ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c
ecpg: $(OBJ) type.o: ../include/ecpgtype.h
$(CC) -o ecpg $(OBJ) $(LEXLIB) $(LDFLAGS) pgc.o: ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c preproc.h
pgc.c: pgc.l
$(LEX) $<
sed -e 's/#define YY_BUF_SIZE .*/#define YY_BUF_SIZE 65536/' \
<lex.yy.c >pgc.c
rm -f lex.yy.c
preproc.o : preproc.h ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c
type.o : ../include/ecpgtype.h
pgc.o : ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c preproc.h
keywords.o: ../include/ecpgtype.h preproc.h keywords.o: ../include/ecpgtype.h preproc.h
c_keywords.o: ../include/ecpgtype.h preproc.h c_keywords.o: ../include/ecpgtype.h preproc.h
ecpg_keywords.o: ../include/ecpgtype.h preproc.h ecpg_keywords.o: ../include/ecpgtype.h preproc.h
......
src/interfaces/ecpg/preproc/pgc.l
+ 96
− 71
View file @ 45c00246
/* This is a modified version of src/backend/parser/scan.l */
%{ %{
/*-------------------------------------------------------------------------
*
* pgc.l
* lexical scanner for ecpg
*
* This is a modified version of src/backend/parser/scan.l
*
*
* Copyright (c) 1994, Regents of the University of California
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/pgc.l,v 1.45 1999/10/22 23:14:50 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#include <ctype.h> #include <ctype.h>
#include <sys/types.h> #include <sys/types.h>
#include <limits.h> #include <limits.h>
#include <errno.h> #include <errno.h>
#include "postgres.h" #include "postgres.h"
#ifndef PATH_MAX #ifndef PATH_MAX
#include <sys/param.h> #include <sys/param.h>
#define PATH_MAX MAXPATHLEN #define PATH_MAX MAXPATHLEN
#endif #endif
#include "miscadmin.h" #include "miscadmin.h"
#include "nodes/pg_list.h"
#include "nodes/parsenodes.h" #include "nodes/parsenodes.h"
#include "nodes/pg_list.h"
#include "parser/gramparse.h" #include "parser/gramparse.h"
#include "parser/scansup.h" #include "parser/scansup.h"
#include "extern.h" #include "extern.h"
#include "preproc.h" #include "preproc.h"
#include "utils/builtins.h" #include "utils/builtins.h"
#ifdef YY_READ_BUF_SIZE
#undef YY_READ_BUF_SIZE
#endif
#define YY_READ_BUF_SIZE MAX_PARSE_BUFFER
/* some versions of lex define this as a macro */ /* some versions of lex define this as a macro */
#if defined(yywrap) #if defined(yywrap)
#undef yywrap #undef yywrap
#endif /* yywrap */ #endif /* yywrap */
extern YYSTYPE yylval; extern YYSTYPE yylval;
int llen;
char literal[MAX_PARSE_BUFFER]; /*
* literalbuf is used to accumulate literal values when multiple rules
* are needed to parse a single literal. Call startlit to reset buffer
* to empty, addlit to add text. Note that the buffer is permanently
* malloc'd to the largest size needed so far in the current run.
*/
static char *literalbuf = NULL; /* expandable buffer */
static int literallen; /* actual current length */
static int literalalloc; /* current allocated buffer size */
#define startlit() (literalbuf[0] = '\0', literallen = 0)
static void addlit(char *ytext, int yleng);
int before_comment; int before_comment;
struct _yy_buffer { YY_BUFFER_STATE buffer; struct _yy_buffer { YY_BUFFER_STATE buffer;
...@@ -142,16 +165,14 @@ self [,()\[\].;$\:\+\-\*\/\%\^\<\>\=\|] ...@@ -142,16 +165,14 @@ self [,()\[\].;$\:\+\-\*\/\%\^\<\>\=\|]
op_and_self [\~\!\@\#\^\&\|\`\?\$\:\+\-\*\/\%\<\>\=] op_and_self [\~\!\@\#\^\&\|\`\?\$\:\+\-\*\/\%\<\>\=]
operator {op_and_self}+ operator {op_and_self}+
/* we do not allow unary minus in numbers. /* we no longer allow unary minus in numbers.
* instead we pass it verbatim to parser. there it gets * instead we pass it separately to parser. there it gets
* coerced via doNegate() -- Leon aug 20 1999 * coerced via doNegate() -- Leon aug 20 1999
*/ */
integer {digit}+ integer {digit}+
decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*)) decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
real ((({digit}*\.{digit}+)|({digit}+\.{digit}*)|({digit}+))([Ee][-+]?{digit}+)) real ((({digit}*\.{digit}+)|({digit}+\.{digit}*)|({digit}+))([Ee][-+]?{digit}+))
/*
real (((({digit}*\.{digit}+)|({digit}+\.{digit}*))([Ee][-+]?{digit}+)?)|({digit}+[Ee][-+]?{digit}+))
*/
param \${integer} param \${integer}
...@@ -200,25 +221,21 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -200,25 +221,21 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xbstart} { <SQL>{xbstart} {
BEGIN(xb); BEGIN(xb);
llen = 0; startlit();
*literal = '\0';
} }
<xb>{xbstop} { <xb>{xbstop} {
char* endptr; char* endptr;
BEGIN(SQL); BEGIN(SQL);
errno = 0; errno = 0;
yylval.ival = strtol((char *)literal,&endptr,2); yylval.ival = strtol(literalbuf, &endptr, 2);
if (*endptr != '\0' || errno == ERANGE) if (*endptr != '\0' || errno == ERANGE)
yyerror("ERROR: Bad binary integer input!"); yyerror("ERROR: Bad binary integer input!");
return ICONST; return ICONST;
} }
<xh>{xhinside} | <xh>{xhinside} |
<xb>{xbinside} { <xb>{xbinside} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<xh>{xhcat} | <xh>{xhcat} |
<xb>{xbcat} { <xb>{xbcat} {
...@@ -226,15 +243,14 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -226,15 +243,14 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xhstart} { <SQL>{xhstart} {
BEGIN(xh); BEGIN(xh);
llen = 0; startlit();
*literal = '\0';
} }
<xh>{xhstop} { <xh>{xhstop} {
char* endptr; char* endptr;
BEGIN(SQL); BEGIN(SQL);
errno = 0; errno = 0;
yylval.ival = strtol((char *)literal,&endptr,16); yylval.ival = strtol(literalbuf, &endptr, 16);
if (*endptr != '\0' || errno == ERANGE) if (*endptr != '\0' || errno == ERANGE)
yyerror("ERROR: Bad hexadecimal integer input"); yyerror("ERROR: Bad hexadecimal integer input");
return ICONST; return ICONST;
...@@ -242,21 +258,17 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -242,21 +258,17 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xqstart} { <SQL>{xqstart} {
BEGIN(xq); BEGIN(xq);
llen = 0; startlit();
*literal = '\0';
} }
<xq>{xqstop} { <xq>{xqstop} {
BEGIN(SQL); BEGIN(SQL);
yylval.str = mm_strdup(literal); yylval.str = mm_strdup(literalbuf);
return SCONST; return SCONST;
} }
<xq>{xqdouble} | <xq>{xqdouble} |
<xq>{xqinside} | <xq>{xqinside} |
<xq>{xqliteral} { <xq>{xqliteral} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<xq>{xqcat} { <xq>{xqcat} {
} }
...@@ -264,35 +276,27 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -264,35 +276,27 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xdstart} { <SQL>{xdstart} {
BEGIN(xd); BEGIN(xd);
llen = 0; startlit();
*literal = '\0';
} }
<xd>{xdstop} { <xd>{xdstop} {
BEGIN(SQL); BEGIN(SQL);
yylval.str = mm_strdup(literal); yylval.str = mm_strdup(literalbuf);
return CSTRING; return CSTRING;
} }
<xd>{xdinside} { <xd>{xdinside} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
{xdstart} { {xdstart} {
BEGIN(xdc); BEGIN(xdc);
llen = 0; startlit();
*literal = '\0';
} }
<xdc>{xdstop} { <xdc>{xdstop} {
BEGIN(C); BEGIN(C);
yylval.str = mm_strdup(literal); yylval.str = mm_strdup(literalbuf);
return CSTRING; return CSTRING;
} }
<xdc>{xdcinside} { <xdc>{xdcinside} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<SQL>{typecast} { return TYPECAST; } <SQL>{typecast} { return TYPECAST; }
<SQL>{self} { /* <SQL>{self} { /*
...@@ -324,24 +328,24 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -324,24 +328,24 @@ cppline {space}*#.*(\\{space}*\n)*\n*
{ {
errno = 0; errno = 0;
yylval.str = mm_strdup((char*)yytext); yylval.str = mm_strdup((char*)yytext);
return SCONST; return SCONST;
} }
return ICONST; return ICONST;
} }
{decimal} { {decimal} {
char* endptr; char* endptr;
if (strlen((char *)yytext) <= 17) if (strlen((char *)yytext) <= 17)
{ {
errno = 0; errno = 0;
yylval.dval = strtod((char *)yytext,&endptr); yylval.dval = strtod((char *)yytext,&endptr);
if (*endptr != '\0' || errno == ERANGE) if (*endptr != '\0' || errno == ERANGE)
yyerror("ERROR: Bad float8 input"); yyerror("ERROR: Bad float8 input");
return FCONST; return FCONST;
} }
yylval.str = mm_strdup((char*)yytext); yylval.str = mm_strdup((char*)yytext);
return SCONST; return SCONST;
} }
<C,SQL>{real} { <C,SQL>{real} {
char* endptr; char* endptr;
...@@ -420,7 +424,7 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -420,7 +424,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
{ {
errno = 0; errno = 0;
yylval.str = mm_strdup((char*)yytext); yylval.str = mm_strdup((char*)yytext);
return SCONST; return SCONST;
} }
return ICONST; return ICONST;
} }
...@@ -486,8 +490,7 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -486,8 +490,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<def_ident>{identifier} { <def_ident>{identifier} {
old = mm_strdup(yytext); old = mm_strdup(yytext);
BEGIN(def); BEGIN(def);
llen = 0; startlit();
*literal = '\0';
} }
<def>{space} /* eat the whitespace */ <def>{space} /* eat the whitespace */
<def>";" { <def>";" {
...@@ -498,8 +501,8 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -498,8 +501,8 @@ cppline {space}*#.*(\\{space}*\n)*\n*
if (strcmp(old, ptr->old) == 0) if (strcmp(old, ptr->old) == 0)
{ {
free(ptr->new); free(ptr->new);
/* ptr->new = mm_strdup(scanstr(literal));*/ /* ptr->new = mm_strdup(scanstr(literalbuf));*/
ptr->new = mm_strdup(literal); ptr->new = mm_strdup(literalbuf);
} }
} }
if (ptr == NULL) if (ptr == NULL)
...@@ -508,8 +511,8 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -508,8 +511,8 @@ cppline {space}*#.*(\\{space}*\n)*\n*
/* initial definition */ /* initial definition */
this->old = old; this->old = old;
/* this->new = mm_strdup(scanstr(literal));*/ /* this->new = mm_strdup(scanstr(literalbuf));*/
this->new = mm_strdup(literal); this->new = mm_strdup(literalbuf);
this->next = defines; this->next = defines;
defines = this; defines = this;
} }
...@@ -517,10 +520,7 @@ cppline {space}*#.*(\\{space}*\n)*\n* ...@@ -517,10 +520,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
BEGIN(C); BEGIN(C);
} }
<def>[^";"] { <def>[^";"] {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: define statement parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<C>{exec}{space}{sql}{space}{include} { BEGIN(incl); } <C>{exec}{space}{sql}{space}{include} { BEGIN(incl); }
<incl>{space} /* eat the whitespace */ <incl>{space} /* eat the whitespace */
...@@ -602,9 +602,34 @@ void ...@@ -602,9 +602,34 @@ void
lex_init(void) lex_init(void)
{ {
braces_open = 0; braces_open = 0;
/* initialize literal buffer to a reasonable but expansible size */
if (literalbuf == NULL)
{
literalalloc = 128;
literalbuf = (char *) malloc(literalalloc);
}
startlit();
BEGIN C; BEGIN C;
} }
static void
addlit(char *ytext, int yleng)
{
/* enlarge buffer if needed */
if ((literallen+yleng) >= literalalloc)
{
do {
literalalloc *= 2;
} while ((literallen+yleng) >= literalalloc);
literalbuf = (char *) realloc(literalbuf, literalalloc);
}
/* append data --- note we assume ytext is null-terminated */
memcpy(literalbuf+literallen, ytext, yleng+1);
literallen += yleng;
}
int yywrap(void) int yywrap(void)
{ {
return 1; return 1;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment