function Tokenize: TTokenList;
var
StartLine,StartCol: LongWord;
Kind: TTokenKind;
Lexeme,ContextBasePath: String;
Look: Char;
TempState: TLexerState;
procedure InitFields;
begin
StartLine := Line;
StartCol := Col;
Lexeme := EmptyStr;
end;
procedure HandleText;
begin
case Look of
'{': begin
LexerState := lsPossiblyTagStart;
end;
else begin
Lexeme := Lexeme + Look;
end;
end;
Look := ReadChar;
end;
procedure HandlePossiblyTagStart;
begin
case Look of
'{': begin
if Lexeme <> EmptyStr then begin
AddToken(StartLine,StartCol,Kind,Lexeme,ContextBasePath,Result);
InitFields;
end;
LexerState := lsTagStart;
end;
else begin
Lexeme := Lexeme + '{' + Look;
LexerState := lsText;
end;
end;
Look := ReadChar;
end;
procedure HandleTagStart;
begin
case Look of
'{': begin
LexerState := lsUnescapedVariable;
Kind := tkUnescapedVariable;
end;
'&': begin
LexerState := lsUnescapedVariableAmp;
Kind := tkUnescapedVariable;
end;
'#': begin
LexerState := lsOpenSection;
Kind := tkOpenSection;
end;
'/': begin
LexerState := lsCloseSection;
Kind := tkCloseSection;
end;
'a'..'z','A'..'Z','_',' ': begin
LexerState := lsEscapedVariable;
Kind := tkEscapedVariable;
Lexeme := Look;
end;
else begin
raise ETokenize.Create('Invalid tag char: ' + Look);
end;
end;
Look := ReadChar;
end;
procedure HandlePossiblyTagEnd;
var
p: SizeInt;
begin
case Look of
'}': begin
Lexeme := Trim(Lexeme);
case TempState of
lsOpenSection: ContextBasePath := ContextBasePath + Lexeme + '.';
lsCloseSection: begin
if ContextBasePath = EmptyStr then
raise Exception.Create('Closing unopened section: ' + Lexeme)
else begin
p := RPos('.',ContextBasePath);
Delete(ContextBasePath,p,Length(ContextBasePath) - p);
end;
end;
end;
AddToken(StartLine,StartCol,Kind,Lexeme,ContextBasePath,Result);
InitFields;
LexerState := lsTagEnd;
end;
else begin
Lexeme := Lexeme + '}' + Look;
LexerState := TempState;
end;
end;
Look := ReadChar;
end;
procedure HandleTagEnd;
begin
LexerState := lsText;
Kind := tkText;
end;
procedure HandleVariable;
begin
case Look of
'}': begin
TempState := LexerState;
if LexerState = lsEscapedVariable then
LexerState := lsPossiblyTagEnd
else
LexerState := lsPossiblyUnescapedVariableEnd;
end;
'a'..'z','A'..'Z','0'..'9','_',' ','.': begin
Lexeme := Lexeme + Look;
end;
else begin
raise ETokenize.Create('Invalid tag char: ' + Look);
end;
end;
Look := ReadChar;
end;
procedure HandlePossiblyUnescapedVariableEnd;
begin
case Look of
'}': begin
LexerState := lsPossiblyTagEnd;
end;
else begin
raise ETokenize.Create('Invalid tag char: ' + Look);
end;
end;
Look := ReadChar;
end;
procedure HandleUnescapedVariableAmp;
begin
case Look of
'}': begin
TempState := LexerState;
LexerState := lsPossiblyTagEnd;
end;
'a'..'z','A'..'Z','0'..'9','_',' ','.': begin
Lexeme := Lexeme + Look;
end;
else begin
raise ETokenize.Create('Invalid tag char: ' + Look);
end;
end;
Look := ReadChar;
end;
procedure HandleSection;
begin
case Look of
'}': begin
TempState := LexerState;
LexerState := lsPossiblyTagEnd;
end;
'a'..'z','A'..'Z','0'..'9','_',' ','.': begin
Lexeme := Lexeme + Look;
end;
else begin
raise ETokenize.Create('Invalid tag char: ' + Look);
end;
end;
Look := ReadChar;
end;
begin
Initialize(Result);
LexerState := lsText;
InitFields;
ContextBasePath := EmptyStr;
Look := ReadChar;
Kind := tkText;
while Look <> EOFChar do begin
case LexerState of
lsText : HandleText;
lsPossiblyTagStart : HandlePossiblyTagStart;
lsTagStart : HandleTagStart;
lsPossiblyTagEnd : HandlePossiblyTagEnd;
lsTagEnd : HandleTagEnd;
lsEscapedVariable,
lsUnescapedVariable : HandleVariable;
lsPossiblyUnescapedVariableEnd: HandlePossiblyUnescapedVariableEnd;
lsUnescapedVariableAmp : HandleUnescapedVariableAmp;
lsOpenSection,lsCloseSection : HandleSection;
end;
end;
if Lexeme <> EmptyStr then begin
AddToken(StartLine,StartCol,Kind,Lexeme,ContextBasePath,Result);
end;
end;