%Przkład tokenize_file('therm-rt.pl',Tokeny).

tokenize_file(File,Tokens) :-
	read_file_to_codes(File,Codes,[]),
	token_list(Tokens,Codes,[]).

% Przykiład uzycia: token_list(Lista,"napis",[]).
token_list([T|Rest]) --> blank0,token(T),!,token_list(Rest).
token_list([]) --> blank0.

% blank0 is a series of zero or more blanks.
blank0 --> [C],{c_type(C,blank)},!,blank0.
blank0 --> [C],{c_type(C,space)},!,blank0.
blank0 --> [].


% Several kinds of tokens.

token(T) --> colon(C), {atom_codes(T,C)}.
token(T) --> garbage(G),{atom_codes(T,G)}.


number([N]) -->  [N], {c_type(N,number)}.
letter([L]) --> [L], {c_type(L,letter)}.
special([C]) --> [C], {c_type(C,special)}.
colon([C]) --> [C], {c_type(C,colon)}.

garbage_el(G) --> special([G]).
garbage_el(G) --> letter([G]).
garbage_el(G) --> number([G]).

garbage([G|Rest]) --> garbage_el(G),garbage(Rest).
garbage([G]) --> garbage_el(G).


c_type(Code,Y) :-
	Code =:= 10,
	Y = newline,!.
	
c_type(Code,Y) :-
	Code =:= 32,!,
	Y = space.
	
c_type(Code,Y):-
	Code =:= 58,!,
	Y = colon.
	
c_type(Code,Y):-% digits
	48 =< Code, Code =< 57,!,
	Y = number.
	
char_type(Code,Y):-% lowercase letters
	97 =< Code,
	Code=< 122,!,
	Y = letter.

char_type(Code,Y):-% uppercase letters
	65 =< Code, Code=< 90,!,
	Y = letter.
	
c_type(Code,Y):-
	Code =< 31,!,
	Y = blank.
	
	
c_type(Code,special).