Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Class Members | File Members

tokenizer.h

Go to the documentation of this file.
00001 
00002 //
00003 //    FreeLing - Open Source Language Analyzers
00004 //
00005 //    Copyright (C) 2004   TALP Research Center
00006 //                         Universitat Politecnica de Catalunya
00007 //
00008 //    This library is free software; you can redistribute it and/or
00009 //    modify it under the terms of the GNU Lesser General Public
00010 //    License as published by the Free Software Foundation; either
00011 //    version 2.1 of the License, or (at your option) any later version.
00012 //
00013 //    This library is distributed in the hope that it will be useful,
00014 //    but WITHOUT ANY WARRANTY; without even the implied warranty of
00015 //    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00016 //    Lesser General Public License for more details.
00017 //
00018 //    You should have received a copy of the GNU Lesser General Public
00019 //    License along with this library; if not, write to the Free Software
00020 //    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
00021 //
00022 //    contact: Lluis Padro (padro@lsi.upc.es)
00023 //             TALP Research Center
00024 //             despatx C6.212 - Campus Nord UPC
00025 //             08034 Barcelona.  SPAIN
00026 //
00028 
00029 #ifndef _TOKENIZER
00030 #define _TOKENIZER
00031 
00032 #include <set>
00033 #include <map>
00034 #include <vector>
00035 
00036 #include "language.h"
00037 #include "regexp.h"
00038 
00045 
00046 class tokenizer {
00047    private:
00049         set<string> abrevs;
00051         vector<pair<string,RegEx> > rules;
00053         map<string,int> matches;
00054 
00055    public:
00057        tokenizer(const string &);
00058 
00060        list<word> tokenize(const string &);
00062        list<word> tokenize(const string &, int &);
00063 };
00064 
00065 #endif
00066 
00067 
00068 
00069 
00070 
00071 
00072 
00073 
00074 

Generated on Wed Apr 26 12:55:30 2006 for FreeLing by  doxygen 1.4.4