Subversion Repositories spk

/*

	High level wrapper around parser <Parser>. It acts as a stream 

	dynamically loading and unloading tokens as required.

*/

#ifndef TOKEN_STREAM_INCLUDED

#define TOKEN_STREAM_INCLUDED

#include "../common/stream_base.h"

template <class Parser>

class token_stream : public mystream::stream_base

{

	public:

		typedef Parser parser_type;

		typedef typename Parser::token token;

		typedef typename Parser::TokenList::size_type size_type;

		typedef int offset_type;

		typedef typename Parser::ParseFlags ParseFlags;

		typedef typename Parser::Error Error;

	private:

		typedef typename Parser::TokenList::iterator iterator;

		Parser p;

		iterator m_end;

		iterator m_pos;

		size_type m_left;

		size_type m_right;

		ParseFlags m_parseFlags;

		void advance(iterator &it, offset_type off)

		{

			if(fail()) return;

			if(off > 0){

				for(offset_type i=0; i < off; i++)

					++it;

			}

			else{

				for(offset_type i=off; i < 0; i++)

					--it;

			}

		}

		void parse(size_t limit)

		{

			m_right+=p.parseBuffer(limit);

			if(p.error().code!=Parser::Error::errNone) 

				setstate(failbit);

		}

		void unload(size_t count=0)

		{

			if(count > m_left) count=m_left;

			m_left-=count;

			while(count--){

				delete *(p.tokens.begin());

				p.tokens.erase(p.tokens.begin());

			}

		}

	public:

		static const int tokenTreshold = 200; // how much tokens to unload

		static const int tokenUnloadTreshold = 2 * 200; // when to start unloading tokens

		token_stream(ParseFlags flags) { parseFlags(flags); }

		token_stream() { parseFlags(Parser::none); }

		void rdbuffer(char *pszBuffer, size_t size) 

		{ 

			p.parseFlags(parseFlags());

			p.preParseBuffer(pszBuffer, size); 

			m_left=0;

			m_right=p.parseBuffer(1); 

			m_pos=p.tokens.begin();

			m_end=p.tokens.end();

			if(p.error().code!=Parser::Error::errNone)

				clear(failbit);

			else

				clear(p.tokens.size() ? goodbit : eofbit); 

		}

		token* tok() { return good() ? *m_pos : 0; }

		size_type tell() const { return m_left; }

		size_type avail() const { return m_right; }

		size_type size() const { return p.tokens.size(); }

		token_stream& operator++()	{ advance(); return *this; }

		token_stream& operator--()	{ advance(-1); return *this; }

		Error parseError() const { return p.error(); }

		ParseFlags parseFlags() const { return m_parseFlags; }

		void parseFlags(ParseFlags flags) { m_parseFlags=flags; }

		bool advance(offset_type offset=1)

		{

			if(offset > (offset_type)m_right)

				parse(offset < tokenTreshold ? tokenTreshold : offset);

			if(offset > (offset_type)m_right || (offset_type)m_left + offset < 0)

				setstate(failbit);

			else{

				m_left+=offset; m_right-=offset;

				if(m_right==0) 

					parse(tokenTreshold);

				advance(m_pos, offset);

				if(m_pos==m_end) 

					setstate(eofbit);

				else

					clear((state)(rdstate() & ~eofbit));

			}

			if(m_left > tokenUnloadTreshold)

				unload(tokenTreshold);

			return good();

		}

		token* previous()

		{

			if(m_left==0)

				return 0;

			else{

				iterator it=m_pos;

				return *(--it);

			}

		}

};

#endif // !defined(TOKEN_STREAM_INCLUDED)