Subversion Repositories spk

/*

  defines token_stream - class that acts as high level wrapper around 

  bod_text_parser. It mimics behaviour of stream.

  There is potential hazard when loading strings because only finite amount of 

  tokens is loaded into memory. If number of tokens forming the string is greater 

  than the token unload threshold, seeking back to beginning of string will fail

  see bod_bob_parser::loadString

*/

#ifndef TOKEN_STREAM_INCLUDED

#define TOKEN_STREAM_INCLUDED

#include "bob_dom_stream.h"

#include "bod_text_parser.h"

#include "../common/ext_list.h"

class token_stream : public ext::stream_base

{

	public:

		typedef bod_text_parser::token token;

		typedef bod_text_parser::TokenList::size_type size_type;

		typedef int offset_type;

	private:

		typedef bod_text_parser::TokenList::iterator iterator;

		bod_text_parser p;

		iterator m_end;

		iterator m_pos;

		size_type m_left;

		size_type m_right;

		bool m_bIgnoreRemarks;

		void advance(iterator &it, offset_type off)

		{

			if(off > 0){

				for(offset_type i=0; i < off; i++)

					++it;

			}

			else{

				for(offset_type i=off; i < 0; i++)

					--it;

			}

		}

		void parse(size_t limit)

		{

			m_right+=p.parseBuffer(limit);

		}

		void unload(size_t count)

		{

			if(count > m_left) count=m_left;

			m_left-=count;

			while(count--){

				delete *(p.tokens.begin());

				p.tokens.erase(p.tokens.begin());

			}

		}

	public:

		static const int tokenTreshold = 200; // how much tokens to unload

		static const int tokenUnloadTreshold = 2 * 200; // when to start unloading tokens

		token_stream() { m_bIgnoreRemarks=false; }

		void rdbuffer(char *pszBuffer, size_t size) 

		{ 

			p.ignoreRemarks(ignoreRemarks());

			p.preParseBuffer(pszBuffer, size); 

			m_left=0;

			m_right=p.parseBuffer(1); 

			m_pos=p.tokens.begin();

			m_end=p.tokens.end();

			clear(p.tokens.size() ? goodbit : badbit); 

		}

		token* tok() { return good() ? *m_pos : 0; }

		size_type tell() const { return m_left; }

		size_type avail() const { return m_right; }

		size_type size() const { return p.tokens.size(); }

		token_stream& operator++()	{ advance(); return *this; }

		token_stream& operator--()	{ advance(-1); return *this; }

		bool ignoreRemarks() const { return m_bIgnoreRemarks; } 

		void ignoreRemarks(bool ignore) { m_bIgnoreRemarks=ignore; }

		bool advance(offset_type offset=1)

		{

			if(offset > (offset_type)m_right)

				parse(offset < tokenTreshold ? tokenTreshold : offset);

			if(offset > (offset_type)m_right || (offset_type)m_left + offset < 0)

				setstate(failbit);

			else{

				m_left+=offset; m_right-=offset;

				if(m_right==0) 

					parse(tokenTreshold);

				advance(m_pos, offset);

				if(m_pos==m_end) 

					setstate(eofbit);

				else

					clear((state)(rdstate() & ~eofbit));

			}

			if(m_left > tokenUnloadTreshold)

				unload(tokenTreshold);

			return good();

		}

		token* previous()

		{

			if(m_left==0)

				return 0;

			else

				return *(m_pos-1);

		}

};

#endif // !defined(TOKEN_STREAM_INCLUDED)