Subversion Repositories spk

/*
  defines token_stream - class that acts as high level wrapper around 
  bod_text_parser. It mimics behaviour of stream.
  
  There is potential hazard when loading strings because only finite amount of 
  tokens is loaded into memory. If number of tokens forming the string is greater 
  than the token unload threshold, seeking back to beginning of string will fail
  
  see bod_bob_parser::loadString
  
*/

#ifndef TOKEN_STREAM_INCLUDED
#define TOKEN_STREAM_INCLUDED

#include "bob_dom_stream.h"
#include "bod_text_parser.h"

#include "../common/ext_list.h"

class token_stream : public ext::stream_base
{
        public:
                typedef bod_text_parser::token token;
                typedef bod_text_parser::TokenList::size_type size_type;
                typedef int offset_type;
                
        private:
                typedef bod_text_parser::TokenList::iterator iterator;
                
                bod_text_parser p;
                iterator m_end;
                iterator m_pos;
                size_type m_left;
                size_type m_right;
                bool m_bIgnoreRemarks;
                
                void advance(iterator &it, offset_type off)
                {
                        if(off > 0){
                                for(offset_type i=0; i < off; i++)
                                        ++it;
                        }
                        else{
                                for(offset_type i=off; i < 0; i++)
                                        --it;
                        }
                }
                
                void parse(size_t limit)
                {
                        m_right+=p.parseBuffer(limit);
                }
                
                void unload(size_t count)
                {
                        if(count > m_left) count=m_left;
                        m_left-=count;
                        while(count--){
                                delete *(p.tokens.begin());
                                p.tokens.erase(p.tokens.begin());
                        }
                }
                
        public:
                static const int tokenTreshold = 200; // how much tokens to unload
                static const int tokenUnloadTreshold = 2 * 200; // when to start unloading tokens
                
                token_stream() { m_bIgnoreRemarks=false; }
                
                void rdbuffer(char *pszBuffer, size_t size) 
                { 
                        p.ignoreRemarks(ignoreRemarks());
                        p.preParseBuffer(pszBuffer, size); 
                        m_left=0;
                        m_right=p.parseBuffer(1); 
                        m_pos=p.tokens.begin();
                        m_end=p.tokens.end();
                        clear(p.tokens.size() ? goodbit : badbit); 
                }
                
                token* tok() { return good() ? *m_pos : 0; }
                
                size_type tell() const { return m_left; }
                size_type avail() const { return m_right; }
                size_type size() const { return p.tokens.size(); }
                
                token_stream& operator++()      { advance(); return *this; }
                token_stream& operator--()      { advance(-1); return *this; }
                
                bool ignoreRemarks() const { return m_bIgnoreRemarks; } 
                void ignoreRemarks(bool ignore) { m_bIgnoreRemarks=ignore; }
                
                bool advance(offset_type offset=1)
                {
                        if(offset > (offset_type)m_right)
                                parse(offset < tokenTreshold ? tokenTreshold : offset);
                                
                        if(offset > (offset_type)m_right || (offset_type)m_left + offset < 0)
                                setstate(failbit);
                        else{
                                m_left+=offset; m_right-=offset;
                                if(m_right==0) 
                                        parse(tokenTreshold);
                                
                                advance(m_pos, offset);
                                
                                if(m_pos==m_end) 
                                        setstate(eofbit);
                                else
                                        clear((state)(rdstate() & ~eofbit));
                        }
                        if(m_left > tokenUnloadTreshold)
                                unload(tokenTreshold);
                        return good();
                }
                
                token* previous()
                {
                        if(m_left==0)
                                return 0;
                        else
                                return *(m_pos-1);
                }
};

#endif // !defined(TOKEN_STREAM_INCLUDED)