Subversion Repositories spk

Rev

Rev 1 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 1 Rev 114
Line 1... Line 1...
1 
/*
 1 
/*
2 
  defines token_stream - class that acts as high level wrapper around
- 
 
3 
  bod_text_parser. It mimics behaviour of stream.
 2 
	High level wrapper around parser <Parser>. It acts as a stream
4
- 
 
5 
  There is potential hazard when loading strings because only finite amount of
3 
	dynamically loading and unloading tokens as required.
6 
  tokens is loaded into memory. If number of tokens forming the string is greater
- 
 
7 
  than the token unload threshold, seeking back to beginning of string will fail
 - 
 
8
- 
 
9 
  see bod_bob_parser::loadString
 - 
 
10
- 
 
11 
*/
 4 
*/
12 
 
 5 
 
13 
#ifndef TOKEN_STREAM_INCLUDED
 6 
#ifndef TOKEN_STREAM_INCLUDED
14 
#define TOKEN_STREAM_INCLUDED
 7 
#define TOKEN_STREAM_INCLUDED
15 
 
 8 
 
16 
#include "bob_dom_stream.h"
 9 
#include "../common/stream_base.h"
17 
#include "bod_text_parser.h"
 - 
 
18 
 
 10 
 
19 
#include &quot;../common/ext_list.h&quot;
 11 
template &lt;class Parser&gt;
20 
 
 - 
 
21 
class token_stream : public ext::stream_base
 12 
class token_stream : public mystream::stream_base
22 
{
 13 
{
23 
	public:
 14 
	public:
- 
 
 15 
		typedef Parser parser_type;
24 
		typedef bod_text_parser::token token;
 16 
		typedef typename Parser::token token;
25 
		typedef bod_text_parser::TokenList::size_type size_type;
 17 
		typedef typename Parser::TokenList::size_type size_type;
26 
		typedef int offset_type;
 18 
		typedef int offset_type;
- 
 
 19 
		typedef typename Parser::ParseFlags ParseFlags;
- 
 
 20 
		typedef typename Parser::Error Error;
27
21
28 
	private:
 22 
	private:
29 
		typedef bod_text_parser::TokenList::iterator iterator;
 23 
		typedef typename Parser::TokenList::iterator iterator;
30
24
31 
		bod_text_parser p;
 25 
		Parser p;
32 
		iterator m_end;
 26 
		iterator m_end;
33 
		iterator m_pos;
 27 
		iterator m_pos;
34 
		size_type m_left;
 28 
		size_type m_left;
35 
		size_type m_right;
 29 
		size_type m_right;
- 
 
 30
36 
		bool m_bIgnoreRemarks;
 31 
		ParseFlags m_parseFlags;
37
32
38 
		void advance(iterator &it, offset_type off)
 33 
		void advance(iterator &it, offset_type off)
39 
		{
 34 
		{
- 
 
 35 
			if(fail()) return;
40 
			if(off > 0){
 36 
			if(off > 0){
41 
				for(offset_type i=0; i < off; i++)
 37 
				for(offset_type i=0; i < off; i++)
42 
					++it;
 38 
					++it;
43 
			}
 39 
			}
44 
			else{
 40 
			else{
45 
				for(offset_type i=off; i < 0; i++)
 41 
				for(offset_type i=off; i < 0; i++)
46 
					--it;
 42 
					--it;
47 
			}
 43 
			}
48 
		}
 44 
		}
49
45
50 
		void parse(size_t limit)
 46 
		void parse(size_t limit)
51 
		{
 47 
		{
52 
			m_right+=p.parseBuffer(limit);
 48 
			m_right+=p.parseBuffer(limit);
- 
 
 49 
			if(p.error().code!=Parser::Error::errNone)
- 
 
 50 
				setstate(failbit);
53 
		}
 51 
		}
54
52
55 
		void unload(size_t count)
 53 
		void unload(size_t count=0)
56 
		{
 54 
		{
57 
			if(count > m_left) count=m_left;
 55 
			if(count > m_left) count=m_left;
58 
			m_left-=count;
 56 
			m_left-=count;
59 
			while(count--){
 57 
			while(count--){
60 
				delete *(p.tokens.begin());
 58 
				delete *(p.tokens.begin());
Line 64... Line 62...
64
62
65 
	public:
 63 
	public:
66 
		static const int tokenTreshold = 200; // how much tokens to unload
 64 
		static const int tokenTreshold = 200; // how much tokens to unload
67 
		static const int tokenUnloadTreshold = 2 * 200; // when to start unloading tokens
 65 
		static const int tokenUnloadTreshold = 2 * 200; // when to start unloading tokens
68
66
- 
 
 67 
		token_stream(ParseFlags flags) { parseFlags(flags); }
69 
		token_stream() { m_bIgnoreRemarks=false; }
 68 
		token_stream() { parseFlags(Parser::none); }
70
69
71 
		void rdbuffer(char *pszBuffer, size_t size)
70 
		void rdbuffer(char *pszBuffer, size_t size)
72 
		{
71 
		{
73 
			p.ignoreRemarks(ignoreRemarks());
 72 
			p.parseFlags(parseFlags());
74 
			p.preParseBuffer(pszBuffer, size);
73 
			p.preParseBuffer(pszBuffer, size);
75 
			m_left=0;
 74 
			m_left=0;
76 
			m_right=p.parseBuffer(1);
75 
			m_right=p.parseBuffer(1);
77 
			m_pos=p.tokens.begin();
 76 
			m_pos=p.tokens.begin();
78 
			m_end=p.tokens.end();
 77 
			m_end=p.tokens.end();
- 
 
 78 
			if(p.error().code!=Parser::Error::errNone)
- 
 
 79 
				clear(failbit);
- 
 
 80 
			else
79 
			clear(p.tokens.size() ? goodbit : badbit);
81 
				clear(p.tokens.size() ? goodbit : eofbit);
80 
		}
 82 
		}
81
83
82 
		token* tok() { return good() ? *m_pos : 0; }
 84 
		token* tok() { return good() ? *m_pos : 0; }
83
85
84 
		size_type tell() const { return m_left; }
 86 
		size_type tell() const { return m_left; }
Line 86... Line 88...
86 
		size_type size() const { return p.tokens.size(); }
 88 
		size_type size() const { return p.tokens.size(); }
87
89
88 
		token_stream& operator++()	{ advance(); return *this; }
 90 
		token_stream& operator++()	{ advance(); return *this; }
89 
		token_stream& operator--()	{ advance(-1); return *this; }
 91 
		token_stream& operator--()	{ advance(-1); return *this; }
90
92
- 
 
 93 
		Error parseError() const { return p.error(); }
- 
 
 94
91 
		bool ignoreRemarks() const { return m_bIgnoreRemarks; }
95 
		ParseFlags parseFlags() const { return m_parseFlags; }
92 
		void ignoreRemarks(bool ignore) { m_bIgnoreRemarks=ignore; }
 96 
		void parseFlags(ParseFlags flags) { m_parseFlags=flags; }
93
97
94 
		bool advance(offset_type offset=1)
 98 
		bool advance(offset_type offset=1)
95 
		{
 99 
		{
96 
			if(offset > (offset_type)m_right)
 100 
			if(offset > (offset_type)m_right)
97 
				parse(offset < tokenTreshold ? tokenTreshold : offset);
 101 
				parse(offset < tokenTreshold ? tokenTreshold : offset);
Line 117... Line 121...
117
121
118 
		token* previous()
 122 
		token* previous()
119 
		{
 123 
		{
120 
			if(m_left==0)
 124 
			if(m_left==0)
121 
				return 0;
 125 
				return 0;
122 
			else
 126 
			else{
- 
 
 127 
				iterator it=m_pos;
123 
				return *(m_pos-1);
 128 
				return *(--it);
- 
 
 129 
			}
124 
		}
 130 
		}
125 
};
 131 
};
126 
 
 132 
 
127 
#endif // !defined(TOKEN_STREAM_INCLUDED)
 133 
#endif // !defined(TOKEN_STREAM_INCLUDED)