src/yaml-cpp/stream.cpp
d958d350
 #include "stream.h"
 #include <iostream>
 #include "exp.h"
 
 #ifndef YAML_PREFETCH_SIZE
 #define YAML_PREFETCH_SIZE 2048
 #endif
 
 #define S_ARRAY_SIZE( A ) (sizeof(A)/sizeof(*(A)))
 #define S_ARRAY_END( A ) ((A) + S_ARRAY_SIZE(A))
 
 #define CP_REPLACEMENT_CHARACTER (0xFFFD)
 
 namespace YAML
 {
 	enum UtfIntroState {
 		uis_start,
 		uis_utfbe_b1,
 		uis_utf32be_b2,
 		uis_utf32be_bom3,
 		uis_utf32be,
 		uis_utf16be,
 		uis_utf16be_bom1,
 		uis_utfle_bom1,
 		uis_utf16le_bom2,
 		uis_utf32le_bom3,
 		uis_utf16le,
 		uis_utf32le,
 		uis_utf8_imp,
 		uis_utf16le_imp,
 		uis_utf32le_imp3,
 		uis_utf8_bom1,
 		uis_utf8_bom2,
 		uis_utf8,
 		uis_error
 	};
 
 	enum UtfIntroCharType {
 		uict00,
 		uictBB,
 		uictBF,
 		uictEF,
 		uictFE,
 		uictFF,
 		uictAscii,
 		uictOther,
 		uictMax
 	};
 
 	static bool s_introFinalState[] = {
 		false, //uis_start
 		false, //uis_utfbe_b1
 		false, //uis_utf32be_b2
 		false, //uis_utf32be_bom3
 		true,  //uis_utf32be
 		true,  //uis_utf16be
 		false, //uis_utf16be_bom1
 		false, //uis_utfle_bom1
 		false, //uis_utf16le_bom2
 		false, //uis_utf32le_bom3
 		true,  //uis_utf16le
 		true,  //uis_utf32le
 		false, //uis_utf8_imp
 		false, //uis_utf16le_imp
 		false, //uis_utf32le_imp3
 		false, //uis_utf8_bom1
 		false, //uis_utf8_bom2
 		true,  //uis_utf8
 		true,  //uis_error
 	};
 
 	static UtfIntroState s_introTransitions[][uictMax] = {
 		// uict00,           uictBB,           uictBF,           uictEF,           uictFE,           uictFF,           uictAscii,        uictOther
 		  {uis_utfbe_b1,     uis_utf8,         uis_utf8,         uis_utf8_bom1,    uis_utf16be_bom1, uis_utfle_bom1,   uis_utf8_imp,     uis_utf8},
 		  {uis_utf32be_b2,   uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf16be,      uis_utf8},
 		  {uis_utf32be,      uis_utf8,         uis_utf8,         uis_utf8,         uis_utf32be_bom3, uis_utf8,         uis_utf8,         uis_utf8},
 		  {uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf32be,      uis_utf8,         uis_utf8},
 		  {uis_utf32be,      uis_utf32be,      uis_utf32be,      uis_utf32be,      uis_utf32be,      uis_utf32be,      uis_utf32be,      uis_utf32be},
 		  {uis_utf16be,      uis_utf16be,      uis_utf16be,      uis_utf16be,      uis_utf16be,      uis_utf16be,      uis_utf16be,      uis_utf16be},
 		  {uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf16be,      uis_utf8,         uis_utf8},
 		  {uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf16le_bom2, uis_utf8,         uis_utf8,         uis_utf8},
 		  {uis_utf32le_bom3, uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le},
 		  {uis_utf32le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le},
 		  {uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le},
 		  {uis_utf32le,      uis_utf32le,      uis_utf32le,      uis_utf32le,      uis_utf32le,      uis_utf32le,      uis_utf32le,      uis_utf32le},
 		  {uis_utf16le_imp,  uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8},
 		  {uis_utf32le_imp3, uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le},
 		  {uis_utf32le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le,      uis_utf16le},
 		  {uis_utf8,         uis_utf8_bom2,    uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8},
 		  {uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8},
 		  {uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8,         uis_utf8},
 	};
 
 	static char s_introUngetCount[][uictMax] = {
 		// uict00, uictBB, uictBF, uictEF, uictFE, uictFF, uictAscii, uictOther
 		  {0,      1,      1,      0,      0,      0,      0,         1},
 		  {0,      2,      2,      2,      2,      2,      2,         2},
 		  {3,      3,      3,      3,      0,      3,      3,         3},
 		  {4,      4,      4,      4,      4,      0,      4,         4},
 		  {1,      1,      1,      1,      1,      1,      1,         1},
 		  {1,      1,      1,      1,      1,      1,      1,         1},
 		  {2,      2,      2,      2,      2,      0,      2,         2},
 		  {2,      2,      2,      2,      0,      2,      2,         2},
 		  {0,      1,      1,      1,      1,      1,      1,         1},
 		  {0,      2,      2,      2,      2,      2,      2,         2},
 		  {1,      1,      1,      1,      1,      1,      1,         1},
 		  {1,      1,      1,      1,      1,      1,      1,         1},
 		  {0,      2,      2,      2,      2,      2,      2,         2},
 		  {0,      3,      3,      3,      3,      3,      3,         3},
 		  {4,      4,      4,      4,      4,      4,      4,         4},
 		  {2,      0,      2,      2,      2,      2,      2,         2},
 		  {3,      3,      0,      3,      3,      3,      3,         3},
 		  {1,      1,      1,      1,      1,      1,      1,         1},
 	};
 
 	inline UtfIntroCharType IntroCharTypeOf(std::istream::int_type ch)
 	{
 		if (std::istream::traits_type::eof() == ch) {
 			return uictOther;
 		}
 
 		switch (ch) {
 		case 0: return uict00;
 		case 0xBB: return uictBB;
 		case 0xBF: return uictBF;
 		case 0xEF: return uictEF;
 		case 0xFE: return uictFE;
 		case 0xFF: return uictFF;
 		}
 
 		if ((ch > 0) && (ch < 0xFF)) {
 			return uictAscii;
 		}
 
 		return uictOther;
 	}
 
 	inline char Utf8Adjust(unsigned long ch, unsigned char lead_bits, unsigned char rshift)
 	{
 		const unsigned char header = ((1 << lead_bits) - 1) << (8 - lead_bits);
 		const unsigned char mask = (0xFF >> (lead_bits + 1));
 		return static_cast<char>(static_cast<unsigned char>(
 			header | ((ch >> rshift) & mask)
 			));
 	}
 
 	inline void QueueUnicodeCodepoint(std::deque<char>& q, unsigned long ch)
 	{
 		// We are not allowed to queue the Stream::eof() codepoint, so
 		// replace it with CP_REPLACEMENT_CHARACTER
 		if (static_cast<unsigned long>(Stream::eof()) == ch)
 		{
 			ch = CP_REPLACEMENT_CHARACTER;
 		}
 
 		if (ch < 0x80)
 		{
 			q.push_back(Utf8Adjust(ch, 0, 0));
 		}
 		else if (ch < 0x800)
 		{
 			q.push_back(Utf8Adjust(ch, 2, 6));
 			q.push_back(Utf8Adjust(ch, 1, 0));
 		}
 		else if (ch < 0x10000)
 		{
 			q.push_back(Utf8Adjust(ch, 3, 12));
 			q.push_back(Utf8Adjust(ch, 1, 6));
 			q.push_back(Utf8Adjust(ch, 1, 0));
 		}
 		else
 		{
 			q.push_back(Utf8Adjust(ch, 4, 18));
 			q.push_back(Utf8Adjust(ch, 1, 12));
 			q.push_back(Utf8Adjust(ch, 1, 6));
 			q.push_back(Utf8Adjust(ch, 1, 0));
 		}
 	}
 
 	Stream::Stream(std::istream& input)
 		: m_input(input), m_nPushedBack(0),
 		m_pPrefetched(new unsigned char[YAML_PREFETCH_SIZE]), 
 		m_nPrefetchedAvailable(0), m_nPrefetchedUsed(0)
 	{
 		typedef std::istream::traits_type char_traits;
 
 		if(!input)
 			return;
 
 		// Determine (or guess) the character-set by reading the BOM, if any.  See
 		// the YAML specification for the determination algorithm.
 		char_traits::int_type intro[4];
 		int nIntroUsed = 0;
 		UtfIntroState state = uis_start;
 		for (; !s_introFinalState[state]; ) {
 			std::istream::int_type ch = input.get();
 			intro[nIntroUsed++] = ch;
 			UtfIntroCharType charType = IntroCharTypeOf(ch);
 			UtfIntroState newState = s_introTransitions[state][charType];
 			int nUngets = s_introUngetCount[state][charType];
 			if (nUngets > 0) {
 				for (; nUngets > 0; --nUngets) {
 					if (char_traits::eof() != intro[--nIntroUsed]) {
 						m_bufPushback[m_nPushedBack++] = 
 							char_traits::to_char_type(intro[nIntroUsed]);
 					}
 				}
 			}
 			state = newState;
 		}
 
 		switch (state) {
 		case uis_utf8: m_charSet = utf8; break;
 		case uis_utf16le: m_charSet = utf16le; break;
 		case uis_utf16be: m_charSet = utf16be; break;
 		case uis_utf32le: m_charSet = utf32le; break;
 		case uis_utf32be: m_charSet = utf32be; break;
 		default: m_charSet = utf8; break;
 		}
 
 		ReadAheadTo(0);
 	}
 
 	Stream::~Stream()
 	{
 		delete[] m_pPrefetched;
 	}
 
 	char Stream::peek() const
 	{
 		if (m_readahead.empty())
 		{
 			return Stream::eof();
 		}
 
 		return m_readahead[0];
 	}
 	
 	Stream::operator bool() const
 	{
 		return m_input.good() || (!m_readahead.empty() && m_readahead[0] != Stream::eof());
 	}
 
 	// get
 	// . Extracts a character from the stream and updates our position
 	char Stream::get()
 	{
 		char ch = peek();
 		AdvanceCurrent();
 		m_mark.column++;
 		
 		if(ch == '\n') {
 			m_mark.column = 0;
 			m_mark.line++;
 		}
 		
 		return ch;
 	}
 
 	// get
 	// . Extracts 'n' characters from the stream and updates our position
 	std::string Stream::get(int n)
 	{
 		std::string ret;
 		ret.reserve(n);
 		for(int i=0;i<n;i++)
 			ret += get();
 		return ret;
 	}
 
 	// eat
 	// . Eats 'n' characters and updates our position.
 	void Stream::eat(int n)
 	{
 		for(int i=0;i<n;i++)
 			get();
 	}
 
 	void Stream::AdvanceCurrent()
 	{
 		if (!m_readahead.empty())
 		{
 			m_readahead.pop_front();
 			m_mark.pos++;
 		}
 
 		ReadAheadTo(0);
 	}
 
 	bool Stream::_ReadAheadTo(size_t i) const
 	{
 		while (m_input.good() && (m_readahead.size() <= i))
 		{
 			switch (m_charSet)
 			{
 			case utf8: StreamInUtf8(); break;
 			case utf16le: StreamInUtf16(); break;
 			case utf16be: StreamInUtf16(); break;
 			case utf32le: StreamInUtf32(); break;
 			case utf32be: StreamInUtf32(); break;
 			}
 		}
 		
 		// signal end of stream
 		if(!m_input.good())
 			m_readahead.push_back(Stream::eof());
 
 		return m_readahead.size() > i;
 	}
 
 	void Stream::StreamInUtf8() const
 	{
 		unsigned char b = GetNextByte();
 		if (m_input.good())
 		{
 			m_readahead.push_back(b);
 		}
 	}
 
 	void Stream::StreamInUtf16() const
 	{
 		unsigned long ch = 0;
 		unsigned char bytes[2];
 		int nBigEnd = (m_charSet == utf16be) ? 0 : 1;
 
 		bytes[0] = GetNextByte();
 		bytes[1] = GetNextByte();
 		if (!m_input.good())
 		{
 			return;
 		}
 		ch = (static_cast<unsigned long>(bytes[nBigEnd]) << 8) |
 			static_cast<unsigned long>(bytes[1 ^ nBigEnd]);
 
 		if (ch >= 0xDC00 && ch < 0xE000)
 		{
 			// Trailing (low) surrogate...ugh, wrong order
 			QueueUnicodeCodepoint(m_readahead, CP_REPLACEMENT_CHARACTER);
 			return;
 		}
 		else if (ch >= 0xD800 && ch < 0xDC00)
 		{
 			// ch is a leading (high) surrogate
 
 			// Four byte UTF-8 code point
 
 			// Read the trailing (low) surrogate
 			for (;;)
 			{
 				bytes[0] = GetNextByte();
 				bytes[1] = GetNextByte();
 				if (!m_input.good())
 				{
 					QueueUnicodeCodepoint(m_readahead, CP_REPLACEMENT_CHARACTER);
 					return;
 				}
 				unsigned long chLow = (static_cast<unsigned long>(bytes[nBigEnd]) << 8) |
 					static_cast<unsigned long>(bytes[1 ^ nBigEnd]);
 				if (chLow < 0xDC00 || ch >= 0xE000)
 				{
 					// Trouble...not a low surrogate.  Dump a REPLACEMENT CHARACTER into the stream.
 					QueueUnicodeCodepoint(m_readahead, CP_REPLACEMENT_CHARACTER);
 
 					// Deal with the next UTF-16 unit
 					if (chLow < 0xD800 || ch >= 0xE000)
 					{
 						// Easiest case: queue the codepoint and return
 						QueueUnicodeCodepoint(m_readahead, ch);
 						return;
 					}
 					else
 					{
 						// Start the loop over with the new high surrogate
 						ch = chLow;
 						continue;
 					}
 				}
 
 				// Select the payload bits from the high surrogate
 				ch &= 0x3FF;
 				ch <<= 10;
 
 				// Include bits from low surrogate
 				ch |= (chLow & 0x3FF);
 
 				// Add the surrogacy offset
 				ch += 0x10000;
 			}
 		}
 
 		QueueUnicodeCodepoint(m_readahead, ch);
 	}
 
 	inline char* ReadBuffer(unsigned char* pBuffer)
 	{
 		return reinterpret_cast<char*>(pBuffer);
 	}
 
 	unsigned char Stream::GetNextByte() const
 	{
 		if (m_nPushedBack)
 		{
 			return m_bufPushback[--m_nPushedBack];
 		}
 
 		if (m_nPrefetchedUsed >= m_nPrefetchedAvailable)
 		{
 			std::streambuf *pBuf = m_input.rdbuf();
 			m_nPrefetchedAvailable = pBuf->sgetn(ReadBuffer(m_pPrefetched), 
 				YAML_PREFETCH_SIZE);
 			m_nPrefetchedUsed = 0;
 			if (!m_nPrefetchedAvailable)
 			{
 				m_input.setstate(std::ios_base::eofbit);
 			}
 
 			if (0 == m_nPrefetchedAvailable)
 			{
 				return 0;
 			}
 		}
 
 		return m_pPrefetched[m_nPrefetchedUsed++];
 	}
 
 	void Stream::StreamInUtf32() const
 	{
 		static int indexes[2][4] = {
 			{3, 2, 1, 0},
 			{0, 1, 2, 3}
 		};
 
 		unsigned long ch = 0;
 		unsigned char bytes[4];
 		int* pIndexes = (m_charSet == utf32be) ? indexes[1] : indexes[0];
 
 		bytes[0] = GetNextByte();
 		bytes[1] = GetNextByte();
 		bytes[2] = GetNextByte();
 		bytes[3] = GetNextByte();
 		if (!m_input.good())
 		{
 			return;
 		}
 
 		for (int i = 0; i < 4; ++i)
 		{
 			ch <<= 8;
 			ch |= bytes[pIndexes[i]];
 		}
 
 		QueueUnicodeCodepoint(m_readahead, ch);
 	}
 }