diff --git a/applications/test/dictionary/calcEntry/SimpleCalc.atg b/applications/test/dictionary/calcEntry/SimpleCalc.atg index cdc84cd0c9..57e41ed407 100644 --- a/applications/test/dictionary/calcEntry/SimpleCalc.atg +++ b/applications/test/dictionary/calcEntry/SimpleCalc.atg @@ -14,6 +14,9 @@ COMPILER SimpleCalc +$prefix=calcEntry +$namespace=Foam::functionEntries::calcEntryInternal + // Simple four function calculator for OpenFOAM dictionaries //! with debug diff --git a/applications/test/dictionary/calcEntry/build.sh b/applications/test/dictionary/calcEntry/build.sh index 4bbaf0eeaf..f13e716d92 100755 --- a/applications/test/dictionary/calcEntry/build.sh +++ b/applications/test/dictionary/calcEntry/build.sh @@ -5,7 +5,5 @@ cd ${0%/*} || exit 1 # run from this directory Coco \ -frames $WM_THIRD_PARTY_DIR/coco-r \ - -prefix calcEntry \ - -namespace Foam::functionEntries::calcEntryInternal \ SimpleCalc.atg diff --git a/applications/test/dictionary/calcEntry/calcEntryParser.cpp b/applications/test/dictionary/calcEntry/calcEntryParser.cpp index 412a00c52f..421d8ba7fa 100644 --- a/applications/test/dictionary/calcEntry/calcEntryParser.cpp +++ b/applications/test/dictionary/calcEntry/calcEntryParser.cpp @@ -199,14 +199,12 @@ Parser::Parser(Scanner* scan, Errors* err) : dummyToken(NULL), deleteErrorsDestruct_(!err), - minErrDist(2), errDist(minErrDist), scanner(scan), errors(err), t(NULL), la(NULL) { - maxT = 13; if (!errors) { // add in default error handling errors = new Errors(); @@ -317,8 +315,12 @@ void Errors::Exception(const wchar_t* msg) { ::exit(1); } + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + } // namespace } // namespace } // namespace +// ************************************************************************* // diff --git a/applications/test/dictionary/calcEntry/calcEntryParser.h b/applications/test/dictionary/calcEntry/calcEntryParser.h index d3da4d05fc..66b30a10ff 100644 --- a/applications/test/dictionary/calcEntry/calcEntryParser.h +++ b/applications/test/dictionary/calcEntry/calcEntryParser.h @@ -55,11 +55,12 @@ private: _variable=3, _number=4, }; - int maxT; + static const int maxT = 13; + + static const int minErrDist = 2; //!< min. distance before reporting errors Token *dummyToken; bool deleteErrorsDestruct_; //!< delete the 'errors' member in destructor - int minErrDist; int errDist; void SynErr(int n); //!< Handle syntax error 'n' diff --git a/applications/test/dictionary/calcEntry/calcEntryScanner.cpp b/applications/test/dictionary/calcEntry/calcEntryScanner.cpp index 86a92d3233..ab0d7b78d3 100644 --- a/applications/test/dictionary/calcEntry/calcEntryScanner.cpp +++ b/applications/test/dictionary/calcEntry/calcEntryScanner.cpp @@ -1,9 +1,16 @@ -#include -#include +#include + #include "calcEntryScanner.h" +// values for the file stream buffering +#define MIN_BUFFER_LENGTH 1024 // 1KB +#define MAX_BUFFER_LENGTH (64*MIN_BUFFER_LENGTH) // 64KB +// value for the heap management +#define HEAP_BLOCK_SIZE (64*1024) // 64KB + + namespace Foam { namespace functionEntries { namespace calcEntryInternal { @@ -36,29 +43,6 @@ wchar_t* coco_string_create(const wchar_t* str, int index, int length) { return dest; } -wchar_t* coco_string_create_upper(const wchar_t* str) { - if (!str) { return NULL; } - return coco_string_create_upper(str, 0, wcslen(str)); -} - - -wchar_t* coco_string_create_upper(const wchar_t* str, int index, int len) { - if (!str) { return NULL; } - wchar_t* dest = new wchar_t[len + 1]; - - for (int i = 0; i < len; i++) { - const wchar_t ch = str[index + i]; - if ((L'a' <= ch) && (ch <= L'z')) { - dest[i] = ch + (L'A' - L'a'); - } - else { - dest[i] = ch; - } - } - dest[len] = L'\0'; - return dest; -} - wchar_t* coco_string_create_lower(const wchar_t* str) { if (!str) { return NULL; } @@ -178,7 +162,7 @@ wchar_t* coco_string_create(const char* str) { int len = str ? strlen(str) : 0; wchar_t* dest = new wchar_t[len + 1]; for (int i = 0; i < len; ++i) { - dest[i] = (wchar_t) str[i]; + dest[i] = wchar_t(str[i]); } dest[len] = 0; return dest; @@ -188,7 +172,7 @@ wchar_t* coco_string_create(const char* str, int index, int length) { int len = str ? length : 0; wchar_t* dest = new wchar_t[len + 1]; for (int i = 0; i < len; ++i) { - dest[i] = (wchar_t) str[index + i]; + dest[i] = wchar_t(str[index + i]); } dest[len] = 0; return dest; @@ -200,7 +184,7 @@ char* coco_string_create_char(const wchar_t* str) { char *dest = new char[len + 1]; for (int i = 0; i < len; ++i) { - dest[i] = (char) str[i]; + dest[i] = char(str[i]); } dest[len] = 0; return dest; @@ -213,7 +197,7 @@ char* coco_string_create_char(const wchar_t* str, int index, int length) { } char *dest = new char[len + 1]; for (int i = 0; i < len; ++i) { - dest[i] = (char) str[index + i]; + dest[i] = char(str[index + i]); } dest[len] = 0; return dest; @@ -251,27 +235,61 @@ Token::Token() {} +// Note: this delete may not be correct if the token was actually +// allocated by the internal heap mechanism Token::~Token() { coco_string_delete(val); } -Buffer::Buffer(FILE* s, bool isUserStream) { +// ---------------------------------------------------------------------------- +// Buffer Implementation +// ---------------------------------------------------------------------------- + +Buffer::Buffer(Buffer* b) +: + buf(b->buf), + bufCapacity(b->bufCapacity), + bufLen(b->bufLen), + bufPos(b->bufPos), + bufStart(b->bufStart), + fileLen(b->fileLen), + cStream(b->cStream), + stdStream(b->stdStream), + isUserStream_(b->isUserStream_) +{ + // avoid accidental deletion on any of these members + b->buf = NULL; + b->cStream = NULL; + b->stdStream = NULL; +} + + +Buffer::Buffer(FILE* istr, bool isUserStream) +: + buf(NULL), + bufCapacity(0), + bufLen(0), + bufPos(0), + bufStart(0), + fileLen(0), + cStream(istr), + stdStream(NULL), + isUserStream_(isUserStream) +{ // ensure binary read on windows #if _MSC_VER >= 1300 - _setmode(_fileno(s), _O_BINARY); + _setmode(_fileno(cStream), _O_BINARY); #endif - stream = s; this->isUserStream = isUserStream; + if (CanSeek()) { - fseek(s, 0, SEEK_END); - fileLen = ftell(s); - fseek(s, 0, SEEK_SET); + fseek(cStream, 0, SEEK_END); + fileLen = ftell(cStream); + fseek(cStream, 0, SEEK_SET); bufLen = (fileLen < MAX_BUFFER_LENGTH) ? fileLen : MAX_BUFFER_LENGTH; bufStart = INT_MAX; // nothing in the buffer so far } - else { - fileLen = bufLen = bufStart = 0; - } + bufCapacity = (bufLen > 0) ? bufLen : MIN_BUFFER_LENGTH; buf = new unsigned char[bufCapacity]; if (fileLen > 0) SetPos(0); // setup buffer to position 0 (start) @@ -280,45 +298,74 @@ Buffer::Buffer(FILE* s, bool isUserStream) { } -Buffer::Buffer(Buffer* b) { - buf = b->buf; - bufCapacity = b->bufCapacity; - b->buf = NULL; - bufStart = b->bufStart; - bufLen = b->bufLen; - fileLen = b->fileLen; - bufPos = b->bufPos; - stream = b->stream; - b->stream = NULL; - isUserStream = b->isUserStream; +Buffer::Buffer(std::istream* istr, bool isUserStream) +: + buf(NULL), + bufCapacity(0), + bufLen(0), + bufPos(0), + bufStart(0), + fileLen(0), + cStream(NULL), + stdStream(istr), + isUserStream_(isUserStream) +{ + // ensure binary read on windows +#if _MSC_VER >= 1300 + // TODO +#endif } -Buffer::Buffer(const unsigned char* buf, int len) { - this->buf = new unsigned char[len]; - memcpy(this->buf, buf, len*sizeof(unsigned char)); - bufStart = 0; - bufCapacity = bufLen = len; - fileLen = len; - bufPos = 0; - stream = NULL; +Buffer::Buffer(std::string& str) +: + buf(NULL), + bufCapacity(0), + bufLen(0), + bufPos(0), + bufStart(0), + fileLen(0), + cStream(NULL), + stdStream(new std::istringstream(str)), + isUserStream_(false) +{} + + +Buffer::Buffer(const unsigned char* chars, int len) +: + buf(new unsigned char[len]), + bufCapacity(len), + bufLen(len), + bufPos(0), + bufStart(0), + fileLen(len), + cStream(NULL), + stdStream(NULL), + isUserStream_(false) +{ + memcpy(this->buf, chars, len*sizeof(char)); } -Buffer::Buffer(const char* buf, int len) { - this->buf = new unsigned char[len]; - memcpy(this->buf, buf, len*sizeof(unsigned char)); - bufStart = 0; - bufCapacity = bufLen = len; - fileLen = len; - bufPos = 0; - stream = NULL; +Buffer::Buffer(const char* chars, int len) +: + buf(new unsigned char[len]), + bufCapacity(len), + bufLen(len), + bufPos(0), + bufStart(0), + fileLen(len), + cStream(NULL), + stdStream(NULL), + isUserStream_(false) +{ + memcpy(this->buf, chars, len*sizeof(char)); } Buffer::~Buffer() { Close(); - if (buf != NULL) { + if (buf) { delete [] buf; buf = NULL; } @@ -326,20 +373,36 @@ Buffer::~Buffer() { void Buffer::Close() { - if (!isUserStream && stream != NULL) { - fclose(stream); - stream = NULL; + if (!isUserStream_) { + if (cStream) { + fclose(cStream); + cStream = NULL; + } + else if (stdStream) { + delete stdStream; + stdStream = 0; + } } } int Buffer::Read() { + if (stdStream) + { + int ch = stdStream->get(); + if (stdStream->eof()) + { + return EoF; + } + return ch; + } + if (bufPos < bufLen) { return buf[bufPos++]; } else if (GetPos() < fileLen) { SetPos(GetPos()); // shift buffer start to Pos return buf[bufPos++]; - } else if ((stream != NULL) && !CanSeek() && (ReadNextStreamChunk() > 0)) { + } else if (cStream && !CanSeek() && (ReadNextStreamChunk() > 0)) { return buf[bufPos++]; } else { return EoF; @@ -347,91 +410,6 @@ int Buffer::Read() { } -int Buffer::Peek() { - int curPos = GetPos(); - int ch = Read(); - SetPos(curPos); - return ch; -} - - -wchar_t* Buffer::GetString(int beg, int end) { - int len = 0; - wchar_t *buf = new wchar_t[end - beg]; - int oldPos = GetPos(); - SetPos(beg); - while (GetPos() < end) buf[len++] = (wchar_t) Read(); - SetPos(oldPos); - wchar_t *res = coco_string_create(buf, 0, len); - coco_string_delete(buf); - return res; -} - - -int Buffer::GetPos() { - return bufPos + bufStart; -} - - -void Buffer::SetPos(int value) { - if ((value >= fileLen) && (stream != NULL) && !CanSeek()) { - // Wanted position is after buffer and the stream - // is not seek-able e.g. network or console, - // thus we have to read the stream manually till - // the wanted position is in sight. - while ((value >= fileLen) && (ReadNextStreamChunk() > 0)) - {} - } - - if ((value < 0) || (value > fileLen)) { - wprintf(L"--- buffer out of bounds access, position: %d\n", value); - ::exit(1); - } - - if ((value >= bufStart) && (value < (bufStart + bufLen))) { // already in buffer - bufPos = value - bufStart; - } else if (stream != NULL) { // must be swapped in - fseek(stream, value, SEEK_SET); - bufLen = fread(buf, sizeof(unsigned char), bufCapacity, stream); - bufStart = value; bufPos = 0; - } else { - bufPos = fileLen - bufStart; // make Pos return fileLen - } -} - - -// Read the next chunk of bytes from the stream, increases the buffer -// if needed and updates the fields fileLen and bufLen. -// Returns the number of bytes read. -int Buffer::ReadNextStreamChunk() { - int freeLen = bufCapacity - bufLen; - if (freeLen == 0) { - // in the case of a growing input stream - // we can neither seek in the stream, nor can we - // foresee the maximum length, thus we must adapt - // the buffer size on demand. - bufCapacity = bufLen * 2; - unsigned char *newBuf = new unsigned char[bufCapacity]; - memcpy(newBuf, buf, bufLen*sizeof(unsigned char)); - delete [] buf; - buf = newBuf; - freeLen = bufLen; - } - int read = fread(buf + bufLen, sizeof(unsigned char), freeLen, stream); - if (read > 0) { - fileLen = bufLen = (bufLen + read); - return read; - } - // end of stream reached - return 0; -} - - -bool Buffer::CanSeek() { - return (stream != NULL) && (ftell(stream) != -1); -} - - int UTF8Buffer::Read() { int ch; do { @@ -464,42 +442,143 @@ int UTF8Buffer::Read() { } -Scanner::Scanner(const unsigned char* buf, int len) { - buffer = new Buffer(buf, len); +int Buffer::Peek() { + int curPos = GetPos(); + int ch = Read(); + SetPos(curPos); + return ch; +} + + +int Buffer::GetPos() const { + if (stdStream) + { + return stdStream->tellg(); + } + + return bufPos + bufStart; +} + + +void Buffer::SetPos(int value) { + if (stdStream) + { + stdStream->seekg(value, std::ios::beg); + return; + } + + if ((value >= fileLen) && cStream && !CanSeek()) { + // Wanted position is after buffer and the stream + // is not seek-able e.g. network or console, + // thus we have to read the stream manually till + // the wanted position is in sight. + while ((value >= fileLen) && (ReadNextStreamChunk() > 0)) + {} + } + + if ((value < 0) || (value > fileLen)) { + wprintf(L"--- buffer out of bounds access, position: %d\n", value); + ::exit(1); + } + + if ((value >= bufStart) && (value < (bufStart + bufLen))) { // already in buffer + bufPos = value - bufStart; + } else if (cStream) { // must be swapped in + fseek(cStream, value, SEEK_SET); + bufLen = fread(buf, sizeof(char), bufCapacity, cStream); + bufStart = value; bufPos = 0; + } else { + bufPos = fileLen - bufStart; // make Pos return fileLen + } +} + + +// Read the next chunk of bytes from the stream, increases the buffer +// if needed and updates the fields fileLen and bufLen. +// Returns the number of bytes read. +int Buffer::ReadNextStreamChunk() { + int freeLen = bufCapacity - bufLen; + if (freeLen == 0) { + // in the case of a growing input stream + // we can neither seek in the stream, nor can we + // foresee the maximum length, thus we must adapt + // the buffer size on demand. + bufCapacity = bufLen * 2; + unsigned char *newBuf = new unsigned char[bufCapacity]; + memcpy(newBuf, buf, bufLen*sizeof(char)); + delete [] buf; + buf = newBuf; + freeLen = bufLen; + } + int read = fread(buf + bufLen, sizeof(char), freeLen, cStream); + if (read > 0) { + fileLen = bufLen = (bufLen + read); + return read; + } + // end of stream reached + return 0; +} + + +bool Buffer::CanSeek() const { + return cStream && (ftell(cStream) != -1); +} + + +// ---------------------------------------------------------------------------- +// Scanner Implementation +// ---------------------------------------------------------------------------- + +Scanner::Scanner(FILE* istr) +: + buffer(new Buffer(istr, true)) +{ Init(); } -Scanner::Scanner(const char* buf, int len) { - buffer = new Buffer(buf, len); +Scanner::Scanner(std::istream& istr) +: + buffer(new Buffer(&istr, true)) +{ Init(); } Scanner::Scanner(const wchar_t* fileName) { - FILE* stream; char *chFileName = coco_string_create_char(fileName); - if ((stream = fopen(chFileName, "rb")) == NULL) { + FILE* istr; + if ((istr = fopen(chFileName, "rb")) == NULL) { wprintf(L"--- Cannot open file %ls\n", fileName); ::exit(1); } coco_string_delete(chFileName); - buffer = new Buffer(stream, false); + buffer = new Buffer(istr, false); Init(); } -Scanner::Scanner(FILE* s) { - buffer = new Buffer(s, true); +Scanner::Scanner(const unsigned char* buf, int len) +: + buffer(new Buffer(buf, len)) +{ + Init(); +} + + +Scanner::Scanner(const char* buf, int len) +: + buffer(new Buffer(buf, len)) +{ Init(); } Scanner::~Scanner() { - char* cur = (char*) firstHeap; + char* cur = reinterpret_cast(firstHeap); - while (cur != NULL) { - cur = *(char**) (cur + HEAP_BLOCK_SIZE); + while (cur) { + cur = *(reinterpret_cast(cur + HEAP_BLOCK_SIZE)); free(firstHeap); firstHeap = cur; } @@ -509,14 +588,11 @@ Scanner::~Scanner() { void Scanner::Init() { - maxT = 13; - noSym = 13; - int i; - for (i = 65; i <= 90; ++i) start.set(i, 1); - for (i = 97; i <= 122; ++i) start.set(i, 1); - for (i = 36; i <= 36; ++i) start.set(i, 5); + for (int i = 65; i <= 90; ++i) start.set(i, 1); + for (int i = 97; i <= 122; ++i) start.set(i, 1); + for (int i = 36; i <= 36; ++i) start.set(i, 5); start.set(45, 20); - for (i = 48; i <= 57; ++i) start.set(i, 9); + for (int i = 48; i <= 57; ++i) start.set(i, 9); start.set(34, 2); start.set(46, 7); start.set(123, 14); @@ -526,7 +602,8 @@ void Scanner::Init() { start.set(47, 17); start.set(40, 18); start.set(41, 19); - start.set(Buffer::EoF, -1); + start.set(Buffer::EoF, -1); + tvalLength = 128; @@ -535,7 +612,9 @@ void Scanner::Init() { // HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = malloc(HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; - heapEnd = (void**) (((char*) heap) + HEAP_BLOCK_SIZE); + heapEnd = + reinterpret_cast + (reinterpret_cast(heap) + HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > HEAP_BLOCK_SIZE) { @@ -645,19 +724,25 @@ bool Scanner::Comment1() { void Scanner::CreateHeapBlock() { - void* newHeap; - char* cur = (char*) firstHeap; + char* cur = reinterpret_cast(firstHeap); - while (((char*) tokens < cur) || ((char*) tokens > (cur + HEAP_BLOCK_SIZE))) { - cur = *((char**) (cur + HEAP_BLOCK_SIZE)); + // release unused blocks + while + ( + (reinterpret_cast(tokens) < cur) + || (reinterpret_cast(tokens) > (cur + HEAP_BLOCK_SIZE)) + ) { + cur = *(reinterpret_cast(cur + HEAP_BLOCK_SIZE)); free(firstHeap); firstHeap = cur; } // HEAP_BLOCK_SIZE byte heap + pointer to next heap block - newHeap = malloc(HEAP_BLOCK_SIZE + sizeof(void*)); + void* newHeap = malloc(HEAP_BLOCK_SIZE + sizeof(void*)); *heapEnd = newHeap; - heapEnd = (void**) (((char*) newHeap) + HEAP_BLOCK_SIZE); + heapEnd = + reinterpret_cast + (reinterpret_cast(newHeap) + HEAP_BLOCK_SIZE); *heapEnd = 0; heap = newHeap; heapTop = heap; @@ -665,32 +750,51 @@ void Scanner::CreateHeapBlock() { Token* Scanner::CreateToken() { - Token *t; - if (((char*) heapTop + (int) sizeof(Token)) >= (char*) heapEnd) { + const int reqMem = sizeof(Token); + if + ( + (reinterpret_cast(heapTop) + reqMem) + >= reinterpret_cast(heapEnd) + ) { CreateHeapBlock(); } - t = (Token*) heapTop; - heapTop = (void*) ((char*) heapTop + sizeof(Token)); - t->val = NULL; - t->next = NULL; - return t; + // token 'occupies' heap starting at heapTop + Token* tok = reinterpret_cast(heapTop); + // increment past this part of the heap, which is now used + heapTop = + reinterpret_cast + (reinterpret_cast(heapTop) + reqMem); + tok->val = NULL; + tok->next = NULL; + return tok; } -void Scanner::AppendVal(Token *t) { - int reqMem = (tlen + 1) * sizeof(wchar_t); - if (((char*) heapTop + reqMem) >= (char*) heapEnd) { +void Scanner::AppendVal(Token* tok) { + const int reqMem = (tlen + 1) * sizeof(wchar_t); + if + ( + (reinterpret_cast(heapTop) + reqMem) + >= reinterpret_cast(heapEnd) + ) { if (reqMem > HEAP_BLOCK_SIZE) { wprintf(L"--- Too long token value\n"); ::exit(1); } CreateHeapBlock(); } - t->val = (wchar_t*) heapTop; - heapTop = (void*) ((char*) heapTop + reqMem); - wcsncpy(t->val, tval, tlen); - t->val[tlen] = L'\0'; + // add text value from heap + tok->val = reinterpret_cast(heapTop); + + // increment past this part of the heap, which is now used + heapTop = + reinterpret_cast + (reinterpret_cast(heapTop) + reqMem); + + // copy the currently parsed tval into the token + wcsncpy(tok->val, tval, tlen); + tok->val[tlen] = L'\0'; } @@ -818,8 +922,11 @@ void Scanner::ResetPeek() { } +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + } // namespace } // namespace } // namespace +// ************************************************************************* // diff --git a/applications/test/dictionary/calcEntry/calcEntryScanner.h b/applications/test/dictionary/calcEntry/calcEntryScanner.h index 8847c9fd2e..fdebddeebe 100644 --- a/applications/test/dictionary/calcEntry/calcEntryScanner.h +++ b/applications/test/dictionary/calcEntry/calcEntryScanner.h @@ -3,11 +3,14 @@ #ifndef COCO_calcEntrySCANNER_H__ #define COCO_calcEntrySCANNER_H__ -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include // io.h and fcntl are used to ensure binary read from streams on windows #if _MSC_VER >= 1300 @@ -24,10 +27,8 @@ #define coco_swprintf swprintf #endif + #define COCO_WCHAR_MAX 65535 -#define MIN_BUFFER_LENGTH 1024 -#define MAX_BUFFER_LENGTH (64*MIN_BUFFER_LENGTH) -#define HEAP_BLOCK_SIZE (64*1024) namespace Foam { @@ -48,12 +49,6 @@ wchar_t* coco_string_create(const wchar_t* str); //! Create a substring of str starting at index and length characters long wchar_t* coco_string_create(const wchar_t* str, int index, int length); -//! Create an uppercase string from str -wchar_t* coco_string_create_upper(const wchar_t* str); - -//! Create an uppercase substring from str starting at index and length characters long -wchar_t* coco_string_create_upper(const wchar_t* str, int index, int length); - //! Create a lowercase string from str wchar_t* coco_string_create_lower(const wchar_t* str); @@ -138,7 +133,6 @@ float coco_string_toFloat(const char* str); // * * * * * * * * * End of Wide Character String Routines * * * * * * * * * // - //! Scanner Token class Token { @@ -151,7 +145,7 @@ public: Token *next; //!< Peek tokens are kept in linked list Token(); //!< Construct null - ~Token(); //!< Destructor - cleanup allocated val + ~Token(); //!< Destructor - cleanup allocated val?? }; @@ -166,30 +160,48 @@ class Buffer { private: unsigned char *buf; //!< input buffer int bufCapacity; //!< capacity of buf - int bufStart; //!< position of first byte in buffer relative to input stream int bufLen; //!< length of buffer - int fileLen; //!< length of input stream (may change if the stream is no file) int bufPos; //!< current position in buffer - FILE* stream; //!< input stream (seekable) - bool isUserStream; //!< was the stream opened by the user? + int bufStart; //!< position of first byte in buffer relative to input stream + int fileLen; //!< length of input stream (may change if the stream is no file) + FILE* cStream; //!< input stdio stream (normally seekable) + std::istream* stdStream; //!< STL std stream (seekable) + bool isUserStream_; //!< was the stream opened by the user? int ReadNextStreamChunk(); - bool CanSeek(); //!< true if stream can be seeked otherwise false + bool CanSeek() const; //!< true if stream can be seeked otherwise false + +protected: + Buffer(Buffer*); //!< for the UTF8Buffer public: static const int EoF = COCO_WCHAR_MAX + 1; - Buffer(FILE*, bool isUserStream); - Buffer(const unsigned char* buf, int len); - Buffer(const char* buf, int len); - Buffer(Buffer*); + //! Attach buffer to a stdio stream. + //! User streams are not closed in the destructor + Buffer(FILE*, bool isUserStream = true); + + //! Attach buffer to an STL std stream + //! User streams are not closed in the destructor + explicit Buffer(std::istream*, bool isUserStream = true); + + //! Copy buffer contents from constant string + //! Handled internally as an istringstream + explicit Buffer(std::string&); + + //! Copy buffer contents from constant character string + Buffer(const unsigned char* chars, int len); + //! Copy buffer contents from constant character string + Buffer(const char* chars, int len); + + //! Close stream (but not user streams) and free buf (if any) virtual ~Buffer(); - virtual void Close(); - virtual int Read(); - virtual int Peek(); - virtual wchar_t* GetString(int beg, int end); - virtual int GetPos(); + virtual void Close(); //!< Close stream (but not user streams) + virtual int Read(); //!< Get character from stream or buffer + virtual int Peek(); //!< Peek character from stream or buffer + + virtual int GetPos() const; virtual void SetPos(int value); }; @@ -205,31 +217,31 @@ public: //------------------------------------------------------------------------------ // StartStates //------------------------------------------------------------------------------ -//! maps characters to start states of tokens +//! maps characters (integers) to start states of tokens class StartStates { private: class Elem { public: int key, val; Elem *next; - Elem(int key, int val) { - this->key = key; - this->val = val; - next = NULL; - } + Elem(int k, int v) : + key(k), val(v), next(0) + {} }; Elem **tab; public: - StartStates() { - tab = new Elem*[128]; + StartStates() : + tab(new Elem*[128]) + { memset(tab, 0, 128 * sizeof(Elem*)); } + virtual ~StartStates() { for (int i = 0; i < 128; ++i) { Elem *e = tab[i]; - while (e != NULL) { + while (e) { Elem *next = e->next; delete e; e = next; @@ -240,15 +252,15 @@ public: void set(int key, int val) { Elem *e = new Elem(key, val); - int k = ((unsigned int) key) % 128; + int k = unsigned(key) % 128; e->next = tab[k]; tab[k] = e; } int state(int key) { - Elem *e = tab[((unsigned int) key) % 128]; - while (e != NULL && e->key != key) e = e->next; - return e == NULL ? 0 : e->val; + Elem *e = tab[unsigned(key) % 128]; + while (e && e->key != key) e = e->next; + return e ? e->val : 0; } }; @@ -264,11 +276,9 @@ private: wchar_t *key; int val; Elem *next; - Elem(const wchar_t *key, int val) { - this->key = coco_string_create(key); - this->val = val; - next = NULL; - } + Elem(const wchar_t *k, int v) : + key(coco_string_create(k)), val(v), next(0) + {} virtual ~Elem() { coco_string_delete(key); } @@ -277,14 +287,16 @@ private: Elem **tab; public: - KeywordMap() { - tab = new Elem*[128]; + KeywordMap() : + tab(new Elem*[128]) + { memset(tab, 0, 128 * sizeof(Elem*)); } + virtual ~KeywordMap() { for (int i = 0; i < 128; ++i) { Elem *e = tab[i]; - while (e != NULL) { + while (e) { Elem *next = e->next; delete e; e = next; @@ -295,14 +307,15 @@ public: void set(const wchar_t *key, int val) { Elem *e = new Elem(key, val); - int k = coco_string_hash(key) % 128; - e->next = tab[k]; tab[k] = e; + const int k = coco_string_hash(key) % 128; + e->next = tab[k]; + tab[k] = e; } int get(const wchar_t *key, int defaultVal) { Elem *e = tab[coco_string_hash(key) % 128]; - while (e != NULL && !coco_string_equal(e->key, key)) e = e->next; - return e == NULL ? defaultVal : e->val; + while (e && !coco_string_equal(e->key, key)) e = e->next; + return e ? e->val : defaultVal; } }; @@ -310,24 +323,24 @@ public: //! A Coco/R Scanner class Scanner { private: - static const unsigned char EOL = '\n'; // end-of-line character - static const int eofSym = 0; // end-of-file token id + static const int maxT = 13; + static const int noSym = 13; - void *firstHeap; - void *heap; - void *heapTop; - void **heapEnd; + static const int eofSym = 0; //!< end-of-file token id + static const char EOL = '\n'; //!< end-of-line character - int noSym; //!< noSym gets highest number, set in Parser - int maxT; - int charSetSize; //!< unused? - StartStates start; - KeywordMap keywords; + void *firstHeap; //!< the start of the heap management + void *heap; //!< the currently active block + void *heapTop; //!< the top of the heap + void **heapEnd; //!< the end of the last heap block + + StartStates start; //!< A map of start states for particular characters + KeywordMap keywords; //!< A hash of keyword literals to token kind Token *t; //!< current token wchar_t *tval; //!< text of current token - int tvalLength; //!< length of text of current token - int tlen; //!< length of current token + int tvalLength; //!< maximum capacity (length) for tval + int tlen; //!< length of tval Token *tokens; //!< list of tokens already peeked (first token is a dummy) Token *pt; //!< current peek token @@ -337,36 +350,42 @@ private: int pos; //!< byte position of current character int line; //!< line number of current character int col; //!< column number of current character - int oldEols; //!< EOLs that appeared in a comment; + int oldEols; //!< the number of EOLs that appeared in a comment - void CreateHeapBlock(); - Token* CreateToken(); - void AppendVal(Token*); + void CreateHeapBlock(); //!< add a heap block, freeing unused ones + Token* CreateToken(); //!< fit token on the heap + void AppendVal(Token* tok); //!< adjust tok->val to point to the heap and copy tval into it - void Init(); - void NextCh(); - void AddCh(); + void Init(); //!< complete the initialization for the constructors + void NextCh(); //!< get the next input character into ch + void AddCh(); //!< append the character ch to tval bool Comment0(); bool Comment1(); - Token* NextToken(); + Token* NextToken(); //!< get the next token public: - //! scanner buffer + //! The scanner buffer Buffer *buffer; - //! Attach scanner to an existing character buffer - Scanner(const unsigned char* buf, int len); - //! Attach scanner to an existing character buffer - Scanner(const char* buf, int len); - //! Open a file for reading and attach scanner - Scanner(const wchar_t* fileName); //! Using an existing open file handle for the scanner - Scanner(FILE* s); - ~Scanner(); - Token* Scan(); - Token* Peek(); - void ResetPeek(); + Scanner(FILE*); + + //! Using an existing open STL std stream + explicit Scanner(std::istream&); + + //! Open a file for reading and attach scanner + explicit Scanner(const wchar_t* fileName); + + //! Attach scanner to an existing character buffer + Scanner(const unsigned char* chars, int len); + //! Attach scanner to an existing character buffer + Scanner(const char* chars, int len); + + ~Scanner(); //!< free heap and allocated memory + Token* Scan(); //!< get the next token (possibly a token already seen during peeking) + Token* Peek(); //!< peek for the next token, ignore pragmas + void ResetPeek(); //!< ensure that peeking starts at the current scan position }; // end Scanner diff --git a/applications/test/dictionary/testDict b/applications/test/dictionary/testDict index 42a473a52b..8c9e672e7b 100644 --- a/applications/test/dictionary/testDict +++ b/applications/test/dictionary/testDict @@ -67,7 +67,6 @@ boundaryField // error #remove self; x 5; y 6; - another #calc{x $x; y $y;}; } // this should have no effect diff --git a/applications/test/dictionary/testDictCalc b/applications/test/dictionary/testDictCalc index cfce0e0b52..134a8daaa8 100644 --- a/applications/test/dictionary/testDictCalc +++ b/applications/test/dictionary/testDictCalc @@ -26,7 +26,7 @@ y 20; // + 1 /*100 */ 10 // }; -p #test{ 1 + 2 + 10 * 15 + $x - $y }; +p #calc{ 1 + 2 + 10 * 15 + $x - $y }; foo 30;