//
// CPT operator / SondeerGame 
// This source file is (c) by Deltares.
// - October 2014
//
//=============================================================================
//  SOURCE CODE FROM: http://www.unrealwiki.com/wiki/Tokenizer
// ------------------------------------------------------------
//  Created by Rui Guimaraes (using El Muerte's source)
//  © 2006, Geodelft
//
// Date       Id   Modification
// 2007-02-11 gui  File created
//=============================================================================
class Tokenizer extends Object;

const NEWLINE = 10;

var private array<string> buffer;   // the input buffer
var private byte c;                 // holds the current char
var private int linenr;             // current line in the input buffer
var private int pos;                // position on the current line

var int verbose;                    // for debugging

enum tokenType
{
  TT_None,
  TT_Literal,
  TT_Identifier,
  TT_Integer,
  TT_Float,
  TT_String,
  TT_Operator,
  TT_EOF,
};


var private tokenType curTokenType; // holds the current token
var private string curTokenString;  // holds the current string representation

/**
  Create a tokenizer
*/
function Create(array<string> buf)
{
  buffer.length = 0;
  buffer = buf;
  linenr = 0;
  pos = 0;
  c = 0;
}

/**
  returns the string representation of the current token
*/
function string tokenString()
{
  return curTokenString;
}

/**
  returns the type of the current token
*/
function tokenType currentToken()
{
  return curTokenType;
}

/**
  retreives the next token
*/
function tokenType nextToken()
{
  return _nextToken();
}

/* Private functions */

private function tokenType _nextToken()
{
  local int tokenPos, endPos;
  skipBlanks();
  if (curTokenType == TT_EOF) return curTokenType;
  tokenPos = pos;
  // identifier: [A-Za-z]([A-Za-z0-9_])*
  if (((c >= 65) && (c <= 90)) || ((c >= 97) && (c <= 122)) || (c == 95))
  {
    pos++;
    c = _c();
    while (((c >= 65) && (c <= 90)) || ((c >= 97) && (c <= 122)) || (c == 95) || ((c >= 48) && (c <= 57)))
    {
      pos++;
      c = _c();
    }
    endPos = pos;
    curTokenType = TT_Identifier;
  }
  // number: [0-9]+(\.([0-9])+)?
  else if ((c >= 48) && (c <= 57))
  {
    pos++;
    c = _c();
    while ((c >= 48) && (c <= 57))
    {
      pos++;
      c = _c();
    }
    if (c == 46) // .
    {
      pos++;
      c = _c();
      while ((c >= 48) && (c <= 57))
      {
        pos++;
        c = _c();
      }
      endPos = pos;
      curTokenType = TT_Float;
    }
    else {
      endPos = pos;
      curTokenType = TT_Integer;
    }
  }
  // string: "[^"]*"
  else if (c == 34)
  {
    pos++;
    c = _c();
    while (true)
    {
      if (c == 34) break;
      if (c == 92) // escape char skip one char
      {
        pos++;
      }
      if (c == NEWLINE)
      {
        Warn("Unterminated string @"@linenr$","$pos);
        assert(false);
      }
      pos++;
      c = _c();
    }
    tokenPos++;
    endPos = pos;
    pos++;
    curTokenType = TT_String;
  }
  // operator: [+-*/=><!]+
  // literal
  else if ((c == 33) || (c == 42) || (c == 43) || (c == 45) || (c == 47) || (c == 60) || (c == 61) || (c == 62) || (c == 61))
  {
    pos++;
    c = _c();
    while ((c == 33) || (c == 42) || (c == 43) || (c == 45) || (c == 47) || (c == 60) || (c == 61) || (c == 62) || (c == 61))
    {
      pos++;
      c = _c();
    }
    endPos = pos;
    curTokenType = TT_Operator;
  }
  else {
    pos++;
    endPos = pos;
    curTokenType = TT_Literal;
  }
  // make up result
  if (linenr >= buffer.length) // EOF break
  {
    curTokenType = TT_EOF;
    curTokenString = "";
  }
  else {
    curTokenString = Mid(buffer[linenr], tokenPos, endPos-tokenPos);
  }
  if (verbose > 0) log(curTokenType@curTokenString, 'Tokenizer');
  return curTokenType;
}

/**
  Skip all characters with ascii value < 33 (32 is space)
*/
private function skipBlanks()
{
  c = _c();
  while (c < 33)
  {
    if (c == NEWLINE)
    {
      linenr++;
      pos = 0;
      if (linenr >= buffer.length) // EOF break
      {
        curTokenType = TT_EOF;
        curTokenString = "";
        return;
      }
    }
    else pos++;
    c = _c();
  }
}

/**
  returns the current char
*/
private function byte _c(optional int displacement)
{
  local string t;
  t =  Mid(buffer[linenr], pos+displacement, 1);
  if (t == "") return NEWLINE; // empty string is a newline
  return Asc(t);
}

defaultproperties
{
  verbose=0
}