Writing a small parser / interpreter (Part 1: Scanner)

[Test]
public void CanRecognizeTokens()
{
  const string input = "REPEAT 4 [ FORWARD 100 LEFT 90 ] ";
  var expectedTokens = new[] { Token.REPEAT, Token.NUMBER, Token.LBRACKET, 
Token.FORWARD, Token.NUMBER, Token.LEFT, 
Token.NUMBER, Token.RBRACKET, Token.EOF };
  var scanner = new LogoScanner(input);
  foreach (var expectedToken in expectedTokens)
  {
    var token = scanner.Scan();
    Assert.AreEqual(expectedToken, token);
  }
}

And the accompanying code could look like:

public enum Token
{
  [TokenAsText("REPEAT")]
  REPEAT,
  [TokenAsText("FORWARD")]
  FORWARD,
  [TokenAsText("BACK")]
  BACK,
  [TokenAsText("LEFT")]
  LEFT,
  [TokenAsText("RIGHT")]
  RIGHT,
  [TokenAsText("NUMBER")]
  NUMBER,
  [TokenAsText("[")]
  LBRACKET,
  [TokenAsText("]")]
  RBRACKET,
  EOF
}

public class LogoScanner
{
  readonly Token[] reserved = new[] { Token.FORWARD, Token.BACK, Token.RIGHT, Token.LEFT, Token.REPEAT };

  private readonly string rawContents;
  private string scanBuffer;
  private int idx;
  private char ch;

  public LogoScanner(string input)
  {
    rawContents = input;
  }

  public Token Scan()
  {
    while (idx < rawContents.Length)
    {
      ch = rawContents[idx];
      if (ch == '[')
      {
        idx++;
        return Token.LBRACKET;
      }
      else if (ch == ']')
      {
        idx++;
        return Token.RBRACKET;
      }
      else if (char.IsDigit(ch))
      {
        scanBuffer = ch.ToString();
        idx++;
        while (idx < rawContents.Length)
        {
          ch = rawContents[idx];
          if (char.IsDigit(ch))
          {
            scanBuffer += ch;
            idx++;
          }
          else break;
        }
        return Token.NUMBER;
      }
      else if (char.IsLetter(ch))
      {
        scanBuffer = ch.ToString();
        idx++;
        while (idx < rawContents.Length)
        {
          ch = rawContents[idx];
          if (char.IsLetter(ch))
          {
            scanBuffer += ch;
            idx++;
          }
          else break;
        }
        Token lookup;
        if (LookupReserved(scanBuffer, out lookup))
        {
          return lookup;
        }
        LexicalError();
      }
      else if (char.IsWhiteSpace(ch))
      {
        idx++;
      }
      else
      {
        LexicalError();
      }
    }
    return Token.EOF;
  }

  private void LexicalError()
  {
    throw new LogoScannerException(String.Format("Lexical error at '{0}' ('{1}')", ch, scanBuffer));
  }

  private bool LookupReserved(string s, out Token lookup)
  {
    lookup = TokenHelper.LookupAsText(s);
    return reserved.Contains(lookup);
  }
}

Next we’ll look at writing the parser, utilizing the scanners token recognizing capabilities. Please, see the articles below.