Update pylex tests

src/commands.ts

@@ -124,8 +124,8 @@ function createContextString(context: pl.LexNode[], line: number): string {
         continue;
       }
 
-      if (node.token!.type != pl.PylexSymbol.EMPTY &&
+      if (node.token!.type !== pl.PylexSymbol.EMPTY &&
-        node.token!.type != pl.PylexSymbol.INDENT) {
+        node.token!.type !== pl.PylexSymbol.INDENT) {
         contextString += ' inside ' + node.token!.type.toString();
         if (node.token!.attr) {
           contextString += ' ' + node.token!.attr.toString();

src/pylex/node.ts

@@ -48,8 +48,9 @@ export default class LexNode extends vscode.TreeItem {
    * Adopt child nodes.
    *
    * @param `child` Array of nodes to adopt.
+   * @returns an updated version of itself
    */
-  adopt(children: LexNode[]): void {
+  adopt(children: LexNode[]): LexNode {
     let parentedChildren = children.map(c => new LexNode(
       c.label,
       c.collapsibleState,
@@ -66,6 +67,7 @@ export default class LexNode extends vscode.TreeItem {
       // No....
       this._children = parentedChildren;
     }
+    return this;
   }
 
   /**

src/pylex/parser.ts

@@ -120,7 +120,7 @@ export default class Parser {
    */
   context(lineNumber: number, root?: LexNode): LexNode[] {
     if (!root) {
-      root = this.root
+      root = this.root;
     }
 
     // is this the target?

src/test/suites/pylex/lexer.test.ts

@@ -17,13 +17,18 @@ suite('Lexer Test Suite', () => {
   });
 
   test('Undefined', () => {
-    let l: Lexer = new Lexer('');
+    let l: Lexer = new Lexer(undefined);
     assert.deepStrictEqual(l.currToken(), EOFTOKEN);
   });
 
   test('Whitespace', () => {
     let l: Lexer = new Lexer('  \t\t'.repeat(4).repeat(4));
-    assert.deepStrictEqual(l.currToken(), EOFTOKEN);
+    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EMPTY, 0, 999999));
+  });
+
+  test('Comment', () => {
+    let l: Lexer = new Lexer('# ur mom eats toes');
+    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EMPTY, 0, 999999));
   });
 
   test('Non-Whitespace with no construct', () => {
@@ -120,98 +125,6 @@ suite('Lexer Test Suite', () => {
     assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'is_old()'));
   });
 
-  test('next() ignores empty lines', () => {
-    let lines: string[] = [
-      'if wurst_available():',
-      '',
-      '    eat_wurst()'
-    ];
-    let l: Lexer = new Lexer(lines.join('\n'));
-
-    l.next();
-
-    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 2, 1));
-  });
-
-  test('retract() ignores empty lines', () => {
-    let lines: string[] = [
-      'if wurst_available():',
-      '',
-      '    eat_wurst()'
-    ];
-    let l: Lexer = new Lexer(lines.join('\n'));
-
-    l.next();
-
-    l.retract();
-
-    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'wurst_available()'));
-  });
-
-  test('next() ignores whitespace lines', () => {
-    let lines: string[] = [
-      'if wurst_available():',
-      ' \t \t   ',
-      '    eat_wurst()'
-    ];
-    let l: Lexer = new Lexer(lines.join('\n'));
-
-    l.next();
-
-    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 2, 1));
-  });
-
-  test('retract() ignores whitespace lines', () => {
-    let lines: string[] = [
-      'if wurst_available():',
-      ' \t  \t   ',
-      '    eat_wurst()'
-    ];
-    let l: Lexer = new Lexer(lines.join('\n'));
-
-    // Advance to end of input
-    // Eliminates dependence on next()
-    // skipping whitespace
-    do {} while (l.next() !== EOFTOKEN);
-
-    l.retract(); // retract past EOFTOKEn
-    l.retract();
-
-    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'wurst_available()'));
-  });
-
-  test('next() ignores comment lines', () => {
-    let lines: string[] = [
-      'if wurst_available():',
-      ' \t # I hate testing \t',
-      '    eat_wurst()'
-    ];
-    let l: Lexer = new Lexer(lines.join('\n'));
-
-    l.next();
-
-    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 2, 1));
-  });
-
-  test('retract() ignores comment lines', () => {
-    let lines: string[] = [
-      'if wurst_available():',
-      ' \t # \t',
-      '    eat_wurst()'
-    ];
-    let l: Lexer = new Lexer(lines.join('\n'));
-
-    // Advance to end of input
-    // Eliminates dependence on next()
-    // skipping comment
-    do {} while (l.next() !== EOFTOKEN);
-
-    l.retract(); // retract past EOFTOKEn
-    l.retract();
-
-    assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'wurst_available()'));
-  });
-
   test('next() out of range', () => {
     let l: Lexer = new Lexer('foo = zaboomafoo');
     l.next();

src/test/suites/pylex/parser.test.ts

@@ -1,51 +1,61 @@
 import * as assert from 'assert';
 import * as vscode from 'vscode';
 import { after } from 'mocha';
-import { deparent, root } from '../../util';
 
 import Parser from '../../../pylex/parser';
 import LexNode from '../../../pylex/node';
 import LineToken from '../../../pylex/token';
 import { Symbol } from '../../../pylex/token';
 
+import { root,indent,empty } from '../../util';
+
+/**
+ * Test Descriptor
+*/
 type ParserTest = {
-  name: string,
+  name: string,     // short name for the test
-  input: string[],
+  input: string[],  // input lines for the test
-  output: LexNode,
+  output: LexNode   // expected output. outputs are compared for token equality *only*
 };
 
 const tests: ParserTest[] = [
+  { name: 'No Input', input: [], output: root(null) },
+  { name: 'Single Empty Line', input: [''], output: root(null) },
   {
-    name: 'No Input',
+    name: 'Single Whitespace Only Line',
-    input: [ ],
+    input: ['                 '],
-    output: root(null),
-  },
-
-  {
-    name: 'Single line without construct',
-    input: [ 'foo = "Yellow M&Ms make me angry >:(' ],
-    output: root(null),
-  },
-
-  {
-    name: 'Single line with construct',
-    input: [ 'for x of y:' ],
     output: root([
-      new LexNode(
+      empty(0)
-        'for x of y',
+    ])
-         vscode.TreeItemCollapsibleState.None,
+  },
-         new LineToken(Symbol.FOR, 0, 0, 'x of y')
+  {
-      )
+    name: 'Single Comment Only Line',
-    ]),
+    input: ['# ur mom likes peas'],
+    output: root([
+      empty(0)
+    ])
+  },
+  {
+    name: 'Single Non-Control Line',
+    input: ['my_age = 42'],
+    output: root([
+      indent(0, 0)
+    ])
+  },
+  {
+    name: 'Single Control Line',
+    input: ['while True:'],
+    output: root([
+      new LexNode('', 0, new LineToken(Symbol.WHILE, 0, 0, 'True'))
+    ])
   },
-
   {
     name: 'Sequential lines, without construct',
     input: [
       'bar = "Blue M&Ms make me happy <:)"',
       'reba = "A hard working gal"'
     ],
-    output: root(null),
+    output: root([indent(0,0), indent(1,0)]),
   },
 
   {
@@ -58,7 +68,9 @@ const tests: ParserTest[] = [
     output: root([
       new LexNode('if radioshack',
         vscode.TreeItemCollapsibleState.None,
-        new LineToken(Symbol.IF, 0, 0, 'radioshack'))
+        new LineToken(Symbol.IF, 0, 0, 'radioshack'),
+        [indent(1, 1)]),
+      indent(2, 0)
     ])
   },
 
@@ -70,9 +82,11 @@ const tests: ParserTest[] = [
       '    print radioshack.hours'
     ],
     output: root([
+      indent(0, 0),
       new LexNode('if radioshack',
         vscode.TreeItemCollapsibleState.None,
-        new LineToken(Symbol.IF, 1, 0, 'radioshack'))
+        new LineToken(Symbol.IF, 1, 0, 'radioshack'),
+        [indent(2, 1)])
     ])
   },
 
@@ -89,13 +103,13 @@ const tests: ParserTest[] = [
     output: root([
       new LexNode('if yummy',
         vscode.TreeItemCollapsibleState.None,
-        new LineToken(Symbol.IF, 0, 0, 'yummy')),
+        new LineToken(Symbol.IF, 0, 0, 'yummy'), [indent(1, 1)]),
       new LexNode('elif just_ok',
         vscode.TreeItemCollapsibleState.None,
-        new LineToken(Symbol.ELIF, 2, 0, 'just_ok')),
+        new LineToken(Symbol.ELIF, 2, 0, 'just_ok'), [indent(3, 1)]),
       new LexNode('else',
         vscode.TreeItemCollapsibleState.None,
-        new LineToken(Symbol.ELSE, 4, 0)),
+        new LineToken(Symbol.ELSE, 4, 0), [indent(5,1)]),
     ])
   },
 
@@ -113,8 +127,9 @@ const tests: ParserTest[] = [
         [
           new LexNode('if in_my_tummy',
             vscode.TreeItemCollapsibleState.None,
-            new LineToken(Symbol.IF, 1, 1, 'in_my_tummy'))
+            new LineToken(Symbol.IF, 1, 1, 'in_my_tummy'),
-        ]
+            [indent(2, 2)])
+        ],
       )
     ])
   },
@@ -135,12 +150,14 @@ const tests: ParserTest[] = [
         [
           new LexNode('if in_my_tummy',
             vscode.TreeItemCollapsibleState.None,
-            new LineToken(Symbol.IF, 1, 1, 'in_my_tummy'))
+            new LineToken(Symbol.IF, 1, 1, 'in_my_tummy'),
+            [indent(2, 2)])
         ]
       ),
-        new LexNode('else',
+      new LexNode('else',
         vscode.TreeItemCollapsibleState.None,
         new LineToken(Symbol.ELSE, 3, 0),
+        [indent(4, 1)]
       )
     ])
   },
@@ -166,7 +183,8 @@ const tests: ParserTest[] = [
             [
               new LexNode('if looks_like_a_mummy',
                 vscode.TreeItemCollapsibleState.None,
-                new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy'))
+                new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy'),
+                [indent(3, 3)])
             ]
           )
         ]
@@ -174,12 +192,13 @@ const tests: ParserTest[] = [
         new LexNode('else',
         vscode.TreeItemCollapsibleState.None,
         new LineToken(Symbol.ELSE, 4, 0),
+        [indent(5, 1)]
       )
     ])
   },
 
   {
-    name: 'Doubly Nested Block, with multiple indent resets',
+    name: 'Doubly Nested Block, with multiple indent resets > 1',
     input: [
       'if yummy:',
       '    if in_my_tummy:',
@@ -188,7 +207,7 @@ const tests: ParserTest[] = [
       '        else:',
       '            print("eek! a zombie!)',
       '    elif in_my_mouth:',
-      '        print("ill be in my tummy soon!"',
+      '        print("itll be in my tummy soon!"',
       'else:',
       '    print("Food is food...")'
     ],
@@ -203,25 +222,70 @@ const tests: ParserTest[] = [
             [
               new LexNode('if looks_like_a_mummy',
                 vscode.TreeItemCollapsibleState.None,
-                new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy')),
+                new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy'),
+                [indent(3, 3)]),
               new LexNode('else',
                 vscode.TreeItemCollapsibleState.None,
-                new LineToken(Symbol.ELSE, 4, 2))
+                new LineToken(Symbol.ELSE, 4, 2),
+                [indent(5, 3)])
             ]
           ),
           new LexNode('elif in_my_mouth',
             vscode.TreeItemCollapsibleState.None,
-            new LineToken(Symbol.ELIF, 6, 1, 'in_my_mouth'))
+            new LineToken(Symbol.ELIF, 6, 1, 'in_my_mouth'),
+            [indent(7, 2)]
+          )
         ]
       ),
         new LexNode('else',
         vscode.TreeItemCollapsibleState.None,
-        new LineToken(Symbol.ELSE, 8, 0)
+        new LineToken(Symbol.ELSE, 8, 0),
+        [indent(9, 1)]
       )
     ])
+  },
+  {
+    name: 'Multiline Block',
+    input: [
+      'if yummy:',
+      '    print("you have a spot on your tummy"',
+      '    print("eek! a zombie!)',
+      '    print("itll be in my tummy soon!"',
+      'else:',
+      '    print("Food is food...")'
+    ],
+    output: root([
+      new LexNode('if yummy', 0, new LineToken(Symbol.IF, 0, 0, 'yummy'),
+        [
+          indent(1, 1),
+          indent(2, 1),
+          indent(3, 1),
+        ]
+      ),
+      new LexNode('else', 0, new LineToken(Symbol.ELSE, 4, 0), [indent(5, 1)])
+    ])
   }
 ];
 
+/* Checks for strict equality between the tokens of a lex node tree */
+const checkEq = (reference: LexNode, subject: LexNode) => {
+  if (!reference.children()) {
+    // subject should also have no children
+    assert.deepStrictEqual(subject.children(), null);
+    return;
+  }
+
+  assert.notStrictEqual(subject.children(), null);
+  assert.deepStrictEqual(reference.children()!.length, subject.children()!.length);
+  for (let i = 0; i < subject.children()!.length; i++) {
+    // compare top level nodes
+    assert.deepStrictEqual(reference.children()![i].token, subject.children()![i].token);
+
+    // compare all children
+    checkEq(reference.children()![i], subject.children()![i]);
+  }
+};
+
 suite('Parser Test Suite', () => {
   after(() => {
     vscode.window.showInformationMessage('All tests passed!');
@@ -230,10 +294,8 @@ suite('Parser Test Suite', () => {
   for (var t of tests) {
     let currTest = t; // without this, all test calls get the last test
     test(currTest.name, () => {
-      let result: LexNode = deparent(new Parser(currTest.input.join('\n')).parse());
+      let result = new Parser(currTest.input.join('\n')).parse();
-      process.stdout.write(Object.entries(result).toString());
+      checkEq(currTest.output, result);
-
-      assert.deepStrictEqual(result, currTest.output);
     });
   }
 });

src/test/util.ts

@@ -1,6 +1,5 @@
 import * as vscode from 'vscode';
-
+import { LineToken, PylexSymbol, LexNode} from '../pylex';
-import LexNode from '../pylex/node';
 
 /**
  * TODO: Eliminate need for me.
@@ -50,8 +49,20 @@ function root(nodes: LexNode[] | null): LexNode {
   );
 }
 
+/* short hand for returning an indentation token for a certain line and indentation */
+function indent(linenr: number, indentLevel: number): LexNode {
+  return new LexNode('INDENT', 0, new LineToken(PylexSymbol.INDENT, linenr, indentLevel));
+}
+
+/* short hand for returning an empty token for a certain line*/
+function empty(linenr: number): LexNode {
+  return new LexNode('EMPTY', 0, new LineToken(PylexSymbol.EMPTY, linenr, 999999));
+}
+
 
 export {
   deparent,
   root,
+  indent,
+  empty
 };