From a67750c30ecd6f7d115f54bb8aec7a0db3a25d89 Mon Sep 17 00:00:00 2001 From: Jacob Grossman Date: Fri, 5 Nov 2021 15:56:36 -0500 Subject: [PATCH] Update pylex tests --- src/commands.ts | 4 +- src/pylex/node.ts | 4 +- src/pylex/parser.ts | 2 +- src/test/suites/pylex/lexer.test.ts | 101 ++---------------- src/test/suites/pylex/parser.test.ts | 154 +++++++++++++++++++-------- src/test/util.ts | 15 ++- 6 files changed, 134 insertions(+), 146 deletions(-) diff --git a/src/commands.ts b/src/commands.ts index b587629..72c5d57 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -124,8 +124,8 @@ function createContextString(context: pl.LexNode[], line: number): string { continue; } - if (node.token!.type != pl.PylexSymbol.EMPTY && - node.token!.type != pl.PylexSymbol.INDENT) { + if (node.token!.type !== pl.PylexSymbol.EMPTY && + node.token!.type !== pl.PylexSymbol.INDENT) { contextString += ' inside ' + node.token!.type.toString(); if (node.token!.attr) { contextString += ' ' + node.token!.attr.toString(); diff --git a/src/pylex/node.ts b/src/pylex/node.ts index a3e00e5..2386ec0 100644 --- a/src/pylex/node.ts +++ b/src/pylex/node.ts @@ -48,8 +48,9 @@ export default class LexNode extends vscode.TreeItem { * Adopt child nodes. * * @param `child` Array of nodes to adopt. + * @returns an updated version of itself */ - adopt(children: LexNode[]): void { + adopt(children: LexNode[]): LexNode { let parentedChildren = children.map(c => new LexNode( c.label, c.collapsibleState, @@ -66,6 +67,7 @@ export default class LexNode extends vscode.TreeItem { // No.... this._children = parentedChildren; } + return this; } /** diff --git a/src/pylex/parser.ts b/src/pylex/parser.ts index 277451a..9a2fb18 100644 --- a/src/pylex/parser.ts +++ b/src/pylex/parser.ts @@ -120,7 +120,7 @@ export default class Parser { */ context(lineNumber: number, root?: LexNode): LexNode[] { if (!root) { - root = this.root + root = this.root; } // is this the target? diff --git a/src/test/suites/pylex/lexer.test.ts b/src/test/suites/pylex/lexer.test.ts index e0ce5a3..14bebd0 100644 --- a/src/test/suites/pylex/lexer.test.ts +++ b/src/test/suites/pylex/lexer.test.ts @@ -17,13 +17,18 @@ suite('Lexer Test Suite', () => { }); test('Undefined', () => { - let l: Lexer = new Lexer(''); + let l: Lexer = new Lexer(undefined); assert.deepStrictEqual(l.currToken(), EOFTOKEN); }); test('Whitespace', () => { let l: Lexer = new Lexer(' \t\t'.repeat(4).repeat(4)); - assert.deepStrictEqual(l.currToken(), EOFTOKEN); + assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EMPTY, 0, 999999)); + }); + + test('Comment', () => { + let l: Lexer = new Lexer('# ur mom eats toes'); + assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EMPTY, 0, 999999)); }); test('Non-Whitespace with no construct', () => { @@ -120,98 +125,6 @@ suite('Lexer Test Suite', () => { assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'is_old()')); }); - test('next() ignores empty lines', () => { - let lines: string[] = [ - 'if wurst_available():', - '', - ' eat_wurst()' - ]; - let l: Lexer = new Lexer(lines.join('\n')); - - l.next(); - - assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 2, 1)); - }); - - test('retract() ignores empty lines', () => { - let lines: string[] = [ - 'if wurst_available():', - '', - ' eat_wurst()' - ]; - let l: Lexer = new Lexer(lines.join('\n')); - - l.next(); - - l.retract(); - - assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'wurst_available()')); - }); - - test('next() ignores whitespace lines', () => { - let lines: string[] = [ - 'if wurst_available():', - ' \t \t ', - ' eat_wurst()' - ]; - let l: Lexer = new Lexer(lines.join('\n')); - - l.next(); - - assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 2, 1)); - }); - - test('retract() ignores whitespace lines', () => { - let lines: string[] = [ - 'if wurst_available():', - ' \t \t ', - ' eat_wurst()' - ]; - let l: Lexer = new Lexer(lines.join('\n')); - - // Advance to end of input - // Eliminates dependence on next() - // skipping whitespace - do {} while (l.next() !== EOFTOKEN); - - l.retract(); // retract past EOFTOKEn - l.retract(); - - assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'wurst_available()')); - }); - - test('next() ignores comment lines', () => { - let lines: string[] = [ - 'if wurst_available():', - ' \t # I hate testing \t', - ' eat_wurst()' - ]; - let l: Lexer = new Lexer(lines.join('\n')); - - l.next(); - - assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 2, 1)); - }); - - test('retract() ignores comment lines', () => { - let lines: string[] = [ - 'if wurst_available():', - ' \t # \t', - ' eat_wurst()' - ]; - let l: Lexer = new Lexer(lines.join('\n')); - - // Advance to end of input - // Eliminates dependence on next() - // skipping comment - do {} while (l.next() !== EOFTOKEN); - - l.retract(); // retract past EOFTOKEn - l.retract(); - - assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'wurst_available()')); - }); - test('next() out of range', () => { let l: Lexer = new Lexer('foo = zaboomafoo'); l.next(); diff --git a/src/test/suites/pylex/parser.test.ts b/src/test/suites/pylex/parser.test.ts index a11271b..b92a5ae 100644 --- a/src/test/suites/pylex/parser.test.ts +++ b/src/test/suites/pylex/parser.test.ts @@ -1,51 +1,61 @@ import * as assert from 'assert'; import * as vscode from 'vscode'; import { after } from 'mocha'; -import { deparent, root } from '../../util'; import Parser from '../../../pylex/parser'; import LexNode from '../../../pylex/node'; import LineToken from '../../../pylex/token'; import { Symbol } from '../../../pylex/token'; +import { root,indent,empty } from '../../util'; + +/** + * Test Descriptor +*/ type ParserTest = { - name: string, - input: string[], - output: LexNode, + name: string, // short name for the test + input: string[], // input lines for the test + output: LexNode // expected output. outputs are compared for token equality *only* }; const tests: ParserTest[] = [ + { name: 'No Input', input: [], output: root(null) }, + { name: 'Single Empty Line', input: [''], output: root(null) }, { - name: 'No Input', - input: [ ], - output: root(null), - }, - - { - name: 'Single line without construct', - input: [ 'foo = "Yellow M&Ms make me angry >:(' ], - output: root(null), - }, - - { - name: 'Single line with construct', - input: [ 'for x of y:' ], + name: 'Single Whitespace Only Line', + input: [' '], output: root([ - new LexNode( - 'for x of y', - vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.FOR, 0, 0, 'x of y') - ) - ]), + empty(0) + ]) + }, + { + name: 'Single Comment Only Line', + input: ['# ur mom likes peas'], + output: root([ + empty(0) + ]) + }, + { + name: 'Single Non-Control Line', + input: ['my_age = 42'], + output: root([ + indent(0, 0) + ]) + }, + { + name: 'Single Control Line', + input: ['while True:'], + output: root([ + new LexNode('', 0, new LineToken(Symbol.WHILE, 0, 0, 'True')) + ]) }, - { name: 'Sequential lines, without construct', input: [ 'bar = "Blue M&Ms make me happy <:)"', 'reba = "A hard working gal"' ], - output: root(null), + output: root([indent(0,0), indent(1,0)]), }, { @@ -58,7 +68,9 @@ const tests: ParserTest[] = [ output: root([ new LexNode('if radioshack', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 0, 0, 'radioshack')) + new LineToken(Symbol.IF, 0, 0, 'radioshack'), + [indent(1, 1)]), + indent(2, 0) ]) }, @@ -70,9 +82,11 @@ const tests: ParserTest[] = [ ' print radioshack.hours' ], output: root([ + indent(0, 0), new LexNode('if radioshack', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 1, 0, 'radioshack')) + new LineToken(Symbol.IF, 1, 0, 'radioshack'), + [indent(2, 1)]) ]) }, @@ -89,13 +103,13 @@ const tests: ParserTest[] = [ output: root([ new LexNode('if yummy', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 0, 0, 'yummy')), + new LineToken(Symbol.IF, 0, 0, 'yummy'), [indent(1, 1)]), new LexNode('elif just_ok', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.ELIF, 2, 0, 'just_ok')), + new LineToken(Symbol.ELIF, 2, 0, 'just_ok'), [indent(3, 1)]), new LexNode('else', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.ELSE, 4, 0)), + new LineToken(Symbol.ELSE, 4, 0), [indent(5,1)]), ]) }, @@ -113,8 +127,9 @@ const tests: ParserTest[] = [ [ new LexNode('if in_my_tummy', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 1, 1, 'in_my_tummy')) - ] + new LineToken(Symbol.IF, 1, 1, 'in_my_tummy'), + [indent(2, 2)]) + ], ) ]) }, @@ -135,12 +150,14 @@ const tests: ParserTest[] = [ [ new LexNode('if in_my_tummy', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 1, 1, 'in_my_tummy')) + new LineToken(Symbol.IF, 1, 1, 'in_my_tummy'), + [indent(2, 2)]) ] ), - new LexNode('else', + new LexNode('else', vscode.TreeItemCollapsibleState.None, new LineToken(Symbol.ELSE, 3, 0), + [indent(4, 1)] ) ]) }, @@ -166,7 +183,8 @@ const tests: ParserTest[] = [ [ new LexNode('if looks_like_a_mummy', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy')) + new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy'), + [indent(3, 3)]) ] ) ] @@ -174,12 +192,13 @@ const tests: ParserTest[] = [ new LexNode('else', vscode.TreeItemCollapsibleState.None, new LineToken(Symbol.ELSE, 4, 0), + [indent(5, 1)] ) ]) }, { - name: 'Doubly Nested Block, with multiple indent resets', + name: 'Doubly Nested Block, with multiple indent resets > 1', input: [ 'if yummy:', ' if in_my_tummy:', @@ -188,7 +207,7 @@ const tests: ParserTest[] = [ ' else:', ' print("eek! a zombie!)', ' elif in_my_mouth:', - ' print("ill be in my tummy soon!"', + ' print("itll be in my tummy soon!"', 'else:', ' print("Food is food...")' ], @@ -203,25 +222,70 @@ const tests: ParserTest[] = [ [ new LexNode('if looks_like_a_mummy', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy')), + new LineToken(Symbol.IF, 2, 2, 'looks_like_a_mummy'), + [indent(3, 3)]), new LexNode('else', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.ELSE, 4, 2)) + new LineToken(Symbol.ELSE, 4, 2), + [indent(5, 3)]) ] ), new LexNode('elif in_my_mouth', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.ELIF, 6, 1, 'in_my_mouth')) + new LineToken(Symbol.ELIF, 6, 1, 'in_my_mouth'), + [indent(7, 2)] + ) ] ), new LexNode('else', vscode.TreeItemCollapsibleState.None, - new LineToken(Symbol.ELSE, 8, 0) + new LineToken(Symbol.ELSE, 8, 0), + [indent(9, 1)] ) ]) + }, + { + name: 'Multiline Block', + input: [ + 'if yummy:', + ' print("you have a spot on your tummy"', + ' print("eek! a zombie!)', + ' print("itll be in my tummy soon!"', + 'else:', + ' print("Food is food...")' + ], + output: root([ + new LexNode('if yummy', 0, new LineToken(Symbol.IF, 0, 0, 'yummy'), + [ + indent(1, 1), + indent(2, 1), + indent(3, 1), + ] + ), + new LexNode('else', 0, new LineToken(Symbol.ELSE, 4, 0), [indent(5, 1)]) + ]) } ]; +/* Checks for strict equality between the tokens of a lex node tree */ +const checkEq = (reference: LexNode, subject: LexNode) => { + if (!reference.children()) { + // subject should also have no children + assert.deepStrictEqual(subject.children(), null); + return; + } + + assert.notStrictEqual(subject.children(), null); + assert.deepStrictEqual(reference.children()!.length, subject.children()!.length); + for (let i = 0; i < subject.children()!.length; i++) { + // compare top level nodes + assert.deepStrictEqual(reference.children()![i].token, subject.children()![i].token); + + // compare all children + checkEq(reference.children()![i], subject.children()![i]); + } +}; + suite('Parser Test Suite', () => { after(() => { vscode.window.showInformationMessage('All tests passed!'); @@ -230,10 +294,8 @@ suite('Parser Test Suite', () => { for (var t of tests) { let currTest = t; // without this, all test calls get the last test test(currTest.name, () => { - let result: LexNode = deparent(new Parser(currTest.input.join('\n')).parse()); - process.stdout.write(Object.entries(result).toString()); - - assert.deepStrictEqual(result, currTest.output); + let result = new Parser(currTest.input.join('\n')).parse(); + checkEq(currTest.output, result); }); } }); diff --git a/src/test/util.ts b/src/test/util.ts index 8967c81..b7c0f12 100644 --- a/src/test/util.ts +++ b/src/test/util.ts @@ -1,6 +1,5 @@ import * as vscode from 'vscode'; - -import LexNode from '../pylex/node'; +import { LineToken, PylexSymbol, LexNode} from '../pylex'; /** * TODO: Eliminate need for me. @@ -50,8 +49,20 @@ function root(nodes: LexNode[] | null): LexNode { ); } +/* short hand for returning an indentation token for a certain line and indentation */ +function indent(linenr: number, indentLevel: number): LexNode { + return new LexNode('INDENT', 0, new LineToken(PylexSymbol.INDENT, linenr, indentLevel)); +} + +/* short hand for returning an empty token for a certain line*/ +function empty(linenr: number): LexNode { + return new LexNode('EMPTY', 0, new LineToken(PylexSymbol.EMPTY, linenr, 999999)); +} + export { deparent, root, + indent, + empty };