import * as assert from 'assert'; import * as vscode from 'vscode'; import { after } from 'mocha'; import Lexer from '../../../pylex/lexer'; import LineToken from '../../../pylex/token'; import { EOFTOKEN, Symbol } from '../../../pylex/token'; suite('Lexer Test Suite', () => { after(() => { vscode.window.showInformationMessage('All tests passed!'); }); test('Empty String', () => { let l: Lexer = new Lexer(undefined); assert.deepStrictEqual(l.currToken(), EOFTOKEN); }); test('Undefined', () => { let l: Lexer = new Lexer(undefined); assert.deepStrictEqual(l.currToken(), EOFTOKEN); }); test('Whitespace', () => { let l: Lexer = new Lexer(' \t\t'.repeat(4).repeat(4)); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EMPTY, 0, 999999)); }); test('Comment', () => { let l: Lexer = new Lexer('# ur mom eats toes'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EMPTY, 0, 999999)); }); test('Non-Whitespace with no construct', () => { let l: Lexer = new Lexer('foobar'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.INDENT, 0, 0)); }); test('getIndent() accuracy, spaces', () => { for (var i = 0; i < 100; i++) { let l: Lexer = new Lexer(' '.repeat(i) + 'foobar'); assert.strictEqual(l.currToken().indentLevel, i); } }); test('getIndent() accuracy, tabs', () => { for (var i = 0; i < 100; i++) { let l: Lexer = new Lexer('\t'.repeat(i) + 'foobar', {size: 4, hard: true}); assert.strictEqual(l.currToken().indentLevel, i); } }); test('getIndent() accuracy, spaces with incomplete tab', () => { for (var i = 0; i < 100; i++) { for (var j = 1; j <= 3; j++) { let l: Lexer = new Lexer(' '.repeat(i) + ' '.repeat(j) + 'foobar', {size: 4, hard: false}); assert.strictEqual(l.currToken().indentLevel, i+1); } } }); test('class definition', () => { let l: Lexer = new Lexer('class Foobar(object):'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.CLASS, 0, 0, 'Foobar')); }); test('function definition', () => { let l: Lexer = new Lexer('def Barbaz(this, that, andTheOther):'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.FUNCTION, 0, 0, 'Barbaz')); }); test('if statement', () => { let l: Lexer = new Lexer('if True and bar == baz:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'True and bar == baz')); }); test('elif statement', () => { let l: Lexer = new Lexer('elif name == "bar" and True:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.ELIF, 0, 0, 'name == "bar" and True')); }); test('else statement', () => { let l: Lexer = new Lexer('else:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.ELSE, 0, 0)); }); test('for loop', () => { let l: Lexer = new Lexer('for pickle in pickleJars:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.FOR, 0, 0, 'pickle in pickleJars')); }); test('while loop', () => { let l: Lexer = new Lexer('while numCookies < capacity:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.WHILE, 0, 0, 'numCookies < capacity')); }); test('try statement', () => { let l: Lexer = new Lexer('try:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.TRY, 0, 0)); }); test('except statement with attr', () => { let l: Lexer = new Lexer('except NameError:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EXCEPT, 0, 0, 'NameError')); }); test('except statement with no attr', () => { let l: Lexer = new Lexer('except:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.EXCEPT, 0, 0)); }); test('finally statement', () => { let l: Lexer = new Lexer('finally:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.FINALLY, 0, 0)); }); test('with statement', () => { let l: Lexer = new Lexer('with open(file) as f:'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.WITH, 0, 0, 'open(file) as f')); }); test('restart()', () => { let l: Lexer = new Lexer('with open(file as f:'); l.restart('if is_old():'); assert.deepStrictEqual(l.currToken(), new LineToken(Symbol.IF, 0, 0, 'is_old()')); }); test('next() out of range', () => { let l: Lexer = new Lexer('foo = zaboomafoo'); l.next(); assert.throws(() => l.next()); }); test('retract() out of range', () => { let l: Lexer = new Lexer('next_token = lexer.next()'); assert.throws(() => l.retract()); }); test('retract() validate argument', () => { let l: Lexer = new Lexer(); // Negative assert.throws(() => l.retract(-1)); // Zero, it doesn't make sense to retract 0 :P assert.throws(() => l.retract(0)); }); test('retract() 1-100', () => { let lines: string[] = Array.from(Array(100), (_, i) => 'line' + i); let reference: LineToken[] = lines.map((_, i) => { return new LineToken(Symbol.INDENT, i, 0); }); for (var i = 0; i < 100; i++) { let l: Lexer = new Lexer(lines.join('\n')); // advance to EOF do {} while (l.next() !== EOFTOKEN); // try retract l.retract(i+1); assert.deepStrictEqual(l.currToken(), reference[99-i]); } }); test('2 full lex and retract passes', () => { let lines: string[] = Array.from(Array(100), (_, i)=> 'line' + i); let reference: LineToken[] = lines.map((_, i) => { return new LineToken(Symbol.INDENT, i, 0); }); let l: Lexer = new Lexer(lines.join('\n')); // Twice for (var _ of [0,1]) { // advance to EOF for (var i = 0; i < lines.length; i++) { assert.deepStrictEqual(l.currToken(), reference[i]); l.next(); } // retract to start for (var i = lines.length - 1; i >= 0; i--) { l.retract(); assert.deepStrictEqual(l.currToken(), reference[i]); } } }); });