Add H5 & H4: Listen to Cursor/Line Context (#7)

* Update parser

Implied nodes caused ambiguity when querying for context. Now every line
is considered, even blank lines, to make the process much, *much* easier.

* Update pylex tests

* Add runCursorContext

src/pylex/index.ts

@@ -4,4 +4,4 @@ export {default as LineToken} from './token';
 export {default as Lexer} from './lexer';
 export {default as LexNode} from './node';
 export {TabInfo as TabInfo} from './token';
-
+export {Symbol as PylexSymbol} from './token';

src/pylex/lexer.ts

@@ -159,14 +159,15 @@ export default class Lexer {
       }
       // No rules matched
 
-      // Skip this line if it is whitespace, comment, or empty
+      // TODO: move to rules
       if (/^\s*(#.*)?$/.test(line)) {
-        this.pos++;
-        continue;
+        // "empty" line
+        token = new LineToken(Symbol.EMPTY, this.pos, 999999);
+      } else {
+        // This is an INDENT token
+        token = new LineToken(Symbol.INDENT, this.pos, indent);
       }
 
-      // This is an INDENT token
-      token = new LineToken(Symbol.INDENT, this.pos, indent);
       this._currToken = token;
       this.pos++;
       return this.currToken();

src/pylex/node.ts

@@ -2,6 +2,8 @@ import * as vscode from 'vscode';
 
 import LineToken from './token';
 
+/* TODO: make accessing children and parent less tedious */
+/* TODO: 'root.children()![i])' */
 /**
  * A node in a Parse tree.
  */
@@ -46,8 +48,9 @@ export default class LexNode extends vscode.TreeItem {
    * Adopt child nodes.
    *
    * @param `child` Array of nodes to adopt.
+   * @returns an updated version of itself
    */
-  adopt(children: LexNode[]): void {
+  adopt(children: LexNode[]): LexNode {
     let parentedChildren = children.map(c => new LexNode(
       c.label,
       c.collapsibleState,
@@ -64,6 +67,7 @@ export default class LexNode extends vscode.TreeItem {
       // No....
       this._children = parentedChildren;
     }
+    return this;
   }
 
   /**

src/pylex/parser.ts

@@ -3,6 +3,7 @@ import * as vscode from 'vscode';
 import { EOFTOKEN, Symbol, TabInfo } from './token';
 import Lexer from './lexer';
 import LexNode from './node';
+/* TODO: update design doc */
 
 /**
  * A parse tree generator
@@ -71,8 +72,20 @@ export default class Parser {
         // go up 1 level of recursion at a time to unravel properly
         this.currIndent--;
         return children;
-      } else if (this.lexer.currToken().type === Symbol.INDENT) {
+      }
+
+      if (this.lexer.currToken().type === Symbol.INDENT ||
+          this.lexer.currToken().type === Symbol.EMPTY) {
+        const label = this.lexer.currToken().type;
         // regular code, advance and stay in same block
+        children.push(new LexNode(
+          label,
+          vscode.TreeItemCollapsibleState.None,
+          this.lexer.currToken(),
+          null,
+          parent)
+        );
+
         this.lexer.next();
         continue;
       } else {
@@ -105,29 +118,28 @@ export default class Parser {
    * @param `lineNumber` The line number to query context for.
    * @return An array of LexNodes for the root path containing `lineNumber`
    */
-  context(lineNumber: number): LexNode[] {
-    if (!this.root.children()) {
-      return [];
+  context(lineNumber: number, root?: LexNode): LexNode[] {
+    if (!root) {
+      root = this.root;
     }
 
-    // Returns the LexNode that is the parent
-    // of the queried line number
-    let find = (root: LexNode): LexNode | undefined => {
-      let prevChild: LexNode;
-      for (var child of root.children()!) {
-        if (lineNumber < child.token!.linenr) {
-          if (prevChild!.children()) {
-            return find(prevChild!);
-          } else {
-            return prevChild!;
-          }
-        } else {
-          prevChild = child;
+    // is this the target?
+    if (root.token && root.token!.linenr === lineNumber) {
+      // match
+      return root.rootPath();
+    }
+
+    if (root.children()) {
+      // recursive call
+      for (let i = 0; i < root.children()!.length; i++) {
+        let ctx = this.context(lineNumber, root.children()![i]);
+        if (ctx.length > 0) {
+          // a rootpath was returned
+          return ctx;
         }
       }
-    };
-
-    let target = find(this.root);
-    return target!.rootPath();
+    }
+    // no matches
+    return [];
   }
 }

src/pylex/token.ts

@@ -17,6 +17,7 @@ export enum Symbol {
   FINALLY = "finally",
   WITH = "with",
   INDENT = "INDENT", // Indent token, default if not EOF, only contains indent information
+  EMPTY = "EMPTY", // empty line, used only to associate with the previous line
   EOF = "EOF"
 }