Name improvements, new tokens, new behavior.

Name improvements:
- runLineContext changed to "Get Line Scope".
- runCursorContext changed to "Get Words Under Cursor".

New tokens:
- STATEMENT: Replaces INDENT. Stores entire line of python statement, not including comments, in attr.
- COMMENT: Stores entire comment line in attr.
- INVALID: Invalid token, used when parser can't figure out a line. Should never be seen.

New behavior:
- Get Line Scope now reads out the comment your cursor is in.
- Implemented fractional indentation notifications (disabled)

src/commands/text.ts

@@ -19,11 +19,11 @@ export const textCommands: CommandEntry[] = [
         callback: getLeadingSpaces,
     },
     {
-        name: 'mind-reader.runLineContext',
+        name: 'mind-reader.getLineScope',
         callback: runLineContext,
     },
     {
-        name: 'mind-reader.runCursorContext',
+        name: 'mind-reader.getWordsUnderCursor',
         callback: runCursorContext
     }
 ];
@@ -168,16 +168,16 @@ function createContextString(context: pl.LexNode[], line: number): string {
     // Print the current line
     if (context[0].token && context[0].token.attr) {
         let tokenTypeString: string = `${context[0].token.type.toString()}`;
-        contextString += `: ${tokenTypeString !== 'INDENT'?tokenTypeString:""
+        contextString += `: ${tokenTypeString !== pl.PylexSymbol.STATEMENT?tokenTypeString:""
         } ${context[0].token.attr.toString()}`;
     }
 
     for (let i: number = 1; i < context.length; i++) {
         const node: pl.LexNode = context[i];
-        const inside: string = "in";
+        const inside: string = "inside";
         // Node contains information relevant to the current line
         if (node.token && node.token.type !== pl.PylexSymbol.EMPTY &&
-            node.token.type !== pl.PylexSymbol.INDENT) {
+            node.token.type !== pl.PylexSymbol.STATEMENT) {
             contextString += ` ${inside} ${node.token.type.toString()}`;
             if (node.token.attr) {
                 contextString += ` ${node.token.attr.toString()}`;

src/pylex/lexer.ts

@@ -57,12 +57,16 @@ const rules: Rule[] = [
         type: Symbol.WITH
     },
     {
-        pattern: /^\s*(#.*)?$/,
+        pattern: /^\s*#+\s*(?<attr>.*)\s*$/,
+        type: Symbol.COMMENT
+    },
+    {
+        pattern: /^\s*$/,
         type: Symbol.EMPTY
     },
     {
-        pattern: /\s*(?<attr>[^#]+)?$/,
-        type: Symbol.INDENT
+        pattern: /^\s*(?<attr>[^#]+)+\s*$/,
+        type: Symbol.STATEMENT
     }
 ];
 
@@ -151,7 +155,7 @@ export default class Lexer {
                 }
             }
             // No rules matched
-            token = new LineToken(Symbol.EMPTY, this.pos, 999999);
+            token = new LineToken(Symbol.INVALID, this.pos, 999999);
             this._currToken = token;
             this.pos++;
 
@@ -218,7 +222,8 @@ export default class Lexer {
             indent = leadingSpace;
         }
         else {
-            // use spaces
+            // used spaces
+            //? indent = Math.round(leadingSpace / tabFmt.size! * 10) / 10; // fractional indentation support?
             indent = Math.ceil(leadingSpace / tabFmt.size!);
         }
 

src/pylex/parser.ts

@@ -74,8 +74,9 @@ export default class Parser {
         return children;
       }
 
-      if (this.lexer.currToken().type === Symbol.INDENT ||
-          this.lexer.currToken().type === Symbol.EMPTY) {
+      if (this.lexer.currToken().type === Symbol.STATEMENT ||
+          this.lexer.currToken().type === Symbol.EMPTY     ||
+          this.lexer.currToken().type === Symbol.INVALID) {
         const label = this.lexer.currToken().type;
         // regular code, advance and stay in same block
         children.push(new LexNode(

src/pylex/token.ts

@@ -16,8 +16,10 @@ export enum Symbol {
   EXCEPT = "except",
   FINALLY = "finally",
   WITH = "with",
-  INDENT = "INDENT", // Indent token, default if not EOF, only contains indent information
+  STATEMENT = "statement", // Indent token, contains non-empty code lines
+  COMMENT = "comment",
   EMPTY = "EMPTY", // empty line, used only to associate with the previous line
+  INVALID = "INVALID",
   EOF = "EOF"
 }