Hub communication (#12)

* Add persistent accessibility pane

This will facilitate more extensive usage of the menu than the
context menu.

* Fixed missing files

* Add missing semicolon

* Implement HubController

* Parse messages on arrival

Rather than queuing inbound messages, the HubController now saves
pending promises/rejects for each pending request. Each inbound packet
is checked at the time of arrival, and if the ID matches a pending
response, the corresponding promise is called.

This fixes a problem where the longer the time between reads, the more
garbage responses queue up that are guaranteed to get thrown away the
next time the next response was gathered.

* Add clarification comment to send

* Add logger, output

* Use stat+stream instead of reading entire file on upload

* Split MindReader view into accessability and hub sub-views

* Add missing comma from conflict resolution

* Fix issues, split commands into sub-lists

* Add rebuild instructions

* More accurate

* Add tools for native modules instructions to README.md

* Move commands to correct spot

* Remove automatic connection

I did not heed the warning where 'only the path is guaranteed' when
listing open serial ports and made the assumption that the manufacturer
would be known (hint: it wasn't).

* Use device specific language for output title

src/commandNodeProvider.ts

@@ -0,0 +1,43 @@
+import * as vscode from 'vscode';
+
+import { CommandEntry } from './commands';
+
+export class CommandItem extends vscode.TreeItem {
+  constructor(
+    public readonly label: string,
+    public readonly command: vscode.Command
+  ) {
+    super(label, vscode.TreeItemCollapsibleState.None);
+  }
+};
+
+export default class CommandNodeProvider implements vscode.TreeDataProvider<CommandItem> {
+  private items: CommandItem[] = [];
+
+  public constructor(commands: CommandEntry[]) {
+    // build and cache command items
+    for (const c of commands) {
+      let humanReadable = c.name.replace(/^mind-reader\./, ''); // strip extensions name
+      // Convert camelCaseText to Title Case Text
+      humanReadable = humanReadable.replace(/([A-Z])/g, ' $1');
+      humanReadable = humanReadable.charAt(0).toUpperCase() + humanReadable.slice(1);
+
+      this.items.push(new CommandItem(
+        humanReadable,
+        {
+          title: humanReadable,
+          command: c.name,
+          tooltip: humanReadable
+        }
+      ));
+    }
+  }
+
+  public getTreeItem(item: CommandItem): vscode.TreeItem {
+    return item;
+  }
+
+  public async getChildren(): Promise<CommandItem[]> {
+    return Promise.resolve(this.items);
+  }
+}

src/commands.ts

@@ -1,19 +1,22 @@
 import * as vscode from 'vscode';
 import * as pl from './pylex';
+import * as path from 'path';
 import * as fs from 'fs';
 
+import HubManager from './hubManager';
+
 /**
  * @type {Object} Command // Command to register with the VS Code Extension API
  * @prop {string} command // Name of the command; e.g., 'mind-reader.selectTheme'
  * @prop {callback} callback // Callback to register when `command` is invoked
  */
-type Command = {
+export type CommandEntry = {
   name: string,
   callback: () => void
 };
 
-// The list of commands to register in the extension
-const commands: Command[] = [
+// Accessibility Commands
+export const accessCommands: CommandEntry[] = [
   {
     name: 'mind-reader.selectTheme',
 
@@ -51,6 +54,23 @@ const commands: Command[] = [
     callback: resetEditorScale,
   },
 
+  {
+    name: 'mind-reader.getIndent',
+    callback: getIndent,
+  },
+
+  {
+    name: 'mind-reader.runLineContext',
+    callback: runLineContext,
+  },
+
+  {
+    name: 'mind-reader.runCursorContext',
+    callback: runCursorContext
+  }
+];
+
+export const navCommands: CommandEntry[] = [
   {
     name: 'mind-reader.openWebview',
     callback: openWebview,
@@ -117,7 +137,7 @@ const commands: Command[] = [
   },
 
   {
-    name: 'mind-reader.getuickInputBack',
+    name: 'mind-reader.getQuickInputBack',
     callback: () => vscode.commands.executeCommand('workbench.action.quickInputBack'),
   },
 
@@ -125,18 +145,38 @@ const commands: Command[] = [
     name: 'mind-reader.navigateForward',
     callback: () => vscode.commands.executeCommand('workbench.action.navigateForward'),
   },
+];
+
+export const hubCommands: CommandEntry[] = [
   {
-    name: 'mind-reader.runLineContext',
-    callback: runLineContext,
+    name: 'mind-reader.connectHub',
+    callback: connectHub
   },
+
   {
-    name: 'mind-reader.runCursorContext',
-    callback: runCursorContext
+    name: 'mind-reader.disconnectHub',
+    callback: disconnectHub
   },
+
   {
-    name: 'mind-reader.getIndent',
-    callback: getIndent
-  }
+    name: 'mind-reader.uploadCurrentFile',
+    callback: uploadCurrentFile
+  },
+
+  {
+    name: 'mind-reader.runProgram',
+    callback: runProgram
+  },
+
+  {
+    name: 'mind-reader.stopExecution',
+    callback: stopExecution
+  },
+
+  {
+    name: 'mind-reader.deleteProgram',
+    callback: deleteProgram
+  },
 ];
 
 // COMMAND CALLBACK IMPLEMENTATIONS
@@ -278,17 +318,17 @@ function createContextString(context: pl.LexNode[], line: number): string {
 }
 
 // find up to `n` words around the cursor, where `n` is
-// the value of `#mindreader.reader.contextWindow`
+// the value of `#mindReader.reader.contextWindow`
 function runCursorContext(): void {
   let editor = vscode.window.activeTextEditor;
   if (!editor) {
-    vscode.window.showErrorMessage("RunCursorContext: No Active Editor");
+    vscode.window.showErrorMessage('RunCursorContext: No Active Editor');
     return;
   }
 
   const cursorPos: vscode.Position = editor.selection.active;
   const text: string = editor.document.lineAt(cursorPos).text;
-  const windowSize: number = vscode.workspace.getConfiguration('mindreader').get('reader.contextWindow')!;
+  const windowSize: number = vscode.workspace.getConfiguration('mindReader').get('reader.contextWindow')!;
 
   let trimmedText = text.trimStart(); // trim leading whitespace
   let leadingWS = text.length - trimmedText.length; // # of characters of leading whitespace
@@ -343,4 +383,140 @@ function runCursorContext(): void {
   }
 }
 
-export default commands;
+// Current connected hub
+let hub: HubManager | null = null;
+
+// TODO: port option
+async function connectHub(): Promise<void> {
+  if (hub) {
+    vscode.window.showWarningMessage('LEGO Hub is already connected, reconnecting...');
+    disconnectHub();
+  }
+
+  try {
+      const ports = await HubManager.queryPorts();
+
+      if (ports.length === 0) {
+        vscode.window.showErrorMessage('No ports found. Is the LEGO Hub connected?');
+        return;
+      }
+
+      let portPath: string | undefined = vscode.workspace.getConfiguration('mindReader.connection').get('portPath');
+
+      if (!portPath) {
+        let slots: vscode.QuickPickItem[] = [];
+        for (const port of ports) {
+          slots.push({ label: port.path });
+        }
+
+        let picked = await vscode.window.showQuickPick(slots);
+
+        if (!picked) {
+          return;
+        }
+
+        portPath = picked.label;
+      }
+      hub = await HubManager.create(portPath);
+      vscode.window.showInformationMessage('LEGO Hub connected');
+  } catch (err) {
+    vscode.window.showErrorMessage('Could not connect to LEGO Hub');
+  }
+}
+
+async function disconnectHub(): Promise<void> {
+  if (!hub || !hub.isOpen()) {
+    vscode.window.showErrorMessage('LEGO Hub is not connected');
+    return;
+  }
+
+  await hub.close();
+  hub = null;
+  vscode.window.showInformationMessage('LEGO Hub disconnected');
+}
+
+async function uploadCurrentFile(): Promise<void> {
+  if (!hub || !hub.isOpen()) {
+    vscode.window.showErrorMessage('LEGO Hub is not connected!');
+    return;
+  }
+
+  if (!vscode.window.activeTextEditor) {
+    vscode.window.showErrorMessage('No active text editor');
+    return;
+  }
+
+  const currentFilePath = vscode.window.activeTextEditor.document.fileName;
+
+  if (currentFilePath) {
+    // construct quickpick
+    const slots: vscode.QuickPickItem[] = [];
+    for (let i = 0; i < 10; i++) {
+      slots.push({ label: i.toString() });
+    }
+    const slotID = await vscode.window.showQuickPick(slots, { canPickMany: false });
+
+    if (!slotID) {
+      return;
+    }
+
+    // TODO: progress bar?
+    vscode.window.showInformationMessage('Uploading current file');
+    await hub.uploadFile(currentFilePath, parseInt(slotID.label), path.basename(currentFilePath));
+    vscode.window.showInformationMessage(path.basename(currentFilePath) + ' uploaded to slot ' + slotID.label);
+  }
+}
+
+// TODO: find empty slots
+async function runProgram(): Promise<void> {
+  if (!hub || !hub.isOpen()) {
+    vscode.window.showErrorMessage('LEGO Hub is not connected!');
+    return;
+  }
+
+  const slots: vscode.QuickPickItem[] = [];
+  // construct quickpick
+  for (let i = 0; i < 10; i++) {
+    slots.push({ label: i.toString() });
+  }
+  const slotID = await vscode.window.showQuickPick(slots, { canPickMany: false });
+
+  if (!slotID) {
+    return;
+  }
+
+  vscode.window.showInformationMessage('Running program ' + slotID.label);
+  await hub.programExecute(parseInt(slotID.label));
+}
+
+async function stopExecution(): Promise<void> {
+  if (!hub || !hub.isOpen()) {
+    vscode.window.showErrorMessage('LEGO Hub is not connected!');
+    return;
+  }
+
+  await hub.programTerminate();
+  vscode.window.showInformationMessage('Execution stopped');
+}
+
+// TODO: find slots from status
+async function deleteProgram(): Promise<void> {
+  if (!hub || !hub.isOpen()) {
+    vscode.window.showErrorMessage('LEGO Hub is not connected!');
+    return;
+  }
+
+  const slots: vscode.QuickPickItem[] = [];
+  // construct quickpick
+  for (let i = 0; i < 10; i++) {
+    slots.push({ label: i.toString() });
+  }
+  const slotID = await vscode.window.showQuickPick(slots, { canPickMany: false });
+
+  if (!slotID) {
+    return;
+  }
+
+  await hub.deleteProgram(parseInt(slotID.label));
+  vscode.window.showInformationMessage('Deleted program ' + slotID.label);
+}

src/extension.ts

@@ -1,20 +1,27 @@
 import * as vscode from 'vscode';
-
 import * as pl from './pylex';
-import commands from './commands';
 
-import AccessNodeProvider from './accessNodeProvider';
+import { accessCommands, hubCommands, navCommands } from './commands';
+
+import CommandNodeProvider from './commandNodeProvider';
+import Logger from './log';
+
+// Output Logger
+const product: string = vscode.workspace.getConfiguration('mindReader').get('productType')!;
+const outputChannel = vscode.window.createOutputChannel(product + " Output");
+export const logger = new Logger(outputChannel);
 
 let parser: pl.Parser = new pl.Parser();
 
 export function activate(context: vscode.ExtensionContext) {
-  console.log('Congratulations, your extension "mind-reader" is now active!');
   vscode.window.showInformationMessage('Mind_Reader is loaded!');
 
   parser.parse('Beep Boop');
 
+  let allCommands = accessCommands.concat(hubCommands).concat(navCommands);
+
   // Register Commands
-  commands.forEach(command => {
+  allCommands.forEach(command => {
     let disposable = vscode.commands.registerCommand(
       command.name,
       command.callback
@@ -22,9 +29,11 @@ export function activate(context: vscode.ExtensionContext) {
     context.subscriptions.push(disposable);
   });
 
-  let provider = new AccessNodeProvider();
-  vscode.window.registerTreeDataProvider('accessActions', provider);
+  let accessProvider = new CommandNodeProvider(accessCommands);
+  vscode.window.registerTreeDataProvider('accessActions', accessProvider);
 
+  let hubProvider = new CommandNodeProvider(hubCommands);
+  vscode.window.registerTreeDataProvider('hubActions', hubProvider);
 }
 
 export function deactivate() {}

src/hubManager.ts

@@ -0,0 +1,391 @@
+import * as SerialPort from 'serialport';
+import * as fs from 'fs';
+
+import { logger } from './extension';
+
+/**
+ * @type RPCRequest an RPC request message
+ *
+ * @prop {string} `'m'` method to invoke.
+ * @prop {Object?} `'p'` optional parameters for method.
+ * @prop {string|null?} `'i'` optional request ID.
+ */
+type RPCRequest = {
+  'm': string;
+  'p'?: Object;
+  'i'?: string;
+};
+
+/**
+ * @type RPCResponse an RPC response message
+ *
+ * @prop {any?} `'r'` RPC response body
+ * @prop {Object?} `'e'` RPC error body
+ * @prop {string|null} `'i'` required response ID.
+ */
+type RPCResponse = {
+  'r'?: any;
+  'e'?: Object;
+  'i': string | null;
+};
+
+/**
+ * Manages sending and receiving of JSON Remote Procedure Call (JSON-RPC) protocol messages
+ * to the Hub.
+ */
+export default class HubManager {
+  private port: SerialPort;
+  private receiveBuffer: string = ''; // buffer for in-flight messages
+  private pendingRequests = new Map<string, [(result: any) => void, (error: string) => void]>(); // lists of requests that are still pending
+
+  // ======================== INSTANCE METHODS ========================
+
+  /**
+   * Private constructor, use static `create` + `init`
+   */
+  private constructor(public portPath: string) { }
+
+  public isOpen(): boolean {
+    return this.port.isOpen;
+  }
+
+  /**
+   * Handle a received data chunk from the serial port
+   *
+   * @param `data` Data received from serial port
+   */
+  private async receiveData(data: string) {
+    // add data to buffer
+    this.receiveBuffer += data;
+
+    // get full lines in buffer
+
+    let msgs = this.receiveBuffer.split(/\r/); // split by newline
+    this.receiveBuffer = msgs.pop()!; // store unhandled data
+
+    msgs = msgs.filter(x => !x.startsWith('{"m":0,"p":')); // drop sensor broadcast response spam
+
+    for (const msg of msgs) {
+      // check if this msg is a response to a pending request
+      try {
+        let json: { [key: string]: any };
+
+        json = JSON.parse(msg);
+
+        let id = json['i'];
+        if (id && this.pendingRequests.has(id)) {
+          // a request is waiting on this response
+          let [resolve, reject] = this.pendingRequests.get(id) ?? [];
+
+          if (json['e'] && reject) {
+            // error
+            reject(Buffer.from(json['e'], 'base64').toString());
+          } else if (resolve) {
+            resolve(json['r']);
+          }
+
+          this.pendingRequests.delete(id);
+        } else if (json['m']) {
+          // Print errors
+          const params = json['p'];
+          switch (json['m']) {
+            case 'user_program_error':
+              logger.error(Buffer.from(params[3], 'base64').toString());
+              logger.error(Buffer.from(params[4], 'base64').toString());
+              break;
+            case 'runtime_error':
+              logger.error(Buffer.from(params[3], 'base64').toString());
+              break;
+          }
+        }
+      } catch (err) {
+        console.log('Could not parse JSON:', msg);
+      }
+    }
+  }
+
+  /**
+   * Initializes a created HubManager with the current option settings
+   */
+  public async init(): Promise<void> {
+    try {
+      this.port = new SerialPort(
+        this.portPath,
+        {
+          autoOpen: true,
+          baudRate: 112500,
+        }
+      );
+    } catch (err) {
+      // error during port opening
+      throw err;
+    }
+
+    this.port.setEncoding('utf-8');
+
+    // push lines received to data queue
+
+    let mgr = this;
+    this.port.on('data', data => {
+      mgr.receiveData(data);
+    });
+  }
+
+  public async close(): Promise<void> {
+    this.port.close(console.error);
+  }
+
+  /**
+   * Send an RPC message. The corresponding Promise resolve
+   * is saved into the `pendingRequests` map. When an inbound
+   * message is found that matches an ID in `pendingRequests`,
+   * the corresponding resolve is called. So, even though
+   * the `resolve` call does not appear explicitly here, it *does*
+   * get resolved at the appropriate time.
+   *
+   * @param `proc` Procedure to execute
+   * @param `params` Optional parameters for the procedure
+   * @param `id` The ID to use for the RPC message. Use null to indicate no ID/notification message.
+   *             If neither a string or `null` is passed, an ID is automatically generated.
+   */
+  public async send(request: RPCRequest): Promise<RPCResponse> {
+    return new Promise((resolve, reject) => {
+      if (request['i'] === undefined) {
+        // generate an ID
+        request['i'] = 'mind-reader-' + HubManager.randomID();
+      }
+
+      // write JSON to port
+
+      this.pendingRequests.set(request['i'], [resolve, reject]);
+      this.port.write(JSON.stringify(request));
+      this.port.write('\r');
+      this.port.drain();
+    });
+  }
+
+  public async uploadFile(file: string, slotid: number, name?: string, autoStart: boolean = false) {
+    const fileStats = fs.statSync(file);
+    name = name || file;
+
+    const ack: {[key: string]: any} = await this.startWriteProgram(
+      name,
+      fileStats.size,
+      slotid,
+      fileStats.birthtime.getTime(),
+      fileStats.mtime.getTime()
+    );
+
+    const blockSize = ack.blocksize;
+    const transferid = ack.transferid;
+
+    let dataStream = fs.createReadStream(file, { highWaterMark: blockSize });
+    for await (const data of dataStream) {
+      await this.writePackage(data, transferid);
+    }
+
+    if (autoStart) {
+      return Promise.resolve(await this.programExecute(slotid));
+    }
+  }
+
+  // ========================= Hub Methods =========================
+  //
+  // These methods each handle a single RPC method's communication.
+
+  /**
+   * Execute a program that is saved on the hub
+   *
+   * @param `slotid` Slot ID of the program to run
+   */
+  public async programExecute(slotid: number): Promise<RPCResponse> {
+    return new Promise(async (resolve) => {
+        let response = await this.send({
+          'm': 'program_execute',
+          'p': {
+            'slotid': slotid
+          }
+        });
+
+
+        resolve(response);
+      });
+  }
+
+  /**
+   * Terminate a currently running program
+   */
+  public async programTerminate(): Promise<RPCResponse> {
+    return Promise.resolve(
+      await this.send({ 'm': 'program_terminate' })
+    );
+  }
+
+  /**
+   * Get the storage status of the hub
+   *
+   * TODO: fill with actual example
+   * slot
+   * decoded name
+   * size
+   * last modified
+   * project_id
+   * type
+   */
+  public async getStorageStatus(): Promise<RPCResponse> {
+    return Promise.resolve(
+      await this.send({ 'm': 'get_storage_status' })
+    );
+  }
+
+  /**
+   * Notify the Hub that a program write is about to occur.
+   *
+   * @param `name` Name of the program.
+   * @param `name` Size of the program TODO: in bytes?
+   * @param `slotID` Slot ID to write the program to
+   * @param `created` Creation timestamp
+   * @param `modified` Modified timestamp
+   */
+  public async startWriteProgram(
+    name: string,
+    size: number,
+    slotID: number,
+    created: number,
+    modified: number
+  ): Promise<RPCResponse> {
+    // file meta data
+    let stat  = {
+      'created': created,
+      'modified': modified,
+      'name': name,
+      'type': 'python', // always python
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      'project_id': HubManager.randomID(16),
+    };
+
+    return Promise.resolve(
+      await this.send({
+        'm': 'start_write_program',
+        'p': {
+          'slotid': slotID,
+          'size': size,
+          'meta': stat
+        }
+      })
+    );
+  }
+
+  /**
+   * Write a package chunk of a file
+   *
+   * @param `data` Chunk of data to write
+   * @param `transferID` The transfer id for the current transfer
+   */
+  public async writePackage(data: string, transferID: string): Promise<RPCResponse> {
+    return Promise.resolve(
+      await this.send({
+        'm': 'write_package',
+        'p': {
+          'data': Buffer.from(data, 'utf-8').toString('base64'),
+          'transferid': transferID
+        }
+      })
+    );
+  }
+
+  /**
+   * Move a program from slot `fromSlot` to slot `toSlot`.
+   * TODO: verify
+   * If the destination already has a program stored in it, the two programs
+   * are swapped
+   *
+   * @param `fromSlot` slot to move from
+   * @param `toSlot` slot to move to
+   */
+  public async moveProgram(oldSlotID: number, newSlotID: number) {
+    return Promise.resolve(
+      await this.send({
+        'm': 'move_project',
+        'p': {
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          'old_slotid': oldSlotID,
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          'new_slotid': newSlotID
+        }
+      })
+    );
+  }
+
+  /**
+   * Delete a program from the slot on the Hub
+   *
+   * @param `slotID` the slot to delete
+   */
+  public async deleteProgram(slotID: number) {
+    return Promise.resolve(
+      await this.send({
+        'm': 'remove_project',
+        'p': {
+          'slotid': slotID
+        }
+      })
+    );
+  }
+
+  /**
+   * Get firmware info for the connected device
+   */
+  public async getHubInfo() {
+    return Promise.resolve(
+      await this.send({
+        'm': 'get_hub_info'
+      })
+    );
+  }
+
+  // ======================== INSTANCE METHODS ========================
+
+  public static async create(portPath: string): Promise<HubManager> {
+    return new Promise(async (resolve) => {
+      try {
+        let mgr = new HubManager(portPath);
+        await mgr.init();
+
+        return resolve(mgr);
+      } catch (err) {
+
+        // could not connect to port
+        throw err;
+      }
+
+    });
+  }
+
+  /**
+   * Returns a list of serial devices
+   */
+  public static async queryPorts() {
+    // get all ports
+    let ports = await SerialPort.list();
+
+    // filter by manufacturer
+    return Promise.resolve(ports);
+  }
+
+  /**
+   * Generate a random identifier consisting of lowercase
+   * letters, numbers, and underscores
+   *
+   * @param `len` The length of the string to generate
+   */
+  static randomID(len: number = 4): string {
+    let result = '';
+    let characters = 'abcdefghijklmnopqrstuvwxyz0123456789_';
+    for (let i = 0; i < len; i++) {
+      result += characters[Math.floor(Math.random() * characters.length)];
+    }
+
+    return result;
+  };
+}

src/log.ts

@@ -0,0 +1,23 @@
+import * as vscode from 'vscode';
+
+export default class Logger {
+  constructor(
+    public readonly outputChannel: vscode.OutputChannel
+  ) { }
+
+  public log(text: string): void {
+    this.outputChannel.appendLine(text);
+  }
+
+  public info(text: string): void {
+    this.outputChannel.appendLine('[INFO]\r' + text);
+  }
+
+  public warn(text: string): void {
+    this.outputChannel.appendLine('[WARNING]\r' + text);
+  }
+
+  public error(text: string): void {
+    this.outputChannel.appendLine('[ERROR]\r' + text);
+  }
+}