Extractor.watch()

src/background/caches.ts

@@ -1,8 +1,15 @@
 import { logger } from "./common";
+import { Actions } from "../common";
+import { messageSubscribers } from "./messaging";
 
 export class Caches {
     private _state: string = "";
-    constructor() { }
+    constructor() {
+        messageSubscribers.addListener(Actions.UPLOAD_STATE, (request, sender, sendResponse) => {
+            sendResponse('recieved!');
+            this.setState(request.fileName, request.state)
+        });
+    }
     get state(): string {
         let s = this._state;
         this._state = "";

src/background/extractor.ts

@@ -54,11 +54,22 @@ export class Extractor {
     start() {
         return this._startTasks(0);
     }
-    stop() {
+    stop(id?: number) {
+        if (id !== undefined) {
+            id = this._checkTaskId(id);
+            if (id < 0) return;
+            this._tasks[id].stop();
+            return;
+        }
         for (let i = 0; i < this._tasks.length; i++) {
             this._tasks[i].stop();
         }
     }
+    watch(id: number) {
+        id = this._checkTaskId(id);
+        if (id < 0) return;
+        this._tasks[id].watch();
+    }
     /**
      * restart from specified task, but don't restart the previous tasks.
      * @param {number} from where to restart the tasks, begins with 0
@@ -138,7 +149,7 @@ ${exResults.toString(50) || "- Empty -"}
             saveFile(exResults.toString(), "text/csv");
         }
     }
-    _checkTaskId(id: number, defaultId: number) {
+    _checkTaskId(id: number, defaultId?: number) {
         if (!this._tasks.length) {
             logger.info("No task found.");
             return -1;

src/background/messaging.ts

@@ -1,6 +1,6 @@
 import { Request, Actions } from "../common";
 import { getTabByID } from "./actions";
-import { caches, logger } from "./common";
+import { logger } from "./common";
 
 /**
  *  Sending a message to target tab repeatedly until the response is not undefined.
@@ -78,14 +78,41 @@ export function sendMessage<T>(
     });
 }
 
-chrome.runtime.onMessage.addListener(function (request: Request, sender, sendResponse) {
+export type ActionSubscriber = (request: Request, sender: chrome.runtime.MessageSender, sendResponse: (response?: any) => void) => void | Promise<void>;
-    switch (request.action) {
+class MessageSubscribers {
-        case Actions.UPLOAD_STATE:
+    private listeners: { [key: number]: ActionSubscriber[] } = {};
-            sendResponse('recieved!');
+    addListener(action: Actions, subscriber: ActionSubscriber) {
-            caches.setState(request.fileName, request.state)
+        this.listeners[action] || (this.listeners[action] = []);
-            break;
+        this.listeners[action].push(subscriber);
-        default:
-            sendResponse("Request not supported.");
-            break;
     }
+    removeListener(action: Actions, subscriber: ActionSubscriber) {
+        this.listeners[action] || (this.listeners[action] = []);
+        for (let i = 0; i < this.listeners[action].length; i++) {
+            if (this.listeners[action][i] == subscriber) {
+                this.listeners[action].splice(i, 1);
+                i--;
+            }
+        }
+        logger.debug(`${this.listeners[action].length} subscriber(s) remained for action ${Actions[action]}`);
+    }
+    getListeners(action: Actions): ActionSubscriber[] {
+        return this.listeners[action]
+    }
+}
+export const messageSubscribers = new MessageSubscribers();
+
+chrome.runtime.onMessage.addListener(function (request: Request, sender, sendResponse) {
+    let subscribers = messageSubscribers.getListeners(request.action);
+    if (!subscribers || !subscribers.length) {
+        sendResponse("Request not supported.");
+        return;
+    }
+    let promises: Promise<any>[] = [];
+    for (let subscriber of subscribers) {
+        let p = subscriber(request, sender, sendResponse);
+        if (p instanceof Promise) promises.push(p);
+    }
+    if (promises.length)
+        return Promise.all(promises);
+    return;
 });

src/background/task.ts

@@ -2,6 +2,9 @@ import { parseUrls } from "./tools";
 import { queryUrl, redirectTab, scrollToBottom, extractTabData } from "./actions";
 import { testArgs, signitures } from "./signiture";
 import { ExtractResult } from "./result";
+import { messageSubscribers, ActionSubscriber } from "./messaging";
+import { Actions } from "../common";
+import { logger } from "./common";
 
 export class Task {
     private _data: { [key: string]: string[][] } = {};
@@ -46,12 +49,22 @@ export class Task {
         return this._fieldSelectors;
     }
     clean(): Task {
+        this.stop();
         this._data = {};
         this._data_keys = [];
         return this;
     }
     stop() {
         this._running = false;
+        messageSubscribers.removeListener(Actions.REPORT_NEW_PAGE, this.listener);
+    }
+    watch() {
+        if (this._running) {
+            logger.info("The task is running. Please wait...");
+            return;
+        }
+        this._running = true;
+        messageSubscribers.addListener(Actions.REPORT_NEW_PAGE, this.listener);
     }
     async execute(tab: chrome.tabs.Tab, upstreamData?: ExtractResult): Promise<void> {
         if (!tab) return Promise.reject("No tab to execute the task.");
@@ -65,35 +78,26 @@ export class Task {
                 urls = [await queryUrl(tab)];
             }
         }
-        let saveResult = (results, key) => {
-            this._data[key] = results;
-            this._data_keys.push(key);
-        }
-        let runningCheck = (fn: () => Promise<any>): Promise<any> => {
-            if (!this._running) return Promise.reject("The task is stopped by user.");
-            return fn();
-        }
         return urls.reduce((p, url, i) => p.then(
             results => {
                 if (i > 0 && results instanceof Array) {
                     let lastURL = urls[i - 1];
-                    saveResult(results, lastURL);
+                    this.saveResult(results, lastURL);
                 }
                 if (this._data[url]) return;
 
-                let pms: Promise<any> = runningCheck(() => redirectTab(tab, url));
+                let pms: Promise<any> = this.runningCheck(() => redirectTab(tab, url));
-                if (this._options["scrollToBottom"]) {
+                return pms
-                    pms = pms.then(() => runningCheck(() => scrollToBottom(tab)));
+                    .then(() => this.makeOptionalTasks(tab))
-                }
+                    .then(
-                return pms.then(
+                        () => this.runningCheck(() => extractTabData(tab, this._itemsSelector, this._fieldSelectors))
-                    () => runningCheck(() => extractTabData(tab, this._itemsSelector, this._fieldSelectors))
+                    );
-                );
             }
         ), Promise.resolve<string[][]>(null)).then(
             results => {
                 if (results && results.length) {
                     let lastURL = urls[urls.length - 1];
-                    saveResult(results, lastURL);
+                    this.saveResult(results, lastURL);
                     this._running = false;
                 }
             }
@@ -104,4 +108,38 @@ export class Task {
             }
         );
     }
+    private listener: ActionSubscriber = (request, sender, sendResponse) => {
+        sendResponse('recieved!');
+        let pm = this.makeOptionalTasks(sender.tab);
+        return pm.then(
+            () => extractTabData(sender.tab, this._itemsSelector, this._fieldSelectors)
+        ).then(
+            results => {
+                if (results && results.length) {
+                    this.saveResult(results, sender.tab.url);
+                }
+            }
+        ).catch(
+            e => logger.error(e)
+        )
+    }
+    private makeOptionalTasks(tab: chrome.tabs.Tab): Promise<any> {
+        let pm: Promise<any>;
+        if (this._options["scrollToBottom"]) {
+            pm = this.runningCheck(() => scrollToBottom(tab));
+        }
+        return pm;
+    }
+    private runningCheck(fn: () => Promise<any>): Promise<any> {
+        if (!this._running) return Promise.reject("The task is stopped by user.");
+        return fn();
+    }
+    private saveResult(results, key) {
+        if (this._data[key] === undefined) {
+            // do not add keys again
+            this._data_keys.push(key);
+        }
+        this._data[key] = results;
+        logger.info(`${results.length} items found.`)
+    }
 }

src/common.ts

@@ -1,12 +1,16 @@
 export enum Actions {
+    // from background to content script
     EXTRACT = 1,
     GOTO_URL,
     PING,
     QUERY_URL,
     SCROLL_BOTTOM,
-    UPLOAD_STATE,
     SLEEP,
     WAKEUP,
+    // from popup to background script
+    UPLOAD_STATE,
+    // from content to background script
+    REPORT_NEW_PAGE,
 }
 
 export interface Request {

src/content/index.ts

@@ -16,6 +16,10 @@ chrome.runtime.onMessage.addListener(
     }
 );
 
+chrome.runtime.sendMessage(<Request>{
+    action: Actions.REPORT_NEW_PAGE,
+});
+
 async function doAction(request: Request, sender: chrome.runtime.MessageSender) {
     switch (request.action) {
         case Actions.EXTRACT: