improvements

* chance to continue on mismatch url for redirectTab
* support empty field selectors
* add Extractor.results()
* add ExtractResult.walk(), ExtractResult.visit()
* add ! directive to click elements
* code optimize

readme.md

@@ -17,6 +17,8 @@ Extract current page
 ```js
 $('.item', ['a', 'a@href']);
 new Extractor().task('.item', ['a', 'a@href']).start();
+// fieldSelectors can be empty strings if items have no child to select
+new Extractor().task('.item a', ['', '@href']).start();
 ```
 
 > `$(...args)` is the short form of `new Extractor().task(...args).start();`, which is introduced later.
@@ -67,7 +69,9 @@ job = new Extractor().task('.search-list-item', ['a@href'], ["http://sample.com/
 job.stop();
 ```
 
-## Extract Attributes.
+> Next time you call `job.start();`, the job will continues from where it stopped.
+
+## Extract Attributes
 
 e.g.: link text and target (use 'selector@attribute')
 
@@ -75,6 +79,14 @@ e.g.: link text and target (use 'selector@attribute')
 new Extractor().task('.item', ['a', 'a@href']).start();
 ```
 
+## Click Selected Elements
+
+The following clicks selected links and extracts link `text` and `href`
+
+```js
+new Extractor().task('.item', ['!a', 'a@href']).start();
+```
+
 ## Advanced Usage
 
 ### Use Task Chain.
@@ -185,6 +197,43 @@ e.start();
 
 > The uploaded state will be cleaned in 30 seconds, if you don't load it.
 
+## Watch Mode
+
+Watch mode tries to exract data from every page you visit **in current window**.
+
+```js
+e = new Extractor();
+e.task('.search-list-item', ['a@href'], ["http://sample.com/abc"])
+    .task('list-item', ["a.title", "p.content"]);
+e.watch(1); // start watching for first task
+```
+
+To stop watching, you can either `close current window`, or:
+
+```js
+e.stop();
+```
+
+## Results Operation
+
+To get the results of a task:
+
+```js
+let results = job.results(0);
+```
+
+Visit URLs (if any) in the results one by one:
+
+```js
+results.visit();
+```
+
+Walk through all results one by one:
+
+```js
+results.walk((row,col,value)=>{console.log(value)});
+```
+
 ## Developpment
 
 Clone this project and execute:

src/background/actions.ts

@@ -1,6 +1,6 @@
 import { Actions, Request } from "../common";
 import { sendMessage, ResponseChecker } from "./messaging";
-import { logger } from "./logger";
+import { logger } from "../common/logger";
 
 /**
  * redirect tab to url.
@@ -8,25 +8,27 @@ import { logger } from "./logger";
  * @param {string} url target URL
  * @returns {Promise<string[]>} a promise of target URL
  */
-export function redirectTab(tab: chrome.tabs.Tab, url: string) {
+export function redirectTab(tab: chrome.tabs.Tab, url: string, check?: boolean) {
     return queryUrl(tab).then(u => {
         if (url !== u) {
             let req: Request = {
                 action: Actions.GOTO_URL,
                 url: url
             }
-            let checker: ResponseChecker<string> = async (r, err, tryCount): Promise<string> => {
+            let checker: ResponseChecker<string> = !check ? undefined : async (r, err, tryCount): Promise<string> => {
                 let queryErr: any;
                 let newURL = await queryUrl(tab).catch(e => queryErr = e);
                 if (queryErr) {
-                    return Promise.reject(queryErr);
+                    throw queryErr;
                 }
                 if (newURL == url) return url;
                 if (
-                    tryCount % 1 == 0 &&
-                    !confirm('Cannot navigate to target url. \nPress OK to continue, Cancel to stop.')
+                    confirm(`Cannot navigate to target url. 
+expected: ${url}\n
+actual:   ${newURL}\n
+Press OK to continue, Cancel to retry. Close the tab to stop`)
                 ) {
-                    return Promise.reject("Tasks stopped by user.");
+                    return newURL;
                 }
                 return undefined;
             }
@@ -82,7 +84,7 @@ export async function ping(tab, count = 1) {
     let checker: ResponseChecker<string> = (r, e, c) =>
         r.result == "pong" ? r.result : undefined;
 
-    let pong = await sendMessage<string>(tab, req, 'Check tab availability...', checker, 1000, count).catch(() => { });
+    let pong = await sendMessage<string>(tab, req, 'Check tab availability...', checker, 1000, 1000, count).catch(() => { });
     return pong == "pong";
 }
 
@@ -111,7 +113,7 @@ export function scrollToBottom(tab: chrome.tabs.Tab) {
     return sendMessage(tab, req, 'Scroll to page bottom...');
 }
 
-export async function createTab(url: string, active: boolean) {
+export async function createTab(url: string, active: boolean): Promise<chrome.tabs.Tab> {
     return new Promise((resolve, reject) => {
         findIncognitoWindow().then(
             incognitoWindow => {
@@ -179,7 +181,7 @@ export async function CreateIncognitoWindow() {
     });
 }
 
-export async function getActiveTab(currentWindow: boolean) {
+export async function getActiveTab(currentWindow: boolean): Promise<chrome.tabs.Tab> {
     return new Promise((resolve, reject) => {
         chrome.tabs.query({
             active: true,
@@ -190,7 +192,7 @@ export async function getActiveTab(currentWindow: boolean) {
     })
 }
 
-export async function getTabByID(id: number) {
+export async function getTabByID(id: number): Promise<chrome.tabs.Tab> {
     return new Promise((resolve, reject) => {
         chrome.tabs.get(id, function (tab) {
             chrome.runtime.lastError;

src/background/caches.ts

@@ -1,4 +1,4 @@
-import { logger } from "./logger";
+import { logger } from "../common/logger";
 import { Actions } from "../common";
 import { messageSubscribers } from "./messaging";
 

src/background/extractor.ts

@@ -1,9 +1,9 @@
 import { Task } from "./task";
-import { saveFile } from "./tools";
-import { createTab, getActiveTab, ping } from "./actions";
-import { ExtractResult } from "./result";
-import { logger } from "./logger";
+import { parseUrls, saveFile } from "./tools";
+import { createTab, getActiveTab, ping, redirectTab } from "./actions";
+import { logger } from "../common/logger";
 import { caches } from "./caches";
+import { ExtractResult } from "./result";
 
 export class Extractor {
     private _tasks: Task[] = [];
@@ -12,6 +12,14 @@ export class Extractor {
     constructor(options?) {
         if (options) this._options = options;
     }
+    static async ping(count: number = 1) {
+        let tab = await getActiveTab(true) || await getActiveTab(false);
+        let succ = await ping(tab, count);
+        if (!succ) {
+            logger.error('Cannot contact with active tab.');
+            return;
+        }
+    }
     /**
      * Save current state, in case we restore it later.
      */
@@ -42,6 +50,17 @@ export class Extractor {
         this._tasks.push(new Task(this._options, ...args));
         return this;
     }
+    /**
+     * Add a task to Extractor. \n
+     * One Extractor could has multiple tasks, which orgnized in a task chian.
+     * If url arguments not given within later tasks, they will use previous task result as input (target url list).
+     * @param {...any} args itemsSelector, fieldSelectors, and more args to specify target urls.
+     */
+    results(id?: number): ExtractResult {
+        id = this._checkTaskId(id);
+        if (id < 0) return;
+        return this._tasks[id].results;
+    }
     /**
      * Clear tasks and task caches.
      */
@@ -113,7 +132,7 @@ export class Extractor {
                     if (i < from) return;
                     if (i > 0) {
                         let prevTask = this._tasks[i - 1];
-                        return task.execute(tab, new ExtractResult(prevTask.results));
+                        return task.execute(tab, prevTask.results);
                     }
                     return task.execute(tab);
                 });
@@ -135,22 +154,22 @@ export class Extractor {
         let id = this._checkTaskId(taskid, this._tasks.length - 1);
         if (id < 0) return;
         let results = this._tasks[id].results
-        if (!results.length) {
+        let count = results.data.length
+        if (!count) {
             logger.info(`No result for task #${id}. Forget to call ".start()"?`);
             return;
         }
-        results.unshift(this._tasks[id].fieldSelectors);
-        let exResults = new ExtractResult(results);
+        results.header = this._tasks[id].fieldSelectors;
         let msg = `
-Please confirm to download (${results.length - 1} items)：
+Please confirm to download (${count} items)：
 
-${exResults.toString(50) || "- Empty -"}
+${results.toString(50) || "- Empty -"}
 `.trim();
         if (confirm(msg)) {
-            saveFile(exResults.toString(), "text/csv");
+            saveFile(results.toString(), "text/csv");
         }
     }
-    _checkTaskId(id: number, defaultId?: number) {
+    private _checkTaskId(id: number, defaultId?: number) {
         if (!this._tasks.length) {
             logger.info("No task found.");
             return -1;

src/background/messaging.ts

@@ -1,6 +1,6 @@
 import { Request, Actions, Response } from "../common";
 import { getTabByID } from "./actions";
-import { logger } from "./logger";
+import { logger } from "../common/logger";
 
 
 export type ResponseCheckerSync<T> = (r: Response<T>, err: chrome.runtime.LastError, count: number) => T;
@@ -21,12 +21,14 @@ export type ResponseChecker<T> = ResponseCheckerSync<T> | ResponseCheckerAsync<T
  */
 export function sendMessage<T>(
     tab: chrome.tabs.Tab,
-    req,
+    req: Request,
     log?: string,
     dataChecker?: ResponseChecker<T>,
+    timeout?: number,
     interval?: number,
     limit?: number
 ) {
+    timeout = timeout || 10;
     interval = interval || 500;
     limit = isNaN(limit) ? 0 : limit;
     let count = 0;
@@ -47,43 +49,23 @@ export function sendMessage<T>(
                 return;
             }
             count++;
+            let timeout = setTimeout(() => { reject(`${Actions[req.action]} requset timeout after ${timeout}s`) }, 10000);
             chrome.tabs.sendMessage(tab.id, req, async (r: Response<T>) => {
+                clearTimeout(timeout);
                 // check error but do nothing until dataChecker.
                 let err = chrome.runtime.lastError;
-                let result: T;
-                // r could be undefined if the content script is interrupted.
-                if (r) {
-                    result = r.result;
-
-                    if (dataChecker) {
-                        let pms: T | Promise<T>;
-                        try {
-                            pms = dataChecker(r, err, count);
-                        } catch (error) {
-                            reject(error);
-                            return;
-                        }
-                        // don't catch if it's not a Promise
-                        if (pms instanceof Promise) {
-                            let checkerError: any;
-                            pms = pms.catch(e => checkerError = e);
-                            result = await pms;
-                            if (checkerError) {
-                                reject(checkerError);
-                                return;
-                            }
-                        } else {
-                            result = pms;
-                        }
-                    }
+                let [result, error] = await checkResponse(dataChecker, r, err, count);
+                if (error) {
+                    reject(error);
+                    return;
                 }
-
-                let flag = result !== undefined && result !== null;
+                let flag = result !== undefined;
                 if (log) logger.info(log, flag ? '(OK)' : '(failed)');
                 if (flag) {
                     resolve(result);
                 } else {
                     setTimeout(() => {
+                        logger.debug('Invalid response', r, 'retry...');
                         loop();
                     }, interval);
                 }
@@ -92,6 +74,39 @@ export function sendMessage<T>(
     });
 }
 
+async function checkResponse<T>(
+    dataChecker: ResponseChecker<T>,
+    response: Response<T>,
+    error: chrome.runtime.LastError,
+    tryCount: number
+): Promise<[T, string]> {
+    // response could be undefined if the content script is interrupted.
+    // don't check, tell sendMessage to retry.
+    if (!response) return [undefined, undefined];
+    if (!dataChecker) {
+        return [response.result, response.error];
+    }
+    let result: T;
+    let pms: T | Promise<T>;
+    try {
+        pms = dataChecker(response, error, tryCount);
+    } catch (err) {
+        return [undefined, err];
+    }
+    // don't catch if it's not a Promise
+    if (pms instanceof Promise) {
+        let checkerError: any;
+        pms = pms.catch(e => checkerError = e);
+        result = await pms;
+        if (checkerError) {
+            return [undefined, checkerError];
+        }
+    } else {
+        result = pms;
+    }
+    return [result, undefined];
+}
+
 export type ActionSubscriberSync = (request: Request, sender: chrome.runtime.MessageSender, sendResponse: (response?: any) => void) => void;
 export type ActionSubscriberAsync = (request: Request, sender: chrome.runtime.MessageSender, sendResponse: (response?: any) => void) => Promise<void>;
 export type ActionSubscriber = ActionSubscriberSync | ActionSubscriberAsync;

src/background/result.ts

@@ -1,6 +1,11 @@
+import { logger } from "../common/logger";
+import { getActiveTab, ping, redirectTab } from "./actions";
+import { parseUrls } from "./tools";
+
 export class ExtractResult {
+    private _header: string[];
     private _data: string[][] = [];
-    constructor(data) {
+    constructor(data: string[][]) {
         this._data = data || [];
     }
     row(index: number): string[] {
@@ -14,11 +19,17 @@ export class ExtractResult {
     squash(): string[] {
         return this._data.reduce((p, c) => p.concat(c), []);
     }
+    set header(h: string[]) {
+        this._header = h
+    }
     get data(): string[][] {
         return this._data;
     }
     toString(rowsCount: number = 0): string {
         let data = rowsCount > 0 ? this._data.slice(0, rowsCount) : this._data;
+        if (this._header && this._header.length) {
+            data.unshift(this._header);
+        }
         return data.slice().reduce(
             (csv, lineCells) => {
                 if (!lineCells || !lineCells.length) {
@@ -26,6 +37,7 @@ export class ExtractResult {
                 }
                 let line = lineCells.reduce(
                     (lineText, cell, idx) => {
+                        cell = cell || "";
                         cell = '"' + cell.trim().replace(/"/g, '""') + '"';
                         return lineText + cell + (idx == lineCells.length - 1 ? "" : ",")
                     }, "");
@@ -34,4 +46,40 @@ export class ExtractResult {
             ""
         );
     }
+    async walk(fn: (row: number, col: number, value: string) => void) {
+        let pms = Promise.resolve(null);
+        for (let i = 0; i < this._data.length; i++) {
+            let cells = this._data[i];
+            for (let j = 0; j < cells.length; j++) {
+                let row = i;
+                let col = j;
+                let value = cells[j];
+                pms = pms.then(
+                    () => fn(row, col, value)
+                )
+            }
+        }
+        return pms.catch(err => {
+            logger.error(err);
+        });
+    }
+    async visit() {
+        let urls = parseUrls(this);
+        let tab = await getActiveTab(true) || await getActiveTab(false);
+        let succ = await ping(tab);
+        if (!succ) {
+            logger.error('Cannot contact with active tab.');
+            return;
+        }
+        return urls.reduce(
+            (pms, url: string, i: number) => {
+                return pms.then(
+                    async () => {
+                        return redirectTab(tab, url, false);
+                    });
+            }, Promise.resolve<void>(undefined)
+        ).catch(err => {
+            logger.error(err);
+        });
+    }
 }

src/background/signiture.ts

@@ -20,7 +20,7 @@ function(itemsSelector:string, fieldSelectors:string[], urls:string[]);
 $(".item", ["a", "a@href"]);
 
 ## See Detailed Help:
-https://git.jebbs.co/jebbs/data-extracter-extesion
+https://git.qjebbs.com/jebbs/data-extracter-extesion
 `.trim();
 
 export function testArgs(...args: any) {

src/background/task.ts

@@ -4,7 +4,7 @@ import { testArgs, signitures } from "./signiture";
 import { ExtractResult } from "./result";
 import { messageSubscribers, ActionSubscriber } from "./messaging";
 import { Actions } from "../common";
-import { logger } from "./logger";
+import { logger } from "../common/logger";
 
 export class Task {
     private _data: { [key: string]: string[][] } = {};
@@ -41,10 +41,11 @@ export class Task {
     get urls(): string[] {
         return this._urls;
     }
-    get results(): string[][] {
-        return this._data_keys.reduce((p, c) => {
+    get results(): ExtractResult {
+        let rs: string[][] = this._data_keys.reduce((p, c) => {
             return p.concat(this._data[c]);
         }, []);
+        return new ExtractResult(rs);
     }
     get fieldSelectors(): string[] {
         return this._fieldSelectors;
@@ -87,7 +88,7 @@ export class Task {
             logger.info(`Watcher #${taskID} starts.`);
             let pm = this.makeOptionalTasks(sender.tab);
             return pm.then(
-                () => extractTabData(sender.tab, this._itemsSelector, this._fieldSelectors, sender.tab.url, false)
+                () => extractTabData(sender.tab, this._itemsSelector, this._fieldSelectors, sender.tab.url, true)
             ).then(
                 results => {
                     if (results && results.length) {
@@ -103,15 +104,26 @@ export class Task {
         messageSubscribers.addListener(Actions.REPORT_NEW_PAGE, listener);
     }
     async execute(tab: chrome.tabs.Tab, upstreamData?: ExtractResult): Promise<void> {
-        if (!tab) return Promise.reject("No tab to execute the task.");
-        if (this._running) return Promise.reject("The task is running. Please wait...");
+        if (!tab) throw "No tab to execute the task.";
+        if (this._running) throw "The task is running. Please wait...";
         this._running = true;
         let urls = this._urls
         if (!urls.length) {
             if (upstreamData) {
                 urls = parseUrls(upstreamData);
             } else {
-                urls = [await queryUrl(tab)];
+                let tabURL: string;
+                await queryUrl(tab)
+                    .then(u => {
+                        tabURL = u;
+                    })
+                    .catch(() => {
+                        e => {
+                            this._running = false;
+                            return Promise.reject(e);
+                        }
+                    });
+                urls = [tabURL];
             }
         }
         return urls.reduce((p, url, i) => p.then(
@@ -134,13 +146,13 @@ export class Task {
                 if (results && results.length) {
                     let lastURL = urls[urls.length - 1];
                     this.saveResult(results, lastURL);
-                    this._running = false;
                 }
+                this._running = false;
             }
         ).catch(
             e => {
                 this._running = false;
-                return Promise.reject(e);
+                throw e;
             }
         );
     }
@@ -152,7 +164,7 @@ export class Task {
         return pm;
     }
     private runningCheck(fn: () => Promise<any>): Promise<any> {
-        if (!this._running) return Promise.reject("The task is stopped by user.");
+        if (!this._running) throw "The task is stopped by user.";
         return fn();
     }
     private saveResult(results, key) {

src/background/tools.ts

@@ -2,7 +2,7 @@ import { ExtractResult } from "./result";
 
 const URL_REG = /^\s*(https?):\/\//im;
 
-export function parseUrls(...args) {
+export function parseUrls(...args): string[] {
     if (!args.length) return [];
     let arg = args.shift();
     if (arg instanceof Array) {

src/common/logger.ts

@@ -13,7 +13,7 @@ export class Logger {
     constructor(logLevel, notifyLevel) {
         if (logLevel) this._log_level = logLevel;
         if (notifyLevel) this._notify_level = notifyLevel;
-        chrome.notifications.onClosed.addListener((id, byUser) => { this._notify_level = undefined });
+        if (chrome.notifications) chrome.notifications.onClosed.addListener((id, byUser) => { this._notify_level = undefined });
     }
     get logLevel() {
         return this._log_level;

src/content/actions.ts

@@ -1,3 +1,5 @@
+import { logger } from "../common/logger";
+
 export function extract(itemsSelector: string, fieldSelectors: string[], expectedURL: string): string[][] {
     if (expectedURL && location.href != expectedURL) {
         throw 'Target tab URL changed, aborting...';
@@ -14,19 +16,42 @@ export function extract(itemsSelector: string, fieldSelectors: string[], expecte
         item => {
             return fieldSelectors.map(
                 selector => {
+                    let doClick = false;
+                    if (selector.startsWith("!")) {
+                        doClick = true;
+                        selector = selector.substring(1);
+                    }
                     let [cls, attr] = selector.split('@').slice(0, 2);
-                    let fieldVals = Array.from(item.querySelectorAll(cls));
-                    if (!fieldVals.length) {
+                    let fieldElements: Element[];
+                    cls = cls.trim()
+                    if (cls != "") {
+                        fieldElements = Array.from(item.querySelectorAll(cls));
+                    } else {
+                        fieldElements = [item];
+                    }
+                    if (!fieldElements.length) {
                         return;
                     }
                     fieldFound[selector] = true;
-                    return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
+                    return fieldElements.map(find => {
+                        if (doClick) {
+                            let e = document.createEvent("MouseEvents");
+                            e.initEvent("click", true, true);
+                            find.dispatchEvent(e);
+                        }
+                        return attr ? find[attr] : find.textContent.trim();
+                    }).join('\n')
                 }
             )
         }
     );
+    // TODO: configurable wait logic
     // if it exists a field, which is not found in any row, the sender should retry.
-    let shouldWait = fieldSelectors.reduce((p, c) => p || !fieldFound[c], false);
+    let notFoundFields = fieldSelectors.filter(f => !fieldFound[f]);
+    let shouldWait = notFoundFields.length > 0;
+    if (shouldWait) {
+        logger.debug('should wait for:', fieldSelectors.filter(f => !fieldFound[f]).join(','));
+    }
     return shouldWait ? [] : results;
 }
 

src/popup/index.ts

@@ -10,7 +10,7 @@ window.onload = function () {
     document.querySelector('#link-document')
         .addEventListener('click', () => {
             chrome.tabs.create({
-                'url': `https://git.jebbs.co/jebbs/data-extracter-extesion`
+                'url': `https://git.qjebbs.com/jebbs/data-extracter-extesion`
             });
         })
     document.querySelector('#state-input')

template/html/popup.html

@@ -52,7 +52,7 @@
                     <p>
                         <b>Full document at:</b>
                         <br>
-                        <a href="#" id="link-document">https://git.jebbs.co/jebbs/data-extracter-extesion</a>
+                        <a href="#" id="link-document">https://git.qjebbs.com/jebbs/data-extracter-extesion</a>
                     </p>
                 </div>
             </div>