migrate to typescript, with fixes

.gitignore

@@ -1,2 +1,154 @@
+# File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig
+
+# Created by https://www.gitignore.io/api/visualstudiocode,macos,node
+# Edit at https://www.gitignore.io/?templates=visualstudiocode,macos,node
+
+### macOS ###
+# General
 .DS_Store
-Thumbs.db
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build output
+.nuxt
+
+# rollup.js default build output
+dist/
+
+# Uncomment the public line if your project uses Gatsby
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# https://create-react-app.dev/docs/using-the-public-folder/#docsNav
+# public
+
+# Storybook build outputs
+.out
+.storybook-out
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# Temporary folders
+tmp/
+temp/
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+
+# End of https://www.gitignore.io/api/visualstudiocode,macos,node
+
+# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
+

manifest.json

@@ -1,45 +0,0 @@
-{
-  "manifest_version": 2,
-  "name": "Data Extracter",
-  "version": "0.5.0",
-  "author": "jebbs",
-  "description": "Extract data from web page elements as sheet.",
-  "icons": {
-    "16": "icon.png",
-    "48": "icon.png",
-    "128": "icon.png"
-  },
-  "browser_action": {
-    "default_icon": "icon.png",
-    "default_popup": "popup/tip.html",
-    "default_title": "Data Extracter"
-  },
-  "background": {
-    "scripts": [
-      "scripts/shared/tools.js",
-      "scripts/shared/common.js",
-      "scripts/background/logger.js",
-      "scripts/background/messaging.js",
-      "scripts/background/result.js",
-      "scripts/background/signiture.js",
-      "scripts/background/actions.js",
-      "scripts/background/task.js",
-      "scripts/background/extractor.js",
-      "scripts/background/helpers.js"
-    ],
-    "persistent": false
-  },
-  "content_scripts": [{
-    "matches": ["*://*/*"],
-    "js": [
-      "scripts/shared/tools.js",
-      "scripts/shared/common.js",
-      "scripts/content/content.js"
-    ],
-    "run_at": "document_idle"
-  }],
-  "permissions": [
-    "activeTab",
-    "notifications"
-  ]
-}

package.json

@@ -0,0 +1,17 @@
+{
+    "name": "data-extractor",
+    "scripts": {
+        "dev": "webpack --mode=development --devtool=inline-source-map --watch",
+        "prod": "webpack --mode=production"
+    },
+    "devDependencies": {
+        "@types/chrome": "0.0.91",
+        "@types/node": "^13.1.6",
+        "copy-webpack-plugin": "^5.1.1",
+        "ts-loader": "^6.2.1",
+        "tslint": "^5.20.1",
+        "typescript": "^3.7.4",
+        "webpack": "^4.41.5",
+        "webpack-cli": "^3.3.10"
+    }
+}

readme.md

@@ -170,4 +170,15 @@ Open the popup window, upload the saved state file. Then, and in the backgoud co
 
 ```js
 e = new Extractor().load();
+```
+
+## Developpment
+
+Clone this project and execute:
+
+```sh
+npm i
+npm run prod
+# or
+npm run dev
 ```

scripts/background/helpers.js

@@ -1,3 +0,0 @@
-function $(...args) {
-    return new Extractor().task(...args).start();
-}

scripts/content/content.js

@@ -1,114 +0,0 @@
-(function () {
-    let asleep = false;
-    chrome.runtime.onMessage.addListener(
-        function (request, sender, sendResponse) {
-            if (!request.action) return;
-            if (asleep && ACTION_WAKEUP != request.action) {
-                sendResponse && sendResponse(undefined);
-                return;
-            }
-            // console.log("Recieved request:",request);
-            doAction(request, sender).then(r => sendResponse && sendResponse(r));
-            // return true to indicate you wish to send a response asynchronously
-            return true;
-        }
-    );
-
-    async function doAction(request, sender) {
-        switch (request.action) {
-            case ACTION_EXTRACT:
-                let data = extract(request.itemsSelector, request.fieldSelectors);
-                return data;
-            case ACTION_GOTO_URL:
-                window.location.replace(request.url);
-                // should not recieve any request until the page & script reload
-                asleep = true;
-                return request.url;
-            case ACTION_REPORT_IN:
-                return request.action;
-            case ACTION_QUERY_URL:
-                return window.location.href;
-            case ACTION_SCROLL_BOTTOM:
-                return executeUntil(
-                    () => window.scrollTo(0, document.body.clientHeight),
-                    () => document.body.clientHeight - window.scrollY - window.innerHeight < 20,
-                    "Scroll to page bottom...",
-                    1000,
-                    10
-                )
-            case ACTION_SLEEP:
-                asleep = true;
-                return "Content script is sleeping.";
-            case ACTION_WAKEUP:
-                asleep = false;
-                return "Content script is available.";
-            default:
-                break;
-        }
-    }
-
-    function extract(itemsSelector, fieldSelectors) {
-        // since some elements may be loaded asynchronously.
-        // if one field is never found, we should return undefined,
-        // so that senders can detect to retry until elements loaded.
-        // If user writes wrong selectors, the task retries infinitely.
-        let fieldFound = {};
-        let items = Array.from(document.querySelectorAll(itemsSelector));
-        // items may not loaded yet, tell the sender to retry.
-        if (!items.length) return MSG_ELEMENT_NOT_FOUND;
-        let results = items.map(
-            item => {
-                return fieldSelectors.map(
-                    selector => {
-                        let [cls, attr] = selector.split('@').slice(0, 2);
-                        let fieldVals = Array.from(item.querySelectorAll(cls));
-                        if (!fieldVals.length) {
-                            return;
-                        }
-                        fieldFound[selector] = true;
-                        return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
-                    }
-                )
-            }
-        );
-        // if it exists a field, which is not found in any row, the sender should retry.
-        let shouldWait = fieldSelectors.reduce((p, c) => p || !fieldFound[c], false);
-        return shouldWait ? MSG_ELEMENT_NOT_FOUND : results
-    }
-
-    /**
-     * Repeatedly execute an function until the the detector returns true.
-     * @param {object} fn the function to execute
-     * @param {object} detector the detector.
-     * @param {string} log messages logged to console.
-     * @param {number} interval interval for detecting
-     * @param {number} limit max execute times of a function
-     * @return {Promise} a promise of the response.
-     */
-    function executeUntil(fn, detector, log, interval, limit) {
-        interval = interval || 500;
-        let count = 0;
-        return new Promise((resolve, reject) => {
-
-            loop();
-
-            async function loop() {
-                fn();
-                limit++;
-                if (limit && count >= limit) {
-                    reject(false);
-                }
-                setTimeout(() => {
-                    let flag = !detector || detector();
-                    if (log) console.log(log, flag ? '(OK)' : '(failed)');
-                    if (flag) {
-                        resolve(true);
-                    } else {
-                        loop();
-                    }
-                }, interval);
-            }
-        });
-    }
-
-})();

scripts/shared/common.js

@@ -1,10 +0,0 @@
-const EXT_NAME = "DataExtracter";
-
-const ACTION_EXTRACT = `${EXT_NAME}:Extract`;
-const ACTION_GOTO_URL = `${EXT_NAME}:GoToTUL`;
-const ACTION_REPORT_IN = `${EXT_NAME}:ReportIn`;
-const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
-const ACTION_SCROLL_BOTTOM = `${EXT_NAME}:ScrollToBottom`;
-const ACTION_UPLOAD_STATE = `${EXT_NAME}:UploadStateFile`;
-const ACTION_SLEEP = `${EXT_NAME}:Sleep`;
-const ACTION_WAKEUP = `${EXT_NAME}:WakeUp`;

src/background/actions.ts

@@ -1,49 +1,35 @@
-function parseUrls(...args) {
+import { ACTION_GOTO_URL, ACTION_EXTRACT, ACTION_PING as ACTION_PING, ACTION_QUERY_URL, ACTION_SCROLL_BOTTOM } from "../common";
-    if (!args.length) return [];
+import { sendMessage } from "./messaging";
-    let arg = args.shift();
-    if (arg instanceof Array) {
-        return arg;
-    } else if (arg instanceof ExtractResult) {
-        return arg.squash().filter(v => URL_REG.test(v));
-    } else {
-        let urlTempl = arg;
-        if (urlTempl) {
-            if (args[0] instanceof Array) {
-                return args[0].map(p => urlTempl.replace("${page}", p));
-            } else if (args.length >= 3) {
-                let urls = [];
-                let from = args.shift();
-                let to = args.shift();
-                let interval = args.shift();
-                for (let i = from; i <= to; i += interval) {
-                    urls.push(urlTempl.replace("${page}", i));
-                }
-                return urls;
-            }
-        }
-    }
-    return [];
-}
 
-function redirectTab(tab, url) {
+/**
+ * redirect tab to url.
+ * @param {any} tab target tab
+ * @param {string} url target URL
+ * @returns {Promise<string[]>} a promise of target URL
+ */
+export function redirectTab(tab: chrome.tabs.Tab, url: string) {
     return queryUrl(tab).then(u => {
         if (url !== u) {
             let req = {
                 action: ACTION_GOTO_URL,
                 url: url
             }
-            let checker = async (url, err, tryCount) => {
+            let checker = async (u, err, tryCount): Promise<string> => {
-                let newURL = await queryUrl(tab).catch(() => { });
+                let queryErr: any;
+                let newURL = await queryUrl(tab).catch(e => queryErr = e);
+                if (queryErr) {
+                    return Promise.reject(queryErr);
+                }
                 if (newURL == url) return url;
                 if (
                     tryCount % 5 == 0 &&
                     !confirm('Cannot navigate to target url. \nPress OK to continue, Cancel to stop.')
                 ) {
-                    return MSG_USER_ABORT;
+                    return Promise.reject("Tasks stopped by user.");
                 }
                 return undefined;
             }
-            return sendMessage(tab, req, `Goto url: ${url}`, checker);
+            return sendMessage<string>(tab, req, `Goto url: ${url}`, checker);
         }
     });
 }
@@ -55,25 +41,23 @@ function redirectTab(tab, url) {
  * @param {Array<string>} fieldSelectors fields selectors for selecting fields (data columns) under each item
  * @returns {Promise<string[]>} a promise of extracted data
  */
-function extractTabData(tab, itemsSelector, fieldSelectors) {
+export function extractTabData(tab, itemsSelector, fieldSelectors) {
     let req = {
         action: ACTION_EXTRACT,
         itemsSelector: itemsSelector,
         fieldSelectors: fieldSelectors
     }
     let checker = (result, err, tryCount) => {
-        if (MSG_ELEMENT_NOT_FOUND.isEqual(result)) {
+        if (!result || !result.length) {
-            if (tryCount % 20 == 0) {
+            if (tryCount % 20 == 0 && confirm('No data found in current page. \n\nContinue to next page?')) {
-                if (confirm('No data found in current page. \n\nContinue to next page?')) {
+                return [];
-                    return [];
-                }
             } else {
                 return undefined;
             }
         }
         return result;
     };
-    return sendMessage(tab, req, 'Extract data from the tab...', checker);
+    return sendMessage<string[][]>(tab, req, 'Extract data from the tab...', checker);
 }
 
 /**
@@ -81,13 +65,13 @@ function extractTabData(tab, itemsSelector, fieldSelectors) {
  * @param {any} tab target tab
  * @returns {Promise<boolean>} a promise of boolean value indicates if ping success
  */
-async function ping(tab, count = 1) {
+export async function ping(tab, count = 1) {
     let req = {
-        action: ACTION_REPORT_IN
+        action: ACTION_PING
     }
-    let checker = r => r == req.action ? req.action : undefined;
+    let checker = (r: string, e, c) => r == "pong" ? r : undefined;
-    let pong = await sendMessage(tab, req, 'Check tab availability...', checker, 1000, count).catch(() => { });
+    let pong = await sendMessage<string>(tab, req, 'Check tab availability...', checker, 1000, count).catch(() => { });
-    return pong == ACTION_REPORT_IN;
+    return pong == "pong";
 }
 
 /**
@@ -95,11 +79,11 @@ async function ping(tab, count = 1) {
  * @param {any} tab target tab
  * @returns {Promise<string>} a promise of the url
  */
-function queryUrl(tab) {
+export function queryUrl(tab: chrome.tabs.Tab) {
     let req = {
         action: ACTION_QUERY_URL
     }
-    return sendMessage(tab, req);
+    return sendMessage<string>(tab, req);
 }
 
 /**
@@ -108,14 +92,14 @@ function queryUrl(tab) {
  * @param {string} expected if specified, queryUrl resolves only when tab url equals to expected
  * @returns {Promise<string>} a promise of the url
  */
-function scrollToBottom(tab) {
+export function scrollToBottom(tab: chrome.tabs.Tab) {
     let req = {
         action: ACTION_SCROLL_BOTTOM
     }
     return sendMessage(tab, req, 'Scroll to page bottom...');
 }
 
-async function createTab(url, active) {
+export async function createTab(url: string, active: boolean) {
     return new Promise((resolve, reject) => {
         chrome.tabs.create({
             'url': url,
@@ -126,7 +110,7 @@ async function createTab(url, active) {
     })
 }
 
-async function getActiveTab(currentWindow) {
+export async function getActiveTab(currentWindow: boolean) {
     return new Promise((resolve, reject) => {
         chrome.tabs.query({
             active: true,
@@ -137,7 +121,7 @@ async function getActiveTab(currentWindow) {
     })
 }
 
-async function getTabByID(id) {
+export async function getTabByID(id: number) {
     return new Promise((resolve, reject) => {
         chrome.tabs.get(id, function (tab) {
             chrome.runtime.lastError;

src/background/caches.ts

@@ -0,0 +1,15 @@
+import { logger } from "./common";
+
+export class Caches {
+    private _state: string = "";
+    constructor() { }
+    get state(): string {
+        let s = this._state;
+        this._state = "";
+        return s;
+    }
+    setState(name: string, content: string) {
+        this._state = content;
+        logger.info(`State (${name}) recieved. To load it: some_var = new Extractor().load()`);
+    }
+}

src/background/common.ts

@@ -0,0 +1,6 @@
+import { Logger, LOGGER_LEVEL } from "./logger";
+import { Caches } from "./caches";
+
+export const caches = new Caches();
+export const logger = new Logger(LOGGER_LEVEL.DEBUG, LOGGER_LEVEL.DISABLED);
+export const URL_REG = /^\s*(https?):\/\//im;

src/background/extractor.ts

@@ -1,10 +1,15 @@
-var __EXTRACTOR_STATE__ = "";
+import { Task } from "./task";
+import { saveFile } from "./tools";
+import { createTab, getActiveTab, ping } from "./actions";
+import { logger, caches } from "./common";
+import { ExtractResult } from "./result";
 
-class Extractor {
+export class Extractor {
-    constructor(options) {
+    private _tasks: Task[] = [];
-        this._tasks = [];
+    private _running = false;
-        this._running = false;
+    private _options: any = {};
-        this._options = options;
+    constructor(options?) {
+        if (options) this._options = options;
     }
     /**
      * Save current state, in case we restore it later.
@@ -16,12 +21,12 @@ class Extractor {
      * Restore previous state by loading from saved state.
      */
     load() {
-        if (!__EXTRACTOR_STATE__) {
+        let content = caches.state;
+        if (!content) {
             logger.info('No state found. Please upload a saved state from the popup window first.');
             return;
         }
-        let state = JSON.parse(__EXTRACTOR_STATE__);
+        let state = JSON.parse(content);
-        __EXTRACTOR_STATE__ = "";
         this._options = state._options;
         this._tasks = state._tasks.map(t => new Task(this._options, 'whaterver', ['whaterver']).load(t));
         return this;
@@ -32,7 +37,7 @@ class Extractor {
      * If url arguments not given within later tasks, they will use previous task result as input (target url list).
      * @param {...any} args itemsSelector, fieldSelectors, and more args to specify target urls.
      */
-    task(...args) {
+    task(...args: any) {
         this._tasks.push(new Task(this._options, ...args));
         return this;
     }
@@ -53,7 +58,7 @@ class Extractor {
      * restart from specified task, but don't restart the previous tasks.
      * @param {number} from where to restart the tasks, begins with 0
      */
-    async restart(from = 0) {
+    async restart(from: number = 0) {
         let id = this._checkTaskId(from, 0);
         if (id < 0) return;
         for (let i = id; i < this._tasks.length; i++) {
@@ -61,7 +66,7 @@ class Extractor {
         }
         return this._startTasks(0);
     }
-    async _startTasks(from) {
+    async _startTasks(from: number) {
         if (this._running) {
             logger.info('The Extractor is running. Please wait..');
             return;
@@ -85,7 +90,7 @@ class Extractor {
             }
         }
         this._running = true;
-        return this._tasks.reduce((pms, task, i) => {
+        return this._tasks.reduce((pms, task: Task, i: number) => {
             return pms.then(
                 () => {
                     if (i < from) return;
@@ -93,9 +98,9 @@ class Extractor {
                         let prevTask = this._tasks[i - 1];
                         return task.execute(tab, new ExtractResult(prevTask.results));
                     }
-                    return task.execute(tab, undefined);
+                    return task.execute(tab);
                 });
-        }, Promise.resolve(undefined)).then(
+        }, Promise.resolve<void>(undefined)).then(
             () => {
                 this._running = false;
                 this.export();
@@ -109,7 +114,7 @@ class Extractor {
      * export result of a task to CSV
      * @param {number} taskid which task id to save, begins with 0
      */
-    export(taskid) {
+    export(taskid?: number) {
         let id = this._checkTaskId(taskid, this._tasks.length - 1);
         if (id < 0) return;
         let results = this._tasks[id].results
@@ -125,10 +130,10 @@ Please confirm to download (${results.length - 1} items)：
 ${exResults.toString(50) || "- Empty -"}
 `.trim();
         if (confirm(msg)) {
-            saveFile(exResults, "text/csv");
+            saveFile(exResults.toString(), "text/csv");
         }
     }
-    _checkTaskId(id, defaultId) {
+    _checkTaskId(id: number, defaultId: number) {
         if (!this._tasks.length) {
             logger.info("No task found.");
             return -1;

src/background/index.ts

@@ -0,0 +1,14 @@
+import { Extractor } from "./extractor";
+
+declare global {
+    interface Window {
+        $: (...args: any) => void;
+        Extractor: any;
+    }
+}
+
+window.$ = function (...args) {
+    return new Extractor().task(...args).start();
+}
+
+window.Extractor = Extractor;

src/background/logger.ts

@@ -1,21 +1,15 @@
-const LOGGER_LEVEL = {
+export enum LOGGER_LEVEL {
-    DEBUG: 1,
+    DEBUG = 1,
-    INFO: 2,
+    INFO,
-    WARNING: 3,
+    WARN,
-    ERROR: 4,
+    ERROR,
-    DISABLED: 100,
+    DISABLED,
-    properties: {
-        1: { name: "debug", value: 1, prefix: "DEBUG" },
-        2: { name: "info", value: 2, prefix: "INFO" },
-        3: { name: "warning", value: 3, prefix: "WARN" },
-        4: { name: "error", value: 3, prefix: "ERROR" }
-    }
 };
 
-class Logger {
+export class Logger {
-    _notificationId = undefined;
+    private _notificationId = undefined;
-    _log_level = LOGGER_LEVEL.INFO;
+    private _log_level = LOGGER_LEVEL.INFO;
-    _notify_level = LOGGER_LEVEL.ERROR;
+    private _notify_level = LOGGER_LEVEL.ERROR;
     constructor(logLevel, notifyLevel) {
         if (logLevel) this._log_level = logLevel;
         if (notifyLevel) this._notify_level = notifyLevel;
@@ -24,19 +18,19 @@ class Logger {
     get logLevel() {
         return this._log_level;
     }
-    set logLevel(val) {
+    set logLevel(val: LOGGER_LEVEL) {
         this._log_level = val;
     }
     get notifyLevel() {
         return this._notify_level;
     }
-    set notifyLevel(val) {
+    set notifyLevel(val: LOGGER_LEVEL) {
         this._notify_level = val;
     }
-    log(level, loggerFn, ...msgs) {
+    log(level: LOGGER_LEVEL, loggerFn: Function, ...msgs) {
         if (level < this._log_level) return;
         let time = new Date().toLocaleString();
-        loggerFn(`${time} [${LOGGER_LEVEL.properties[level].prefix}]`, ...msgs);
+        loggerFn(`${time} [${LOGGER_LEVEL[level]}]`, ...msgs);
         if (level < this._notify_level) return;
         this.notify(...msgs);
     }
@@ -47,7 +41,7 @@ class Logger {
         this.log(LOGGER_LEVEL.INFO, console.info, ...msgs);
     }
     warn(...msgs) {
-        this.log(LOGGER_LEVEL.WARNING, console.info, ...msgs);
+        this.log(LOGGER_LEVEL.WARN, console.info, ...msgs);
     }
     error(...msgs) {
         this.log(LOGGER_LEVEL.ERROR, console.info, ...msgs);
@@ -77,5 +71,3 @@ class Logger {
         );
     }
 }
-
-const logger = new Logger(LOGGER_LEVEL.DEBUG, LOGGER_LEVEL.DISABLED);

src/background/messaging.ts

@@ -1,3 +1,6 @@
+import { EXT_NAME, ACTION_UPLOAD_STATE } from "../common";
+import { getTabByID } from "./actions";
+import { caches, logger } from "./common";
 
 /**
  *  Sending a message to target tab repeatedly until the response is not undefined.
@@ -12,11 +15,18 @@
  * @param {string} log messages logged to console.
  * @return {Promise} a promise of the response.
  */
-function sendMessage(tab, req, log, dataChecker, interval, limit = 0) {
+export function sendMessage<T>(
+    tab: chrome.tabs.Tab,
+    req,
+    log?: string,
+    dataChecker?: (r: T, err: chrome.runtime.LastError, count: number) => T | Promise<T>,
+    interval?: number,
+    limit?: number
+) {
     interval = interval || 500;
-    limit = limit && !isNaN(limit) ? limit : 0;
+    limit = isNaN(limit) ? 0 : limit;
     let count = 0;
-    return new Promise((resolve, reject) => {
+    return new Promise<T>((resolve, reject) => {
 
         loop();
 
@@ -33,18 +43,27 @@ function sendMessage(tab, req, log, dataChecker, interval, limit = 0) {
                 return;
             }
             count++;
-            chrome.tabs.sendMessage(tab.id, req, async r => {
+            chrome.tabs.sendMessage(tab.id, req, async (r: T) => {
-                // check error but do nothing.
+                // check error but do nothing until dataChecker.
-                // do not interrupt promise chains even if error, or the task always fail when:
-                // a tab is newly created, and the content scripts won't have time to initialize
                 let err = chrome.runtime.lastError;
-                let result = r;
+                let result: T = r;
+
                 if (dataChecker) {
-                    result = await dataChecker(r, err, count);
+                    let pms = dataChecker(r, err, count);
-                    if (MSG_USER_ABORT.isEqual(result)) {
+                    // don't catch if it's not a Promise
-                        reject(MSG_USER_ABORT.message);
+                    if (pms instanceof Promise) {
+                        let checkerError: any;
+                        pms = pms.catch(e => checkerError = e);
+                        result = await pms;
+                        if (checkerError) {
+                            reject(checkerError);
+                            return;
+                        }
+                    } else {
+                        result = pms;
                     }
                 }
+
                 let flag = result !== undefined && result !== null;
                 if (log) logger.info(log, flag ? '(OK)' : '(failed)');
                 if (flag) {
@@ -66,8 +85,7 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
     switch (request.action) {
         case ACTION_UPLOAD_STATE:
             sendResponse('recieved!');
-            __EXTRACTOR_STATE__ = request.state;
+            caches.setState(request.name, request.state)
-            logger.info(`State (${request.name}) recieved. To load it: some_var = new Extractor().load()`);
             break;
         default:
             sendResponse("Request not supported.");

src/background/result.ts

@@ -1,23 +1,23 @@
-class ExtractResult {
+export class ExtractResult {
+    private _data: string[][] = [];
     constructor(data) {
         this._data = data || [];
-
     }
-    row(index) {
+    row(index: number): string[] {
         return this._data[index];
     }
-    column(index) {
+    column(index: number): string[] {
         return [...new Array(this._data.length).keys()].map(
             i => this._data[i][index]
         );
     }
-    squash() {
+    squash(): string[] {
         return this._data.reduce((p, c) => p.concat(c), []);
     }
-    get data() {
+    get data(): string[][] {
         return this._data;
     }
-    toString(rowsCount) {
+    toString(rowsCount: number = 0): string {
         let data = rowsCount > 0 ? this._data.slice(0, rowsCount) : this._data;
         return data.slice().reduce(
             (csv, lineCells) => {

src/background/signiture.ts

@@ -1,4 +1,6 @@
-const signitures = `
+import { ExtractResult } from "./result";
+
+export const signitures = `
 ## Usage
 // single task
 $(...args);
@@ -21,9 +23,10 @@ $(".item", ["a", "a@href"]);
 https://git.jebbs.co/jebbs/data-extracter-extesion
 `.trim();
 
-function testArgs(...args) {
+export function testArgs(...args: any) {
     switch (args.length) {
-        case 0, 1:
+        case 0:
+        case 1:
             return false;
         case 2:
             return args[0] && args[1] &&
@@ -66,7 +69,3 @@ function testArgs(...args) {
         return arr.reduce((p, c) => p && tester(c), true);
     }
 }
-
-function argsToString(...args) {
-    return args.map(v => (v instanceof Array ? `[${v.join(', ')}]` : v.toString())).join(', ');
-}

src/background/task.ts

@@ -1,14 +1,21 @@
-class Task {
+import { parseUrls } from "./tools";
-    _data = {};
+import { queryUrl, redirectTab, scrollToBottom, extractTabData } from "./actions";
-    _data_keys = [];
+import { testArgs, signitures } from "./signiture";
-    /**
+import { ExtractResult } from "./result";
-     * Create a task.
+
-     * constructor(itemsSelector:string, fieldSelectors:string[])
+export class Task {
-     * constructor(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
+    private _data: { [key: string]: string[][] } = {};
-     * constructor(itemsSelector:string, fieldSelectors:string[], url:string, pages:number[])
+    private _data_keys: string[] = [];
-     * constructor(itemsSelector:string, fieldSelectors:string[], urls:string[])
+    private _options: any;
-     * @param {...any} args
+    private _itemsSelector: string;
-     */
+    private _fieldSelectors: string[];
+    private _urls: string[] = [];
+
+    constructor(options: any, ...arg: any);
+    constructor(options: any, itemsSelector: string, fieldSelectors: string[]);
+    constructor(options: any, itemsSelector: string, fieldSelectors: string[], url: string, from: number, to: number, interval: number);
+    constructor(options: any, itemsSelector: string, fieldSelectors: string[], url: string, pages: number[]);
+    constructor(options: any, itemsSelector: string, fieldSelectors: string[], urls: string[]);
     constructor(options, ...args) {
         if (!testArgs(...args))
             throw new Error(`Invalid call arguments.\n\n${signitures}\n\n`);
@@ -17,7 +24,7 @@ class Task {
         this._fieldSelectors = args.shift();
         this._urls = parseUrls(...args);
     }
-    load(state) {
+    load(state: any): Task {
         this._itemsSelector = state._itemsSelector;
         this._data = state._data;
         this._data_keys = state._data_keys;
@@ -26,25 +33,23 @@ class Task {
         this._urls = state._urls;
         return this;
     }
-    get urls() {
+    get urls(): string[] {
         return this._urls;
     }
-    get data() {
+    get results(): string[][] {
-        return this._data;
-    }
-    get results() {
         return this._data_keys.reduce((p, c) => {
             return p.concat(this._data[c]);
         }, []);
     }
-    get fieldSelectors() {
+    get fieldSelectors(): string[] {
         return this._fieldSelectors;
     }
-    clean() {
+    clean(): Task {
         this._data = {};
         this._data_keys = [];
+        return this;
     }
-    async execute(tab, upstreamData) {
+    async execute(tab: chrome.tabs.Tab, upstreamData?: ExtractResult): Promise<void> {
         if (!tab) return Promise.reject("No tab to execute the task.");
         let urls = this._urls
         if (!urls.length) {
@@ -60,14 +65,12 @@ class Task {
         }
         return urls.reduce((p, url, i) => p.then(
             results => {
-                if (i > 0) {
+                if (i > 0 && results instanceof Array) {
-                    if (!MSG_URL_SKIPPED.isEqual(results)) {
+                    let lastURL = urls[i - 1];
-                        let lastURL = urls[i - 1];
+                    saveResult(results, lastURL);
-                        saveResult(results, lastURL);
-                    }
                 }
-                if (this._data[url]) return MSG_URL_SKIPPED;
+                if (this._data[url]) return;
-                let pms = redirectTab(tab, url);
+                let pms: Promise<any> = redirectTab(tab, url);
                 if (this._options["scrollToBottom"]) {
                     pms = pms.then(() => scrollToBottom(tab));
                 }
@@ -75,9 +78,9 @@ class Task {
                     () => extractTabData(tab, this._itemsSelector, this._fieldSelectors)
                 );
             }
-        ), Promise.resolve(null)).then(
+        ), Promise.resolve<string[][]>(null)).then(
             results => {
-                if (!MSG_URL_SKIPPED.isEqual(results)) {
+                if (results && results.length) {
                     let lastURL = urls[urls.length - 1];
                     saveResult(results, lastURL);
                     return;

src/background/tools.ts

@@ -0,0 +1,61 @@
+import { URL_REG } from "./common";
+import { ExtractResult } from "./result";
+
+export function parseUrls(...args) {
+    if (!args.length) return [];
+    let arg = args.shift();
+    if (arg instanceof Array) {
+        return arg;
+    } else if (arg instanceof ExtractResult) {
+        return arg.squash().filter(v => URL_REG.test(v));
+    } else {
+        let urlTempl = arg;
+        if (urlTempl) {
+            if (args[0] instanceof Array) {
+                return args[0].map(p => urlTempl.replace("${page}", p));
+            } else if (args.length >= 3) {
+                let urls = [];
+                let from = args.shift();
+                let to = args.shift();
+                let interval = args.shift();
+                for (let i = from; i <= to; i += interval) {
+                    urls.push(urlTempl.replace("${page}", i));
+                }
+                return urls;
+            }
+        }
+    }
+    return [];
+}
+
+export function saveFile(data: string, mimeType: string, fileName?: string) {
+    fileName = fileName || document.title || "result";
+    let blob: Blob;
+    if (typeof window.Blob == "function") {
+        blob = new Blob([data], {
+            type: mimeType
+        })
+    } else {
+        var BlobBuiler = window.MSBlobBuilder;
+        var builer = new BlobBuiler();
+        builer.append(data);
+        blob = builer.getBlob(mimeType)
+    }
+    var URL = window.URL || window.webkitURL;
+    var url = URL.createObjectURL(blob);
+    var link = document.createElement("a");
+    if ('download' in link) {
+        link.style.visibility = "hidden";
+        link.href = url;
+        link.download = fileName;
+        document.body.appendChild(link);
+        var j = document.createEvent("MouseEvents");
+        j.initEvent("click", true, true);
+        link.dispatchEvent(j);
+        document.body.removeChild(link)
+    } else if (navigator.msSaveBlob) {
+        navigator.msSaveBlob(blob, fileName)
+    } else {
+        location.href = url
+    }
+}

src/common.ts

@@ -0,0 +1,11 @@
+
+export const EXT_NAME = "DataExtracter";
+
+export const ACTION_EXTRACT = `${EXT_NAME}:Extract`;
+export const ACTION_GOTO_URL = `${EXT_NAME}:GoToTUL`;
+export const ACTION_PING = `${EXT_NAME}:ReportIn`;
+export const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
+export const ACTION_SCROLL_BOTTOM = `${EXT_NAME}:ScrollToBottom`;
+export const ACTION_UPLOAD_STATE = `${EXT_NAME}:UploadStateFile`;
+export const ACTION_SLEEP = `${EXT_NAME}:Sleep`;
+export const ACTION_WAKEUP = `${EXT_NAME}:WakeUp`;

src/content/actions.ts

@@ -0,0 +1,73 @@
+export function extract(itemsSelector: string, fieldSelectors: string[]): string[][] {
+    // since some elements may be loaded asynchronously.
+    // if one field is never found, we should return undefined,
+    // so that senders can detect to retry until elements loaded.
+    // If user writes wrong selectors, the task retries infinitely.
+    let fieldFound: { [key: string]: boolean } = {};
+    let items: Element[] = Array.from(document.querySelectorAll(itemsSelector));
+    // items may not loaded yet, tell the sender to retry.
+    if (!items.length) return [];
+    let results: string[][] = items.map(
+        item => {
+            return fieldSelectors.map(
+                selector => {
+                    let [cls, attr] = selector.split('@').slice(0, 2);
+                    let fieldVals = Array.from(item.querySelectorAll(cls));
+                    if (!fieldVals.length) {
+                        return;
+                    }
+                    fieldFound[selector] = true;
+                    return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
+                }
+            )
+        }
+    );
+    // if it exists a field, which is not found in any row, the sender should retry.
+    let shouldWait = fieldSelectors.reduce((p, c) => p || !fieldFound[c], false);
+    return shouldWait ? [] : results;
+}
+
+export function scrollToBottom() {
+    return executeUntil(
+        () => window.scrollTo(0, document.body.clientHeight),
+        () => document.body.clientHeight - window.scrollY - window.innerHeight < 20,
+        "Scroll to page bottom...",
+        1000,
+        10
+    );
+}
+
+/**
+ * Repeatedly execute an function until the the detector returns true.
+ * @param {object} fn the function to execute
+ * @param {object} detector the detector.
+ * @param {string} log messages logged to console.
+ * @param {number} interval interval for detecting
+ * @param {number} limit max execute times of a function
+ * @return {Promise} a promise of the response.
+ */
+function executeUntil(fn: () => void, detector: () => boolean, log: string, interval: number, limit: number) {
+    interval = interval || 500;
+    let count = 0;
+    return new Promise<boolean>((resolve, reject) => {
+
+        loop();
+
+        async function loop() {
+            fn();
+            limit++;
+            if (limit && count >= limit) {
+                reject(false);
+            }
+            setTimeout(() => {
+                let flag = !detector || detector();
+                if (log) console.log(log, flag ? '(OK)' : '(failed)');
+                if (flag) {
+                    resolve(true);
+                } else {
+                    loop();
+                }
+            }, interval);
+        }
+    });
+}

src/content/index.ts

@@ -0,0 +1,45 @@
+import { ACTION_WAKEUP, ACTION_EXTRACT, ACTION_GOTO_URL, ACTION_PING, ACTION_QUERY_URL, ACTION_SCROLL_BOTTOM, ACTION_SLEEP } from '../common';
+import { scrollToBottom, extract } from './actions';
+
+let asleep = false;
+chrome.runtime.onMessage.addListener(
+    function (request, sender: chrome.runtime.MessageSender, sendResponse: (r: any) => void) {
+        if (!request.action) return;
+        if (asleep && ACTION_WAKEUP != request.action) {
+            sendResponse && sendResponse(undefined);
+            return;
+        }
+        // console.log("Recieved request:",request);
+        doAction(request, sender).then(r => sendResponse && sendResponse(r));
+        // return true to indicate you wish to send a response asynchronously
+        return true;
+    }
+);
+
+async function doAction(request: any, sender: chrome.runtime.MessageSender) {
+    switch (request.action) {
+        case ACTION_EXTRACT:
+            let data = extract(request.itemsSelector, request.fieldSelectors);
+            return data;
+        case ACTION_GOTO_URL:
+            window.location.replace(request.url);
+            // should not recieve any request until the page & script reload
+            asleep = true;
+            return request.url;
+        case ACTION_PING:
+            return "pong";
+        case ACTION_QUERY_URL:
+            return window.location.href;
+        case ACTION_SCROLL_BOTTOM:
+            return scrollToBottom();
+        case ACTION_SLEEP:
+            asleep = true;
+            return "Content script is sleeping.";
+        case ACTION_WAKEUP:
+            asleep = false;
+            return "Content script is available.";
+        default:
+            break;
+    }
+}
+

src/popup/index.ts

@@ -1,3 +1,5 @@
+import { ACTION_UPLOAD_STATE } from '../common';
+
 window.onload = function () {
     document.querySelector('#link-extension-detail')
         .addEventListener('click', () => {

template/html/popup.html

@@ -3,10 +3,9 @@
 <link>
 <meta charset="utf-8">
 <title>Data Extractor</title>
-<script charset="UTF-8" type="text/javascript" src="../scripts/shared/common.js"></script>
+<script charset="UTF-8" type="text/javascript" src="../scripts/popup.bundle.js"></script>
-<script charset="UTF-8" type="text/javascript" src="tip.js"></script>
 
-<link rel="stylesheet" href="styles/bootstrap.min.css">
+<link rel="stylesheet" href="../assets/bootstrap.min.css">
 </head>
 
 <body style="margin: 20px 10px;">
@@ -19,13 +18,12 @@
         <div class="row">
             <div class="col">
                 <div class="alert alert-info small">
-                    <!-- <h6>Usage:</h6> -->
                     <p>
                         Goto <a href="#" id="link-extension-detail">Extension Detail</a>, click "backgroud page",
                         and type your scripts in the console.
                     </p>
                     <p>
-                        <img src="../images/console.png" alt=""
+                        <img src="../assets/console.png" alt=""
                             style="max-width: 489px; width: 100%; border-radius: 5px">
                     </p>
 
@@ -66,7 +64,7 @@
         </div>
         <div class="row">
             <div class="col">
-               <input type="file" name="state" id="state-input">
+                <input type="file" name="state" id="state-input">
             </div>
         </div>
     </div>

template/manifest.json

@@ -0,0 +1,34 @@
+{
+  "manifest_version": 2,
+  "name": "Data Extracter",
+  "version": "0.5.1",
+  "author": "jebbs",
+  "description": "Extract data from web page elements as sheet.",
+  "icons": {
+    "16": "icon.png",
+    "48": "icon.png",
+    "128": "icon.png"
+  },
+  "browser_action": {
+    "default_icon": "icon.png",
+    "default_popup": "html/popup.html",
+    "default_title": "Data Extracter"
+  },
+  "background": {
+    "scripts": [
+      "scripts/background.bundle.js"
+    ],
+    "persistent": false
+  },
+  "content_scripts": [{
+    "matches": ["*://*/*"],
+    "js": [
+      "scripts/content.bundle.js"
+    ],
+    "run_at": "document_idle"
+  }],
+  "permissions": [
+    "activeTab",
+    "notifications"
+  ]
+}

tsconfig.json

@@ -0,0 +1,12 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "es6",
+        "noImplicitAny": false,
+        "sourceMap": true,
+        "rootDir": "src",
+        "outDir": "dist/js",
+        "noEmitOnError": true,
+        "typeRoots": [ "node_modules/@types" ]
+    }
+}

webpack.config.js

@@ -0,0 +1,33 @@
+const path = require('path');
+const CopyPlugin = require('copy-webpack-plugin');
+
+module.exports = {
+    mode: 'production',
+    entry: {
+        background: './src/background/index.ts',
+        content: './src/content/index.ts',
+        popup: './src/popup/index.ts',
+    },
+    // devtool: 'inline-source-map',
+    output: {
+        path: path.resolve(__dirname, 'dist'),
+        filename: 'scripts/[name].bundle.js'
+    },
+    module: {
+        rules: [
+            {
+                test: /\.tsx?$/,
+                use: 'ts-loader',
+                exclude: /node_modules/
+            }
+        ]
+    },
+    resolve: {
+        extensions: ['.tsx', '.ts', '.js']
+    },
+    plugins: [
+        new CopyPlugin([
+            { from: '**/*', to: '.', toType: "dir" },
+        ], { context: 'template', logLevel: 'warn' }),
+    ]
+};