init

scripts/background.js

@@ -0,0 +1,6 @@
+chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) {
+    if (message.from === "doExtractRequest")
+        extract(...message.args).catch(
+            err => console.log(err)
+        );
+});

scripts/content.js

@@ -0,0 +1,64 @@
+chrome.runtime.onMessage.addListener(
+    function (request, sender, sendResponse) {
+        // console.log(request);
+        if (request.from == "doExtractRequest") {
+            // console.log(request);
+            let data = extractData(request.itemsSelector, request.fieldSelectors);
+            // console.log(data);
+            sendResponse(data);
+        } else if (request.from == "doExtractGotoUrl") {
+            // console.log(request);
+            window.location.replace(request.url);
+        } else if (request.from == "doExtractReportIn") {
+            // console.log("doExtractReportIn");
+            sendResponse(request.from);
+        }
+    }
+);
+
+function extractData(itemsSelector, fieldSelectors) {
+    return $(itemsSelector).toArray().map(
+        item => fieldSelectors.map(
+            cls => $(item).find(cls).toArray().map(find => find.textContent.trim()).join('\n')
+        )
+    );
+}
+
+function extract(...args) {
+    let sig = `Invalid call args.
+function extract(itemsSelector:string, fieldSelectors:string[])
+function extract(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
+function extract(itemsSelector:string, fieldSelectors:string, url:string, pages:number[])`;
+    if (!testArgs(...args)) {
+        console.log(sig);
+        return;
+    }
+    if (args.length == 2) {
+        saveFileAsk(extractData(args[0], args[1]));
+        return;
+    }
+    let message = {
+        from: "doExtractRequest",
+        args: args
+    }
+    chrome.runtime.sendMessage(message, r => {
+        if (r) {
+            console.log(r);
+            alert(r);
+        }
+    });
+}
+
+function testArgs(...args) {
+
+    if (args.length < 2) return false;
+
+    if (args.length == 2)
+        return (args[0] && args[1] && (typeof args[0] == "string") && (args[1] instanceof Array))
+
+    let urls = [];
+    if (args.length > 2) return (typeof args[2] == "string") && (
+        (args[3] instanceof Array) ||
+        (!isNaN(args[3]) && !isNaN(args[4]) && !isNaN(args[5]))
+    )
+}

scripts/extract.js

@@ -0,0 +1,121 @@
+// function extract(itemsSelector, fieldSelectors, url, from, to, interval)
+// function extract(itemsSelector, fieldSelectors, url, pages)
+function extract(itemsSelector, fieldSelectors, url, ...args) {
+    let urls = [];
+    if (url) {
+        if (args[0] instanceof Array) {
+            urls = args[0].map(p => url.replace("${page}", p));
+        } else if (args.length >= 3) {
+            let from = args.shift();
+            let to = args.shift();
+            let interval = args.shift();
+            for (let i = from; i <= to; i += interval) {
+                urls.push(url.replace("${page}", i));
+            }
+        }
+    }
+    let data = [];
+    return new Promise((resolve, reject) => {
+        chrome.tabs.query({
+            active: true,
+            currentWindow: true
+        }, function (tabs) {
+            let pms;
+            let tab = tabs[0];
+            if (urls.length) {
+                pms = urls.reduce((p, url) => p.then(
+                    results => {
+                        data.push(...results);
+                        return redirectTab(tab, url).then(
+                            tab => extractTabData(tab, itemsSelector, fieldSelectors)
+                        );
+                    },
+                    () => p
+                ), Promise.resolve([]));
+            } else {
+                pms = extractTabData(tab, itemsSelector, fieldSelectors);
+            }
+            pms.then(
+                results => {
+                    data.push(...results);
+                    data.unshift(fieldSelectors);
+                    saveFileAsk(data);
+                    resolve("save done.")
+                },
+                err => reject(err)
+            );
+        });
+    });
+}
+
+function extractTabData(tab, itemsSelector, fieldSelectors) {
+    let done = false;
+    return new Promise((resolve, reject) => {
+        chrome.tabs.sendMessage(
+            tab.id, {
+                from: "doExtractRequest",
+                itemsSelector: itemsSelector,
+                fieldSelectors: fieldSelectors
+            },
+            response => {
+                done = true;
+                resolve(response);
+            }
+        );
+        setTimeout(() => {
+            if (!done) reject(`extractTabData failed after 2 second.`);
+        }, 2000);
+    });
+}
+
+function redirectTab(tab, url) {
+    let done = false;
+    let timeOut;
+    return new Promise((resolve, reject) => {
+        let req = {
+            from: "doExtractGotoUrl",
+            url: url
+        }
+        chrome.tabs.sendMessage(tab.id, req, r => {
+            // if (r != req.from) return;
+            // detectLoop();
+            setTimeout(() => {
+                detectLoop();
+            }, 500);
+        });
+        setTimeout(() => {
+            if (!done) {
+                reject(`redirectTab failed after 10 second.`);
+                clearTimeout(timeOut);
+            }
+        }, 10000);
+
+        function detectLoop() {
+            let req = {
+                from: "doExtractReportIn"
+            }
+            chrome.tabs.sendMessage(tab.id, req, r => {
+                if (r != req.from) return;
+                resolve(tab)
+                done = true;
+            });
+            if (!done) {
+                clearTimeout(timeOut);
+                timeOut = setTimeout(() => {
+                    detectLoop();
+                }, 100);
+            }
+        }
+    });
+}
+
+function createTab(url) {
+    return new Promise((resolve, reject) => {
+        chrome.tabs.create({
+            active: true,
+            url: url
+        }, function (tab) {
+            resolve(tab);
+        });
+    });
+}

scripts/tools.js

@@ -0,0 +1,53 @@
+function formatCSV(data) {
+    return data.reduce(
+        (csv, lineCells) => {
+            let line = lineCells.reduce(
+                (lineText, cell, idx) => {
+                    cell = '"' + cell.trim().replace(/"/g, '""') + '"';
+                    return lineText + cell + (idx == lineCells.length - 1 ? "" : ",")
+                }, "");
+            return csv + line + "\n";
+        },
+        ""
+    );
+}
+
+function saveFile(data, mimeType, fileName) {
+    fileName = fileName || document.title || "result";
+    var blob;
+    if (typeof window.Blob == "function") {
+        blob = new Blob([data], {
+            type: mimeType
+        })
+    } else {
+        var BlobBuiler = window.BlobBuilder || window.MozBlobBuilder || window.WebKitBlobBuilder || window.MSBlobBuilder;
+        var builer = new BlobBuiler();
+        builer.append(data);
+        blob = builer.getBlob(mimeType)
+    }
+    var URL = window.URL || window.webkitURL;
+    var url = URL.createObjectURL(blob);
+    var link = document.createElement("a");
+    if ('download' in link) {
+        link.style.visibility = "hidden";
+        link.href = url;
+        link.download = fileName;
+        document.body.appendChild(link);
+        var j = document.createEvent("MouseEvents");
+        j.initEvent("click", true, true);
+        link.dispatchEvent(j);
+        document.body.removeChild(link)
+    } else if (navigator.msSaveBlob) {
+        navigator.msSaveBlob(blob, fileName)
+    } else {
+        location.href = url
+    }
+}
+
+function saveFileAsk(data) {
+    let csv = formatCSV(data.slice(1, 50)).trim() || "- Empty -";
+    if (confirm(`Click confirm to download if the sample data looks good (${data.length-1} items)：\n\n${csv}`)) {
+        csv = formatCSV(data);
+        saveFile(csv, "text/csv");
+    }
+}