jebbs/data-extracter-extesion
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
jebbs
2020-01-10 12:07:21 +08:00
parent 51da68fee5
commit 26c6c1159e
15 changed files with 175 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
manifest.json
View File

@ -16,24 +16,24 @@
},
"background": {
"scripts": [
"scripts/background.js",
"scripts/result.js",
"scripts/tools.js",
"scripts/extract.js",
"scripts/extractor.js"
"scripts/shared.js",
"scripts/background/messaging.js",
"scripts/background/result.js",
"scripts/background/signiture.js",
"scripts/background/actions.js",
"scripts/background/extractor.js"
],
"persistent": false
},
"content_scripts": [{
"matches": ["*://*/*"],
"js": [
"scripts/jquery.min.js",
"scripts/content.js"
"scripts/shared.js",
"scripts/content/content.js"
],
"run_at": "document_idle"
}],
"permissions": [
"activeTab",
"storage"
"activeTab"
]
}

0
styles/bootstrap.min.css → popup/styles/bootstrap.min.css vendored
View File

9
popup/tip.html
View File

@ -3,11 +3,9 @@
<link>
<meta charset="utf-8">
<title>Data Extractor</title>
<script charset="UTF-8" type="text/javascript" src="../scripts/jquery.min.js"></script>
<script charset="UTF-8" type="text/javascript" src="../styles/bootstrap.min.js"></script>
<script charset="UTF-8" type="text/javascript" src="./tip.js"></script>
<script charset="UTF-8" type="text/javascript" src="tip.js"></script>
<link rel="stylesheet" href="../styles/bootstrap.min.css">
<link rel="stylesheet" href="styles/bootstrap.min.css">
</head>
<body style="margin: 20px 10px;">
@ -26,7 +24,8 @@
and type your scripts in the console.
</p>
<p>
<img src="../images/console.png" alt="" style="max-width: 489px; width: 100%; border-radius: 5px">
<img src="../images/console.png" alt=""
style="max-width: 489px; width: 100%; border-radius: 5px">
</p>
</div>

12
popup/tip.js
View File

@ -1,14 +1,14 @@
$().ready(
() => {
$("#link-extension-detail").on('click', () => {
window.onload = function () {
document.querySelector('#link-extension-detail')
.addEventListener('click', () => {
chrome.tabs.create({
'url': `chrome://extensions/?id=${chrome.runtime.id}`
});
})
$("#link-document").on('click', () => {
document.querySelector('#link-document')
.addEventListener('click', () => {
chrome.tabs.create({
'url': `https://git.jebbs.co/jebbs/data-extracter-extesion`
});
})
}
);
}

14
scripts/background.js
View File

@ -1,14 +0,0 @@
chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) {
if (message.from === "DataExtracter:Extract") {
if (!testArgs(...message.args)) {
sendResponse(signitures);
return;
}
extract(...message.args).catch(
err => {
console.log(err);
alert(err);
}
);
}
});

71
scripts/extract.js → scripts/background/actions.js
View File

@ -1,18 +1,3 @@
/**
* Extract data from current tab / multiple urls.
* @param {string} itemsSelector items selectors for selecting items (data rows)
* @param {Array<string>} fieldSelectors fields selectors for selecting fields (data columns) under each item
* @param {...any} args url list / url templates, page numers, either [from, to, interval] or [...pages]
*/
async function extract(itemsSelector, fieldSelectors, ...args) {
let result = await getData(itemsSelector, fieldSelectors, ...args);
if (confirm(
`Click confirm to download if the sample data looks good (${result.data.length} items)：\n\n${result.toString(50) || "- Empty -"}`
)) {
saveFile(result, "text/csv");
}
}
/**
* Extract data from current page / multiple urls.
* getData(tab, itemsSelector:string, fieldSelectors:string[])
@ -43,7 +28,7 @@ async function getData(...args) {
if (urls.length) {
pms = urls.reduce((p, url) => p.then(
results => {
data.push(...results);
if (results) data.push(...results);
return redirectTab(tab, url).then(
() => extractTabData(tab, itemsSelector, fieldSelectors)
);
@ -55,7 +40,8 @@ async function getData(...args) {
}
pms.then(
results => {
data.push(...results);
if (results) data.push(...results);
data.unshift(fieldSelectors);
resolve(new ExtractResult(data));
},
err => reject(err)
@ -97,7 +83,7 @@ function redirectTab(tab, url) {
if (url !== u) {
curUrl = u;
let req = {
from: "GotoUrl",
action: ACTION_GOTO_URL,
url: url
}
sendMessage(tab, req, `Goto url: ${url}`);
@ -115,11 +101,11 @@ function redirectTab(tab, url) {
*/
function extractTabData(tab, itemsSelector, fieldSelectors) {
let req = {
from: "Extract",
action: ACTION_EXTRACT,
itemsSelector: itemsSelector,
fieldSelectors: fieldSelectors
}
let cond = r => r && r.length;
let cond = r => r !== undefined;
return sendMessage(tab, req, 'Extract data from the tab...', cond);
}
@ -130,9 +116,9 @@ function extractTabData(tab, itemsSelector, fieldSelectors) {
*/
function reportIn(tab) {
let req = {
from: "ReportIn"
action: ACTION_REPORT_IN
}
let cond = r => r == req.from;
let cond = r => r == req.action;
return sendMessage(tab, req, 'Check tab availability...', cond);
}
@ -144,51 +130,12 @@ function reportIn(tab) {
*/
function queryUrl(tab, urlExcluded, log) {
let req = {
from: "QueryUrl"
action: ACTION_QUERY_URL
}
let cond = url => url && (!urlExcluded || (urlExcluded && urlExcluded != url));
return sendMessage(tab, req, log, cond);
}
/**
* Repeatedly sending a message to target tab until the response is detected good.
* @param {object} tab the table where to send the message
* @param {object} req the request data.
* @param {function} cond success condition function, r:any=>boolean
* @param {number} interval interval for detecting
* @param {string} log messages logged to console.
* @return {Promise} a promise of the response.
*/
function sendMessage(tab, req, log, cond, interval) {
req.from = "DataExtracter:" + req.from;
interval = interval || 500;
return new Promise((resolve, reject) => {
loop();
async function loop() {
// console.log("request for", req.from);
let tabAvailable = await getTabByID(tab.id);
if (!tabAvailable) {
reject("Task interrupted due to the target tab is closed.");
return;
}
chrome.tabs.sendMessage(tab.id, req, r => {
let flag = !cond || cond(r);
if (log) console.log(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(r);
} else {
setTimeout(() => {
loop();
}, interval);
}
});
}
});
}
async function createTab(url, active) {
return new Promise((resolve, reject) => {
chrome.tabs.create({

6
scripts/extractor.js → scripts/background/extractor.js
View File

@ -14,9 +14,9 @@ class Extractor {
*/
task(...args) {
if (!testArgs(...args)) {
console.log(`Invalid call arguments.\n\n${argsToString(...args)}\n${signitures}\n`);
console.log(`Invalid task arguments: ${argsToString(...args)}\n\n${signitures}\n`);
// break call chain to avoid unexpected task running
return undefined;
return this;
}
// given >2 arguments means the task specifies target page,
// so it won't accept last task result as url list.
@ -119,7 +119,7 @@ class Extractor {
if (!taskid) return;
const result = this._results[this._tasks[taskid - 1]];
if (!result) {
console.log(`No task result for id (${taskid}). Forget to call ".start()"?`);
console.log(`No result for task #${taskid}. Forget to call ".start()"?`);
return;
}
if (confirm(

50
scripts/background/messaging.js Normal file
View File

@ -0,0 +1,50 @@
/**
* Repeatedly sending a message to target tab until the response is detected good.
* @param {object} tab the table where to send the message
* @param {object} req the request data.
* @param {function} cond success condition function, r:any=>boolean
* @param {number} interval interval for detecting
* @param {string} log messages logged to console.
* @return {Promise} a promise of the response.
*/
function sendMessage(tab, req, log, cond, interval) {
interval = interval || 500;
return new Promise((resolve, reject) => {
loop();
async function loop() {
// console.log("request for", req.action);
let tabAvailable = await getTabByID(tab.id);
if (!tabAvailable) {
reject("Task interrupted due to the target tab is closed.");
return;
}
chrome.tabs.sendMessage(tab.id, req, r => {
if (chrome.runtime.lastError) {
reject(chrome.runtime.lastError.message);
return;
}
let flag = !cond || cond(r);
if (log) console.log(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(r);
} else {
setTimeout(() => {
loop();
}, interval);
}
});
}
});
}
chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) {
if (!message.action || !message.action.startsWith(EXT_NAME)) {
return;
}
sendResponse("Calling from user pages is not allowed.");
return;
});

0
scripts/result.js → scripts/background/result.js
View File

47
scripts/tools.js → scripts/background/signiture.js
View File

@ -3,51 +3,20 @@ const signitures = `
new Extractor().task(...args).task(...args).start();
## Extractor.task() Signitures:
----------------------------
function(itemsSelector:string, fieldSelectors:string[])
function(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
function(itemsSelector:string, fieldSelectors:string[], url:string, pages:number[])
function(itemsSelector:string, fieldSelectors:string[], urls:string[])
task(itemsSelector:string, fieldSelectors:string[])
task(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
task(itemsSelector:string, fieldSelectors:string, url:string, pages:number[])
task(itemsSelector:string, fieldSelectors:string[], urls:string[])
task(itemsSelector:string, fieldSelectors:string[], urls:ExtractResult)
## Example:
// extract all links text & url under '.item' elements
// use 'selector@attr' to get attribute of the field elements
new Extractor().task(".item", ["a", "a@href"]).start();
## See Detailed Help:
https://git.jebbs.co/jebbs/data-extracter-extesion
`.trim();
function saveFile(data, mimeType, fileName) {
fileName = fileName || document.title || "result";
var blob;
if (typeof window.Blob == "function") {
blob = new Blob([data], {
type: mimeType
})
} else {
var BlobBuiler = window.BlobBuilder || window.MozBlobBuilder || window.WebKitBlobBuilder || window.MSBlobBuilder;
var builer = new BlobBuiler();
builer.append(data);
blob = builer.getBlob(mimeType)
}
var URL = window.URL || window.webkitURL;
var url = URL.createObjectURL(blob);
var link = document.createElement("a");
if ('download' in link) {
link.style.visibility = "hidden";
link.href = url;
link.download = fileName;
document.body.appendChild(link);
var j = document.createEvent("MouseEvents");
j.initEvent("click", true, true);
link.dispatchEvent(j);
document.body.removeChild(link)
} else if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, fileName)
} else {
location.href = url
}
}
function testArgs(...args) {
switch (args.length) {
case 0, 1:

58
scripts/content.js
View File

@ -1,58 +0,0 @@
function extract(...args) {
let message = {
from: "DataExtracter:Extract",
args: args
}
chrome.runtime.sendMessage(message, r => {
if (r) console.log(r);
});
}
chrome.runtime.onMessage.addListener(
function (request, sender, sendResponse) {
if (!request.from) return;
let [ext, act] = request.from.split(":");
if (ext.toLowerCase() !== 'dataextracter') return;
// console.log(request);
switch (act.toLowerCase()) {
case "extract":
let data = extractTabData(request.itemsSelector, request.fieldSelectors);
if (sendResponse) sendResponse(data);
break;
case "gotourl":
window.location.replace(request.url);
if (sendResponse) sendResponse(request.url);
break;
case "reportin":
if (sendResponse) sendResponse(request.from);
break;
case "queryurl":
if (sendResponse) sendResponse(window.location.href);
break;
default:
break;
}
}
);
function extractTabData(itemsSelector, fieldSelectors) {
let fieldNotFound = false;
let results = $(itemsSelector).toArray().map(
item => {
return fieldSelectors.map(
selector => {
let [cls, attr] = selector.split('@').slice(0, 2);
// TODO: close tab to cancel task tip
if (fieldNotFound) return;
let fieldVals = $(item).find(cls).toArray();
if (!fieldVals.length) {
fieldNotFound = true;
return;
}
return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
}
)
}
);
return fieldNotFound ? [] : results
}

49
scripts/content/content.js Normal file
View File

@ -0,0 +1,49 @@
chrome.runtime.onMessage.addListener(
function (request, sender, sendResponse) {
if (!request.action) return;
// console.log("Recieved request:",request);
switch (request.action) {
case ACTION_EXTRACT:
let data = extract(request.itemsSelector, request.fieldSelectors);
if (sendResponse) sendResponse(data);
break;
case ACTION_GOTO_URL:
window.location.replace(request.url);
if (sendResponse) sendResponse(request.url);
break;
case ACTION_REPORT_IN:
if (sendResponse) sendResponse(request.action);
break;
case ACTION_QUERY_URL:
if (sendResponse) sendResponse(window.location.href);
break;
default:
break;
}
}
);
function extract(itemsSelector, fieldSelectors) {
// let fieldNotFound = false;
let items = Array.from(document.querySelectorAll(itemsSelector));
if (!items.length) return [];
let results = items.map(
item => {
return fieldSelectors.map(
selector => {
let [cls, attr] = selector.split('@').slice(0, 2);
// TODO: close tab to cancel task tip
// if (fieldNotFound) return;
let fieldVals = Array.from(item.querySelectorAll(cls));
// if (!fieldVals.length) {
// fieldNotFound = true;
// return;
// }
return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
}
)
}
);
// return fieldNotFound ? [] : results
return results;
}

4
scripts/jquery.min.js vendored
View File

File diff suppressed because one or more lines are too long

37
scripts/shared.js Normal file
View File

@ -0,0 +1,37 @@
const EXT_NAME = "DataExtracter";
const ACTION_EXTRACT = `${EXT_NAME}:Extract`;
const ACTION_GOTO_URL = `${EXT_NAME}:GoToTUL`;
const ACTION_REPORT_IN = `${EXT_NAME}:ReportIn`;
const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
function saveFile(data, mimeType, fileName) {
fileName = fileName || document.title || "result";
var blob;
if (typeof window.Blob == "function") {
blob = new Blob([data], {
type: mimeType
})
} else {
var BlobBuiler = window.BlobBuilder || window.MozBlobBuilder || window.WebKitBlobBuilder || window.MSBlobBuilder;
var builer = new BlobBuiler();
builer.append(data);
blob = builer.getBlob(mimeType)
}
var URL = window.URL || window.webkitURL;
var url = URL.createObjectURL(blob);
var link = document.createElement("a");
if ('download' in link) {
link.style.visibility = "hidden";
link.href = url;
link.download = fileName;
document.body.appendChild(link);
var j = document.createEvent("MouseEvents");
j.initEvent("click", true, true);
link.dispatchEvent(j);
document.body.removeChild(link)
} else if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, fileName)
} else {
location.href = url
}
}

7
styles/bootstrap.min.js vendored
View File

File diff suppressed because one or more lines are too long