jebbs/data-extracter-extesion
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
jebbs
2020-01-10 12:07:21 +08:00
parent 51da68fee5
commit 26c6c1159e
15 changed files with 175 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
manifest.json
View File

@ -16,24 +16,24 @@
}, },
"background": { "background": {
"scripts": [ "scripts": [
"scripts/background.js", "scripts/shared.js",
"scripts/result.js", "scripts/background/messaging.js",
"scripts/tools.js", "scripts/background/result.js",
"scripts/extract.js", "scripts/background/signiture.js",
"scripts/extractor.js" "scripts/background/actions.js",
"scripts/background/extractor.js"
], ],
"persistent": false "persistent": false
}, },
"content_scripts": [{ "content_scripts": [{
"matches": ["*://*/*"], "matches": ["*://*/*"],
"js": [ "js": [
"scripts/jquery.min.js", "scripts/shared.js",
"scripts/content.js" "scripts/content/content.js"
], ],
"run_at": "document_idle" "run_at": "document_idle"
}], }],
"permissions": [ "permissions": [
"activeTab", "activeTab"
"storage"
] ]
} }

0
styles/bootstrap.min.css → popup/styles/bootstrap.min.css vendored
View File

9
popup/tip.html
View File

@ -3,11 +3,9 @@
<link> <link>
<meta charset="utf-8"> <meta charset="utf-8">
<title>Data Extractor</title> <title>Data Extractor</title>
<script charset="UTF-8" type="text/javascript" src="../scripts/jquery.min.js"></script> <script charset="UTF-8" type="text/javascript" src="tip.js"></script>
<script charset="UTF-8" type="text/javascript" src="../styles/bootstrap.min.js"></script>
<script charset="UTF-8" type="text/javascript" src="./tip.js"></script>
<link rel="stylesheet" href="../styles/bootstrap.min.css"> <link rel="stylesheet" href="styles/bootstrap.min.css">
</head> </head>
<body style="margin: 20px 10px;"> <body style="margin: 20px 10px;">
@ -26,7 +24,8 @@
and type your scripts in the console. and type your scripts in the console.
</p> </p>
<p> <p>
<img src="../images/console.png" alt="" style="max-width: 489px; width: 100%; border-radius: 5px"> <img src="../images/console.png" alt=""
style="max-width: 489px; width: 100%; border-radius: 5px">
</p> </p>
</div> </div>

12
popup/tip.js
View File

@ -1,14 +1,14 @@
$().ready( window.onload = function () {
() => { document.querySelector('#link-extension-detail')
$("#link-extension-detail").on('click', () => { .addEventListener('click', () => {
chrome.tabs.create({ chrome.tabs.create({
'url': `chrome://extensions/?id=${chrome.runtime.id}` 'url': `chrome://extensions/?id=${chrome.runtime.id}`
}); });
}) })
$("#link-document").on('click', () => { document.querySelector('#link-document')
.addEventListener('click', () => {
chrome.tabs.create({ chrome.tabs.create({
'url': `https://git.jebbs.co/jebbs/data-extracter-extesion` 'url': `https://git.jebbs.co/jebbs/data-extracter-extesion`
}); });
}) })
} }
);

14
scripts/background.js
View File

@ -1,14 +0,0 @@
chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) {
if (message.from === "DataExtracter:Extract") {
if (!testArgs(...message.args)) {
sendResponse(signitures);
return;
}
extract(...message.args).catch(
err => {
console.log(err);
alert(err);
}
);
}
});

71
scripts/extract.js → scripts/background/actions.js
View File

@ -1,18 +1,3 @@
/**
* Extract data from current tab / multiple urls.
* @param {string} itemsSelector items selectors for selecting items (data rows)
* @param {Array<string>} fieldSelectors fields selectors for selecting fields (data columns) under each item
* @param {...any} args url list / url templates, page numers, either [from, to, interval] or [...pages]
*/
async function extract(itemsSelector, fieldSelectors, ...args) {
let result = await getData(itemsSelector, fieldSelectors, ...args);
if (confirm(
`Click confirm to download if the sample data looks good (${result.data.length} items)：\n\n${result.toString(50) || "- Empty -"}`
)) {
saveFile(result, "text/csv");
}
}
/** /**
* Extract data from current page / multiple urls. * Extract data from current page / multiple urls.
* getData(tab, itemsSelector:string, fieldSelectors:string[]) * getData(tab, itemsSelector:string, fieldSelectors:string[])
@ -43,7 +28,7 @@ async function getData(...args) {
if (urls.length) { if (urls.length) {
pms = urls.reduce((p, url) => p.then( pms = urls.reduce((p, url) => p.then(
results => { results => {
data.push(...results); if (results) data.push(...results);
return redirectTab(tab, url).then( return redirectTab(tab, url).then(
() => extractTabData(tab, itemsSelector, fieldSelectors) () => extractTabData(tab, itemsSelector, fieldSelectors)
); );
@ -55,7 +40,8 @@ async function getData(...args) {
} }
pms.then( pms.then(
results => { results => {
data.push(...results); if (results) data.push(...results);
data.unshift(fieldSelectors);
resolve(new ExtractResult(data)); resolve(new ExtractResult(data));
}, },
err => reject(err) err => reject(err)
@ -97,7 +83,7 @@ function redirectTab(tab, url) {
if (url !== u) { if (url !== u) {
curUrl = u; curUrl = u;
let req = { let req = {
from: "GotoUrl", action: ACTION_GOTO_URL,
url: url url: url
} }
sendMessage(tab, req, `Goto url: ${url}`); sendMessage(tab, req, `Goto url: ${url}`);
@ -115,11 +101,11 @@ function redirectTab(tab, url) {
*/ */
function extractTabData(tab, itemsSelector, fieldSelectors) { function extractTabData(tab, itemsSelector, fieldSelectors) {
let req = { let req = {
from: "Extract", action: ACTION_EXTRACT,
itemsSelector: itemsSelector, itemsSelector: itemsSelector,
fieldSelectors: fieldSelectors fieldSelectors: fieldSelectors
} }
let cond = r => r && r.length; let cond = r => r !== undefined;
return sendMessage(tab, req, 'Extract data from the tab...', cond); return sendMessage(tab, req, 'Extract data from the tab...', cond);
} }
@ -130,9 +116,9 @@ function extractTabData(tab, itemsSelector, fieldSelectors) {
*/ */
function reportIn(tab) { function reportIn(tab) {
let req = { let req = {
from: "ReportIn" action: ACTION_REPORT_IN
} }
let cond = r => r == req.from; let cond = r => r == req.action;
return sendMessage(tab, req, 'Check tab availability...', cond); return sendMessage(tab, req, 'Check tab availability...', cond);
} }
@ -144,51 +130,12 @@ function reportIn(tab) {
*/ */
function queryUrl(tab, urlExcluded, log) { function queryUrl(tab, urlExcluded, log) {
let req = { let req = {
from: "QueryUrl" action: ACTION_QUERY_URL
} }
let cond = url => url && (!urlExcluded || (urlExcluded && urlExcluded != url)); let cond = url => url && (!urlExcluded || (urlExcluded && urlExcluded != url));
return sendMessage(tab, req, log, cond); return sendMessage(tab, req, log, cond);
} }
/**
* Repeatedly sending a message to target tab until the response is detected good.
* @param {object} tab the table where to send the message
* @param {object} req the request data.
* @param {function} cond success condition function, r:any=>boolean
* @param {number} interval interval for detecting
* @param {string} log messages logged to console.
* @return {Promise} a promise of the response.
*/
function sendMessage(tab, req, log, cond, interval) {
req.from = "DataExtracter:" + req.from;
interval = interval || 500;
return new Promise((resolve, reject) => {
loop();
async function loop() {
// console.log("request for", req.from);
let tabAvailable = await getTabByID(tab.id);
if (!tabAvailable) {
reject("Task interrupted due to the target tab is closed.");
return;
}
chrome.tabs.sendMessage(tab.id, req, r => {
let flag = !cond || cond(r);
if (log) console.log(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(r);
} else {
setTimeout(() => {
loop();
}, interval);
}
});
}
});
}
async function createTab(url, active) { async function createTab(url, active) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
chrome.tabs.create({ chrome.tabs.create({

6
scripts/extractor.js → scripts/background/extractor.js
View File

@ -14,9 +14,9 @@ class Extractor {
*/ */
task(...args) { task(...args) {
if (!testArgs(...args)) { if (!testArgs(...args)) {
console.log(`Invalid call arguments.\n\n${argsToString(...args)}\n${signitures}\n`); console.log(`Invalid task arguments: ${argsToString(...args)}\n\n${signitures}\n`);
// break call chain to avoid unexpected task running // break call chain to avoid unexpected task running
return undefined; return this;
} }
// given >2 arguments means the task specifies target page, // given >2 arguments means the task specifies target page,
// so it won't accept last task result as url list. // so it won't accept last task result as url list.
@ -119,7 +119,7 @@ class Extractor {
if (!taskid) return; if (!taskid) return;
const result = this._results[this._tasks[taskid - 1]]; const result = this._results[this._tasks[taskid - 1]];
if (!result) { if (!result) {
console.log(`No task result for id (${taskid}). Forget to call ".start()"?`); console.log(`No result for task #${taskid}. Forget to call ".start()"?`);
return; return;
} }
if (confirm( if (confirm(

50
scripts/background/messaging.js Normal file
View File

@ -0,0 +1,50 @@
/**
* Repeatedly sending a message to target tab until the response is detected good.
* @param {object} tab the table where to send the message
* @param {object} req the request data.
* @param {function} cond success condition function, r:any=>boolean
* @param {number} interval interval for detecting
* @param {string} log messages logged to console.
* @return {Promise} a promise of the response.
*/
function sendMessage(tab, req, log, cond, interval) {
interval = interval || 500;
return new Promise((resolve, reject) => {
loop();
async function loop() {
// console.log("request for", req.action);
let tabAvailable = await getTabByID(tab.id);
if (!tabAvailable) {
reject("Task interrupted due to the target tab is closed.");
return;
}
chrome.tabs.sendMessage(tab.id, req, r => {
if (chrome.runtime.lastError) {
reject(chrome.runtime.lastError.message);
return;
}
let flag = !cond || cond(r);
if (log) console.log(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(r);
} else {
setTimeout(() => {
loop();
}, interval);
}
});
}
});
}
chrome.runtime.onMessage.addListener(function (message, sender, sendResponse) {
if (!message.action || !message.action.startsWith(EXT_NAME)) {
return;
}
sendResponse("Calling from user pages is not allowed.");
return;
});

0
scripts/result.js → scripts/background/result.js
View File

47
scripts/tools.js → scripts/background/signiture.js
View File

@ -3,51 +3,20 @@ const signitures = `
new Extractor().task(...args).task(...args).start(); new Extractor().task(...args).task(...args).start();
## Extractor.task() Signitures: ## Extractor.task() Signitures:
---------------------------- function(itemsSelector:string, fieldSelectors:string[])
function(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
function(itemsSelector:string, fieldSelectors:string[], url:string, pages:number[])
function(itemsSelector:string, fieldSelectors:string[], urls:string[])
task(itemsSelector:string, fieldSelectors:string[]) ## Example:
task(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number) // extract all links text & url under '.item' elements
task(itemsSelector:string, fieldSelectors:string, url:string, pages:number[]) // use 'selector@attr' to get attribute of the field elements
task(itemsSelector:string, fieldSelectors:string[], urls:string[]) new Extractor().task(".item", ["a", "a@href"]).start();
task(itemsSelector:string, fieldSelectors:string[], urls:ExtractResult)
## See Detailed Help: ## See Detailed Help:
https://git.jebbs.co/jebbs/data-extracter-extesion https://git.jebbs.co/jebbs/data-extracter-extesion
`.trim(); `.trim();
function saveFile(data, mimeType, fileName) {
fileName = fileName || document.title || "result";
var blob;
if (typeof window.Blob == "function") {
blob = new Blob([data], {
type: mimeType
})
} else {
var BlobBuiler = window.BlobBuilder || window.MozBlobBuilder || window.WebKitBlobBuilder || window.MSBlobBuilder;
var builer = new BlobBuiler();
builer.append(data);
blob = builer.getBlob(mimeType)
}
var URL = window.URL || window.webkitURL;
var url = URL.createObjectURL(blob);
var link = document.createElement("a");
if ('download' in link) {
link.style.visibility = "hidden";
link.href = url;
link.download = fileName;
document.body.appendChild(link);
var j = document.createEvent("MouseEvents");
j.initEvent("click", true, true);
link.dispatchEvent(j);
document.body.removeChild(link)
} else if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, fileName)
} else {
location.href = url
}
}
function testArgs(...args) { function testArgs(...args) {
switch (args.length) { switch (args.length) {
case 0, 1: case 0, 1:

58
scripts/content.js
View File

@ -1,58 +0,0 @@
function extract(...args) {
let message = {
from: "DataExtracter:Extract",
args: args
}
chrome.runtime.sendMessage(message, r => {
if (r) console.log(r);
});
}
chrome.runtime.onMessage.addListener(
function (request, sender, sendResponse) {
if (!request.from) return;
let [ext, act] = request.from.split(":");
if (ext.toLowerCase() !== 'dataextracter') return;
// console.log(request);
switch (act.toLowerCase()) {
case "extract":
let data = extractTabData(request.itemsSelector, request.fieldSelectors);
if (sendResponse) sendResponse(data);
break;
case "gotourl":
window.location.replace(request.url);
if (sendResponse) sendResponse(request.url);
break;
case "reportin":
if (sendResponse) sendResponse(request.from);
break;
case "queryurl":
if (sendResponse) sendResponse(window.location.href);
break;
default:
break;
}
}
);
function extractTabData(itemsSelector, fieldSelectors) {
let fieldNotFound = false;
let results = $(itemsSelector).toArray().map(
item => {
return fieldSelectors.map(
selector => {
let [cls, attr] = selector.split('@').slice(0, 2);
// TODO: close tab to cancel task tip
if (fieldNotFound) return;
let fieldVals = $(item).find(cls).toArray();
if (!fieldVals.length) {
fieldNotFound = true;
return;
}
return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
}
)
}
);
return fieldNotFound ? [] : results
}

49
scripts/content/content.js Normal file
View File

@ -0,0 +1,49 @@
chrome.runtime.onMessage.addListener(
function (request, sender, sendResponse) {
if (!request.action) return;
// console.log("Recieved request:",request);
switch (request.action) {
case ACTION_EXTRACT:
let data = extract(request.itemsSelector, request.fieldSelectors);
if (sendResponse) sendResponse(data);
break;
case ACTION_GOTO_URL:
window.location.replace(request.url);
if (sendResponse) sendResponse(request.url);
break;
case ACTION_REPORT_IN:
if (sendResponse) sendResponse(request.action);
break;
case ACTION_QUERY_URL:
if (sendResponse) sendResponse(window.location.href);
break;
default:
break;
}
}
);
function extract(itemsSelector, fieldSelectors) {
// let fieldNotFound = false;
let items = Array.from(document.querySelectorAll(itemsSelector));
if (!items.length) return [];
let results = items.map(
item => {
return fieldSelectors.map(
selector => {
let [cls, attr] = selector.split('@').slice(0, 2);
// TODO: close tab to cancel task tip
// if (fieldNotFound) return;
let fieldVals = Array.from(item.querySelectorAll(cls));
// if (!fieldVals.length) {
// fieldNotFound = true;
// return;
// }
return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
}
)
}
);
// return fieldNotFound ? [] : results
return results;
}

4
scripts/jquery.min.js vendored
View File

File diff suppressed because one or more lines are too long

37
scripts/shared.js Normal file
View File

@ -0,0 +1,37 @@
const EXT_NAME = "DataExtracter";
const ACTION_EXTRACT = `${EXT_NAME}:Extract`;
const ACTION_GOTO_URL = `${EXT_NAME}:GoToTUL`;
const ACTION_REPORT_IN = `${EXT_NAME}:ReportIn`;
const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
function saveFile(data, mimeType, fileName) {
fileName = fileName || document.title || "result";
var blob;
if (typeof window.Blob == "function") {
blob = new Blob([data], {
type: mimeType
})
} else {
var BlobBuiler = window.BlobBuilder || window.MozBlobBuilder || window.WebKitBlobBuilder || window.MSBlobBuilder;
var builer = new BlobBuiler();
builer.append(data);
blob = builer.getBlob(mimeType)
}
var URL = window.URL || window.webkitURL;
var url = URL.createObjectURL(blob);
var link = document.createElement("a");
if ('download' in link) {
link.style.visibility = "hidden";
link.href = url;
link.download = fileName;
document.body.appendChild(link);
var j = document.createEvent("MouseEvents");
j.initEvent("click", true, true);
link.dispatchEvent(j);
document.body.removeChild(link)
} else if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, fileName)
} else {
location.href = url
}
}

7
styles/bootstrap.min.js vendored
View File

File diff suppressed because one or more lines are too long