jebbs/data-extracter-extesion
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

allow user decision when some action fails

Browse Source
This commit is contained in:
jebbs
2020-01-13 16:47:52 +08:00
parent 21d3dfb247
commit 13e233fbe7
5 changed files with 70 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
scripts/background/actions.js
View File

@ -1,54 +1,3 @@
/**
* Extract data from current page / multiple urls.
* getData(tab, itemsSelector:string, fieldSelectors:string[])
* getData(tab, itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
* getData(tab, itemsSelector:string, fieldSelectors:string, url:string, pages:number[])
* getData(tab, itemsSelector:string, fieldSelectors:string[], urls:string[])
* getData(tab, itemsSelector:string, fieldSelectors:string[], urls:ExtractResult)
* getData(itemsSelector:string, fieldSelectors:string[])
* getData(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
* getData(itemsSelector:string, fieldSelectors:string, url:string, pages:number[])
* getData(itemsSelector:string, fieldSelectors:string[], urls:string[])
* getData(itemsSelector:string, fieldSelectors:string[], urls:ExtractResult)
* @param {...any} args
*/
async function getData(...args) {
let tab;
if (typeof args[0] !== 'string') tab = args.shift();
if (!testArgs(...args))
throw new Error(`Invalid call arguments.\n\n${signitures}\n\n`);
itemsSelector = args.shift();
fieldSelectors = args.shift();
let urls = parseUrls(...args);
let data = [];
if (!tab) tab = await getActiveTab(true) || await getActiveTab(false);
if (!tab) throw new Error("Cannot find active tab.");
return new Promise((resolve, reject) => {
let pms;
if (urls.length) {
pms = urls.reduce((p, url) => p.then(
results => {
if (results) data.push(...results);
return redirectTab(tab, url).then(
() => extractTabData(tab, itemsSelector, fieldSelectors)
);
},
() => p
), Promise.resolve([]));
} else {
pms = extractTabData(tab, itemsSelector, fieldSelectors);
}
pms.then(
results => {
if (results) data.push(...results);
data.unshift(fieldSelectors);
resolve(new ExtractResult(data));
},
err => reject(err)
);
});
}
function parseUrls(...args) {
if (!args.length) return [];
let arg = args.shift();
@ -77,19 +26,26 @@ function parseUrls(...args) {
}
function redirectTab(tab, url) {
let curUrl = "";
return queryUrl(tab, undefined, 'Query current url...')
.then(u => {
if (url !== u) {
curUrl = u;
let req = {
action: ACTION_GOTO_URL,
url: url
}
return sendMessage(tab, req, `Goto url: ${url}`);
return queryUrl(tab).then(u => {
if (url !== u) {
let req = {
action: ACTION_GOTO_URL,
url: url
}
})
.then(() => queryUrl(tab, url, 'Check if tab url matches expected...'))
let checker = async (url, err, tryCount) => {
let newURL = await queryUrl(tab).catch(() => { });
if (newURL == url) return url;
if (
tryCount % 5 == 0 &&
!confirm('Cannot navigate to target url. \nPress OK to continue, Cancel to stop.')
) {
return MSG_USER_ABORT;
}
return undefined;
}
return sendMessage(tab, req, `Goto url: ${url}`, checker);
}
});
}
/**
@ -105,8 +61,19 @@ function extractTabData(tab, itemsSelector, fieldSelectors) {
itemsSelector: itemsSelector,
fieldSelectors: fieldSelectors
}
let cond = r => !MSG_ELEMENT_NOT_FOUND.isEqual(r);
return sendMessage(tab, req, 'Extract data from the tab...', cond);
let checker = (result, err, tryCount) => {
if (MSG_ELEMENT_NOT_FOUND.isEqual(result)) {
if (tryCount % 20 == 0) {
if (confirm('No data found in current page. \n\nContinue to next page?')) {
return [];
}
} else {
return undefined;
}
}
return result;
};
return sendMessage(tab, req, 'Extract data from the tab...', checker);
}
/**
@ -118,23 +85,21 @@ async function ping(tab, count = 1) {
let req = {
action: ACTION_REPORT_IN
}
let cond = r => r == req.action;
let pong = await sendMessage(tab, req, 'Check tab availability...', cond, 1000, count).catch(() => { });
let checker = r => r == req.action ? req.action : undefined;
let pong = await sendMessage(tab, req, 'Check tab availability...', checker, 1000, count).catch(() => { });
return pong == ACTION_REPORT_IN;
}
/**
* get the url of the target tab
* @param {any} tab target tab
* @param {string} expected if specified, queryUrl resolves only when tab url equals to expected
* @returns {Promise<string>} a promise of the url
*/
function queryUrl(tab, expected, log) {
function queryUrl(tab) {
let req = {
action: ACTION_QUERY_URL
}
let cond = url => url && (!expected || (expected && expected == url));
return sendMessage(tab, req, log, cond);
return sendMessage(tab, req);
}
/**

27
scripts/background/messaging.js
View File

@ -1,18 +1,21 @@
/**
* Repeatedly sending a message to target tab until the response is detected good.
* Sending a message to target tab repeatedly until the response is not undefined.
* @param {object} tab the table where to send the message
* @param {object} req the request data.
* @param {function} cond success condition function, r:any=>boolean
* @param {function} dataChecker (reulst:any, err:error, tryCount:number) => any.
* Check and decide what value finally returns.
* Return undefined to make 'sendMessage' retry.
* Return MSG_USER_ABORT to cancel this promise.
* @param {number} interval retry interval, default: 500ms.
* @param {number} limit retry limit, default: 0, no limit.
* @param {string} log messages logged to console.
* @return {Promise} a promise of the response.
*/
function sendMessage(tab, req, log, cond, interval, limit = 0) {
function sendMessage(tab, req, log, dataChecker, interval, limit = 0) {
interval = interval || 500;
limit = limit && !isNaN(limit) ? limit : 0;
count = 0;
let count = 0;
return new Promise((resolve, reject) => {
loop();
@ -30,16 +33,22 @@ function sendMessage(tab, req, log, cond, interval, limit = 0) {
return;
}
count++;
chrome.tabs.sendMessage(tab.id, req, r => {
chrome.tabs.sendMessage(tab.id, req, async r => {
// check error but do nothing.
// do not interrupt promise chains even if error, or the task always fail when:
// a tab is newly created, and the content scripts won't have time to initialize
chrome.runtime.lastError;
let flag = !cond || cond(r);
let err = chrome.runtime.lastError;
let result = r;
if (dataChecker) {
result = await dataChecker(r, err, count);
if (MSG_USER_ABORT.isEqual(result)) {
reject(MSG_USER_ABORT.message);
}
}
let flag = result !== undefined && result !== null;
if (log) logger.info(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(r);
resolve(result);
} else {
setTimeout(() => {
loop();

13
scripts/content/content.js
View File

@ -1,7 +1,12 @@
(function () {
let asleep = false;
chrome.runtime.onMessage.addListener(
function (request, sender, sendResponse) {
if (!request.action) return;
if (asleep && ACTION_WAKEUP != request.action) {
sendResponse && sendResponse(undefined);
return;
}
// console.log("Recieved request:",request);
doAction(request, sender).then(r => sendResponse && sendResponse(r));
// return true to indicate you wish to send a response asynchronously
@ -16,6 +21,8 @@
return data;
case ACTION_GOTO_URL:
window.location.replace(request.url);
// should not recieve any request until the page & script reload
asleep = true;
return request.url;
case ACTION_REPORT_IN:
return request.action;
@ -29,6 +36,12 @@
1000,
10
)
case ACTION_SLEEP:
asleep = true;
return "Content script is sleeping.";
case ACTION_WAKEUP:
asleep = false;
return "Content script is available.";
default:
break;
}

2
scripts/shared/common.js
View File

@ -6,3 +6,5 @@ const ACTION_REPORT_IN = `${EXT_NAME}:ReportIn`;
const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
const ACTION_SCROLL_BOTTOM = `${EXT_NAME}:ScrollToBottom`;
const ACTION_UPLOAD_STATE = `${EXT_NAME}:UploadStateFile`;
const ACTION_SLEEP = `${EXT_NAME}:Sleep`;
const ACTION_WAKEUP = `${EXT_NAME}:WakeUp`;

1
scripts/shared/tools.js
View File

@ -12,6 +12,7 @@ class ConstMessage {
const URL_REG = getWebUrl();
const MSG_ELEMENT_NOT_FOUND = new ConstMessage(1, "No element found for at least one selector, maybe it's not loaded yet");
const MSG_URL_SKIPPED = new ConstMessage(100, "Skipped current URL");
const MSG_USER_ABORT = new ConstMessage(100, "Tasks stopped by user.");
function saveFile(data, mimeType, fileName) {
fileName = fileName || document.title || "result";