jebbs/data-extracter-extesion
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

better task tab management

Browse Source
This commit is contained in:
jebbs
2018-09-29 14:37:24 +08:00
parent ecd8a0d3d8
commit 7e782c633a
2 changed files with 71 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
scripts/extract.js
View File

@ -14,39 +14,29 @@ async function extract(itemsSelector, fieldSelectors, ...args) {
}
/**
* Extract data from current tab / multiple urls.
* @param {string} itemsSelector items selectors for selecting items (data rows)
* @param {Array<string>} fieldSelectors fields selectors for selecting fields (data columns) under each item
* @param {...any} args url list / url templates, page numers, either [from, to, interval] or [...pages]
* Extract data from current page / multiple urls.
* getData(tab, itemsSelector:string, fieldSelectors:string[])
* getData(tab, itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
* getData(tab, itemsSelector:string, fieldSelectors:string, url:string, pages:number[])
* getData(tab, itemsSelector:string, fieldSelectors:string[], urls:string[])
* getData(tab, itemsSelector:string, fieldSelectors:string[], urls:ExtractResult)
* getData(itemsSelector:string, fieldSelectors:string[])
* getData(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
* getData(itemsSelector:string, fieldSelectors:string, url:string, pages:number[])
* getData(itemsSelector:string, fieldSelectors:string[], urls:string[])
* getData(itemsSelector:string, fieldSelectors:string[], urls:ExtractResult)
* @param {...any} args
*/
async function getData(itemsSelector, fieldSelectors, ...args) {
if (!testArgs(itemsSelector, fieldSelectors, ...args))
async function getData(...args) {
let tab;
if (typeof args[0] !== 'string') tab = args.shift();
if (!testArgs(...args))
throw new Error(`Invalid call arguments.\n\n${signitures}\n\n`);
let urls = [];
if (args.length) {
let arg = args.shift();
if (arg instanceof Array) {
urls = arg;
} else if (arg instanceof ExtractResult) {
urls = arg.squash().filter(v => !!v);
} else {
let urlTempl = arg;
if (urlTempl) {
if (args[0] instanceof Array) {
urls = args[0].map(p => urlTempl.replace("${page}", p));
} else if (args.length >= 3) {
let from = args.shift();
let to = args.shift();
let interval = args.shift();
for (let i = from; i <= to; i += interval) {
urls.push(urlTempl.replace("${page}", i));
}
}
}
}
}
itemsSelector = args.shift();
fieldSelectors = args.shift();
let urls = parseUrls(...args);
let data = [];
let tab = await getActiveTab(true) || await getActiveTab(false);
if (!tab) tab = await getActiveTab(true) || await getActiveTab(false);
if (!tab) throw new Error("Cannot find active tab.");
return new Promise((resolve, reject) => {
let pms;
@ -73,6 +63,33 @@ async function getData(itemsSelector, fieldSelectors, ...args) {
});
}
function parseUrls(...args) {
if (!args.length) return [];
let arg = args.shift();
if (arg instanceof Array) {
return arg;
} else if (arg instanceof ExtractResult) {
return arg.squash().filter(v => !!v);
} else {
let urlTempl = arg;
if (urlTempl) {
if (args[0] instanceof Array) {
return args[0].map(p => urlTempl.replace("${page}", p));
} else if (args.length >= 3) {
let urls = [];
let from = args.shift();
let to = args.shift();
let interval = args.shift();
for (let i = from; i <= to; i += interval) {
urls.push(urlTempl.replace("${page}", i));
}
return urls;
}
}
}
return [];
}
function redirectTab(tab, url) {
let curUrl = "";
return queryUrl(tab)
@ -170,6 +187,17 @@ function sendMessage(tab, req, cond, interval) {
});
}
async function createTab(url, active) {
return new Promise((resolve, reject) => {
chrome.tabs.create({
'url': url,
'active': active
}, function (tab) {
resolve(tab);
})
})
}
async function getActiveTab(currentWindow) {
return new Promise((resolve, reject) => {
chrome.tabs.query({

16
scripts/extractor.js
View File

@ -1,6 +1,7 @@
class Extractor {
constructor() {
this._tasks = [];
this._tab = undefined;
this._results = {};
}
/**
@ -38,12 +39,20 @@ class Extractor {
console.log('No task to run.');
return;
}
let firstTaskArgs = this._tasks[0];
if (firstTaskArgs.length > 2) {
// task specifies target urls, create new tab with first url for it
let urls = parseUrls(...firstTaskArgs.slice(2, firstTaskArgs.length));
this._tab = await createTab(urls[0], false);
} else {
this._tab = await getActiveTab(false);
}
return this._tasks.reduce((pms, args, i, tasks) => {
return pms.then(
result => {
if (result === undefined) return getData(...args);
if (result === undefined) return getData(this._tab, ...args);
this._results[tasks[i - 1]] = result;
return getData(...args, result);
return getData(this._tab, ...args, result);
});
}, Promise.resolve(undefined)).then(
result => {
@ -68,11 +77,12 @@ class Extractor {
console.log(`No result cache for task (id ${taskid}). \nMake sure call ".start()" before ".restart()"?`);
return;
}
this._tab = await createTab(parseUrls(cache)[0], false)
return this._tasks.slice(taskid - 1).reduce((pms, args, i, tasks) => {
return pms.then(
result => {
this._results[tasks[i - 1]] = result;
return getData(...args, result);
return getData(this._tab, ...args, result);
});
}, Promise.resolve(cache)).then(
result => {