jebbs/data-extracter-extesion
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Extractor.watch()

Browse Source
This commit is contained in:
jebbs
2020-01-15 17:53:23 +08:00
parent 3338f78d91
commit 7644a1363f
6 changed files with 122 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/background/caches.ts
View File

@ -1,8 +1,15 @@
import { logger } from "./common";
import { Actions } from "../common";
import { messageSubscribers } from "./messaging";
export class Caches {
private _state: string = "";
constructor() { }
constructor() {
messageSubscribers.addListener(Actions.UPLOAD_STATE, (request, sender, sendResponse) => {
sendResponse('recieved!');
this.setState(request.fileName, request.state)
});
}
get state(): string {
let s = this._state;
this._state = "";

15
src/background/extractor.ts
View File

@ -54,11 +54,22 @@ export class Extractor {
start() {
return this._startTasks(0);
}
stop() {
stop(id?: number) {
if (id !== undefined) {
id = this._checkTaskId(id);
if (id < 0) return;
this._tasks[id].stop();
return;
}
for (let i = 0; i < this._tasks.length; i++) {
this._tasks[i].stop();
}
}
watch(id: number) {
id = this._checkTaskId(id);
if (id < 0) return;
this._tasks[id].watch();
}
/**
* restart from specified task, but don't restart the previous tasks.
* @param {number} from where to restart the tasks, begins with 0
@ -138,7 +149,7 @@ ${exResults.toString(50) || "- Empty -"}
saveFile(exResults.toString(), "text/csv");
}
}
_checkTaskId(id: number, defaultId: number) {
_checkTaskId(id: number, defaultId?: number) {
if (!this._tasks.length) {
logger.info("No task found.");
return -1;

47
src/background/messaging.ts
View File

@ -1,6 +1,6 @@
import { Request, Actions } from "../common";
import { getTabByID } from "./actions";
import { caches, logger } from "./common";
import { logger } from "./common";
/**
* Sending a message to target tab repeatedly until the response is not undefined.
@ -78,14 +78,41 @@ export function sendMessage<T>(
});
}
chrome.runtime.onMessage.addListener(function (request: Request, sender, sendResponse) {
switch (request.action) {
case Actions.UPLOAD_STATE:
sendResponse('recieved!');
caches.setState(request.fileName, request.state)
break;
default:
sendResponse("Request not supported.");
break;
export type ActionSubscriber = (request: Request, sender: chrome.runtime.MessageSender, sendResponse: (response?: any) => void) => void | Promise<void>;
class MessageSubscribers {
private listeners: { [key: number]: ActionSubscriber[] } = {};
addListener(action: Actions, subscriber: ActionSubscriber) {
this.listeners[action] || (this.listeners[action] = []);
this.listeners[action].push(subscriber);
}
removeListener(action: Actions, subscriber: ActionSubscriber) {
this.listeners[action] || (this.listeners[action] = []);
for (let i = 0; i < this.listeners[action].length; i++) {
if (this.listeners[action][i] == subscriber) {
this.listeners[action].splice(i, 1);
i--;
}
}
logger.debug(`${this.listeners[action].length} subscriber(s) remained for action ${Actions[action]}`);
}
getListeners(action: Actions): ActionSubscriber[] {
return this.listeners[action]
}
}
export const messageSubscribers = new MessageSubscribers();
chrome.runtime.onMessage.addListener(function (request: Request, sender, sendResponse) {
let subscribers = messageSubscribers.getListeners(request.action);
if (!subscribers || !subscribers.length) {
sendResponse("Request not supported.");
return;
}
let promises: Promise<any>[] = [];
for (let subscriber of subscribers) {
let p = subscriber(request, sender, sendResponse);
if (p instanceof Promise) promises.push(p);
}
if (promises.length)
return Promise.all(promises);
return;
});

72
src/background/task.ts
View File

@ -2,6 +2,9 @@ import { parseUrls } from "./tools";
import { queryUrl, redirectTab, scrollToBottom, extractTabData } from "./actions";
import { testArgs, signitures } from "./signiture";
import { ExtractResult } from "./result";
import { messageSubscribers, ActionSubscriber } from "./messaging";
import { Actions } from "../common";
import { logger } from "./common";
export class Task {
private _data: { [key: string]: string[][] } = {};
@ -46,12 +49,22 @@ export class Task {
return this._fieldSelectors;
}
clean(): Task {
this.stop();
this._data = {};
this._data_keys = [];
return this;
}
stop() {
this._running = false;
messageSubscribers.removeListener(Actions.REPORT_NEW_PAGE, this.listener);
}
watch() {
if (this._running) {
logger.info("The task is running. Please wait...");
return;
}
this._running = true;
messageSubscribers.addListener(Actions.REPORT_NEW_PAGE, this.listener);
}
async execute(tab: chrome.tabs.Tab, upstreamData?: ExtractResult): Promise<void> {
if (!tab) return Promise.reject("No tab to execute the task.");
@ -65,35 +78,26 @@ export class Task {
urls = [await queryUrl(tab)];
}
}
let saveResult = (results, key) => {
this._data[key] = results;
this._data_keys.push(key);
}
let runningCheck = (fn: () => Promise<any>): Promise<any> => {
if (!this._running) return Promise.reject("The task is stopped by user.");
return fn();
}
return urls.reduce((p, url, i) => p.then(
results => {
if (i > 0 && results instanceof Array) {
let lastURL = urls[i - 1];
saveResult(results, lastURL);
this.saveResult(results, lastURL);
}
if (this._data[url]) return;
let pms: Promise<any> = runningCheck(() => redirectTab(tab, url));
if (this._options["scrollToBottom"]) {
pms = pms.then(() => runningCheck(() => scrollToBottom(tab)));
}
return pms.then(
() => runningCheck(() => extractTabData(tab, this._itemsSelector, this._fieldSelectors))
);
let pms: Promise<any> = this.runningCheck(() => redirectTab(tab, url));
return pms
.then(() => this.makeOptionalTasks(tab))
.then(
() => this.runningCheck(() => extractTabData(tab, this._itemsSelector, this._fieldSelectors))
);
}
), Promise.resolve<string[][]>(null)).then(
results => {
if (results && results.length) {
let lastURL = urls[urls.length - 1];
saveResult(results, lastURL);
this.saveResult(results, lastURL);
this._running = false;
}
}
@ -104,4 +108,38 @@ export class Task {
}
);
}
private listener: ActionSubscriber = (request, sender, sendResponse) => {
sendResponse('recieved!');
let pm = this.makeOptionalTasks(sender.tab);
return pm.then(
() => extractTabData(sender.tab, this._itemsSelector, this._fieldSelectors)
).then(
results => {
if (results && results.length) {
this.saveResult(results, sender.tab.url);
}
}
).catch(
e => logger.error(e)
)
}
private makeOptionalTasks(tab: chrome.tabs.Tab): Promise<any> {
let pm: Promise<any>;
if (this._options["scrollToBottom"]) {
pm = this.runningCheck(() => scrollToBottom(tab));
}
return pm;
}
private runningCheck(fn: () => Promise<any>): Promise<any> {
if (!this._running) return Promise.reject("The task is stopped by user.");
return fn();
}
private saveResult(results, key) {
if (this._data[key] === undefined) {
// do not add keys again
this._data_keys.push(key);
}
this._data[key] = results;
logger.info(`${results.length} items found.`)
}
}

6
src/common.ts
View File

@ -1,12 +1,16 @@
export enum Actions {
// from background to content script
EXTRACT = 1,
GOTO_URL,
PING,
QUERY_URL,
SCROLL_BOTTOM,
UPLOAD_STATE,
SLEEP,
WAKEUP,
// from popup to background script
UPLOAD_STATE,
// from content to background script
REPORT_NEW_PAGE,
}
export interface Request {

4
src/content/index.ts
View File

@ -16,6 +16,10 @@ chrome.runtime.onMessage.addListener(
}
);
chrome.runtime.sendMessage(<Request>{
action: Actions.REPORT_NEW_PAGE,
});
async function doAction(request: Request, sender: chrome.runtime.MessageSender) {
switch (request.action) {
case Actions.EXTRACT: