jebbs/data-extracter-extesion
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

migrate to typescript, with fixes

Browse Source
This commit is contained in:
jebbs
2020-01-14 16:37:50 +08:00
parent 3d375261df
commit f06a6f4e78
32 changed files with 5071 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
.gitignore vendored
View File

@ -1,2 +1,154 @@
# File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig
# Created by https://www.gitignore.io/api/visualstudiocode,macos,node
# Edit at https://www.gitignore.io/?templates=visualstudiocode,macos,node
### macOS ###
# General
.DS_Store
Thumbs.db
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# next.js build output
.next
# nuxt.js build output
.nuxt
# rollup.js default build output
dist/
# Uncomment the public line if your project uses Gatsby
# https://nextjs.org/blog/next-9-1#public-directory-support
# https://create-react-app.dev/docs/using-the-public-folder/#docsNav
# public
# Storybook build outputs
.out
.storybook-out
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# Temporary folders
tmp/
temp/
### VisualStudioCode ###
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
# End of https://www.gitignore.io/api/visualstudiocode,macos,node
# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)

BIN
images/extnsion.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 24 KiB

45
manifest.json
View File

@ -1,45 +0,0 @@
{
"manifest_version": 2,
"name": "Data Extracter",
"version": "0.5.0",
"author": "jebbs",
"description": "Extract data from web page elements as sheet.",
"icons": {
"16": "icon.png",
"48": "icon.png",
"128": "icon.png"
},
"browser_action": {
"default_icon": "icon.png",
"default_popup": "popup/tip.html",
"default_title": "Data Extracter"
},
"background": {
"scripts": [
"scripts/shared/tools.js",
"scripts/shared/common.js",
"scripts/background/logger.js",
"scripts/background/messaging.js",
"scripts/background/result.js",
"scripts/background/signiture.js",
"scripts/background/actions.js",
"scripts/background/task.js",
"scripts/background/extractor.js",
"scripts/background/helpers.js"
],
"persistent": false
},
"content_scripts": [{
"matches": ["*://*/*"],
"js": [
"scripts/shared/tools.js",
"scripts/shared/common.js",
"scripts/content/content.js"
],
"run_at": "document_idle"
}],
"permissions": [
"activeTab",
"notifications"
]
}

4433
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

17
package.json Normal file
View File

@ -0,0 +1,17 @@
{
"name": "data-extractor",
"scripts": {
"dev": "webpack --mode=development --devtool=inline-source-map --watch",
"prod": "webpack --mode=production"
},
"devDependencies": {
"@types/chrome": "0.0.91",
"@types/node": "^13.1.6",
"copy-webpack-plugin": "^5.1.1",
"ts-loader": "^6.2.1",
"tslint": "^5.20.1",
"typescript": "^3.7.4",
"webpack": "^4.41.5",
"webpack-cli": "^3.3.10"
}
}

11
readme.md
View File

@ -171,3 +171,14 @@ Open the popup window, upload the saved state file. Then, and in the backgoud co
```js
e = new Extractor().load();
```
## Developpment
Clone this project and execute:
```sh
npm i
npm run prod
# or
npm run dev
```

3
scripts/background/helpers.js
View File

@ -1,3 +0,0 @@
function $(...args) {
return new Extractor().task(...args).start();
}

114
scripts/content/content.js
View File

@ -1,114 +0,0 @@
(function () {
let asleep = false;
chrome.runtime.onMessage.addListener(
function (request, sender, sendResponse) {
if (!request.action) return;
if (asleep && ACTION_WAKEUP != request.action) {
sendResponse && sendResponse(undefined);
return;
}
// console.log("Recieved request:",request);
doAction(request, sender).then(r => sendResponse && sendResponse(r));
// return true to indicate you wish to send a response asynchronously
return true;
}
);
async function doAction(request, sender) {
switch (request.action) {
case ACTION_EXTRACT:
let data = extract(request.itemsSelector, request.fieldSelectors);
return data;
case ACTION_GOTO_URL:
window.location.replace(request.url);
// should not recieve any request until the page & script reload
asleep = true;
return request.url;
case ACTION_REPORT_IN:
return request.action;
case ACTION_QUERY_URL:
return window.location.href;
case ACTION_SCROLL_BOTTOM:
return executeUntil(
() => window.scrollTo(0, document.body.clientHeight),
() => document.body.clientHeight - window.scrollY - window.innerHeight < 20,
"Scroll to page bottom...",
1000,
10
)
case ACTION_SLEEP:
asleep = true;
return "Content script is sleeping.";
case ACTION_WAKEUP:
asleep = false;
return "Content script is available.";
default:
break;
}
}
function extract(itemsSelector, fieldSelectors) {
// since some elements may be loaded asynchronously.
// if one field is never found, we should return undefined,
// so that senders can detect to retry until elements loaded.
// If user writes wrong selectors, the task retries infinitely.
let fieldFound = {};
let items = Array.from(document.querySelectorAll(itemsSelector));
// items may not loaded yet, tell the sender to retry.
if (!items.length) return MSG_ELEMENT_NOT_FOUND;
let results = items.map(
item => {
return fieldSelectors.map(
selector => {
let [cls, attr] = selector.split('@').slice(0, 2);
let fieldVals = Array.from(item.querySelectorAll(cls));
if (!fieldVals.length) {
return;
}
fieldFound[selector] = true;
return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
}
)
}
);
// if it exists a field, which is not found in any row, the sender should retry.
let shouldWait = fieldSelectors.reduce((p, c) => p || !fieldFound[c], false);
return shouldWait ? MSG_ELEMENT_NOT_FOUND : results
}
/**
* Repeatedly execute an function until the the detector returns true.
* @param {object} fn the function to execute
* @param {object} detector the detector.
* @param {string} log messages logged to console.
* @param {number} interval interval for detecting
* @param {number} limit max execute times of a function
* @return {Promise} a promise of the response.
*/
function executeUntil(fn, detector, log, interval, limit) {
interval = interval || 500;
let count = 0;
return new Promise((resolve, reject) => {
loop();
async function loop() {
fn();
limit++;
if (limit && count >= limit) {
reject(false);
}
setTimeout(() => {
let flag = !detector || detector();
if (log) console.log(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(true);
} else {
loop();
}
}, interval);
}
});
}
})();

10
scripts/shared/common.js
View File

@ -1,10 +0,0 @@
const EXT_NAME = "DataExtracter";
const ACTION_EXTRACT = `${EXT_NAME}:Extract`;
const ACTION_GOTO_URL = `${EXT_NAME}:GoToTUL`;
const ACTION_REPORT_IN = `${EXT_NAME}:ReportIn`;
const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
const ACTION_SCROLL_BOTTOM = `${EXT_NAME}:ScrollToBottom`;
const ACTION_UPLOAD_STATE = `${EXT_NAME}:UploadStateFile`;
const ACTION_SLEEP = `${EXT_NAME}:Sleep`;
const ACTION_WAKEUP = `${EXT_NAME}:WakeUp`;

69
scripts/shared/tools.js
View File

File diff suppressed because one or more lines are too long

82
scripts/background/actions.js → src/background/actions.ts
View File

@ -1,49 +1,35 @@
function parseUrls(...args) {
if (!args.length) return [];
let arg = args.shift();
if (arg instanceof Array) {
return arg;
} else if (arg instanceof ExtractResult) {
return arg.squash().filter(v => URL_REG.test(v));
} else {
let urlTempl = arg;
if (urlTempl) {
if (args[0] instanceof Array) {
return args[0].map(p => urlTempl.replace("${page}", p));
} else if (args.length >= 3) {
let urls = [];
let from = args.shift();
let to = args.shift();
let interval = args.shift();
for (let i = from; i <= to; i += interval) {
urls.push(urlTempl.replace("${page}", i));
}
return urls;
}
}
}
return [];
}
import { ACTION_GOTO_URL, ACTION_EXTRACT, ACTION_PING as ACTION_PING, ACTION_QUERY_URL, ACTION_SCROLL_BOTTOM } from "../common";
import { sendMessage } from "./messaging";
function redirectTab(tab, url) {
/**
* redirect tab to url.
* @param {any} tab target tab
* @param {string} url target URL
* @returns {Promise<string[]>} a promise of target URL
*/
export function redirectTab(tab: chrome.tabs.Tab, url: string) {
return queryUrl(tab).then(u => {
if (url !== u) {
let req = {
action: ACTION_GOTO_URL,
url: url
}
let checker = async (url, err, tryCount) => {
let newURL = await queryUrl(tab).catch(() => { });
let checker = async (u, err, tryCount): Promise<string> => {
let queryErr: any;
let newURL = await queryUrl(tab).catch(e => queryErr = e);
if (queryErr) {
return Promise.reject(queryErr);
}
if (newURL == url) return url;
if (
tryCount % 5 == 0 &&
!confirm('Cannot navigate to target url. \nPress OK to continue, Cancel to stop.')
) {
return MSG_USER_ABORT;
return Promise.reject("Tasks stopped by user.");
}
return undefined;
}
return sendMessage(tab, req, `Goto url: ${url}`, checker);
return sendMessage<string>(tab, req, `Goto url: ${url}`, checker);
}
});
}
@ -55,25 +41,23 @@ function redirectTab(tab, url) {
* @param {Array<string>} fieldSelectors fields selectors for selecting fields (data columns) under each item
* @returns {Promise<string[]>} a promise of extracted data
*/
function extractTabData(tab, itemsSelector, fieldSelectors) {
export function extractTabData(tab, itemsSelector, fieldSelectors) {
let req = {
action: ACTION_EXTRACT,
itemsSelector: itemsSelector,
fieldSelectors: fieldSelectors
}
let checker = (result, err, tryCount) => {
if (MSG_ELEMENT_NOT_FOUND.isEqual(result)) {
if (tryCount % 20 == 0) {
if (confirm('No data found in current page. \n\nContinue to next page?')) {
return [];
}
if (!result || !result.length) {
if (tryCount % 20 == 0 && confirm('No data found in current page. \n\nContinue to next page?')) {
return [];
} else {
return undefined;
}
}
return result;
};
return sendMessage(tab, req, 'Extract data from the tab...', checker);
return sendMessage<string[][]>(tab, req, 'Extract data from the tab...', checker);
}
/**
@ -81,13 +65,13 @@ function extractTabData(tab, itemsSelector, fieldSelectors) {
* @param {any} tab target tab
* @returns {Promise<boolean>} a promise of boolean value indicates if ping success
*/
async function ping(tab, count = 1) {
export async function ping(tab, count = 1) {
let req = {
action: ACTION_REPORT_IN
action: ACTION_PING
}
let checker = r => r == req.action ? req.action : undefined;
let pong = await sendMessage(tab, req, 'Check tab availability...', checker, 1000, count).catch(() => { });
return pong == ACTION_REPORT_IN;
let checker = (r: string, e, c) => r == "pong" ? r : undefined;
let pong = await sendMessage<string>(tab, req, 'Check tab availability...', checker, 1000, count).catch(() => { });
return pong == "pong";
}
/**
@ -95,11 +79,11 @@ async function ping(tab, count = 1) {
* @param {any} tab target tab
* @returns {Promise<string>} a promise of the url
*/
function queryUrl(tab) {
export function queryUrl(tab: chrome.tabs.Tab) {
let req = {
action: ACTION_QUERY_URL
}
return sendMessage(tab, req);
return sendMessage<string>(tab, req);
}
/**
@ -108,14 +92,14 @@ function queryUrl(tab) {
* @param {string} expected if specified, queryUrl resolves only when tab url equals to expected
* @returns {Promise<string>} a promise of the url
*/
function scrollToBottom(tab) {
export function scrollToBottom(tab: chrome.tabs.Tab) {
let req = {
action: ACTION_SCROLL_BOTTOM
}
return sendMessage(tab, req, 'Scroll to page bottom...');
}
async function createTab(url, active) {
export async function createTab(url: string, active: boolean) {
return new Promise((resolve, reject) => {
chrome.tabs.create({
'url': url,
@ -126,7 +110,7 @@ async function createTab(url, active) {
})
}
async function getActiveTab(currentWindow) {
export async function getActiveTab(currentWindow: boolean) {
return new Promise((resolve, reject) => {
chrome.tabs.query({
active: true,
@ -137,7 +121,7 @@ async function getActiveTab(currentWindow) {
})
}
async function getTabByID(id) {
export async function getTabByID(id: number) {
return new Promise((resolve, reject) => {
chrome.tabs.get(id, function (tab) {
chrome.runtime.lastError;

15
src/background/caches.ts Normal file
View File

@ -0,0 +1,15 @@
import { logger } from "./common";
export class Caches {
private _state: string = "";
constructor() { }
get state(): string {
let s = this._state;
this._state = "";
return s;
}
setState(name: string, content: string) {
this._state = content;
logger.info(`State (${name}) recieved. To load it: some_var = new Extractor().load()`);
}
}

6
src/background/common.ts Normal file
View File

@ -0,0 +1,6 @@
import { Logger, LOGGER_LEVEL } from "./logger";
import { Caches } from "./caches";
export const caches = new Caches();
export const logger = new Logger(LOGGER_LEVEL.DEBUG, LOGGER_LEVEL.DISABLED);
export const URL_REG = /^\s*(https?):\/\//im;

41
scripts/background/extractor.js → src/background/extractor.ts
View File

@ -1,10 +1,15 @@
var __EXTRACTOR_STATE__ = "";
import { Task } from "./task";
import { saveFile } from "./tools";
import { createTab, getActiveTab, ping } from "./actions";
import { logger, caches } from "./common";
import { ExtractResult } from "./result";
class Extractor {
constructor(options) {
this._tasks = [];
this._running = false;
this._options = options;
export class Extractor {
private _tasks: Task[] = [];
private _running = false;
private _options: any = {};
constructor(options?) {
if (options) this._options = options;
}
/**
* Save current state, in case we restore it later.
@ -16,12 +21,12 @@ class Extractor {
* Restore previous state by loading from saved state.
*/
load() {
if (!__EXTRACTOR_STATE__) {
let content = caches.state;
if (!content) {
logger.info('No state found. Please upload a saved state from the popup window first.');
return;
}
let state = JSON.parse(__EXTRACTOR_STATE__);
__EXTRACTOR_STATE__ = "";
let state = JSON.parse(content);
this._options = state._options;
this._tasks = state._tasks.map(t => new Task(this._options, 'whaterver', ['whaterver']).load(t));
return this;
@ -32,7 +37,7 @@ class Extractor {
* If url arguments not given within later tasks, they will use previous task result as input (target url list).
* @param {...any} args itemsSelector, fieldSelectors, and more args to specify target urls.
*/
task(...args) {
task(...args: any) {
this._tasks.push(new Task(this._options, ...args));
return this;
}
@ -53,7 +58,7 @@ class Extractor {
* restart from specified task, but don't restart the previous tasks.
* @param {number} from where to restart the tasks, begins with 0
*/
async restart(from = 0) {
async restart(from: number = 0) {
let id = this._checkTaskId(from, 0);
if (id < 0) return;
for (let i = id; i < this._tasks.length; i++) {
@ -61,7 +66,7 @@ class Extractor {
}
return this._startTasks(0);
}
async _startTasks(from) {
async _startTasks(from: number) {
if (this._running) {
logger.info('The Extractor is running. Please wait..');
return;
@ -85,7 +90,7 @@ class Extractor {
}
}
this._running = true;
return this._tasks.reduce((pms, task, i) => {
return this._tasks.reduce((pms, task: Task, i: number) => {
return pms.then(
() => {
if (i < from) return;
@ -93,9 +98,9 @@ class Extractor {
let prevTask = this._tasks[i - 1];
return task.execute(tab, new ExtractResult(prevTask.results));
}
return task.execute(tab, undefined);
return task.execute(tab);
});
}, Promise.resolve(undefined)).then(
}, Promise.resolve<void>(undefined)).then(
() => {
this._running = false;
this.export();
@ -109,7 +114,7 @@ class Extractor {
* export result of a task to CSV
* @param {number} taskid which task id to save, begins with 0
*/
export(taskid) {
export(taskid?: number) {
let id = this._checkTaskId(taskid, this._tasks.length - 1);
if (id < 0) return;
let results = this._tasks[id].results
@ -125,10 +130,10 @@ Please confirm to download (${results.length - 1} items)：
${exResults.toString(50) || "- Empty -"}
`.trim();
if (confirm(msg)) {
saveFile(exResults, "text/csv");
saveFile(exResults.toString(), "text/csv");
}
}
_checkTaskId(id, defaultId) {
_checkTaskId(id: number, defaultId: number) {
if (!this._tasks.length) {
logger.info("No task found.");
return -1;

14
src/background/index.ts Normal file
View File

@ -0,0 +1,14 @@
import { Extractor } from "./extractor";
declare global {
interface Window {
$: (...args: any) => void;
Extractor: any;
}
}
window.$ = function (...args) {
return new Extractor().task(...args).start();
}
window.Extractor = Extractor;

38
scripts/background/logger.js → src/background/logger.ts
View File

@ -1,21 +1,15 @@
const LOGGER_LEVEL = {
DEBUG: 1,
INFO: 2,
WARNING: 3,
ERROR: 4,
DISABLED: 100,
properties: {
1: { name: "debug", value: 1, prefix: "DEBUG" },
2: { name: "info", value: 2, prefix: "INFO" },
3: { name: "warning", value: 3, prefix: "WARN" },
4: { name: "error", value: 3, prefix: "ERROR" }
}
export enum LOGGER_LEVEL {
DEBUG = 1,
INFO,
WARN,
ERROR,
DISABLED,
};
class Logger {
_notificationId = undefined;
_log_level = LOGGER_LEVEL.INFO;
_notify_level = LOGGER_LEVEL.ERROR;
export class Logger {
private _notificationId = undefined;
private _log_level = LOGGER_LEVEL.INFO;
private _notify_level = LOGGER_LEVEL.ERROR;
constructor(logLevel, notifyLevel) {
if (logLevel) this._log_level = logLevel;
if (notifyLevel) this._notify_level = notifyLevel;
@ -24,19 +18,19 @@ class Logger {
get logLevel() {
return this._log_level;
}
set logLevel(val) {
set logLevel(val: LOGGER_LEVEL) {
this._log_level = val;
}
get notifyLevel() {
return this._notify_level;
}
set notifyLevel(val) {
set notifyLevel(val: LOGGER_LEVEL) {
this._notify_level = val;
}
log(level, loggerFn, ...msgs) {
log(level: LOGGER_LEVEL, loggerFn: Function, ...msgs) {
if (level < this._log_level) return;
let time = new Date().toLocaleString();
loggerFn(`${time} [${LOGGER_LEVEL.properties[level].prefix}]`, ...msgs);
loggerFn(`${time} [${LOGGER_LEVEL[level]}]`, ...msgs);
if (level < this._notify_level) return;
this.notify(...msgs);
}
@ -47,7 +41,7 @@ class Logger {
this.log(LOGGER_LEVEL.INFO, console.info, ...msgs);
}
warn(...msgs) {
this.log(LOGGER_LEVEL.WARNING, console.info, ...msgs);
this.log(LOGGER_LEVEL.WARN, console.info, ...msgs);
}
error(...msgs) {
this.log(LOGGER_LEVEL.ERROR, console.info, ...msgs);
@ -77,5 +71,3 @@ class Logger {
);
}
}
const logger = new Logger(LOGGER_LEVEL.DEBUG, LOGGER_LEVEL.DISABLED);

44
scripts/background/messaging.js → src/background/messaging.ts
View File

@ -1,3 +1,6 @@
import { EXT_NAME, ACTION_UPLOAD_STATE } from "../common";
import { getTabByID } from "./actions";
import { caches, logger } from "./common";
/**
* Sending a message to target tab repeatedly until the response is not undefined.
@ -12,11 +15,18 @@
* @param {string} log messages logged to console.
* @return {Promise} a promise of the response.
*/
function sendMessage(tab, req, log, dataChecker, interval, limit = 0) {
export function sendMessage<T>(
tab: chrome.tabs.Tab,
req,
log?: string,
dataChecker?: (r: T, err: chrome.runtime.LastError, count: number) => T | Promise<T>,
interval?: number,
limit?: number
) {
interval = interval || 500;
limit = limit && !isNaN(limit) ? limit : 0;
limit = isNaN(limit) ? 0 : limit;
let count = 0;
return new Promise((resolve, reject) => {
return new Promise<T>((resolve, reject) => {
loop();
@ -33,18 +43,27 @@ function sendMessage(tab, req, log, dataChecker, interval, limit = 0) {
return;
}
count++;
chrome.tabs.sendMessage(tab.id, req, async r => {
// check error but do nothing.
// do not interrupt promise chains even if error, or the task always fail when:
// a tab is newly created, and the content scripts won't have time to initialize
chrome.tabs.sendMessage(tab.id, req, async (r: T) => {
// check error but do nothing until dataChecker.
let err = chrome.runtime.lastError;
let result = r;
let result: T = r;
if (dataChecker) {
result = await dataChecker(r, err, count);
if (MSG_USER_ABORT.isEqual(result)) {
reject(MSG_USER_ABORT.message);
let pms = dataChecker(r, err, count);
// don't catch if it's not a Promise
if (pms instanceof Promise) {
let checkerError: any;
pms = pms.catch(e => checkerError = e);
result = await pms;
if (checkerError) {
reject(checkerError);
return;
}
} else {
result = pms;
}
}
let flag = result !== undefined && result !== null;
if (log) logger.info(log, flag ? '(OK)' : '(failed)');
if (flag) {
@ -66,8 +85,7 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
switch (request.action) {
case ACTION_UPLOAD_STATE:
sendResponse('recieved!');
__EXTRACTOR_STATE__ = request.state;
logger.info(`State (${request.name}) recieved. To load it: some_var = new Extractor().load()`);
caches.setState(request.name, request.state)
break;
default:
sendResponse("Request not supported.");

14
scripts/background/result.js → src/background/result.ts
View File

@ -1,23 +1,23 @@
class ExtractResult {
export class ExtractResult {
private _data: string[][] = [];
constructor(data) {
this._data = data || [];
}
row(index) {
row(index: number): string[] {
return this._data[index];
}
column(index) {
column(index: number): string[] {
return [...new Array(this._data.length).keys()].map(
i => this._data[i][index]
);
}
squash() {
squash(): string[] {
return this._data.reduce((p, c) => p.concat(c), []);
}
get data() {
get data(): string[][] {
return this._data;
}
toString(rowsCount) {
toString(rowsCount: number = 0): string {
let data = rowsCount > 0 ? this._data.slice(0, rowsCount) : this._data;
return data.slice().reduce(
(csv, lineCells) => {

13
scripts/background/signiture.js → src/background/signiture.ts
View File

@ -1,4 +1,6 @@
const signitures = `
import { ExtractResult } from "./result";
export const signitures = `
## Usage
// single task
$(...args);
@ -21,9 +23,10 @@ $(".item", ["a", "a@href"]);
https://git.jebbs.co/jebbs/data-extracter-extesion
`.trim();
function testArgs(...args) {
export function testArgs(...args: any) {
switch (args.length) {
case 0, 1:
case 0:
case 1:
return false;
case 2:
return args[0] && args[1] &&
@ -66,7 +69,3 @@ function testArgs(...args) {
return arr.reduce((p, c) => p && tester(c), true);
}
}
function argsToString(...args) {
return args.map(v => (v instanceof Array ? `[${v.join(', ')}]` : v.toString())).join(', ');
}

61
scripts/background/task.js → src/background/task.ts
View File

@ -1,14 +1,21 @@
class Task {
_data = {};
_data_keys = [];
/**
* Create a task.
* constructor(itemsSelector:string, fieldSelectors:string[])
* constructor(itemsSelector:string, fieldSelectors:string[], url:string, from:number, to:number, interval:number)
* constructor(itemsSelector:string, fieldSelectors:string[], url:string, pages:number[])
* constructor(itemsSelector:string, fieldSelectors:string[], urls:string[])
* @param {...any} args
*/
import { parseUrls } from "./tools";
import { queryUrl, redirectTab, scrollToBottom, extractTabData } from "./actions";
import { testArgs, signitures } from "./signiture";
import { ExtractResult } from "./result";
export class Task {
private _data: { [key: string]: string[][] } = {};
private _data_keys: string[] = [];
private _options: any;
private _itemsSelector: string;
private _fieldSelectors: string[];
private _urls: string[] = [];
constructor(options: any, ...arg: any);
constructor(options: any, itemsSelector: string, fieldSelectors: string[]);
constructor(options: any, itemsSelector: string, fieldSelectors: string[], url: string, from: number, to: number, interval: number);
constructor(options: any, itemsSelector: string, fieldSelectors: string[], url: string, pages: number[]);
constructor(options: any, itemsSelector: string, fieldSelectors: string[], urls: string[]);
constructor(options, ...args) {
if (!testArgs(...args))
throw new Error(`Invalid call arguments.\n\n${signitures}\n\n`);
@ -17,7 +24,7 @@ class Task {
this._fieldSelectors = args.shift();
this._urls = parseUrls(...args);
}
load(state) {
load(state: any): Task {
this._itemsSelector = state._itemsSelector;
this._data = state._data;
this._data_keys = state._data_keys;
@ -26,25 +33,23 @@ class Task {
this._urls = state._urls;
return this;
}
get urls() {
get urls(): string[] {
return this._urls;
}
get data() {
return this._data;
}
get results() {
get results(): string[][] {
return this._data_keys.reduce((p, c) => {
return p.concat(this._data[c]);
}, []);
}
get fieldSelectors() {
get fieldSelectors(): string[] {
return this._fieldSelectors;
}
clean() {
clean(): Task {
this._data = {};
this._data_keys = [];
return this;
}
async execute(tab, upstreamData) {
async execute(tab: chrome.tabs.Tab, upstreamData?: ExtractResult): Promise<void> {
if (!tab) return Promise.reject("No tab to execute the task.");
let urls = this._urls
if (!urls.length) {
@ -60,14 +65,12 @@ class Task {
}
return urls.reduce((p, url, i) => p.then(
results => {
if (i > 0) {
if (!MSG_URL_SKIPPED.isEqual(results)) {
let lastURL = urls[i - 1];
saveResult(results, lastURL);
}
if (i > 0 && results instanceof Array) {
let lastURL = urls[i - 1];
saveResult(results, lastURL);
}
if (this._data[url]) return MSG_URL_SKIPPED;
let pms = redirectTab(tab, url);
if (this._data[url]) return;
let pms: Promise<any> = redirectTab(tab, url);
if (this._options["scrollToBottom"]) {
pms = pms.then(() => scrollToBottom(tab));
}
@ -75,9 +78,9 @@ class Task {
() => extractTabData(tab, this._itemsSelector, this._fieldSelectors)
);
}
), Promise.resolve(null)).then(
), Promise.resolve<string[][]>(null)).then(
results => {
if (!MSG_URL_SKIPPED.isEqual(results)) {
if (results && results.length) {
let lastURL = urls[urls.length - 1];
saveResult(results, lastURL);
return;

61
src/background/tools.ts Normal file
View File

@ -0,0 +1,61 @@
import { URL_REG } from "./common";
import { ExtractResult } from "./result";
export function parseUrls(...args) {
if (!args.length) return [];
let arg = args.shift();
if (arg instanceof Array) {
return arg;
} else if (arg instanceof ExtractResult) {
return arg.squash().filter(v => URL_REG.test(v));
} else {
let urlTempl = arg;
if (urlTempl) {
if (args[0] instanceof Array) {
return args[0].map(p => urlTempl.replace("${page}", p));
} else if (args.length >= 3) {
let urls = [];
let from = args.shift();
let to = args.shift();
let interval = args.shift();
for (let i = from; i <= to; i += interval) {
urls.push(urlTempl.replace("${page}", i));
}
return urls;
}
}
}
return [];
}
export function saveFile(data: string, mimeType: string, fileName?: string) {
fileName = fileName || document.title || "result";
let blob: Blob;
if (typeof window.Blob == "function") {
blob = new Blob([data], {
type: mimeType
})
} else {
var BlobBuiler = window.MSBlobBuilder;
var builer = new BlobBuiler();
builer.append(data);
blob = builer.getBlob(mimeType)
}
var URL = window.URL || window.webkitURL;
var url = URL.createObjectURL(blob);
var link = document.createElement("a");
if ('download' in link) {
link.style.visibility = "hidden";
link.href = url;
link.download = fileName;
document.body.appendChild(link);
var j = document.createEvent("MouseEvents");
j.initEvent("click", true, true);
link.dispatchEvent(j);
document.body.removeChild(link)
} else if (navigator.msSaveBlob) {
navigator.msSaveBlob(blob, fileName)
} else {
location.href = url
}
}

11
src/common.ts Normal file
View File

@ -0,0 +1,11 @@
export const EXT_NAME = "DataExtracter";
export const ACTION_EXTRACT = `${EXT_NAME}:Extract`;
export const ACTION_GOTO_URL = `${EXT_NAME}:GoToTUL`;
export const ACTION_PING = `${EXT_NAME}:ReportIn`;
export const ACTION_QUERY_URL = `${EXT_NAME}:QueryURL`;
export const ACTION_SCROLL_BOTTOM = `${EXT_NAME}:ScrollToBottom`;
export const ACTION_UPLOAD_STATE = `${EXT_NAME}:UploadStateFile`;
export const ACTION_SLEEP = `${EXT_NAME}:Sleep`;
export const ACTION_WAKEUP = `${EXT_NAME}:WakeUp`;

73
src/content/actions.ts Normal file
View File

@ -0,0 +1,73 @@
export function extract(itemsSelector: string, fieldSelectors: string[]): string[][] {
// since some elements may be loaded asynchronously.
// if one field is never found, we should return undefined,
// so that senders can detect to retry until elements loaded.
// If user writes wrong selectors, the task retries infinitely.
let fieldFound: { [key: string]: boolean } = {};
let items: Element[] = Array.from(document.querySelectorAll(itemsSelector));
// items may not loaded yet, tell the sender to retry.
if (!items.length) return [];
let results: string[][] = items.map(
item => {
return fieldSelectors.map(
selector => {
let [cls, attr] = selector.split('@').slice(0, 2);
let fieldVals = Array.from(item.querySelectorAll(cls));
if (!fieldVals.length) {
return;
}
fieldFound[selector] = true;
return fieldVals.map(find => attr ? find[attr] : find.textContent.trim()).join('\n')
}
)
}
);
// if it exists a field, which is not found in any row, the sender should retry.
let shouldWait = fieldSelectors.reduce((p, c) => p || !fieldFound[c], false);
return shouldWait ? [] : results;
}
export function scrollToBottom() {
return executeUntil(
() => window.scrollTo(0, document.body.clientHeight),
() => document.body.clientHeight - window.scrollY - window.innerHeight < 20,
"Scroll to page bottom...",
1000,
10
);
}
/**
* Repeatedly execute an function until the the detector returns true.
* @param {object} fn the function to execute
* @param {object} detector the detector.
* @param {string} log messages logged to console.
* @param {number} interval interval for detecting
* @param {number} limit max execute times of a function
* @return {Promise} a promise of the response.
*/
function executeUntil(fn: () => void, detector: () => boolean, log: string, interval: number, limit: number) {
interval = interval || 500;
let count = 0;
return new Promise<boolean>((resolve, reject) => {
loop();
async function loop() {
fn();
limit++;
if (limit && count >= limit) {
reject(false);
}
setTimeout(() => {
let flag = !detector || detector();
if (log) console.log(log, flag ? '(OK)' : '(failed)');
if (flag) {
resolve(true);
} else {
loop();
}
}, interval);
}
});
}

45
src/content/index.ts Normal file
View File

@ -0,0 +1,45 @@
import { ACTION_WAKEUP, ACTION_EXTRACT, ACTION_GOTO_URL, ACTION_PING, ACTION_QUERY_URL, ACTION_SCROLL_BOTTOM, ACTION_SLEEP } from '../common';
import { scrollToBottom, extract } from './actions';
let asleep = false;
chrome.runtime.onMessage.addListener(
function (request, sender: chrome.runtime.MessageSender, sendResponse: (r: any) => void) {
if (!request.action) return;
if (asleep && ACTION_WAKEUP != request.action) {
sendResponse && sendResponse(undefined);
return;
}
// console.log("Recieved request:",request);
doAction(request, sender).then(r => sendResponse && sendResponse(r));
// return true to indicate you wish to send a response asynchronously
return true;
}
);
async function doAction(request: any, sender: chrome.runtime.MessageSender) {
switch (request.action) {
case ACTION_EXTRACT:
let data = extract(request.itemsSelector, request.fieldSelectors);
return data;
case ACTION_GOTO_URL:
window.location.replace(request.url);
// should not recieve any request until the page & script reload
asleep = true;
return request.url;
case ACTION_PING:
return "pong";
case ACTION_QUERY_URL:
return window.location.href;
case ACTION_SCROLL_BOTTOM:
return scrollToBottom();
case ACTION_SLEEP:
asleep = true;
return "Content script is sleeping.";
case ACTION_WAKEUP:
asleep = false;
return "Content script is available.";
default:
break;
}
}

2
popup/tip.js → src/popup/index.ts
View File

@ -1,3 +1,5 @@
import { ACTION_UPLOAD_STATE } from '../common';
window.onload = function () {
document.querySelector('#link-extension-detail')
.addEventListener('click', () => {

0
popup/styles/bootstrap.min.css → template/assets/bootstrap.min.css vendored
View File

0
images/console.png → template/assets/console.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

10
popup/tip.html → template/html/popup.html
View File

@ -3,10 +3,9 @@
<link>
<meta charset="utf-8">
<title>Data Extractor</title>
<script charset="UTF-8" type="text/javascript" src="../scripts/shared/common.js"></script>
<script charset="UTF-8" type="text/javascript" src="tip.js"></script>
<script charset="UTF-8" type="text/javascript" src="../scripts/popup.bundle.js"></script>
<link rel="stylesheet" href="styles/bootstrap.min.css">
<link rel="stylesheet" href="../assets/bootstrap.min.css">
</head>
<body style="margin: 20px 10px;">
@ -19,13 +18,12 @@
<div class="row">
<div class="col">
<div class="alert alert-info small">
<!-- <h6>Usage:</h6> -->
<p>
Goto <a href="#" id="link-extension-detail">Extension Detail</a>, click "backgroud page",
and type your scripts in the console.
</p>
<p>
<img src="../images/console.png" alt=""
<img src="../assets/console.png" alt=""
style="max-width: 489px; width: 100%; border-radius: 5px">
</p>
@ -66,7 +64,7 @@
</div>
<div class="row">
<div class="col">
<input type="file" name="state" id="state-input">
<input type="file" name="state" id="state-input">
</div>
</div>
</div>

0
icon.png → template/icon.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.1 KiB

After

Width:  |  Height:  |  Size: 4.1 KiB

34
template/manifest.json Executable file
View File

@ -0,0 +1,34 @@
{
"manifest_version": 2,
"name": "Data Extracter",
"version": "0.5.1",
"author": "jebbs",
"description": "Extract data from web page elements as sheet.",
"icons": {
"16": "icon.png",
"48": "icon.png",
"128": "icon.png"
},
"browser_action": {
"default_icon": "icon.png",
"default_popup": "html/popup.html",
"default_title": "Data Extracter"
},
"background": {
"scripts": [
"scripts/background.bundle.js"
],
"persistent": false
},
"content_scripts": [{
"matches": ["*://*/*"],
"js": [
"scripts/content.bundle.js"
],
"run_at": "document_idle"
}],
"permissions": [
"activeTab",
"notifications"
]
}

12
tsconfig.json Normal file
View File

@ -0,0 +1,12 @@
{
"compilerOptions": {
"module": "commonjs",
"target": "es6",
"noImplicitAny": false,
"sourceMap": true,
"rootDir": "src",
"outDir": "dist/js",
"noEmitOnError": true,
"typeRoots": [ "node_modules/@types" ]
}
}

33
webpack.config.js Normal file
View File

@ -0,0 +1,33 @@
const path = require('path');
const CopyPlugin = require('copy-webpack-plugin');
module.exports = {
mode: 'production',
entry: {
background: './src/background/index.ts',
content: './src/content/index.ts',
popup: './src/popup/index.ts',
},
// devtool: 'inline-source-map',
output: {
path: path.resolve(__dirname, 'dist'),
filename: 'scripts/[name].bundle.js'
},
module: {
rules: [
{
test: /\.tsx?$/,
use: 'ts-loader',
exclude: /node_modules/
}
]
},
resolve: {
extensions: ['.tsx', '.ts', '.js']
},
plugins: [
new CopyPlugin([
{ from: '**/*', to: '.', toType: "dir" },
], { context: 'template', logLevel: 'warn' }),
]
};