loke/userflow-predictor
1
0
Fork 0
Code Issues Pull requests 12 Projects Releases Packages Wiki Activity Actions

Feat: bot

Browse source
This commit is contained in:
Lorenzo Iovino 2019-04-26 19:20:40 +02:00
parent b3e47f098f
commit 1c8645039e
24 changed files with 1474 additions and 376 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Predictor/PredictorWebService/package-lock.json generated
View file

@ -988,8 +988,7 @@
},
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"optional": true
"bundled": true
},
"concat-map": {
"version": "0.0.1",
@ -1100,8 +1099,7 @@
},
"inherits": {
"version": "2.0.3",
"bundled": true,
"optional": true
"bundled": true
},
"ini": {
"version": "1.3.5",
@ -1111,7 +1109,6 @@
"is-fullwidth-code-point": {
"version": "1.0.0",
"bundled": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -1223,8 +1220,7 @@
},
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"optional": true
"bundled": true
},
"object-assign": {
"version": "4.1.1",
@ -1339,7 +1335,6 @@
"string-width": {
"version": "1.0.2",
"bundled": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -2346,6 +2341,11 @@
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
},
"query-string-parser": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/query-string-parser/-/query-string-parser-0.2.3.tgz",
"integrity": "sha512-t8us5B2sj4pnMXxRkRE0B5NM7YeevFPew1ijr3uiv8AXVY4I6LwQRECxe6ZRmMJarmihhzgcjJmjbPGrMWg7iA=="
},
"range-parser": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",

10
Predictor/PredictorWebService/package.json
View file

@ -10,6 +10,7 @@
"cors": "^2.8.5",
"express": "^4.16.4",
"nodemon": "^1.18.10",
"query-string-parser": "^0.2.3",
"ws": "^6.2.1"
},
"devDependencies": {
@ -20,10 +21,11 @@
"typescript-eslint-parser": "^22.0.0"
},
"scripts": {
"build": "tsc -w",
"dev": "ts-node ./src/main.ts",
"start": "nodemon ./dist/main.js",
"prod": "npm run build && npm run start"
"build:ws": "tsc -w",
"dev:ws": "ts-node src/predictor-web-service/main.ts",
"dev:imageCreator": "ts-node src/image-creator/main.ts",
"start:ws": "nodemon ./dist/main.js",
"prod:ws": "npm run build:ws && npm run start:ws"
},
"author": "",
"license": "ISC"

158
Predictor/PredictorWebService/src/image-creator/ImageCreatorService.ts Normal file
View file

@ -0,0 +1,158 @@
import {Data} from "../../../../DataGatherer/src/shared/Data";
import {createCanvas, Image} from "canvas";
import fs from "fs";
export class ImageCreatorService {
createImageFromData(data: Array<Data>, name: string) {
return new Promise((resolve, reject) => {
const htmlElements = data.filter( val => {
return val.name !== 'screen';
});
const mouseClicks = data.filter(val => {
return val.name === 'click';
});
const mouseMovements = data.filter(val => {
return val.name === 'mousemove';
});
const keyboard = data.filter(val => {
return val.name === 'keydown';
});
//todo da cambiare le dimensioni e renderle dinamiche
const canvas = createCanvas(1000, 800);
const ctx = canvas.getContext('2d');
this.printHTMLElements(htmlElements, ctx);
this.printMouseClick(mouseClicks, ctx);
this.printMouseMove(mouseMovements, ctx);
this.printKeyboard(keyboard, ctx);
const base64data = canvas.toBuffer();
resolve({data: base64data, name: name});
});
}
saveImage(base64datas: any, path: string) {
fs.writeFile(path, base64datas, 'base64', function (err) {
if (err) throw err;
console.log('Image saved');
});
}
applyDataToImage(data: Array<Data>, name: string) {
return new Promise((resolve, reject) => {
const mouseClicks = data.filter(val => {
return val.name === 'click';
});
const mouseMovements = data.filter(val => {
return val.name === 'mousemove';
});
const keyboard = data.filter(val => {
return val.name === 'keydown';
});
const image: any = data.filter(val => {
return val.name === 'screen';
})[0];
if (image) {
const canvas = createCanvas(image.size.width, image.size.height);
const ctx = canvas.getContext('2d');
const img = new Image();
img.onload = () => {
this.printImage(img, ctx);
this.printMouseClick(mouseClicks, ctx);
this.printMouseMove(mouseMovements, ctx);
this.printKeyboard(keyboard, ctx);
const base64data = canvas.toBuffer();
resolve({data: base64data, name: name});
};
img.onerror = err => {
throw err
};
img.src = image.data;
} else {
resolve(null);
}
});
}
printMouseClick(mouseclicks: Array<Data>, ctx: CanvasRenderingContext2D) {
ctx.strokeStyle = 'rgba(219, 10, 91, 0.5)';
ctx.lineWidth = 5;
ctx.beginPath();
for(const move of mouseclicks){
ctx.strokeRect(move.data.x, move.data.y,10, 10);
}
ctx.stroke();
}
printHTMLElements(htmlElements: Array<Data>, ctx: CanvasRenderingContext2D) {
ctx.strokeStyle = 'rgba(33, 123, 22, 0.5)';
ctx.lineWidth = 2;
ctx.beginPath();
for(const element of htmlElements){
ctx.fillStyle = element.data.target.style.backgroundColor !== '' ? element.data.target.style.backgroundColor : 'transparent';
ctx.fillRect(element.data.target.rect.x, element.data.target.rect.y, element.data.target.rect.width, element.data.target.rect.height);
ctx.strokeRect(element.data.target.rect.x, element.data.target.rect.y, element.data.target.rect.width, element.data.target.rect.height);
ctx.font = "10px Arial";
ctx.fillStyle = 'black';
ctx.textAlign = "center";
ctx.fillText(element.data.target.id, element.data.target.rect.x + element.data.target.rect.width/2, element.data.target.rect.y + element.data.target.rect.height/2);
}
ctx.stroke();
}
printMouseMove(mousemoves: Array<Data>, ctx: CanvasRenderingContext2D) {
ctx.strokeStyle = 'rgba(0,0,0,0.8)';
ctx.lineWidth = 5;
ctx.beginPath();
for(const move of mousemoves){
ctx.lineTo(move.data.x, move.data.y);
}
ctx.stroke();
}
printKeyboard(keyboard: Array<Data>, ctx: CanvasRenderingContext2D) {
}
printImage(image: any, ctx: CanvasRenderingContext2D){
ctx.drawImage(image, 0, 0);
}
public startProcessing(srcPath: string, destPath: string) {
const that = this;
fs.readdir(srcPath, function(err, items) {
for (var i=0; i<items.length; i++) {
const data = fs.readFileSync(srcPath + '/' + items[i]);
try {
const jsonData = JSON.parse(data.toString('utf8'));
fs.mkdirSync('./trainingImages/virtual/' + destPath, {recursive: true});
that.createImageFromData(jsonData, items[i].split('.')[0])
.then(res => {
if(res) {
console.log('virtual image ' + ' processed');
that.saveImage(res['data'], './trainingImages/virtual/' + destPath + '/' + res['name'] + '.png');
}
});
that.applyDataToImage(jsonData, items[i].split('.')[0])
.then(res => {
if(res) {
console.log('real image ' + ' processed');
that.saveImage(res['data'], './trainingImages/real/' + destPath + '/' + res['name'] + '.png');
}
});
} catch (e) {
console.log('file not valid');
}
}
});
}
}

11
Predictor/PredictorWebService/src/image-creator/main.ts Normal file
View file

@ -0,0 +1,11 @@
import {ImageCreatorService} from "./ImageCreatorService";
function main() {
const imageCreatorService = new ImageCreatorService();
imageCreatorService.startProcessing('./trainingDatas/flow1', 'flow1');
imageCreatorService.startProcessing('./trainingDatas/flow2', 'flow2');
imageCreatorService.startProcessing('./trainingDatas/flow3', 'flow3');
}
main();

9
Predictor/PredictorWebService/src/main.ts
View file

@ -1,9 +0,0 @@
import { PredictorWebService } from "./predictor-web-service/PredictorWebService";
function main() {
const predictorWebService = new PredictorWebService('/', 4000, 4100);
predictorWebService.startExpress();
predictorWebService.startWebSocket();
}
main();

127
Predictor/PredictorWebService/src/predictor-web-service/PredictorWebService.ts
View file

@ -4,8 +4,8 @@ import * as http from 'http';
import cors from 'cors';
import * as bodyParser from "body-parser";
import fs from 'fs';
import {Data} from "../../../../DataGatherer/src/data/Data";
import {createCanvas, Image} from "canvas";
import {Data} from "../../../../DataGatherer/src/shared/Data";
import parser from 'query-string-parser';
export class PredictorWebService {
@ -16,44 +16,40 @@ export class PredictorWebService {
private httpServer: http.Server;
private wss: WebSocket.Server;
private counter: number = 0;
private oldImageData: Data;
constructor(url: string, portApi: number, portWebSocket: number) {
this.url = url;
this.portApi = portApi;
this.portWebSocket = portWebSocket;
}
public startWebSocket() {
public startTrainer() {
this.httpServer = http.createServer(this.app);
this.wss = new WebSocket.Server({port: this.portWebSocket});
this.wss.on('connection', (ws: WebSocket) => {
this.wss.on('connection', (ws: WebSocket, req: any) => {
let url = require('url').parse(req.url).query;
const queryObj = parser.fromQuery(url);
const that = this;
ws.on('message', (message: string) => {
const jsonMsg = JSON.parse(message);
const data: Array<Data> = jsonMsg.map(d => new Data(d.name, d.data, d.size, d.timestamp));
let image = data.find(val => {
return val.getName() === 'screen';
});
if(image) {
that.saveImage(image);
that.applyDataToImage(image, data)
.then(val => {
ws.send(val);
});
} else {
that.applyDataToImage(that.oldImageData, data)
.then(val => {
ws.send(val);
});
}
that.counter++;
that.saveData(data, queryObj.flowName, queryObj.agentName)
.then( (msg) => {
if(msg === 'ok') {
ws.send('trainer - Data saved');
}
})
.catch( (err) => {
ws.send('trainer - Error while saving data');
});
});
});
}
public startExpress() {
const that = this;
public startPredictor() {
this.app = express();
this.app.use(cors());
this.app.use(bodyParser.json());
@ -63,83 +59,38 @@ export class PredictorWebService {
function generateFakeResponse() {
return Array.from({length: 20}, () => Math.random().toPrecision(2));
}
console.log('Request prediction');
console.log('predictor - Request prediction');
res.send(generateFakeResponse());
});
this.app.listen(this.portApi, () => {
console.log('PredictorWebService is up and running on port: %d', this.portApi);
console.log('predictor - up and running on port: %d', this.portApi);
});
}
printMouseClick(mouseclicks: Array<Data>, ctx: any) {
ctx.strokeStyle = 'rgba(219, 10, 91, 0.5)';
ctx.lineWidth = 5;
ctx.beginPath();
for(const move of mouseclicks){
ctx.strokeRect(move.getData().x, move.getData().y,10, 10);
}
ctx.stroke();
}
printMouseMove(mousemoves: Array<Data>, ctx: any) {
ctx.strokeStyle = 'rgba(0,0,0,0.8)';
ctx.lineWidth = 5;
ctx.beginPath();
for(const move of mousemoves){
ctx.lineTo(move.getData().x, move.getData().y);
}
ctx.stroke();
}
printKeyboard(keyboard: Array<Data>, ctx: any) {
}
printImage(image: Image, ctx: any){
ctx.drawImage(image, 0, 0);
}
applyDataToImage(image: Data, data: Array<Data>) {
saveData(data: Array<Data>, flowName: string, agentName: string) {
return new Promise((resolve, reject) => {
const mouseClicks = data.filter(val => {
return val.getName() === 'click';
});
const mouseMovements = data.filter(val => {
return val.getName() === 'mousemove';
});
const keyboard = data.filter(val => {
return val.getName() === 'keydown';
});
if (image) {
const canvas = createCanvas(image.getSize().width, image.getSize().height);
const ctx = canvas.getContext('2d');
const img = new Image();
img.onload = () => {
this.printImage(img, ctx);
this.printMouseClick(mouseClicks, ctx);
this.printMouseMove(mouseMovements, ctx);
this.printKeyboard(keyboard, ctx);
const base64data = canvas.toBuffer();
fs.writeFile('./trainingDatas/' + image.getTimestamp() + '-' + Date.now() + '.png', base64data, 'base64', function (err) {
if (err) throw err;
console.log('File saved.');
resolve('ok - image received');
})
};
img.onerror = err => {
throw err
};
img.src = image.getData();
console.log(flowName);
console.log(agentName);
if(flowName && agentName) {
fs.mkdirSync('./trainingDatas/' + flowName + '/', { recursive: true });
} else {
resolve('ok - only datas');
fs.mkdirSync('./trainingDatas/undefined', { recursive: true });
}
if(!flowName && !agentName) {
fs.writeFile('./trainingDatas/undefined/' + this.counter + '.json', JSON.stringify(data), 'utf8', function (err) {
if (err) reject('err');
console.log('Data saved');
resolve('ok');
});
} else {
fs.writeFile('./trainingDatas/' + flowName + '/' + agentName + '_' + this.counter + '.json', JSON.stringify(data), 'utf8', function (err) {
if (err) reject('err');
console.log('Data saved');
resolve('ok');
});
}
});
}
saveImage(image: Data) {
if(image) {
this.oldImageData = image;
}
}
}

9
Predictor/PredictorWebService/src/predictor-web-service/main.ts Normal file
View file

@ -0,0 +1,9 @@
import { PredictorWebService } from "./PredictorWebService";
function main() {
const predictorWebService = new PredictorWebService('/', 4000, 4100);
predictorWebService.startPredictor();
predictorWebService.startTrainer();
}
main();