Feat: bot
This commit is contained in:
Lorenzo Iovino
parent
b3e47f098f
commit
1c8645039e
24 changed files with 1474 additions and 376 deletions
|
|
@ -0,0 +1,158 @@
|
|||
import {Data} from "../../../../DataGatherer/src/shared/Data";
|
||||
import {createCanvas, Image} from "canvas";
|
||||
import fs from "fs";
|
||||
|
||||
export class ImageCreatorService {
|
||||
|
||||
createImageFromData(data: Array<Data>, name: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const htmlElements = data.filter( val => {
|
||||
return val.name !== 'screen';
|
||||
});
|
||||
const mouseClicks = data.filter(val => {
|
||||
return val.name === 'click';
|
||||
});
|
||||
|
||||
const mouseMovements = data.filter(val => {
|
||||
return val.name === 'mousemove';
|
||||
});
|
||||
|
||||
const keyboard = data.filter(val => {
|
||||
return val.name === 'keydown';
|
||||
});
|
||||
|
||||
//todo da cambiare le dimensioni e renderle dinamiche
|
||||
const canvas = createCanvas(1000, 800);
|
||||
const ctx = canvas.getContext('2d');
|
||||
this.printHTMLElements(htmlElements, ctx);
|
||||
this.printMouseClick(mouseClicks, ctx);
|
||||
this.printMouseMove(mouseMovements, ctx);
|
||||
this.printKeyboard(keyboard, ctx);
|
||||
const base64data = canvas.toBuffer();
|
||||
resolve({data: base64data, name: name});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
saveImage(base64datas: any, path: string) {
|
||||
fs.writeFile(path, base64datas, 'base64', function (err) {
|
||||
if (err) throw err;
|
||||
console.log('Image saved');
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
applyDataToImage(data: Array<Data>, name: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const mouseClicks = data.filter(val => {
|
||||
return val.name === 'click';
|
||||
});
|
||||
const mouseMovements = data.filter(val => {
|
||||
return val.name === 'mousemove';
|
||||
});
|
||||
const keyboard = data.filter(val => {
|
||||
return val.name === 'keydown';
|
||||
});
|
||||
const image: any = data.filter(val => {
|
||||
return val.name === 'screen';
|
||||
})[0];
|
||||
|
||||
if (image) {
|
||||
const canvas = createCanvas(image.size.width, image.size.height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
this.printImage(img, ctx);
|
||||
this.printMouseClick(mouseClicks, ctx);
|
||||
this.printMouseMove(mouseMovements, ctx);
|
||||
this.printKeyboard(keyboard, ctx);
|
||||
const base64data = canvas.toBuffer();
|
||||
resolve({data: base64data, name: name});
|
||||
};
|
||||
img.onerror = err => {
|
||||
throw err
|
||||
};
|
||||
img.src = image.data;
|
||||
} else {
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
printMouseClick(mouseclicks: Array<Data>, ctx: CanvasRenderingContext2D) {
|
||||
ctx.strokeStyle = 'rgba(219, 10, 91, 0.5)';
|
||||
ctx.lineWidth = 5;
|
||||
ctx.beginPath();
|
||||
for(const move of mouseclicks){
|
||||
ctx.strokeRect(move.data.x, move.data.y,10, 10);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
printHTMLElements(htmlElements: Array<Data>, ctx: CanvasRenderingContext2D) {
|
||||
ctx.strokeStyle = 'rgba(33, 123, 22, 0.5)';
|
||||
ctx.lineWidth = 2;
|
||||
ctx.beginPath();
|
||||
for(const element of htmlElements){
|
||||
ctx.fillStyle = element.data.target.style.backgroundColor !== '' ? element.data.target.style.backgroundColor : 'transparent';
|
||||
ctx.fillRect(element.data.target.rect.x, element.data.target.rect.y, element.data.target.rect.width, element.data.target.rect.height);
|
||||
ctx.strokeRect(element.data.target.rect.x, element.data.target.rect.y, element.data.target.rect.width, element.data.target.rect.height);
|
||||
ctx.font = "10px Arial";
|
||||
ctx.fillStyle = 'black';
|
||||
ctx.textAlign = "center";
|
||||
ctx.fillText(element.data.target.id, element.data.target.rect.x + element.data.target.rect.width/2, element.data.target.rect.y + element.data.target.rect.height/2);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
printMouseMove(mousemoves: Array<Data>, ctx: CanvasRenderingContext2D) {
|
||||
ctx.strokeStyle = 'rgba(0,0,0,0.8)';
|
||||
ctx.lineWidth = 5;
|
||||
ctx.beginPath();
|
||||
for(const move of mousemoves){
|
||||
ctx.lineTo(move.data.x, move.data.y);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
printKeyboard(keyboard: Array<Data>, ctx: CanvasRenderingContext2D) {
|
||||
|
||||
}
|
||||
|
||||
printImage(image: any, ctx: CanvasRenderingContext2D){
|
||||
ctx.drawImage(image, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
public startProcessing(srcPath: string, destPath: string) {
|
||||
const that = this;
|
||||
fs.readdir(srcPath, function(err, items) {
|
||||
|
||||
for (var i=0; i<items.length; i++) {
|
||||
const data = fs.readFileSync(srcPath + '/' + items[i]);
|
||||
try {
|
||||
const jsonData = JSON.parse(data.toString('utf8'));
|
||||
fs.mkdirSync('./trainingImages/virtual/' + destPath, {recursive: true});
|
||||
that.createImageFromData(jsonData, items[i].split('.')[0])
|
||||
.then(res => {
|
||||
if(res) {
|
||||
console.log('virtual image ' + ' processed');
|
||||
that.saveImage(res['data'], './trainingImages/virtual/' + destPath + '/' + res['name'] + '.png');
|
||||
}
|
||||
});
|
||||
|
||||
that.applyDataToImage(jsonData, items[i].split('.')[0])
|
||||
.then(res => {
|
||||
if(res) {
|
||||
console.log('real image ' + ' processed');
|
||||
that.saveImage(res['data'], './trainingImages/real/' + destPath + '/' + res['name'] + '.png');
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.log('file not valid');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
11
Predictor/PredictorWebService/src/image-creator/main.ts
Normal file
11
Predictor/PredictorWebService/src/image-creator/main.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import {ImageCreatorService} from "./ImageCreatorService";
|
||||
|
||||
function main() {
|
||||
const imageCreatorService = new ImageCreatorService();
|
||||
imageCreatorService.startProcessing('./trainingDatas/flow1', 'flow1');
|
||||
imageCreatorService.startProcessing('./trainingDatas/flow2', 'flow2');
|
||||
imageCreatorService.startProcessing('./trainingDatas/flow3', 'flow3');
|
||||
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
import { PredictorWebService } from "./predictor-web-service/PredictorWebService";
|
||||
|
||||
function main() {
|
||||
const predictorWebService = new PredictorWebService('/', 4000, 4100);
|
||||
predictorWebService.startExpress();
|
||||
predictorWebService.startWebSocket();
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -4,8 +4,8 @@ import * as http from 'http';
|
|||
import cors from 'cors';
|
||||
import * as bodyParser from "body-parser";
|
||||
import fs from 'fs';
|
||||
import {Data} from "../../../../DataGatherer/src/data/Data";
|
||||
import {createCanvas, Image} from "canvas";
|
||||
import {Data} from "../../../../DataGatherer/src/shared/Data";
|
||||
import parser from 'query-string-parser';
|
||||
|
||||
export class PredictorWebService {
|
||||
|
||||
|
|
@ -16,44 +16,40 @@ export class PredictorWebService {
|
|||
private httpServer: http.Server;
|
||||
private wss: WebSocket.Server;
|
||||
|
||||
private counter: number = 0;
|
||||
|
||||
private oldImageData: Data;
|
||||
constructor(url: string, portApi: number, portWebSocket: number) {
|
||||
this.url = url;
|
||||
this.portApi = portApi;
|
||||
this.portWebSocket = portWebSocket;
|
||||
}
|
||||
|
||||
public startWebSocket() {
|
||||
public startTrainer() {
|
||||
this.httpServer = http.createServer(this.app);
|
||||
this.wss = new WebSocket.Server({port: this.portWebSocket});
|
||||
|
||||
this.wss.on('connection', (ws: WebSocket) => {
|
||||
this.wss.on('connection', (ws: WebSocket, req: any) => {
|
||||
let url = require('url').parse(req.url).query;
|
||||
const queryObj = parser.fromQuery(url);
|
||||
const that = this;
|
||||
ws.on('message', (message: string) => {
|
||||
const jsonMsg = JSON.parse(message);
|
||||
const data: Array<Data> = jsonMsg.map(d => new Data(d.name, d.data, d.size, d.timestamp));
|
||||
let image = data.find(val => {
|
||||
return val.getName() === 'screen';
|
||||
});
|
||||
if(image) {
|
||||
that.saveImage(image);
|
||||
that.applyDataToImage(image, data)
|
||||
.then(val => {
|
||||
ws.send(val);
|
||||
});
|
||||
} else {
|
||||
that.applyDataToImage(that.oldImageData, data)
|
||||
.then(val => {
|
||||
ws.send(val);
|
||||
});
|
||||
}
|
||||
that.counter++;
|
||||
that.saveData(data, queryObj.flowName, queryObj.agentName)
|
||||
.then( (msg) => {
|
||||
if(msg === 'ok') {
|
||||
ws.send('trainer - Data saved');
|
||||
}
|
||||
})
|
||||
.catch( (err) => {
|
||||
ws.send('trainer - Error while saving data');
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public startExpress() {
|
||||
const that = this;
|
||||
public startPredictor() {
|
||||
this.app = express();
|
||||
this.app.use(cors());
|
||||
this.app.use(bodyParser.json());
|
||||
|
|
@ -63,83 +59,38 @@ export class PredictorWebService {
|
|||
function generateFakeResponse() {
|
||||
return Array.from({length: 20}, () => Math.random().toPrecision(2));
|
||||
}
|
||||
console.log('Request prediction');
|
||||
console.log('predictor - Request prediction');
|
||||
res.send(generateFakeResponse());
|
||||
});
|
||||
|
||||
this.app.listen(this.portApi, () => {
|
||||
console.log('PredictorWebService is up and running on port: %d', this.portApi);
|
||||
console.log('predictor - up and running on port: %d', this.portApi);
|
||||
});
|
||||
}
|
||||
|
||||
printMouseClick(mouseclicks: Array<Data>, ctx: any) {
|
||||
ctx.strokeStyle = 'rgba(219, 10, 91, 0.5)';
|
||||
ctx.lineWidth = 5;
|
||||
ctx.beginPath();
|
||||
for(const move of mouseclicks){
|
||||
ctx.strokeRect(move.getData().x, move.getData().y,10, 10);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
printMouseMove(mousemoves: Array<Data>, ctx: any) {
|
||||
ctx.strokeStyle = 'rgba(0,0,0,0.8)';
|
||||
ctx.lineWidth = 5;
|
||||
ctx.beginPath();
|
||||
for(const move of mousemoves){
|
||||
ctx.lineTo(move.getData().x, move.getData().y);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
printKeyboard(keyboard: Array<Data>, ctx: any) {
|
||||
|
||||
}
|
||||
|
||||
printImage(image: Image, ctx: any){
|
||||
ctx.drawImage(image, 0, 0);
|
||||
}
|
||||
|
||||
applyDataToImage(image: Data, data: Array<Data>) {
|
||||
saveData(data: Array<Data>, flowName: string, agentName: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const mouseClicks = data.filter(val => {
|
||||
return val.getName() === 'click';
|
||||
});
|
||||
const mouseMovements = data.filter(val => {
|
||||
return val.getName() === 'mousemove';
|
||||
});
|
||||
const keyboard = data.filter(val => {
|
||||
return val.getName() === 'keydown';
|
||||
});
|
||||
if (image) {
|
||||
const canvas = createCanvas(image.getSize().width, image.getSize().height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
this.printImage(img, ctx);
|
||||
this.printMouseClick(mouseClicks, ctx);
|
||||
this.printMouseMove(mouseMovements, ctx);
|
||||
this.printKeyboard(keyboard, ctx);
|
||||
const base64data = canvas.toBuffer();
|
||||
fs.writeFile('./trainingDatas/' + image.getTimestamp() + '-' + Date.now() + '.png', base64data, 'base64', function (err) {
|
||||
if (err) throw err;
|
||||
console.log('File saved.');
|
||||
resolve('ok - image received');
|
||||
})
|
||||
};
|
||||
img.onerror = err => {
|
||||
throw err
|
||||
};
|
||||
img.src = image.getData();
|
||||
console.log(flowName);
|
||||
console.log(agentName);
|
||||
if(flowName && agentName) {
|
||||
fs.mkdirSync('./trainingDatas/' + flowName + '/', { recursive: true });
|
||||
} else {
|
||||
resolve('ok - only datas');
|
||||
fs.mkdirSync('./trainingDatas/undefined', { recursive: true });
|
||||
}
|
||||
if(!flowName && !agentName) {
|
||||
fs.writeFile('./trainingDatas/undefined/' + this.counter + '.json', JSON.stringify(data), 'utf8', function (err) {
|
||||
if (err) reject('err');
|
||||
console.log('Data saved');
|
||||
resolve('ok');
|
||||
});
|
||||
} else {
|
||||
fs.writeFile('./trainingDatas/' + flowName + '/' + agentName + '_' + this.counter + '.json', JSON.stringify(data), 'utf8', function (err) {
|
||||
if (err) reject('err');
|
||||
console.log('Data saved');
|
||||
resolve('ok');
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
saveImage(image: Data) {
|
||||
if(image) {
|
||||
this.oldImageData = image;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
import { PredictorWebService } from "./PredictorWebService";
|
||||
|
||||
function main() {
|
||||
const predictorWebService = new PredictorWebService('/', 4000, 4100);
|
||||
predictorWebService.startPredictor();
|
||||
predictorWebService.startTrainer();
|
||||
}
|
||||
|
||||
main();
|
||||
Loading…
Add table
Add a link
Reference in a new issue