From b3e47f098f55a63cb50ebbf82088f722a75e9aea Mon Sep 17 00:00:00 2001 From: Lorenzo Iovino Date: Thu, 11 Apr 2019 16:13:54 +0200 Subject: [PATCH] Feat: Added websocket comm. for DataGatherer and PredictorWebService --- .idea/workspace.xml | 262 ++++++++++-------- DataGatherer/src/main.ts | 8 +- DataGatherer/src/sender/Sender.ts | 37 ++- DataGatherer/src/source/screen/Screen.ts | 3 +- .../PredictorWebService.ts | 109 +++++--- 5 files changed, 245 insertions(+), 174 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index a2f005d..fcf8bb0 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -5,20 +5,11 @@ - - - - - + - - - - - - - + + @@ -30,12 +21,12 @@ - + - - + + @@ -43,11 +34,20 @@ + + + + + + + + + - - + + @@ -55,11 +55,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -79,8 +103,8 @@ - - + + @@ -91,8 +115,8 @@ - - + + @@ -100,6 +124,15 @@ + + + + + + + + + @@ -108,6 +141,7 @@ new Data stroke window + req @@ -118,12 +152,13 @@ @@ -139,10 +174,10 @@ - @@ -239,8 +274,9 @@ + - + @@ -267,25 +303,22 @@ - - + - - + - - - - - + @@ -293,6 +326,11 @@ + + + + + @@ -309,13 +347,6 @@ - - - - - - - @@ -354,50 +385,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -405,27 +392,10 @@ - - - - - - - - - - - - - - - - - - - + + @@ -444,10 +414,78 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/DataGatherer/src/main.ts b/DataGatherer/src/main.ts index 17aca91..e7e1fcc 100644 --- a/DataGatherer/src/main.ts +++ b/DataGatherer/src/main.ts @@ -12,14 +12,14 @@ function main() { ]); gatherer.start(); - startPrediction(gatherer); + //startPrediction(gatherer); startGathering(gatherer); } function startPrediction(gatherer: Gatherer) { const sender: Sender = new Sender(() => gatherer.getData(), 'localhost:4000/predict', 10000); - sender.start() + sender.start('http') .subscribe( val => { console.log(val); @@ -29,8 +29,8 @@ function startPrediction(gatherer: Gatherer) { function startGathering(gatherer: Gatherer) { - const sender: Sender = new Sender(() => gatherer.getData(), 'localhost:4000/trainData', 3000); - sender.start() + const sender: Sender = new Sender(() => gatherer.getData(), 'localhost:4100/', 3000); + sender.start('ws') .subscribe( val => { console.log(val); diff --git a/DataGatherer/src/sender/Sender.ts b/DataGatherer/src/sender/Sender.ts index 981635b..0316844 100644 --- a/DataGatherer/src/sender/Sender.ts +++ b/DataGatherer/src/sender/Sender.ts @@ -1,11 +1,13 @@ import { interval, Observable, of } from 'rxjs'; import { filter, flatMap, delay } from 'rxjs/operators'; import { Rxios } from 'rxios'; +import {Data} from "../data/Data"; export class Sender { private url: string; private interval: number; private dataSourceFn: any; + private wsConnect; constructor(dataSourceFn: any, url: string, interval: number) { this.url = url; @@ -13,22 +15,35 @@ export class Sender { this.dataSourceFn = dataSourceFn; } - public start() { + public start(protocol: string) { + if(protocol === 'ws') { + this.wsConnect = new WebSocket('ws://' + this.url); + } return interval(this.interval) .pipe( - flatMap(() => this.send(this.dataSourceFn())) + flatMap(() => this.send(this.dataSourceFn(), protocol)) ) } - private send(data: any) { - const http: Rxios = new Rxios(); - /* var wsConnect = new WebSocket('ws://' + this.url, "protocolOne"); - wsConnect.send(data); - wsConnect.onmessage((msg) => { - console.log(msg) + private send(data: Array, protocol: string) { + if(data.length > 0) { + if (protocol === 'http') { + const http: Rxios = new Rxios(); + return http.post('http://' + this.url, data); } - //gestire con rxjs o un wrapper di websocket client - )*/ - return http.post('http://' + this.url, data); + + if (protocol === 'ws') { + this.wsConnect.send(JSON.stringify(data)); + return new Observable((observer) => { + this.wsConnect.onmessage = (msg) => { + console.log(msg); + observer.next(msg); + }; + }); + + } + } else { + return of(null) + } } } diff --git a/DataGatherer/src/source/screen/Screen.ts b/DataGatherer/src/source/screen/Screen.ts index 86762be..9bc8c9a 100644 --- a/DataGatherer/src/source/screen/Screen.ts +++ b/DataGatherer/src/source/screen/Screen.ts @@ -13,11 +13,10 @@ export class Screen extends Source { if (document.body) { Html2CanvasStatic(document.body, {logging: false}) .then((canvas) => { - console.log(canvas); const imgData = canvas.toDataURL("image/png"); this.data.push(new Data('screen', imgData, {width: window.innerWidth, height: window.innerHeight})) }); } - }, 3000); + }, 5000); } } diff --git a/Predictor/PredictorWebService/src/predictor-web-service/PredictorWebService.ts b/Predictor/PredictorWebService/src/predictor-web-service/PredictorWebService.ts index 5bf3fec..cb87072 100644 --- a/Predictor/PredictorWebService/src/predictor-web-service/PredictorWebService.ts +++ b/Predictor/PredictorWebService/src/predictor-web-service/PredictorWebService.ts @@ -16,6 +16,8 @@ export class PredictorWebService { private httpServer: http.Server; private wss: WebSocket.Server; + + private oldImageData: Data; constructor(url: string, portApi: number, portWebSocket: number) { this.url = url; this.portApi = portApi; @@ -26,11 +28,26 @@ export class PredictorWebService { this.httpServer = http.createServer(this.app); this.wss = new WebSocket.Server({port: this.portWebSocket}); - this.wss.on('connection', (ws: WebSocket) => { + const that = this; ws.on('message', (message: string) => { - console.log('received: %s', message); - ws.send(`Hello, you sent -> ${message}`); + const jsonMsg = JSON.parse(message); + const data: Array = jsonMsg.map(d => new Data(d.name, d.data, d.size, d.timestamp)); + let image = data.find(val => { + return val.getName() === 'screen'; + }); + if(image) { + that.saveImage(image); + that.applyDataToImage(image, data) + .then(val => { + ws.send(val); + }); + } else { + that.applyDataToImage(that.oldImageData, data) + .then(val => { + ws.send(val); + }); + } }); }); } @@ -50,47 +67,6 @@ export class PredictorWebService { res.send(generateFakeResponse()); }); - this.app.post('/trainData', function (req, res) { - console.log('Received data'); - const data: Array = req.body.map(d => new Data(d.name, d.data, d.size, d.timestamp)); - console.dir(data); - const image = data.find(val => { - return val.getName() === 'screen'; - }); - const mouseClicks = data.filter(val => { - return val.getName() === 'click'; - }); - const mouseMovements = data.filter(val => { - return val.getName() === 'mousemove'; - }); - const keyboard = data.filter(val => { - return val.getName() === 'keydown'; - }); - if(image){ - const canvas = createCanvas(image.getSize().width, image.getSize().height); - const ctx = canvas.getContext('2d'); - const img = new Image(); - img.onload = () => { - that.printImage(img, ctx); - that.printMouseClick(mouseClicks, ctx); - that.printMouseMove(mouseMovements, ctx); - that.printKeyboard(keyboard, ctx); - const base64data = canvas.toBuffer(); - fs.writeFile('./trainingDatas/' + image.getTimestamp() + '.png', base64data, 'base64', function(err){ - if (err) throw err; - console.log('File saved.'); - res.send('ok - image received'); - }) - }; - img.onerror = err => { throw err }; - img.src = image.getData(); - - - } else { - res.send('ok - only datas'); - } - }); - this.app.listen(this.portApi, () => { console.log('PredictorWebService is up and running on port: %d', this.portApi); }); @@ -107,7 +83,7 @@ export class PredictorWebService { } printMouseMove(mousemoves: Array, ctx: any) { - ctx.strokeStyle = 'rgba(0,0,0,0.5)'; + ctx.strokeStyle = 'rgba(0,0,0,0.8)'; ctx.lineWidth = 5; ctx.beginPath(); for(const move of mousemoves){ @@ -123,4 +99,47 @@ export class PredictorWebService { printImage(image: Image, ctx: any){ ctx.drawImage(image, 0, 0); } + + applyDataToImage(image: Data, data: Array) { + return new Promise((resolve, reject) => { + const mouseClicks = data.filter(val => { + return val.getName() === 'click'; + }); + const mouseMovements = data.filter(val => { + return val.getName() === 'mousemove'; + }); + const keyboard = data.filter(val => { + return val.getName() === 'keydown'; + }); + if (image) { + const canvas = createCanvas(image.getSize().width, image.getSize().height); + const ctx = canvas.getContext('2d'); + const img = new Image(); + img.onload = () => { + this.printImage(img, ctx); + this.printMouseClick(mouseClicks, ctx); + this.printMouseMove(mouseMovements, ctx); + this.printKeyboard(keyboard, ctx); + const base64data = canvas.toBuffer(); + fs.writeFile('./trainingDatas/' + image.getTimestamp() + '-' + Date.now() + '.png', base64data, 'base64', function (err) { + if (err) throw err; + console.log('File saved.'); + resolve('ok - image received'); + }) + }; + img.onerror = err => { + throw err + }; + img.src = image.getData(); + } else { + resolve('ok - only datas'); + } + }); + + } + saveImage(image: Data) { + if(image) { + this.oldImageData = image; + } + } }