Feat: implemented mongo store/retrieve datas for webservice and imagecreator; Feat: parallel execution of Puppets with random flow
This commit is contained in:
Lorenzo Iovino
parent
1c8645039e
commit
ae477fc3df
11 changed files with 506 additions and 223 deletions
186
Predictor/PredictorWebService/package-lock.json
generated
186
Predictor/PredictorWebService/package-lock.json
generated
|
|
@ -13,6 +13,14 @@
|
|||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/bson": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/bson/-/bson-4.0.0.tgz",
|
||||
"integrity": "sha512-pq/rqJwJWkbS10crsG5bgnrisL8pML79KlMKQMoQwLUjlPAkrUHMvHJ3oGwE7WHR61Lv/nadMwXVAD2b+fpD8Q==",
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/connect": {
|
||||
"version": "3.4.32",
|
||||
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.32.tgz",
|
||||
|
|
@ -51,6 +59,24 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz",
|
||||
"integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw=="
|
||||
},
|
||||
"@types/mongodb": {
|
||||
"version": "3.1.26",
|
||||
"resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-3.1.26.tgz",
|
||||
"integrity": "sha512-i7+l95IbqnGLRW+AJ6F2nzqosLUgU022lDuoHhbQquMV7tgek0vNUg9RwC2Fn7ImBSQSFdCWeU394ciPEwSeaQ==",
|
||||
"requires": {
|
||||
"@types/bson": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/mongoose": {
|
||||
"version": "5.3.27",
|
||||
"resolved": "https://registry.npmjs.org/@types/mongoose/-/mongoose-5.3.27.tgz",
|
||||
"integrity": "sha512-1bQcQaYw4kN9mMuJC54FcfjyYe2gtxbUj9OAvrUA/5G2rSv0fuvu2jplizXqyADgBcXzPVEglDNzT3YrztWLoQ==",
|
||||
"requires": {
|
||||
"@types/mongodb": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "11.13.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-11.13.0.tgz",
|
||||
|
|
@ -184,6 +210,14 @@
|
|||
"resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
|
||||
"integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c="
|
||||
},
|
||||
"async": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
|
||||
"integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
|
||||
"requires": {
|
||||
"lodash": "^4.17.10"
|
||||
}
|
||||
},
|
||||
"async-each": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.2.tgz",
|
||||
|
|
@ -259,6 +293,11 @@
|
|||
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
|
||||
"integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw=="
|
||||
},
|
||||
"bluebird": {
|
||||
"version": "3.5.1",
|
||||
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz",
|
||||
"integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA=="
|
||||
},
|
||||
"body-parser": {
|
||||
"version": "1.18.3",
|
||||
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz",
|
||||
|
|
@ -326,6 +365,11 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"bson": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.1.tgz",
|
||||
"integrity": "sha512-jCGVYLoYMHDkOsbwJZBCqwMHyH4c+wzgI9hG7Z6SZJRXWr+x58pdIbm2i9a/jFGCkRJqRUr8eoI7lDWa0hTkxg=="
|
||||
},
|
||||
"buffer-from": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
|
||||
|
|
@ -1809,6 +1853,11 @@
|
|||
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
|
||||
"integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
|
||||
},
|
||||
"kareem": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/kareem/-/kareem-2.3.0.tgz",
|
||||
"integrity": "sha512-6hHxsp9e6zQU8nXsP+02HGWXwTkOEw6IROhF2ZA28cYbUk4eJ6QbtZvdqZOdD9YPKghG3apk5eOCvs+tLl3lRg=="
|
||||
},
|
||||
"kind-of": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
|
||||
|
|
@ -1822,6 +1871,11 @@
|
|||
"package-json": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.11",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
|
||||
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
|
||||
},
|
||||
"lodash.unescape": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz",
|
||||
|
|
@ -1874,6 +1928,12 @@
|
|||
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
|
||||
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
|
||||
},
|
||||
"memory-pager": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
|
||||
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
|
||||
"optional": true
|
||||
},
|
||||
"merge-descriptors": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
|
||||
|
|
@ -1993,6 +2053,85 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.2.3.tgz",
|
||||
"integrity": "sha512-jw8UyPsq4QleZ9z+t/pIVy3L++51vKdaJ2Q/XXeYxk/3cnKioAH8H6f5tkkDivrQL4PUgUOHe9uZzkpRFH1XtQ==",
|
||||
"requires": {
|
||||
"mongodb-core": "^3.2.3",
|
||||
"safe-buffer": "^5.1.2"
|
||||
}
|
||||
},
|
||||
"mongodb-core": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-3.2.3.tgz",
|
||||
"integrity": "sha512-UyI0rmvPPkjOJV8XGWa9VCTq7R4hBVipimhnAXeSXnuAPjuTqbyfA5Ec9RcYJ1Hhu+ISnc8bJ1KfGZd4ZkYARQ==",
|
||||
"requires": {
|
||||
"bson": "^1.1.1",
|
||||
"require_optional": "^1.0.1",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"saslprep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"mongoose": {
|
||||
"version": "5.5.5",
|
||||
"resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.5.5.tgz",
|
||||
"integrity": "sha512-rov43FpXDMoE22q8/iUoeSdg6zBtyTE/ZIIKRpQomASl0JOyD0479Weu9w5g0GjmP8fHmNjsWneoLIbnyGcnDQ==",
|
||||
"requires": {
|
||||
"async": "2.6.1",
|
||||
"bson": "~1.1.1",
|
||||
"kareem": "2.3.0",
|
||||
"mongodb": "3.2.3",
|
||||
"mongodb-core": "3.2.3",
|
||||
"mongoose-legacy-pluralize": "1.0.2",
|
||||
"mpath": "0.5.2",
|
||||
"mquery": "3.2.0",
|
||||
"ms": "2.1.1",
|
||||
"regexp-clone": "0.0.1",
|
||||
"safe-buffer": "5.1.2",
|
||||
"sift": "7.0.1",
|
||||
"sliced": "1.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"ms": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
|
||||
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"mongoose-legacy-pluralize": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz",
|
||||
"integrity": "sha512-Yo/7qQU4/EyIS8YDFSeenIvXxZN+ld7YdV9LqFVQJzTLye8unujAWPZ4NWKfFA+RNjh+wvTWKY9Z3E5XM6ZZiQ=="
|
||||
},
|
||||
"mpath": {
|
||||
"version": "0.5.2",
|
||||
"resolved": "https://registry.npmjs.org/mpath/-/mpath-0.5.2.tgz",
|
||||
"integrity": "sha512-NOeCoW6AYc3hLi30npe7uzbD9b4FQZKH40YKABUCCvaKKL5agj6YzvHoNx8jQpDMNPgIa5bvSZQbQpWBAVD0Kw=="
|
||||
},
|
||||
"mquery": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mquery/-/mquery-3.2.0.tgz",
|
||||
"integrity": "sha512-qPJcdK/yqcbQiKoemAt62Y0BAc0fTEKo1IThodBD+O5meQRJT/2HSe5QpBNwaa4CjskoGrYWsEyjkqgiE0qjhg==",
|
||||
"requires": {
|
||||
"bluebird": "3.5.1",
|
||||
"debug": "3.1.0",
|
||||
"regexp-clone": "0.0.1",
|
||||
"safe-buffer": "5.1.2",
|
||||
"sliced": "1.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
|
||||
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
|
||||
"requires": {
|
||||
"ms": "2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
|
|
@ -2406,6 +2545,11 @@
|
|||
"safe-regex": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"regexp-clone": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-0.0.1.tgz",
|
||||
"integrity": "sha1-p8LgmJH9vzj7sQ03b7cwA+aKxYk="
|
||||
},
|
||||
"registry-auth-token": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz",
|
||||
|
|
@ -2438,6 +2582,20 @@
|
|||
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
|
||||
"integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc="
|
||||
},
|
||||
"require_optional": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz",
|
||||
"integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==",
|
||||
"requires": {
|
||||
"resolve-from": "^2.0.0",
|
||||
"semver": "^5.1.0"
|
||||
}
|
||||
},
|
||||
"resolve-from": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
|
||||
"integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
|
||||
},
|
||||
"resolve-url": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
|
||||
|
|
@ -2474,6 +2632,15 @@
|
|||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"saslprep": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
|
||||
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"sparse-bitfield": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"sax": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
|
||||
|
|
@ -2567,11 +2734,21 @@
|
|||
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
|
||||
"integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
|
||||
},
|
||||
"sift": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/sift/-/sift-7.0.1.tgz",
|
||||
"integrity": "sha512-oqD7PMJ+uO6jV9EQCl0LrRw1OwsiPsiFQR5AR30heR+4Dl7jBBbDLnNvWiak20tzZlSE1H7RB30SX/1j/YYT7g=="
|
||||
},
|
||||
"signal-exit": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
|
||||
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
|
||||
},
|
||||
"sliced": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz",
|
||||
"integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E="
|
||||
},
|
||||
"snapdragon": {
|
||||
"version": "0.8.2",
|
||||
"resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz",
|
||||
|
|
@ -2709,6 +2886,15 @@
|
|||
"resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",
|
||||
"integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM="
|
||||
},
|
||||
"sparse-bitfield": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
|
||||
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"memory-pager": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"split-string": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@
|
|||
"main": "main.js",
|
||||
"dependencies": {
|
||||
"@types/express": "^4.16.1",
|
||||
"@types/mongoose": "^5.3.27",
|
||||
"body-parser": "^1.18.3",
|
||||
"canvas": "^2.4.1",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^4.16.4",
|
||||
"mongodb": "^3.2.3",
|
||||
"mongoose": "^5.5.5",
|
||||
"nodemon": "^1.18.10",
|
||||
"query-string-parser": "^0.2.3",
|
||||
"ws": "^6.2.1"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,25 @@
|
|||
import {Data} from "../../../../DataGatherer/src/shared/Data";
|
||||
import {createCanvas, Image} from "canvas";
|
||||
import fs from "fs";
|
||||
import mongoose from "mongoose";
|
||||
import {DataSchema} from "../predictor-web-service/models/Data";
|
||||
|
||||
export class ImageCreatorService {
|
||||
private mongoEndpoint = 'mongodb://localhost:27017/predictorDatas';
|
||||
|
||||
constructor() {
|
||||
mongoose.Promise = global.Promise;
|
||||
mongoose.connect(this.mongoEndpoint, { useNewUrlParser: true },
|
||||
err => {
|
||||
if (err) {
|
||||
console.log(err);
|
||||
}
|
||||
else {
|
||||
console.log('Connected to MongoDb');
|
||||
this.startProcessing(6, 200);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
createImageFromData(data: Array<Data>, name: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
|
@ -124,35 +141,40 @@ export class ImageCreatorService {
|
|||
}
|
||||
|
||||
|
||||
public startProcessing(srcPath: string, destPath: string) {
|
||||
public startProcessing(flows: number, agents: number, threshold: number = 5) {
|
||||
const that = this;
|
||||
fs.readdir(srcPath, function(err, items) {
|
||||
|
||||
for (var i=0; i<items.length; i++) {
|
||||
const data = fs.readFileSync(srcPath + '/' + items[i]);
|
||||
try {
|
||||
const jsonData = JSON.parse(data.toString('utf8'));
|
||||
fs.mkdirSync('./trainingImages/virtual/' + destPath, {recursive: true});
|
||||
that.createImageFromData(jsonData, items[i].split('.')[0])
|
||||
.then(res => {
|
||||
if(res) {
|
||||
console.log('virtual image ' + ' processed');
|
||||
that.saveImage(res['data'], './trainingImages/virtual/' + destPath + '/' + res['name'] + '.png');
|
||||
}
|
||||
});
|
||||
|
||||
that.applyDataToImage(jsonData, items[i].split('.')[0])
|
||||
.then(res => {
|
||||
if(res) {
|
||||
console.log('real image ' + ' processed');
|
||||
that.saveImage(res['data'], './trainingImages/real/' + destPath + '/' + res['name'] + '.png');
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.log('file not valid');
|
||||
}
|
||||
const orderedByFlows = [];
|
||||
const promises = [];
|
||||
for(let i=0; i<flows; i++) {
|
||||
for (let j = 0; j < agents; j++) {
|
||||
promises.push(DataSchema.find({
|
||||
flowName: 'flow'+(i + 1).toString(),
|
||||
agentName: 'bot_'+(j+1).toString()
|
||||
}, function (error, ids) {
|
||||
orderedByFlows.push(ids);
|
||||
}));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Promise.all(promises)
|
||||
.then(() => {
|
||||
for(let i = 0; i< flows; i++){
|
||||
fs.mkdirSync('./trainingImages/virtual/' + 'flow'+(i+1), {recursive: true});
|
||||
}
|
||||
for(let i = 0; i < orderedByFlows.length; i++) {
|
||||
for (let j of orderedByFlows[i]) {
|
||||
if(j.data.length > threshold) {
|
||||
that.createImageFromData(j.data, j.agentName + '_' + j.counter)
|
||||
.then(res => {
|
||||
if (res) {
|
||||
console.log('virtual image ' + ' processed');
|
||||
that.saveImage(res['data'], './trainingImages/virtual/' + j.flowName + '/' + res['name'] + '.png');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,6 @@ import {ImageCreatorService} from "./ImageCreatorService";
|
|||
|
||||
function main() {
|
||||
const imageCreatorService = new ImageCreatorService();
|
||||
imageCreatorService.startProcessing('./trainingDatas/flow1', 'flow1');
|
||||
imageCreatorService.startProcessing('./trainingDatas/flow2', 'flow2');
|
||||
imageCreatorService.startProcessing('./trainingDatas/flow3', 'flow3');
|
||||
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ import * as WebSocket from 'ws';
|
|||
import * as http from 'http';
|
||||
import cors from 'cors';
|
||||
import * as bodyParser from "body-parser";
|
||||
import fs from 'fs';
|
||||
import {Data} from "../../../../DataGatherer/src/shared/Data";
|
||||
import parser from 'query-string-parser';
|
||||
import mongoose from "mongoose";
|
||||
import {DataSchema} from './models/Data';
|
||||
|
||||
export class PredictorWebService {
|
||||
|
||||
|
|
@ -15,6 +16,7 @@ export class PredictorWebService {
|
|||
private app: express.Application;
|
||||
private httpServer: http.Server;
|
||||
private wss: WebSocket.Server;
|
||||
private mongoEndpoint = 'mongodb://localhost:27017/predictorDatas';
|
||||
|
||||
private counter: number = 0;
|
||||
|
||||
|
|
@ -22,6 +24,18 @@ export class PredictorWebService {
|
|||
this.url = url;
|
||||
this.portApi = portApi;
|
||||
this.portWebSocket = portWebSocket;
|
||||
mongoose.Promise = global.Promise;
|
||||
mongoose.connect(this.mongoEndpoint, { useNewUrlParser: true },
|
||||
err => {
|
||||
if (err) {
|
||||
console.log(err);
|
||||
}
|
||||
else {
|
||||
console.log('Connected to MongoDb');
|
||||
this.startPredictor();
|
||||
this.startTrainer();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public startTrainer() {
|
||||
|
|
@ -38,9 +52,7 @@ export class PredictorWebService {
|
|||
that.counter++;
|
||||
that.saveData(data, queryObj.flowName, queryObj.agentName)
|
||||
.then( (msg) => {
|
||||
if(msg === 'ok') {
|
||||
ws.send('trainer - Data saved');
|
||||
}
|
||||
ws.send('trainer - Data saved');
|
||||
})
|
||||
.catch( (err) => {
|
||||
ws.send('trainer - Error while saving data');
|
||||
|
|
@ -71,26 +83,8 @@ export class PredictorWebService {
|
|||
|
||||
saveData(data: Array<Data>, flowName: string, agentName: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(flowName);
|
||||
console.log(agentName);
|
||||
if(flowName && agentName) {
|
||||
fs.mkdirSync('./trainingDatas/' + flowName + '/', { recursive: true });
|
||||
} else {
|
||||
fs.mkdirSync('./trainingDatas/undefined', { recursive: true });
|
||||
}
|
||||
if(!flowName && !agentName) {
|
||||
fs.writeFile('./trainingDatas/undefined/' + this.counter + '.json', JSON.stringify(data), 'utf8', function (err) {
|
||||
if (err) reject('err');
|
||||
console.log('Data saved');
|
||||
resolve('ok');
|
||||
});
|
||||
} else {
|
||||
fs.writeFile('./trainingDatas/' + flowName + '/' + agentName + '_' + this.counter + '.json', JSON.stringify(data), 'utf8', function (err) {
|
||||
if (err) reject('err');
|
||||
console.log('Data saved');
|
||||
resolve('ok');
|
||||
});
|
||||
}
|
||||
const dataSchema = new DataSchema({flowName: flowName, agentName: agentName, counter: this.counter, data: data});
|
||||
resolve(dataSchema.save());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,7 @@ import { PredictorWebService } from "./PredictorWebService";
|
|||
|
||||
function main() {
|
||||
const predictorWebService = new PredictorWebService('/', 4000, 4100);
|
||||
predictorWebService.startPredictor();
|
||||
predictorWebService.startTrainer();
|
||||
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
import mongoose from "mongoose";
|
||||
import {Data} from "../../../../../DataGatherer/src/shared/Data";
|
||||
import {Schema} from "mongoose";
|
||||
|
||||
const DataSchema_:Schema = new Schema(
|
||||
{
|
||||
flowName: String,
|
||||
agentName: String,
|
||||
counter: Number,
|
||||
data: Array<Data>()
|
||||
}
|
||||
);
|
||||
|
||||
export const DataSchema: any = mongoose.model('Data', DataSchema_, 'datas');
|
||||
Loading…
Add table
Add a link
Reference in a new issue