Feat: generated network for classification problem of cat/dog

Feat: first version of dataGatherer client

.gitignore

@@ -1 +1,3 @@
 datas
+node_modules/
+dist/

DataGatherer/.eslintrc.yaml

@@ -0,0 +1,12 @@
+# .eslintrc.yaml
+---
+  extends: airbnb-base
+  env:
+    node: true
+    mocha: true
+    es6: true
+  parser: typescript-eslint-parser
+  parserOptions:
+    sourceType: module
+    ecmaFeatures: 
+      modules: true

DataGatherer/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "git.ignoreLimitWarning": true
+}

DataGatherer/package-lock.json

@@ -0,0 +1,138 @@
+{
+  "name": "datagatherer",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "axios": {
+      "version": "0.18.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz",
+      "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=",
+      "requires": {
+        "follow-redirects": "^1.3.0",
+        "is-buffer": "^1.1.5"
+      }
+    },
+    "eslint-scope": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz",
+      "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==",
+      "dev": true,
+      "requires": {
+        "esrecurse": "^4.1.0",
+        "estraverse": "^4.1.1"
+      }
+    },
+    "eslint-visitor-keys": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz",
+      "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==",
+      "dev": true
+    },
+    "esrecurse": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
+      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+      "dev": true,
+      "requires": {
+        "estraverse": "^4.1.0"
+      }
+    },
+    "estraverse": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
+      "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=",
+      "dev": true
+    },
+    "follow-redirects": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.7.0.tgz",
+      "integrity": "sha512-m/pZQy4Gj287eNy94nivy5wchN3Kp+Q5WgUPNy5lJSZ3sgkVKSYV/ZChMAQVIgx1SqfZ2zBZtPA2YlXIWxxJOQ==",
+      "requires": {
+        "debug": "^3.2.6"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.6",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+        }
+      }
+    },
+    "is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
+    "lodash.unescape": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz",
+      "integrity": "sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=",
+      "dev": true
+    },
+    "rxios": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/rxios/-/rxios-1.1.2.tgz",
+      "integrity": "sha512-nfQH2PdBqqvX41zDCM3KbXm4lJjUEaSoggT4g6tJTpBe2rrwDvKmVZ6Qaq3mKxs0IO33QT2RPEj1wIrwblhtPQ=="
+    },
+    "rxjs": {
+      "version": "6.4.0",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.4.0.tgz",
+      "integrity": "sha512-Z9Yfa11F6B9Sg/BK9MnqnQ+aQYicPLtilXBp2yUtDt2JRCE0h26d33EnfO3ZxoNxG0T92OUucP3Ct7cpfkdFfw==",
+      "requires": {
+        "tslib": "^1.9.0"
+      }
+    },
+    "rxjs-compat": {
+      "version": "6.4.0",
+      "resolved": "https://registry.npmjs.org/rxjs-compat/-/rxjs-compat-6.4.0.tgz",
+      "integrity": "sha512-eo/O8RS83hJdJukCtA+IF6qnqa8FPOuVo+OPCzgVi+dbTle9KCdNv97IcQO0WwNVik7DJLKmf0F8uwzc0q40vw=="
+    },
+    "semver": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz",
+      "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==",
+      "dev": true
+    },
+    "tslib": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
+      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ=="
+    },
+    "typescript": {
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.2.tgz",
+      "integrity": "sha512-Og2Vn6Mk7JAuWA1hQdDQN/Ekm/SchX80VzLhjKN9ETYrIepBFAd8PkOdOTK2nKt0FCkmMZKBJvQ1dV1gIxPu/A==",
+      "dev": true
+    },
+    "typescript-eslint-parser": {
+      "version": "22.0.0",
+      "resolved": "https://registry.npmjs.org/typescript-eslint-parser/-/typescript-eslint-parser-22.0.0.tgz",
+      "integrity": "sha512-pD8D7oTeRwWvFVxK3PaY6FYAiZsuRXFkIc2+1xkwCT3NduySgCgjeAkR5/dnIWecOiFVcEHf4ypXurF02Q6Z3Q==",
+      "dev": true,
+      "requires": {
+        "eslint-scope": "^4.0.0",
+        "eslint-visitor-keys": "^1.0.0",
+        "typescript-estree": "18.0.0"
+      }
+    },
+    "typescript-estree": {
+      "version": "18.0.0",
+      "resolved": "https://registry.npmjs.org/typescript-estree/-/typescript-estree-18.0.0.tgz",
+      "integrity": "sha512-HxTWrzFyYOPWA91Ij7xL9mNUVpGTKLH2KiaBn28CMbYgX2zgWdJqU9hO7Are+pAPAqY91NxAYoaAyDDZ3rLj2A==",
+      "dev": true,
+      "requires": {
+        "lodash.unescape": "4.0.1",
+        "semver": "5.5.0"
+      }
+    }
+  }
+}

DataGatherer/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "datagatherer",
+  "version": "1.0.0",
+  "description": "",
+  "main": "main.js",
+  "dependencies": {
+    "axios": "^0.18.0",
+    "rxios": "^1.1.2",
+    "rxjs": "^6.4.0",
+    "rxjs-compat": "^6.4.0"
+  },
+  "devDependencies": {
+    "typescript": "^3.4.2",
+    "typescript-eslint-parser": "^22.0.0"
+  },
+  "scripts": {
+    "build": "./node_modules/.bin/tsc && browserify ./dist/main.js > ./dist/bundle.js"
+  },
+  "author": "",
+  "license": "ISC"
+}

DataGatherer/src/data/Data.ts

@@ -0,0 +1,11 @@
+export class Data {
+    private name: string;
+    private data: any;
+    private timestamp: number;
+
+    constructor(name: string, data: any) {
+        this.name = name;
+        this.data = data;
+        this.timestamp = Date.now();
+    }
+}

DataGatherer/src/gatherer/Gatherer.ts

@@ -0,0 +1,24 @@
+import { Source } from "./../source/Source";
+
+export class Gatherer {
+
+    private sources: Array<Source>;
+
+    constructor(sources: Array<Source>) {
+        this.sources = sources;
+    }
+
+    public start() {
+        for(const source of this.sources){
+            source.startCollect();
+        }
+    }
+
+    public getData(): any {
+        let allData = [];
+        for(const source of this.sources){
+            allData = allData.concat(source.getData());
+        }
+        return allData;
+    }
+}

DataGatherer/src/index.html

@@ -0,0 +1,8 @@
+<html>
+    <head>
+        <script src="../dist/bundle.js"></script>
+    </head>
+    <body>
+        
+    </body>
+</html>

DataGatherer/src/main.ts

@@ -0,0 +1,23 @@
+import { Sender } from "./sender/Sender";
+import { Mouse } from "./source/mouse/Mouse";
+import { Gatherer } from "./gatherer/Gatherer";
+import { Keyboard } from "./source/keyboard/Keyboard";
+
+function main() {
+    const gatherer: Gatherer = new Gatherer([
+            new Keyboard('keyboard', ['keydown']), 
+            new Mouse('mouse', ['click', 'mousemove'])
+        ]);
+
+    gatherer.start();
+
+    const sender: Sender = new Sender(() => gatherer.getData(), '/predictor', 4000, 2000);
+    sender.start()
+        .subscribe(
+            val => {
+                console.log(val);
+            }
+        );
+}
+
+main();

DataGatherer/src/sender/Sender.ts

@@ -0,0 +1,34 @@
+import { interval, Observable, of } from 'rxjs';
+import { filter, flatMap, delay } from 'rxjs/operators';
+import { Rxios } from 'rxios';
+
+export class Sender {
+    private url: string;
+    private port: number;
+    private interval: number;
+    private dataSourceFn: any;
+
+    constructor(dataSourceFn: any, url: string, port: number, interval: number) {
+        this.url = url;
+        this.port = port;
+        this.interval = interval;
+        this.dataSourceFn = dataSourceFn;
+    }
+    
+    public start() {
+        return interval(this.interval)
+            .pipe(
+                flatMap(() => this.send(this.dataSourceFn()))
+            )
+    }
+
+    private send(data: any) {
+        const http: Rxios = new Rxios();
+        //return http.post(this.url, data);
+        function generateFakeResponse() {
+            return Array.from({length: 20}, () => Math.random().toPrecision(2));
+        }
+
+        return of(generateFakeResponse()).pipe(delay(200));
+    }
+}

DataGatherer/src/source/Source.ts

@@ -0,0 +1,30 @@
+import { Data } from "./../data/Data";
+
+export class Source {
+    private name: string;
+    public data: Array<any> = [];
+    public events: Array<string>;
+
+    constructor(name: string, events: Array<string>) {
+        this.name = name;
+        this.events = events;
+    }
+
+    public startCollect() {
+        for(const event of this.events){
+            document.addEventListener(event, (e) => {
+                this.data.push(new Data(event, e));
+            });
+        }
+    }
+
+    public getData() {
+        const data = JSON.parse(JSON.stringify(this.data));
+        this.deleteAllData();
+        return data;
+    }
+
+    public deleteAllData() {
+        this.data = [];
+    }
+}

DataGatherer/src/source/keyboard/Keyboard.ts

@@ -0,0 +1,8 @@
+import { Source } from '../Source';
+
+export class Keyboard extends Source {
+
+    constructor(name: string, events: Array<string>) {
+        super(name, events);
+    }
+}

DataGatherer/src/source/mouse/Mouse.ts

@@ -0,0 +1,9 @@
+import { Source } from '../Source';
+
+export class Mouse extends Source {
+
+    constructor(name: string, events: Array<string>) {
+        super(name, events);
+    }
+}
+   

DataGatherer/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "sourceMap": true,
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "outDir": "dist",
+    "experimentalDecorators": true,
+    "target": "es5",
+    "jsx": "react",
+    "lib": [
+      "dom",
+      "es6"
+    ]
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "compileOnSave": true
+}

predictor/Predictor.py

@@ -1,4 +1,3 @@
-
 from keras.models import Sequential
 from keras.layers import Activation, Dropout, Flatten, Dense
 from keras import backend as K
@@ -10,23 +9,12 @@ import matplotlib.image as mpimg
 import cv2
 import numpy as np
 
-import tensorflow as tf
-sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
-
-
-
-#from keras.datasets import imdb
-
-
-
-#https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d
-
 img_width, img_height = 150, 150
-train_data_dir = './datas/train'
+train_data_dir = './UserflowPredictorSystem/predictor/datas/train'
-validation_data_dir = './datas/test'
+validation_data_dir = './UserflowPredictorSystem/predictor/datas/test'
 nb_train_samples = 2000
 nb_validation_samples = 800
-epochs = 10
+epochs = 100
 batch_size = 16
 
 
@@ -35,114 +23,99 @@ if K.image_data_format() == 'channels_first':
 else:
     input_shape = (img_width, img_height, 3)
 
+def create_model():
+    model = Sequential()
+    model.add(Conv2D(32, (3, 3), input_shape=input_shape))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
 
-model = Sequential()
+    model.add(Conv2D(32, (3, 3)))
-model.add(Conv2D(32, (3, 3), input_shape=input_shape))
+    model.add(Activation('relu'))
-model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
-model.add(MaxPooling2D(pool_size=(2, 2)))
 
-model.add(Conv2D(32, (3, 3)))
+    model.add(Conv2D(64, (3, 3)))
-model.add(Activation('relu'))
+    model.add(Activation('relu'))
-model.add(MaxPooling2D(pool_size=(2, 2)))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
 
-model.add(Conv2D(64, (3, 3)))
+    model.add(Flatten())
-model.add(Activation('relu'))
+    model.add(Dense(64))
-model.add(MaxPooling2D(pool_size=(2, 2)))
+    model.add(Activation('relu'))
-
+    model.add(Dropout(0.5))
-model.add(Flatten())
+    model.add(Dense(1))
-model.add(Dense(64))
+    model.add(Activation('sigmoid'))
-model.add(Activation('relu'))
-model.add(Dropout(0.5))
-model.add(Dense(1))
-model.add(Activation('sigmoid'))
 
 
-model.compile(loss='binary_crossentropy',
+    model.compile(loss='binary_crossentropy',
                   optimizer='rmsprop',
                   metrics=['accuracy'])
+    return model
 
-# this is the augmentation configuration we will use for training
+def train_model(model):
-train_datagen = ImageDataGenerator(
+    # this is the augmentation configuration we will use for training
+    train_datagen = ImageDataGenerator(
         rescale=1. / 255,
         shear_range=0.2,
         zoom_range=0.2,
         horizontal_flip=True)
 
-# this is the augmentation configuration we will use for testing:
+    # this is the augmentation configuration we will use for testing:
-# only rescaling
+    # only rescaling
-test_datagen = ImageDataGenerator(rescale=1. / 255)
+    test_datagen = ImageDataGenerator(rescale=1. / 255)
 
-train_generator = train_datagen.flow_from_directory(
+    train_generator = train_datagen.flow_from_directory(
         train_data_dir,
         target_size=(img_width, img_height),
         batch_size=batch_size,
         class_mode='binary')
 
-validation_generator = test_datagen.flow_from_directory(
+    validation_generator = test_datagen.flow_from_directory(
         validation_data_dir,
         target_size=(img_width, img_height),
         batch_size=batch_size,
         class_mode='binary')
 
-model.fit_generator(
+    model.fit_generator(
         train_generator,
         steps_per_epoch=nb_train_samples // batch_size,
         epochs=epochs,
         validation_data=validation_generator,
         validation_steps=nb_validation_samples // batch_size)
 
-from tensorflow.python.client import device_lib
+    model.save_weights('./UserflowPredictorSystem/first_try2.h5')
-print(device_lib.list_local_devices())
+    return model
-
-from keras import backend as K
-K.tensorflow_backend._get_available_gpus()
-
-model.save_weights('first_try.h5')
-model.summary()
-
-img = cv2.imread("./datas/test/27.jpg")
-im = mpimg.imread("./datas/test/27.jpg")
-plt.imshow(im)
-
-img = cv2.resize(img, (img_width,img_height))
-print(img.shape)
-img = img.reshape(1, img_width, img_height, 3)
-
-print(img.shape)
-#print(np.argmax(loaded_model.predict(img)))
-print(model.predict(img))
 
 
 
+def load_trained_model(weights_path):
+   model = create_model()
+   model.load_weights(weights_path)
+   return model
+
+
+def predict(number, model):
+    img = cv2.imread("./UserflowPredictorSystem/predictor/datas/test/" + str(number) + ".jpg")
+    im = mpimg.imread("./UserflowPredictorSystem/predictor/datas/test/" + str(number) + ".jpg")
+    plt.imshow(im)
+    img = cv2.resize(img, (img_width,img_height))
+    img = img.reshape(1, img_width, img_height, 3)
+    res = model.predict(img)
+    if res == 1:
+        print('DOG')
+    else:
+        print('CAT')
+
+model = create_model()
+model = train_model(model)
+
+import os
+os.getcwd()
+
+trained_model = load_trained_model("./UserflowPredictorSystem/first_try2.h5")
+trained_model.summary()
+import random
+predict(random.randint(1,12500), trained_model)
+predict('lolly', model)
 
 
 
-
+print(np.argmax(loaded_model.predict(img)))
-
-K.tensorflow_backend._get_available_gpus()
-
-
-num_words = 10000
-(X_train, y_train), (X_test, y_test) = loadDatas()
-
-maxlen = 500
-
-X_train = pad_sequences(X_train, maxlen=maxlen)
-X_test = pad_sequences(X_test, maxlen=maxlen)
-
-from keras.layers import Flatten, LSTM
-from keras.layers.convolutional import Conv1D, MaxPooling1D
-
-model = Sequential()
-model.add(Embedding(num_words, 50, input_length=500))
-model.add(Conv1D(filters=32, kernel_size=3, padding="same", activation="relu"))
-model.add(MaxPooling1D(pool_size=2))
-model.add(LSTM(32, dropout=0.4))
-model.add(Dense(1, activation="sigmoid"))
-
-model.summary()
-
-model.compile(loss="binary_crossentropy", optimizer="rmsprop", metrics=["accuracy"])
-model.fit(X_train, y_train, batch_size=512, validation_split=0.2, epochs=5)
-
-model.evaluate(X_test, y_test)

predictor/Untitled.ipynb

@@ -1,53 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<tensorflow.python.client.session.Session object at 0x7fe921736b38>\n"
-     ]
-    }
-   ],
-   "source": [
-    "\n",
-    "import tensorflow as tf\n",
-    "\n",
-    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
-    "print(sess)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.2"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}