Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | "use strict";
require("colors");
var _ = require("lodash");
var prettyms = require("pretty-ms");
var utils = require("./utils");
var train = filePath => {
var startTime = new Date();
console.log("Training model...".yellow);
var tests = utils.loadFile(filePath);
var stepsCount = countSteps(tests);
var bias = calcBias(stepsCount);
var bigrams = getNgrams(tests, 2);
var trigrams = getNgrams(tests, 3);
var result = {};
_.merge(result, getWeights(stepsCount, bigrams, bias));
_.merge(result, getWeights(stepsCount, trigrams, bias));
console.log(`Model is trained during ${prettyms(new Date() - startTime)}`.yellow);
return result;
};
var calcBias = stepsCount => {
var counter = {};
for (var i of Object.values(stepsCount)) {
counter[i] = (counter[i] || 0) + 1;
}
var weights = [];
for (var [k, v] of Object.entries(counter)) {
k = parseInt(k);
weights.push([k * v, k, v]);
}
weights.sort((a, b) => a[0] - b[0]);
var bias = weights[0][1] || 1, diff = 0;
for (var j = 0; j < weights.length - 1; j++) {
var a = weights[j], b = weights[j + 1];
var d = b[0] / a[0];
if (d > diff) {
diff = d;
bias = a[1];
}
}
return bias;
};
var calcWeight = x => x / (1 + x);
var getWeights = (stepsCount, ngrams, bias) => {
var dim, del;
var result = {};
for (var [name, ngram] of Object.entries(ngrams)) {
if (!dim) {
dim = ngram.steps.length;
del = Math.pow(2, dim - 2);
}
var x = ngram.count;
for (var s of ngram.steps) {
x += (stepsCount[s] - ngram.count) / (dim * stepsCount[s]);
}
x = x / bias;
result[name] = _.round(calcWeight(x) / del, 3);
}
return result;
};
var getNgrams = (tests, n) => {
var ngrams = {};
for (var steps of Object.values(tests)) {
for (var i = 0; i < steps.length - n + 1; i++) {
var ngram = [];
for (var j = 0; j < n; j++) {
ngram.push(steps[i+j]);
};
var key = ngram.join(" | ");
if (key in ngrams) {
ngrams[key].count += 1;
} else {
ngrams[key] = {
count: 1,
steps: ngram,
};
}
};
};
return ngrams;
};
var countSteps = tests => {
var result = {};
for (var steps of Object.values(tests)) {
for (var step of steps) {
result[step] = (result[step] || 0) + 1;
}
}
return result;
};
module.exports = train;
|