Merge pull request #22 from yoeo/v2.0.1

V2.0.1

docs/contents.rst

@@ -232,7 +232,7 @@ Accuracy
 
 Guesslang deep learning model performs very well.
 It was tested with 12,000 different source code files and correctly
-guessed the programming language of **93.29%** of them.
+guessed the programming language of **93.82%** of them.
 
 Most of the misclassifications come from few languages
 that are **compatible** with each other, like C/C++ or JavaScript/TypeScript.

guesslang/__init__.py

@@ -7,4 +7,4 @@
 from guesslang.guess import Guess, GuesslangError  # noqa: F401
 
 
-__version__ = '2.0.0'
+__version__ = '2.0.1'

guesslang/data/languages.json

@@ -3,29 +3,29 @@
   "C": ["c"],
   "C#": ["cs"],
   "C++": ["cpp", "cc"],
-  "CSS": ["css"],
   "CoffeeScript": ["coffee", "litcoffee"],
+  "CSS": ["css"],
   "Erlang": ["erl", "hrl"],
   "Go": ["go"],
-  "HTML": ["html"],
   "Haskell": ["hs", "lhs"],
+  "HTML": ["html"],
   "Java": ["java"],
   "JavaScript": ["js", "es6"],
   "Jupyter Notebook": ["ipynb"],
   "Lua": ["lua"],
   "Markdown": ["md"],
   "Matlab": ["matlab", "m"],
   "Objective-C": ["mm", "m"],
-  "PHP": ["php"],
   "Perl": ["pl", "pm"],
+  "PHP": ["php"],
   "PowerShell": ["ps1"],
   "Python": ["py"],
   "R": ["r", "rdata", "rds", "rda"],
   "Ruby": ["rb"],
   "Rust": ["rs"],
-  "SQL": ["sql"],
   "Scala": ["scala"],
   "Shell": ["sh"],
+  "SQL": ["sql"],
   "Swift": ["swift"],
   "TeX": ["tex"],
   "TypeScript": ["ts", "tsx"]

guesslang/model.py

@@ -25,11 +25,12 @@
 class HyperParameter:
     """Model hyper parameters"""
     BATCH_SIZE = 100
-    NB_TOKENS = 1000
-    VOCABULARY_SIZE = 20000
-    EMBEDDING_SIZE = int(VOCABULARY_SIZE**0.5)
+    NB_TOKENS = 10000
+    VOCABULARY_SIZE = 5000
+    EMBEDDING_SIZE = max(10, int(VOCABULARY_SIZE**0.5))
     DNN_HIDDEN_UNITS = [512, 32]
     DNN_DROPOUT = 0.5
+    N_GRAM = 2
 
 
 class Training:
@@ -208,7 +209,8 @@ def _preprocess(
 def _preprocess_text(data: tf.Tensor) -> tf.Tensor:
     """Feature engineering"""
     padding = tf.constant(['']*HyperParameter.NB_TOKENS)
-    data = tf.strings.split([data]).values
+    data = tf.strings.bytes_split(data)
+    data = tf.strings.ngrams(data, HyperParameter.N_GRAM)
     data = tf.concat((data, padding), axis=0)
     data = data[:HyperParameter.NB_TOKENS]
     return data