tokenizer rename

Files changed (2) hide show

tokenization_freelm.py → tokenization_flm.py RENAMED Viewed

File without changes

tokenizer_config.json CHANGED Viewed

@@ -3,7 +3,7 @@
   "add_prefix_space": false,
   "auto_map": {
     "AutoTokenizer": [
-      "tokenization_freelm.FLMTokenizer",
       null
     ]
   },

   "add_prefix_space": false,
   "auto_map": {
     "AutoTokenizer": [
+      "tokenization_flm.FLMTokenizer",
       null
     ]
   },