Hugging Face's logo

alasdairforsythe
/
tokenmonster

Model card Files Community
alasdairforsythe commited on
Commit
2a3e12e
•
1 Parent(s): c47476f

Upload 26 files

Browse files
Files changed (27) hide show
  1. .gitattributes +19 -0
  2. vocabs/code-100256-balanced-v1.vocab +3 -0
  3. vocabs/code-50256-clean-v1.vocab +3 -0
  4. vocabs/code-50256-unfiltered-v1.vocab +3 -0
  5. vocabs/code-65536-strict-v1.vocab +3 -0
  6. vocabs/englishcode-100256-balanced-v1.vocab +3 -0
  7. vocabs/englishcode-100256-clean-v1.vocab +3 -0
  8. vocabs/englishcode-100256-consistent-v1.vocab +3 -0
  9. vocabs/englishcode-100256-strict-v1.vocab +3 -0
  10. vocabs/englishcode-100256-unfiltered-v1.vocab +3 -0
  11. vocabs/englishcode-1024-strict-v1.vocab +0 -0
  12. vocabs/englishcode-24000-consistent-v1.vocab +0 -0
  13. vocabs/englishcode-50256-balanced-v1.vocab +3 -0
  14. vocabs/englishcode-50256-clean-v1.vocab +3 -0
  15. vocabs/englishcode-50256-consistent-v1.vocab +3 -0
  16. vocabs/englishcode-50256-unfiltered-v1.vocab +3 -0
  17. vocabs/fiction-100256-balanced-v1.vocab +3 -0
  18. vocabs/fiction-100256-clean-v1.vocab +3 -0
  19. vocabs/fiction-100256-unfiltered-v1.vocab +3 -0
  20. vocabs/fiction-24000-balanced-v1.vocab +0 -0
  21. vocabs/fiction-24000-strict-v1.vocab +0 -0
  22. vocabs/fiction-32000-strict-v1.vocab +0 -0
  23. vocabs/fiction-4096-consistent-v1.vocab +0 -0
  24. vocabs/fiction-4096-strict-v1.vocab +0 -0
  25. vocabs/fiction-50256-balanced-v1.vocab +3 -0
  26. vocabs/fiction-50256-unfiltered-v1.vocab +3 -0
  27. vocabs/fiction-65536-strict-v1.vocab +3 -0
.gitattributes CHANGED
@@ -61,3 +61,22 @@ binaries/windows_x86_64/getalltokens.exe filter=lfs diff=lfs merge=lfs -text
61
  binaries/windows_x86_64/tokenmonsterserver.exe filter=lfs diff=lfs merge=lfs -text
62
  binaries/windows_x86_64/trainvocab.exe filter=lfs diff=lfs merge=lfs -text
63
  vocabs/gpt2.vocab filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  binaries/windows_x86_64/tokenmonsterserver.exe filter=lfs diff=lfs merge=lfs -text
62
  binaries/windows_x86_64/trainvocab.exe filter=lfs diff=lfs merge=lfs -text
63
  vocabs/gpt2.vocab filter=lfs diff=lfs merge=lfs -text
64
+ vocabs/code-100256-balanced-v1.vocab filter=lfs diff=lfs merge=lfs -text
65
+ vocabs/code-50256-clean-v1.vocab filter=lfs diff=lfs merge=lfs -text
66
+ vocabs/code-50256-unfiltered-v1.vocab filter=lfs diff=lfs merge=lfs -text
67
+ vocabs/code-65536-strict-v1.vocab filter=lfs diff=lfs merge=lfs -text
68
+ vocabs/englishcode-100256-balanced-v1.vocab filter=lfs diff=lfs merge=lfs -text
69
+ vocabs/englishcode-100256-clean-v1.vocab filter=lfs diff=lfs merge=lfs -text
70
+ vocabs/englishcode-100256-consistent-v1.vocab filter=lfs diff=lfs merge=lfs -text
71
+ vocabs/englishcode-100256-strict-v1.vocab filter=lfs diff=lfs merge=lfs -text
72
+ vocabs/englishcode-100256-unfiltered-v1.vocab filter=lfs diff=lfs merge=lfs -text
73
+ vocabs/englishcode-50256-balanced-v1.vocab filter=lfs diff=lfs merge=lfs -text
74
+ vocabs/englishcode-50256-clean-v1.vocab filter=lfs diff=lfs merge=lfs -text
75
+ vocabs/englishcode-50256-consistent-v1.vocab filter=lfs diff=lfs merge=lfs -text
76
+ vocabs/englishcode-50256-unfiltered-v1.vocab filter=lfs diff=lfs merge=lfs -text
77
+ vocabs/fiction-100256-balanced-v1.vocab filter=lfs diff=lfs merge=lfs -text
78
+ vocabs/fiction-100256-clean-v1.vocab filter=lfs diff=lfs merge=lfs -text
79
+ vocabs/fiction-100256-unfiltered-v1.vocab filter=lfs diff=lfs merge=lfs -text
80
+ vocabs/fiction-50256-balanced-v1.vocab filter=lfs diff=lfs merge=lfs -text
81
+ vocabs/fiction-50256-unfiltered-v1.vocab filter=lfs diff=lfs merge=lfs -text
82
+ vocabs/fiction-65536-strict-v1.vocab filter=lfs diff=lfs merge=lfs -text
vocabs/code-100256-balanced-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe5675ac8e156773b568a2d8d12c240437a9115ea079a526bd298e71317b65cd
3
+ size 2922856
vocabs/code-50256-clean-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df23a36cd43ad0f15e0b9f714a95ee187d85a3204c67dc51cf31d40ebd2c890f
3
+ size 1382485
vocabs/code-50256-unfiltered-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42632670339b582485fd1fe44f40b80166c903e91713525681d16a52b7bbf000
3
+ size 1448927
vocabs/code-65536-strict-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4702e58ec1ebbb87a66bbada40f8ae2979c23806c557c46489fa95022f884ba5
3
+ size 1781628
vocabs/englishcode-100256-balanced-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6bf0bee4bb3030ae0e0dcbdc6390fc537805093c8e91e1509ec47ee6d36038e
3
+ size 2738511
vocabs/englishcode-100256-clean-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5476cfc63cfb39c58d80780ea620db89ae159b9f86625f08e1c322534c6f6d8a
3
+ size 2706594
vocabs/englishcode-100256-consistent-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b415b7aab06d70021a6eb0d93518caa0917e35a99f47c2a35c944b29e98bd133
3
+ size 2737647
vocabs/englishcode-100256-strict-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a4b9b9ef0d4187ef98e869d69565e64226fb94c864a9df1b471b6c0ffe77d3e
3
+ size 2737624
vocabs/englishcode-100256-unfiltered-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c7188ca8ea9c06a1ee587c70224fe7e458fbc5b08625a925b6dd1c2eeef6dcd
3
+ size 2770844
vocabs/englishcode-1024-strict-v1.vocab ADDED
Binary file (24.9 kB). View file
 
vocabs/englishcode-24000-consistent-v1.vocab ADDED
Binary file (623 kB). View file
 
vocabs/englishcode-50256-balanced-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:497b9ffac79be4c739d492e8012f88decfb76a56f8a30fb888bf81632fb667e9
3
+ size 1337550
vocabs/englishcode-50256-clean-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ed744a63d9c22063bc4f3e9b84a5ec75e4def0bb8a6ab3f51497a150b9f5cdb
3
+ size 1327778
vocabs/englishcode-50256-consistent-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:654d0bacd0ec95cc31049d5d8ca7bacd38271f641ad539a1cd0f43f9c06c00b8
3
+ size 1337753
vocabs/englishcode-50256-unfiltered-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ec231864faf18029a83bbcf14761706a947191fb6d94c0d06a6500207c69fb1
3
+ size 1362837
vocabs/fiction-100256-balanced-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b22570dc89203232ec19787190c2b32aa1f7fa54f1e6f8c95a35dd8c913c65c
3
+ size 2750115
vocabs/fiction-100256-clean-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b409ad180740b31bb8b69e1904a6e775fba409b3acb95c0ef67c478e82d37e78
3
+ size 2748659
vocabs/fiction-100256-unfiltered-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:846d02166b6a56a68d313363bd97732f301df63ac46cfeabcde66b4b33957174
3
+ size 2758710
vocabs/fiction-24000-balanced-v1.vocab ADDED
Binary file (626 kB). View file
 
vocabs/fiction-24000-strict-v1.vocab ADDED
Binary file (624 kB). View file
 
vocabs/fiction-32000-strict-v1.vocab ADDED
Binary file (843 kB). View file
 
vocabs/fiction-4096-consistent-v1.vocab ADDED
Binary file (101 kB). View file
 
vocabs/fiction-4096-strict-v1.vocab ADDED
Binary file (101 kB). View file
 
vocabs/fiction-50256-balanced-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:306e1f967fae7114a191858f234b834a08f54d56ec52300ae72574282f48e69a
3
+ size 1344169
vocabs/fiction-50256-unfiltered-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7f75315c73233c322ae8bbbef1983db5251f73e3508330b29399b48d5da9dc2
3
+ size 1348181
vocabs/fiction-65536-strict-v1.vocab ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:791bf479ff921975fa79533bb385c962f2e80f5b9105f1fb08bed3aaedc3e7ba
3
+ size 1783197