Home Explore Help
Register Sign In
chensr
/
ocr_models
1
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Tree: e5db8fde94
Branches Tags
ocr_models
/
official_models
/
PP-Chart2Table
/
tokenizer_config.json

tokenizer_config.json 1.9 KB
History Raw

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192 
  1. {
    2. 

  2.   "added_tokens_decoder": {
    3. 

  3.     "151851": {
    4. 

  4.       "content": "<ref>",
    5. 

  5.       "lstrip": false,
    6. 

  6.       "normalized": false,
    7. 

  7.       "rstrip": false,
    8. 

  8.       "single_word": false,
    9. 

  9.       "special": true
    10. 

  10.     },
    11. 

  11.     "151852": {
    12. 

  12.       "content": "</ref>",
    13. 

  13.       "lstrip": false,
    14. 

  14.       "normalized": false,
    15. 

  15.       "rstrip": false,
    16. 

  16.       "single_word": false,
    17. 

  17.       "special": true
    18. 

  18.     },
    19. 

  19.     "151853": {
    20. 

  20.       "content": "<box>",
    21. 

  21.       "lstrip": false,
    22. 

  22.       "normalized": false,
    23. 

  23.       "rstrip": false,
    24. 

  24.       "single_word": false,
    25. 

  25.       "special": true
    26. 

  26.     },
    27. 

  27.     "151854": {
    28. 

  28.       "content": "</box>",
    29. 

  29.       "lstrip": false,
    30. 

  30.       "normalized": false,
    31. 

  31.       "rstrip": false,
    32. 

  32.       "single_word": false,
    33. 

  33.       "special": true
    34. 

  34.     },
    35. 

  35.     "151855": {
    36. 

  36.       "content": "<quad>",
    37. 

  37.       "lstrip": false,
    38. 

  38.       "normalized": false,
    39. 

  39.       "rstrip": false,
    40. 

  40.       "single_word": false,
    41. 

  41.       "special": true
    42. 

  42.     },
    43. 

  43.     "151856": {
    44. 

  44.       "content": "</quad>",
    45. 

  45.       "lstrip": false,
    46. 

  46.       "normalized": false,
    47. 

  47.       "rstrip": false,
    48. 

  48.       "single_word": false,
    49. 

  49.       "special": true
    50. 

  50.     },
    51. 

  51.     "151857": {
    52. 

  52.       "content": "<img>",
    53. 

  53.       "lstrip": false,
    54. 

  54.       "normalized": false,
    55. 

  55.       "rstrip": false,
    56. 

  56.       "single_word": false,
    57. 

  57.       "special": true
    58. 

  58.     },
    59. 

  59.     "151858": {
    60. 

  60.       "content": "</img>",
    61. 

  61.       "lstrip": false,
    62. 

  62.       "normalized": false,
    63. 

  63.       "rstrip": false,
    64. 

  64.       "single_word": false,
    65. 

  65.       "special": true
    66. 

  66.     },
    67. 

  67.     "151859": {
    68. 

  68.       "content": "<imgpad>",
    69. 

  69.       "lstrip": false,
    70. 

  70.       "normalized": false,
    71. 

  71.       "rstrip": false,
    72. 

  72.       "single_word": false,
    73. 

  73.       "special": true
    74. 

  74.     }
    75. 

  75.   },
    76. 

  76.   "additional_special_tokens": [
    77. 

  77.     "<ref>",
    78. 

  78.     "</ref>",
    79. 

  79.     "<box>",
    80. 

  80.     "</box>",
    81. 

  81.     "<quad>",
    82. 

  82.     "</quad>",
    83. 

  83.     "<img>",
    84. 

  84.     "</img>",
    85. 

  85.     "<imgpad>"
    86. 

  86.   ],
    87. 

  87.   "clean_up_tokenization_spaces": true,
    88. 

  88.   "model_max_length": 8000,
    89. 

  89.   "pad_token": "<|endoftext|>",
    90. 

  90.   "padding_side": "right",
    91. 

  91.   "tokenizer_class": "QWenTokenizer"
    92. 

  92. }
    93.